Review and analyse this whole Scraperlib library's codes, after this make the Scraperlib the best of the best possible practically feasible client-side only library you can craft or can think of to make this Scraperlib library more powerful, fast, reliable, bug free, use full, effective, efficient, bottleneck free, error free, memory leaks free, resources efficient, practical better than before.

Think really hard and deeply before coming to the conclusion.

Don't use any external libraries. Always consider and handle all of the possible or potential consequences of your modification so that it's won't introduce any new errors or issues or bugs.

Send the fully practical, usable and working full complete improved codes.

/*!
 * ScraperLib.js v11.0.0
 * A fully stable, practical, and robust client-side web scraping library ready for production use
 * License: MIT
 */

(function (global) {
  'use strict';

  /**
   * Utility functions for validation and processing
   */
  const Utils = {
    // Validates if a string is a valid URL
    isValidURL(url) {
      if (typeof url !== 'string' || url.length > 2048) return false;
      try {
        new URL(url);
        return true;
      } catch {
        return false;
      }
    },

    // Sanitizes a string to prevent XSS
    sanitize(str) {
      if (typeof str !== 'string') return '';
      const temp = document.createElement('div');
      temp.textContent = str;
      return temp.innerHTML;
    },

    // Initializes the cache if available
    async initCache() {
      if ('caches' in global) {
        try {
          return await caches.open('scraper-lib-cache');
        } catch (e) {
          console.error('Cache initialization failed:', e);
          return null;
        }
      }
      return null;
    },

    // Generates a cache key based on URL and options
    async generateCacheKey(url, options = {}) {
      const cacheOptions = {
        useProxy: options.useProxy,
        proxyUrl: options.useProxy ? options.proxyUrl : undefined,
        selectorsHash: await Utils.hashSelectors(options.selectors),
        cacheTime: options.cacheTime,
        method: options.method,
        postDataHash: options.method === 'POST' ? await Utils.hashString(JSON.stringify(options.postData)) : undefined,
      };
      // Exclude functions from JSON.stringify
      const cacheKeyString = `${url}-${JSON.stringify(cacheOptions, (key, value) => {
        if (typeof value === 'function') {
          return undefined;
        }
        return value;
      })}`;
      const hash = await Utils.hashString(cacheKeyString);
      return hash;
    },

    // Generates a hash for selectors to include in cache key
    async hashSelectors(selectors) {
      if (!selectors) return '';
      try {
        const selectorsString = JSON.stringify(selectors, (key, value) => {
          if (typeof value === 'function') {
            return undefined; // Exclude functions from JSON.stringify
          }
          return value;
        });
        return await Utils.hashString(selectorsString);
      } catch (e) {
        console.error('Error hashing selectors:', e);
        return '';
      }
    },

    // Generates a cryptographic hash from a string
    async hashString(str) {
      if (typeof crypto !== 'undefined' && crypto.subtle) {
        const encoder = new TextEncoder();
        const data = encoder.encode(str);
        const hashBuffer = await crypto.subtle.digest('SHA-256', data);
        const hashArray = Array.from(new Uint8Array(hashBuffer));
        const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
        return hashHex;
      } else {
        // Fallback to a simple hash function
        let hash = 0,
          i,
          chr;
        if (str.length === 0) return hash.toString();
        for (i = 0; i < str.length; i++) {
          chr = str.charCodeAt(i);
          hash = (hash << 5) - hash + chr;
          hash |= 0; // Convert to 32bit integer
          hash = hash >>> 0; // Convert to unsigned
        }
        return hash.toString(16);
      }
    },

    // Validates and normalizes CSS selectors
    normalizeSelector(selector) {
      if (typeof selector !== 'string') throw new Error('Selector must be a string');
      try {
        document.createDocumentFragment().querySelector(selector);
        return selector;
      } catch {
        throw new Error(`Invalid CSS selector: "${selector}"`);
      }
    },

    // Deep merge of objects
    deepMerge(target, ...sources) {
      target = target || {};
      sources.forEach(src => {
        if (src && typeof src === 'object') {
          Object.keys(src).forEach(key => {
            const value = src[key];
            if (value && typeof value === 'object' && !Array.isArray(value) && !(value instanceof RegExp)) {
              target[key] = Utils.deepMerge(target[key], value);
            } else {
              target[key] = value;
            }
          });
        }
      });
      return target;
    },

    // Extracts text from an element, optionally preserving formatting
    extractText(element, options = {}) {
      if (!element) return '';
      const clone = element.cloneNode(true);
      // Remove scripts, styles, and noscript elements
      clone.querySelectorAll('script, style, noscript').forEach(el => el.remove());
      if (options.preserveFormatting) {
        // Replace <br> with newline characters
        clone.querySelectorAll('br').forEach(br => {
          const textNode = document.createTextNode('\n');
          br.parentNode.replaceChild(textNode, br);
        });
        // Add newline after </p> elements
        clone.querySelectorAll('p').forEach(p => {
          const textNode = document.createTextNode('\n');
          p.appendChild(textNode);
        });
      }
      return clone.textContent.trim();
    },

    // Removes scripts and styles from HTML string
    removeScriptsAndStyles(html) {
      try {
        const doc = new DOMParser().parseFromString(html, 'text/html');
        doc.querySelectorAll('script, style, noscript').forEach(el => el.remove());
        return doc.documentElement.outerHTML;
      } catch (e) {
        console.error('Error parsing HTML for script/style removal:', e);
        return html
          .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
          .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
          .replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, '');
      }
    },

    // Chunks an array into smaller arrays of specified size
    chunkArray(array, size) {
      if (!Array.isArray(array)) throw new Error('Input must be an array');
      if (!Number.isInteger(size) || size <= 0) throw new Error('Chunk size must be a positive integer');
      const result = [];
      for (let i = 0; i < array.length; i += size) {
        result.push(array.slice(i, i + size));
      }
      return result;
    },

    // Parses a string to JSON safely
    safeParseJSON(str) {
      try {
        return JSON.parse(str);
      } catch {
        return null;
      }
    },

    // Converts a NodeList to an Array
    nodeListToArray(nodeList) {
      return Array.from(nodeList);
    },

    // Function to delay execution
    delay(ms) {
      return new Promise(resolve => setTimeout(resolve, ms));
    },

    // Check for anti-scraping mechanisms (e.g., CAPTCHA)
    detectAntiScraping(html) {
      // Simple keyword detection for common anti-scraping messages
      const antiScrapingKeywords = [
        'captcha',
        'verify you are human',
        'access denied',
        'bot detection',
        'unusual traffic',
        'request blocked',
        'please enable cookies',
        'firewall',
       // 'cloudflare',
        'access forbidden',
        'browser check',
      ];
      const lowerHtml = html.toLowerCase();
      return antiScrapingKeywords.some(keyword => lowerHtml.includes(keyword));
    },

    // Built-in filters for common operations
    builtInFilters: {
      trim: value => (typeof value === 'string' ? value.trim() : value),
      toUpperCase: value => (typeof value === 'string' ? value.toUpperCase() : value),
      toLowerCase: value => (typeof value === 'string' ? value.toLowerCase() : value),
      parseInt: value => parseInt(value, 10),
      parseFloat: value => parseFloat(value),
      // Additional built-in filters
      replace: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 2) return value;
        const [pattern, replacement] = args;
        let regex;
        if (pattern instanceof RegExp) {
          regex = pattern;
        } else if (typeof pattern === 'string') {
          regex = new RegExp(pattern, 'g');
        } else {
          return value;
        }
        return value.replace(regex, replacement);
      },
      substring: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args)) return value;
        return value.substring(args[0], args[1]);
      },
      split: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args)) return value;
        return value.split(args[0]);
      },
      join: (value, args) => {
        if (!Array.isArray(value)) return value;
        return value.join(args[0]);
      },
      regexMatch: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 1) return value;
        const [pattern, group = 0] = args;
        let regex;
        if (pattern instanceof RegExp) {
          regex = pattern;
        } else if (typeof pattern === 'string') {
          regex = new RegExp(pattern);
        } else {
          return value;
        }
        const match = value.match(regex);
        return match ? match[group] : null;
      },
      trimChars: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 1) return value;
        const chars = args[0];
        const regex = new RegExp(`^[${chars}]+|[${chars}]+$`, 'g');
        return value.replace(regex, '');
      },
      defaultValue: (value, args) => {
        return value == null || value === '' ? args[0] : value;
      },
      toDate: (value, args) => {
        const date = new Date(value);
        return isNaN(date.getTime()) ? null : date;
      },
      formatDate: (value, args) => {
        if (!(value instanceof Date)) return value;
        const options = args[0] || {};
        return value.toLocaleDateString(undefined, options);
      },
      // New filters
      toBoolean: value => {
        if (typeof value === 'string') {
          const val = value.toLowerCase();
          return val === 'true' || val === 'yes' || val === '1';
        }
        return Boolean(value);
      },
      encodeURI: value => (typeof value === 'string' ? encodeURI(value) : value),
      decodeURI: value => (typeof value === 'string' ? decodeURI(value) : value),
      encodeURIComponent: value => (typeof value === 'string' ? encodeURIComponent(value) : value),
      decodeURIComponent: value => (typeof value === 'string' ? decodeURIComponent(value) : value),
      toJSON: value => {
        try {
          return JSON.parse(value);
        } catch {
          return null;
        }
      },
      // Mathematical operations
      add: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand)) return value;
        return num + operand;
      },
      subtract: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand)) return value;
        return num - operand;
      },
      multiply: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand)) return value;
        return num * operand;
      },
      divide: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand) || operand === 0) return value;
        return num / operand;
      },
      // New filters for arrays
      unique: value => {
        if (Array.isArray(value)) {
          return Array.from(new Set(value));
        }
        return value;
      },
      flatten: value => {
        if (Array.isArray(value)) {
          return value.flat(Infinity);
        }
        return value;
      },
      // String case conversions
      camelCase: value => {
        if (typeof value !== 'string') return value;
        return value
          .replace(/(?:^\w|[A-Z]|\b\w)/g, (word, index) =>
            index === 0 ? word.toLowerCase() : word.toUpperCase()
          )
          .replace(/\s+/g, '');
      },
      snakeCase: value => {
        if (typeof value !== 'string') return value;
        return value
          .replace(/\W+/g, ' ')
          .split(/ |\B(?=[A-Z])/)
          .map(word => word.toLowerCase())
          .join('_');
      },
      // New filter: chaining multiple filters
      chain: (value, args) => {
        if (!Array.isArray(args) || args.length === 0) return value;
        return args.reduce((val, filterStr) => {
          const filterFunc = Utils.builtInFilters[filterStr];
          if (typeof filterFunc === 'function') {
            return filterFunc(val);
          }
          return val;
        }, value);
      },
    },

    // Function to create a DOM element from an HTML string
    createElementFromHTML(htmlString) {
      const div = document.createElement('div');
      div.innerHTML = htmlString.trim();
      return div.firstChild;
    },
  };

  /**
   * Plugin manager for extending functionality
   */
  class PluginManager {
    constructor() {
      this.plugins = new Map();
    }

    registerPlugin(name, plugin) {
      if (typeof plugin !== 'function') {
        throw new Error('Plugin must be a function');
      }
      this.plugins.set(name, plugin);
    }

    getPlugin(name) {
      return this.plugins.get(name);
    }
  }

  /**
   * Handles rate limiting to respect website policies
   */
  class RateLimiter {
    constructor(requestsPerSecond = 2) {
      this.requestsPerSecond = Math.max(1, requestsPerSecond);
      this.interval = 1000 / this.requestsPerSecond;
      this.queue = [];
      this.lastRequestTime = 0;
      this.isProcessing = false;
      this.timeoutId = null;
    }

    async acquireToken() {
      return new Promise(resolve => {
        this.queue.push(resolve);
        this._processQueue();
      });
    }

    _processQueue() {
      if (this.isProcessing || this.queue.length === 0) return;
      this.isProcessing = true;

      const now = Date.now();
      const elapsed = now - this.lastRequestTime;
      const delay = Math.max(this.interval - elapsed, 0);

      this.timeoutId = setTimeout(() => {
        this.lastRequestTime = Date.now();
        const resolve = this.queue.shift();
        if (resolve) resolve();
        this.isProcessing = false;
        if (this.queue.length > 0) {
          this._processQueue();
        }
      }, delay);
    }

    // Cleanup to prevent memory leaks
    clear() {
      clearTimeout(this.timeoutId);
      this.queue = [];
    }
  }

  /**
   * Main ScraperLib class
   */
  class ScraperLib {
    constructor(options = {}) {
      this.defaultSettings = {
        rateLimitRequests: 2,
        maxConcurrent: 5,
        timeout: 30000,
        retries: 3,
        retryDelay: 1000,
        maxRetryDelay: 30000,
        cacheTime: 3600000,
        outputFormat: 'json',
        useCache: false,
        headers: {},
        preserveFormatting: false,
        validateSelectors: true,
        maxResponseSize: 10 * 1024 * 1024,
        useProxy: false,
        proxyUrl: '',
        filters: {}, // Global filters
        concurrencyStrategy: 'auto', // 'auto', 'parallel', 'sequential'
        logger: null, // Custom logger function
        customFetch: null, // Custom fetch function
        politenessDelay: 0, // Delay between requests to the same domain
        maxRetriesOnAntiScraping: 2, // Max retries if anti-scraping measures detected
        enableLogging: true, // Enable or disable logging
        defaultOutputPath: '', // Default output path for non-JSON outputs
        pagination: null, // Pagination config
        retryOnNetworkFailure: true, // Retry on network failures
        method: 'GET', // HTTP method
        postData: null, // Data for POST requests
        customResponseParser: null, // Custom response parser
        plugins: [], // Plugins to use
      };

      // Forbidden headers in browsers
      this.forbiddenHeaders = new Set([
        'accept-charset',
        'accept-encoding',
        'access-control-request-headers',
        'access-control-request-method',
        'connection',
        'content-length',
        'cookie',
        'cookie2',
        'date',
        'dnt',
        'expect',
        'host',
        'keep-alive',
        'origin',
        'referer',
        'te',
        'trailer',
        'transfer-encoding',
        'upgrade',
        'via',
        'proxy-',
        'sec-',
        'user-agent',
        'accept-language',
      ]);

      // Merge and validate default settings with user options
      this.settings = Utils.deepMerge({}, this.defaultSettings, options);
      this._validateSettings(this.settings);

      // Initialize per-URL settings
      this.urlSettings = new Map();

      this.storage = new StorageHandler();
      this.rateLimiter = new RateLimiter(this.settings.rateLimitRequests);
      this.cache = null;
      this.errorHandlers = new Map();

      // Initialize cache
      if (this.settings.useCache) {
        this.cachePromise = Utils.initCache().then(cache => {
          this.cache = cache;
        });
      } else {
        this.cachePromise = Promise.resolve();
      }

      // Initialize logger
      this.logger = this.settings.logger || console;

      // Hostname to last request time mapping for politeness
      this.hostLastRequestTime = new Map();

      // Initialize plugins
      this.pluginManager = new PluginManager();
      this._initializePlugins(this.settings.plugins);
    }

    _initializePlugins(plugins) {
      if (!Array.isArray(plugins)) return;
      plugins.forEach(plugin => {
        if (typeof plugin.name !== 'string' || typeof plugin.plugin !== 'function') {
          throw new Error('Each plugin must have a name and a plugin function');
        }
        this.pluginManager.registerPlugin(plugin.name, plugin.plugin);
      });
    }

    _validateSettings(settings) {
      const positiveNumberProps = [
        'rateLimitRequests',
        'maxConcurrent',
        'retries',
        'retryDelay',
        'maxRetryDelay',
        'cacheTime',
        'timeout',
        'maxResponseSize',
        'politenessDelay',
        'maxRetriesOnAntiScraping',
      ];
      positiveNumberProps.forEach(prop => {
        if (typeof settings[prop] !== 'number' || settings[prop] < 0) {
          throw new Error(`Invalid setting "${prop}": must be a positive number`);
        }
      });

      const allowedStrategies = ['auto', 'parallel', 'sequential'];
      if (!allowedStrategies.includes(settings.concurrencyStrategy)) {
        throw new Error(
          `Invalid concurrency strategy: "${settings.concurrencyStrategy}". Allowed values are ${allowedStrategies.join(
            ', '
          )}`
        );
      }

      if (settings.customFetch && typeof settings.customFetch !== 'function') {
        throw new Error('customFetch must be a function');
      }

      if (settings.pagination && typeof settings.pagination !== 'object') {
        throw new Error('pagination must be an object');
      }

      if (settings.method && !['GET', 'POST'].includes(settings.method.toUpperCase())) {
        throw new Error('method must be either GET or POST');
      }

      if (settings.method.toUpperCase() === 'POST' && !settings.postData) {
        throw new Error('postData must be provided for POST requests');
      }
    }

    /**
     * Main scraping method
     */
    async scrape(configs, globalOptions = {}) {
      await this.cachePromise;

      if (typeof configs !== 'object' || configs === null || Array.isArray(configs)) {
        throw new Error('Configs must be an object with URL keys and configuration values');
      }

      // Merge global options into settings
      const globalSettings = Utils.deepMerge({}, this.settings, globalOptions);
      this._validateSettings(globalSettings);

      const results = {};
      const urls = Object.keys(configs);
      if (urls.length === 0) {
        return results;
      }

      await this._log('Starting scraping process', { urls });

      const scrapeTasks = urls.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results));

      switch (globalSettings.concurrencyStrategy) {
        case 'parallel':
          await Promise.all(scrapeTasks);
          break;

        case 'sequential':
          for (const task of scrapeTasks) {
            await task;
          }
          break;

        case 'auto':
        default:
          const chunks = Utils.chunkArray(scrapeTasks, globalSettings.maxConcurrent);
          for (const chunk of chunks) {
            await Promise.all(chunk);
          }
          break;
      }

      await this._log('Completed scraping process', { results });

      return results;
    }

    /**
     * Wrapper for processing a URL, handles logging and error handling
     */
    async _scrapeUrlWrapper(url, config, globalSettings, results) {
      try {
        const data = await this._processUrl(url, config, globalSettings);
        results[url] = data;
      } catch (error) {
        const handler = this.errorHandlers.get(url) || this.errorHandlers.get('default');
        if (handler) {
          try {
            const data = await handler(error, url);
            results[url] = data;
          } catch (e) {
            results[url] = { error: e.message };
          }
        } else {
          results[url] = { error: error.message };
        }
        await this._log(`Error scraping ${url}: ${error.message}`, { error });
      }
    }

    /**
     * Process a single URL with retries and error handling
     */
    async _processUrl(url, config, globalSettings) {
      if (!Utils.isValidURL(url)) {
        throw new Error(`Invalid URL: ${url}`);
      }

      // Merge global settings with per-URL settings using deep merge
      const settings = Utils.deepMerge({}, globalSettings, config.options);
      this._validateSettings(settings);
      this.urlSettings.set(url, settings);

      if (settings.validateSelectors) {
        this._validateSelectors(config.selectors);
      }

      // Wait for cache initialization if needed
      if (settings.useCache) {
        await this.cachePromise;
        if (this.cache) {
          const cachedData = await this._getFromCache(url, settings);
          if (cachedData) {
            await this._log(`Using cached data for ${url}`);
            return cachedData;
          }
        }
      }

      let attempts = 0;
      let antiScrapingAttempts = 0;
      while (attempts < settings.retries) {
        attempts += 1;
        await this._respectPoliteness(url, settings);

        await this.rateLimiter.acquireToken();
        const options = this._prepareFetchOptions(config, settings);
        try {
          const response = await this._fetchWithTimeout(url, options, settings);

          if (!response.ok) {
            throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
          }

          const contentType = response.headers.get('Content-Type') || '';
          if (!contentType.includes('text/html') && !settings.customResponseParser) {
            throw new Error(`Invalid Content-Type: ${contentType}`);
          }

          let data;
          if (settings.customResponseParser) {
            data = await settings.customResponseParser(response);
          } else {
            data = await this._extractDataFromResponse(response, config.selectors, settings);
          }

          // Apply plugins
          data = await this._applyPlugins(data, settings);

          // Pagination handling
          if (settings.pagination) {
            data = await this._handlePagination(url, config, settings, data);
          }

          if (settings.outputFormat !== 'json') {
            await this.storage.store(url, data, settings.outputFormat, { customStore: settings.customStore });
          }

          if (settings.useCache && this.cache) {
            await this._storeInCache(url, data, settings);
          }

          return data;
        } catch (error) {
          if (
            Utils.detectAntiScraping(error.message) &&
            antiScrapingAttempts < settings.maxRetriesOnAntiScraping
          ) {
            antiScrapingAttempts += 1;
            await this._log(
              `Detected anti-scraping measures on ${url}. Attempt ${antiScrapingAttempts}`,
              { error }
            );
            // Delay before retrying
            await Utils.delay(settings.retryDelay * antiScrapingAttempts);
            continue;
          }

          if (attempts >= settings.retries) {
            throw error;
          }
          // Exponential backoff with jitter
          const delay = Math.min(
            settings.retryDelay * Math.pow(2, attempts - 1) * (0.5 + Math.random() * 0.5),
            settings.maxRetryDelay
          );
          await this._log(`Retrying ${url} after delay of ${delay}ms`, { attempt: attempts, error });
          await Utils.delay(delay);
        }
      }
    }

    /**
     * Respects politeness policies by delaying requests to the same domain
     */
    async _respectPoliteness(url, settings) {
      const urlObj = new URL(url);
      const hostname = urlObj.hostname;
      const now = Date.now();
      const lastRequestTime = this.hostLastRequestTime.get(hostname) || 0;
      const elapsed = now - lastRequestTime;

      if (elapsed < settings.politenessDelay) {
        const delay = settings.politenessDelay - elapsed;
        await this._log(`Delaying request to ${url} by ${delay}ms for politeness`);
        await Utils.delay(delay);
      }

      this.hostLastRequestTime.set(hostname, Date.now());
    }

    /**
     * Validates the CSS selectors
     */
    _validateSelectors(selectors) {
      if (typeof selectors !== 'object' || selectors === null) {
        throw new Error('Selectors must be an object');
      }
      for (const key of Object.keys(selectors)) {
        const sel = selectors[key];
        if (typeof sel === 'string') {
          Utils.normalizeSelector(sel);
        } else if (sel && typeof sel.selector === 'string') {
          Utils.normalizeSelector(sel.selector);
        } else if (sel && typeof sel.customSelector === 'function') {
          // Custom selector function, assume valid
        } else if (sel && typeof sel.xpath === 'string') {
          // Could validate XPath here if needed
        } else {
          throw new Error(`Invalid selector configuration for key "${key}"`);
        }
      }
    }

    /**
     * Prepares fetch options including headers
     */
    _prepareFetchOptions(config, settings) {
      const headersInit = Utils.deepMerge({}, settings.headers, config.headers);

      // Remove forbidden headers
      Object.keys(headersInit).forEach(header => {
        const headerLower = header.toLowerCase();
        if (
          this.forbiddenHeaders.has(headerLower) ||
          Array.from(this.forbiddenHeaders).some(
            fh => fh.endsWith('-') && headerLower.startsWith(fh)
          )
        ) {
          delete headersInit[header];
        }
      });

      const headers = new Headers(headersInit);

      let body = null;
      if (settings.method.toUpperCase() === 'POST') {
        body = settings.postData;
        if (!(body instanceof FormData) && typeof body === 'object') {
          headers.set('Content-Type', 'application/json');
          body = JSON.stringify(body);
        }
      }

      const options = {
        method: settings.method.toUpperCase(),
        headers: headers,
        mode: 'cors',
        credentials: 'omit',
        redirect: 'follow',
        body: body,
      };

      return options;
    }

    /**
     * Performs a fetch request with a timeout, using proxy if configured
     */
    async _fetchWithTimeout(url, options, settings) {
      const controller = new AbortController();
      options.signal = controller.signal;

      const timeoutId = setTimeout(() => {
        controller.abort();
      }, settings.timeout);

      try {
        let fetchUrl = url;
        if (settings.useProxy) {
          if (!settings.proxyUrl) {
            throw new Error('Proxy URL is not set');
          }
          // Correctly construct the fetch URL with the proxy
          if (settings.proxyUrl.includes('${url}')) {
            fetchUrl = settings.proxyUrl.replace('${url}', encodeURIComponent(url));
          } else {
            // Append the encoded URL appropriately
            if (settings.proxyUrl.endsWith('=') || settings.proxyUrl.endsWith('?')) {
              fetchUrl = settings.proxyUrl + encodeURIComponent(url);
            } else {
              // Append '?url=' + encodeURIComponent(url)
              const separator = settings.proxyUrl.includes('?') ? '&' : '?';
              fetchUrl = settings.proxyUrl + separator + 'url=' + encodeURIComponent(url);
            }
          }
        }

        const fetchFunction = settings.customFetch || fetch;

        const response = await fetchFunction(fetchUrl, options);
        clearTimeout(timeoutId);

        // Check if response size exceeds maxResponseSize
        const contentLength = response.headers.get('Content-Length');
        if (contentLength && parseInt(contentLength, 10) > settings.maxResponseSize) {
          throw new Error('Response size exceeds maximum allowed');
        }

        return response;
      } catch (error) {
        clearTimeout(timeoutId);
        if (error.name === 'AbortError') {
          throw new Error('Request timed out');
        }
        if (settings.retryOnNetworkFailure && (error instanceof TypeError || error.message === 'Failed to fetch')) {
          throw new Error('Network failure, will retry');
        }
        throw error;
      }
    }

    /**
     * Enhanced data extraction with advanced filters
     */
    async _extractDataFromResponse(response, selectors, settings) {
      try {
        const text = await response.text();

        // Check for anti-scraping mechanisms
        if (Utils.detectAntiScraping(text)) {
          throw new Error('Detected anti-scraping measures in the response');
        }

        // Remove scripts and styles to prevent XSS and reduce parsing overhead
        const sanitizedHtml = Utils.removeScriptsAndStyles(text);

        const parser = new DOMParser();
        const doc = parser.parseFromString(sanitizedHtml, 'text/html');

        const data = {};

        for (const key of Object.keys(selectors)) {
          const selConfig = selectors[key];

          let elements;
          if (typeof selConfig === 'string') {
            elements = doc.querySelectorAll(selConfig);
          } else if (selConfig && typeof selConfig.selector === 'string') {
            elements = doc.querySelectorAll(selConfig.selector);
          } else if (selConfig && typeof selConfig.customSelector === 'function') {
            elements = selConfig.customSelector(doc);
            if (!Array.isArray(elements)) {
              throw new Error(`customSelector for key "${key}" must return an array of elements`);
            }
          } else if (selConfig && typeof selConfig.xpath === 'string') {
            elements = this._selectXPath(doc, selConfig.xpath);
          } else {
            throw new Error(`Invalid selector configuration for key "${key}"`);
          }

          if (!elements || elements.length === 0) {
            data[key] = selConfig.single ? null : [];
            continue;
          }

          data[key] = await Promise.all(
            Array.from(elements).map(async element => {
              let value;
              if (selConfig.attr) {
                value = element.getAttribute(selConfig.attr) || '';
              } else if (selConfig.html) {
                value = element.innerHTML;
              } else {
                value = Utils.extractText(element, { preserveFormatting: settings.preserveFormatting });
              }

              // Apply per-selector filters (support chaining)
              if (selConfig.filters) {
                value = await this._applyFilters(value, selConfig.filters, element, {
                  key,
                  settings,
                });
              }

              // Apply global filters if no per-selector filters
              else if (settings.filters && settings.filters[key]) {
                value = await this._applyFilters(value, settings.filters[key], element, {
                  key,
                  settings,
                });
              }

              return value;
            })
          );

          data[key] = data[key].filter(val => val !== undefined && val !== null);

          if (selConfig.single) {
            data[key] = data[key][0] || null;
          }
        }

        return data;
      } catch (e) {
        throw new Error(`Failed to extract data: ${e.message}`);
      }
    }

    /**
     * Apply filters to a value, supporting chaining and built-in filters
     */
    async _applyFilters(value, filters, element, context) {
      if (!filters) return value;

      const filtersArray = Array.isArray(filters) ? filters : [filters];

      for (let filter of filtersArray) {
        if (typeof filter === 'string') {
          // Built-in filter by name
          const filterFunc = this._parseFilterString(filter);
          if (typeof filterFunc === 'function') {
            value = filterFunc(value, element, context);
          } else {
            console.warn(`Built-in filter "${filter}" not found`);
          }
        } else if (typeof filter === 'function') {
          try {
            const filterResult = filter(value, element, context);
            value = filterResult instanceof Promise ? await filterResult : filterResult;
          } catch (e) {
            console.error(`Error in filter for key "${context.key}":`, e);
            value = null;
            break;
          }
        } else if (filter instanceof RegExp) {
          // Apply regular expression
          const match = value.match(filter);
          value = match ? match[0] : null;
        } else {
          console.warn(`Invalid filter type for key "${context.key}"`);
        }

        // If value becomes null or undefined, stop processing filters
        if (value == null) {
          break;
        }
      }

      return value;
    }

    /**
     * Parse filter string to support filter arguments safely
     */
    _parseFilterString(filterStr) {
      // Example filter string: 'replace(/foo/g, "bar")'
      const filterPattern = /^(\w+)\((.*)\)$/;
      const match = filterStr.match(filterPattern);
      if (match) {
        const filterName = match[1];
        const argsStr = match[2];
        const args = this._parseFilterArgs(argsStr);

        const filterFunc = Utils.builtInFilters[filterName];
        if (typeof filterFunc === 'function') {
          return (value, element, context) => filterFunc(value, args, element, context);
        }
      } else {
        const filterFunc = Utils.builtInFilters[filterStr];
        if (typeof filterFunc === 'function') {
          return filterFunc;
        }
      }
      return null;
    }

    /**
     * Parse filter arguments from string safely
     */
    _parseFilterArgs(argsStr) {
      try {
        const args = [];
        let currentArg = '';
        let inQuotes = false;
        let quoteChar = '';
        let escapeNext = false;
        let bracketDepth = 0;

        for (let i = 0; i < argsStr.length; i++) {
          const char = argsStr.charAt(i);

          if (escapeNext) {
            currentArg += char;
            escapeNext = false;
            continue;
          }

          if (char === '\\') {
            escapeNext = true;
            continue;
          }

          if (inQuotes) {
            if (char === quoteChar) {
              inQuotes = false;
            }
            currentArg += char;
            continue;
          }

          if (char === '"' || char === "'") {
            inQuotes = true;
            quoteChar = char;
            currentArg += char;
            continue;
          }

          if (char === '(') {
            bracketDepth++;
          } else if (char === ')') {
            bracketDepth--;
          }

          if (char === ',' && !inQuotes && bracketDepth === 0) {
            args.push(this._parseArgument(currentArg.trim()));
            currentArg = '';
            continue;
          }

          currentArg += char;
        }

        if (currentArg) {
          args.push(this._parseArgument(currentArg.trim()));
        }

        return args;
      } catch (e) {
        console.error('Failed to parse filter arguments:', e);
        return [];
      }
    }

    _parseArgument(arg) {
      if ((arg.startsWith('"') && arg.endsWith('"')) || (arg.startsWith("'") && arg.endsWith("'"))) {
        // String argument
        return arg.slice(1, -1).replace(/\\'/g, "'").replace(/\\"/g, '"');
      }

      // Try to parse as number
      const num = Number(arg);
      if (!isNaN(num)) {
        return num;
      }

      // For regex arguments, e.g., /pattern/flags
      if (arg.startsWith('/') && arg.lastIndexOf('/') > 0) {
        const lastSlash = arg.lastIndexOf('/');
        const pattern = arg.slice(1, lastSlash);
        const flags = arg.slice(lastSlash + 1);
        try {
          return new RegExp(pattern, flags);
        } catch (e) {
          console.error('Invalid regular expression:', e);
          return null;
        }
      }

      // Boolean arguments
      if (arg === 'true') return true;
      if (arg === 'false') return false;

      // Otherwise, return the arg as string
      return arg;
    }

    /**
     * Select elements using XPath
     */
    _selectXPath(doc, xpathExpression) {
      const iterator = doc.evaluate(xpathExpression, doc, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
      const nodes = [];
      for (let i = 0; i < iterator.snapshotLength; i++) {
        nodes.push(iterator.snapshotItem(i));
      }
      return nodes;
    }

    /**
     * Handles pagination by recursively scraping next pages
     */
    async _handlePagination(url, config, settings, initialData) {
      const paginationConfig = settings.pagination;
      if (!paginationConfig.selector || !paginationConfig.pageLimit) {
        return initialData;
      }

      let pageData = initialData;
      let allData = [pageData];
      let pageCount = 1;
      let nextPageUrl = this._extractNextPageUrl(pageData, paginationConfig, url);

      while (nextPageUrl && pageCount < paginationConfig.pageLimit) {
        await this.rateLimiter.acquireToken();
        await this._respectPoliteness(nextPageUrl, settings);
        const options = this._prepareFetchOptions(config, settings);

        try {
          const response = await this._fetchWithTimeout(nextPageUrl, options, settings);

          if (!response.ok) {
            throw new Error(`HTTP error on pagination: ${response.status} ${response.statusText}`);
          }

          const contentType = response.headers.get('Content-Type') || '';
          if (!contentType.includes('text/html') && !settings.customResponseParser) {
            throw new Error(`Invalid Content-Type on pagination: ${contentType}`);
          }

          let data;
          if (settings.customResponseParser) {
            data = await settings.customResponseParser(response);
          } else {
            data = await this._extractDataFromResponse(response, config.selectors, settings);
          }

          // Apply plugins
          data = await this._applyPlugins(data, settings);

          allData.push(data);
          pageCount++;
          nextPageUrl = this._extractNextPageUrl(data, paginationConfig, nextPageUrl);
        } catch (error) {
          await this._log(`Pagination error on ${nextPageUrl}: ${error.message}`, { error });
          break;
        }
      }

      // Merge all data
      const mergedData = this._mergePaginatedData(allData);
      return mergedData;
    }

    _extractNextPageUrl(data, paginationConfig, currentUrl) {
      const nextPageLinks = data[paginationConfig.selector];
      if (!nextPageLinks || nextPageLinks.length === 0) {
        return null;
      }
      let nextPageUrl = nextPageLinks[0]; // Assuming the first link is the next page
      if (!Utils.isValidURL(nextPageUrl)) {
        // Handle relative URLs
        try {
          const baseUrl = new URL(currentUrl);
          nextPageUrl = new URL(nextPageUrl, baseUrl).href;
        } catch {
          return null;
        }
      }
      return nextPageUrl;
    }

    _mergePaginatedData(allData) {
      const mergedData = {};
      allData.forEach(data => {
        Object.keys(data).forEach(key => {
          if (!mergedData[key]) {
            mergedData[key] = [];
          }
          if (Array.isArray(data[key])) {
            mergedData[key] = mergedData[key].concat(data[key]);
          } else {
            mergedData[key].push(data[key]);
          }
        });
      });
      return mergedData;
    }

    /**
     * Gets data from the cache if available and not expired
     */
    async _getFromCache(url, settings) {
      if (!this.cache) return null;
      const cacheKey = await Utils.generateCacheKey(url, settings);
      const cachedResponse = await this.cache.match(cacheKey);

      if (cachedResponse) {
        try {
          const cachedData = await cachedResponse.json();
          const age = Date.now() - cachedData.timestamp;
          if (age < settings.cacheTime) {
            return cachedData.data;
          } else {
            // Cache expired, remove it
            await this.cache.delete(cacheKey);
          }
        } catch (e) {
          console.warn('Error reading from cache:', e);
        }
      }

      return null;
    }

    /**
     * Stores data in the cache
     */
    async _storeInCache(url, data, settings) {
      if (!this.cache) return;
      const cacheKey = await Utils.generateCacheKey(url, settings);
      const cachedResponse = new Response(
        JSON.stringify({ data, timestamp: Date.now() }),
        {
          headers: { 'Content-Type': 'application/json' },
        }
      );
      try {
        await this.cache.put(cacheKey, cachedResponse);
      } catch (e) {
        console.error('Failed to store data in cache:', e);
      }
    }

    /**
     * Apply registered plugins to the data
     */
    async _applyPlugins(data, settings) {
      if (!settings.plugins || settings.plugins.length === 0) return data;
      let processedData = data;
      for (const pluginConfig of settings.plugins) {
        const plugin = this.pluginManager.getPlugin(pluginConfig.name);
        if (typeof plugin === 'function') {
          processedData = await plugin(processedData, pluginConfig.options || {});
        }
      }
      return processedData;
    }

    /**
     * Registers a custom error handler
     */
    onError(urlOrDefault, handler) {
      if (typeof handler !== 'function') {
        throw new Error('Error handler must be a function');
      }
      this.errorHandlers.set(urlOrDefault, handler);
    }

    /**
     * Logs messages using the provided logger
     */
    async _log(message, data = {}) {
      if (!this.settings.enableLogging) return;
      if (typeof this.logger === 'function') {
        this.logger(message, data);
      } else if (this.logger && typeof this.logger.log === 'function') {
        this.logger.log(message, data);
      }
    }

    /**
     * Clean up resources to prevent memory leaks
     */
    destroy() {
      this.rateLimiter.clear();
      this.hostLastRequestTime.clear();
      this.errorHandlers.clear();
      this.urlSettings.clear();
      // Additional cleanup if necessary
    }
  }

  /**
   * Handles storing scraped data
   */
  class StorageHandler {
    constructor() {
      this.dbPromise = null;
      this.storeName = 'scrapedData';
      if ('indexedDB' in global) {
        this.storageType = 'indexeddb';
        this.dbPromise = this.initIndexedDB();
      } else if (this._isLocalStorageAccessible()) {
        this.storageType = 'localstorage';
      } else {
        this.storageType = 'memory';
        this.memoryStorage = new Map();
      }
    }

    _isLocalStorageAccessible() {
      try {
        const testKey = '__test__';
        localStorage.setItem(testKey, testKey);
        localStorage.removeItem(testKey);
        return true;
      } catch {
        console.warn('localStorage is not accessible. Using in-memory storage.');
        return false;
      }
    }

    async initIndexedDB(dbName = 'ScraperLibDB') {
      return new Promise((resolve, reject) => {
        const request = indexedDB.open(dbName, 1);

        request.onerror = () => reject(new Error('IndexedDB initialization failed'));

        request.onupgradeneeded = event => {
          const db = event.target.result;
          if (!db.objectStoreNames.contains(this.storeName)) {
            const store = db.createObjectStore(this.storeName, { keyPath: 'id', autoIncrement: true });
            store.createIndex('url', 'url', { unique: false });
            store.createIndex('timestamp', 'timestamp', { unique: false });
          }
        };

        request.onsuccess = event => {
          const db = event.target.result;
          db.onerror = event => {
            console.error('Database error:', event.target.error);
          };
          resolve(db);
        };
      });
    }

    async store(url, data, format, options = {}) {
      const timestamp = Date.now();
      const storeData = { url, data, timestamp };

      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readwrite');
              const store = transaction.objectStore(this.storeName);
              const request = store.add(storeData);

              request.onsuccess = () => resolve(true);
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          try {
            const key = `scraper-${url}-${timestamp}`;
            localStorage.setItem(key, JSON.stringify(storeData));
            return true;
          } catch (e) {
            if (e.name === 'QuotaExceededError' || e.name === 'NS_ERROR_DOM_QUOTA_REACHED') {
              this._cleanupLocalStorage();
              const key = `scraper-${url}-${timestamp}`;
              localStorage.setItem(key, JSON.stringify(storeData));
              return true;
            }
            throw e;
          }

        case 'memory':
          this.memoryStorage.set(`${url}-${timestamp}`, storeData);
          return true;

        case 'custom':
          if (options.customStore && typeof options.customStore.save === 'function') {
            await options.customStore.save(url, data);
            return true;
          }
          throw new Error('Invalid custom storage handler');

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }

    _cleanupLocalStorage() {
      const keys = [];
      for (let i = 0; i < localStorage.length; i++) {
        const key = localStorage.key(i);
        if (key && key.startsWith('scraper-')) {
          keys.push(key);
        }
      }
      if (keys.length > 100) {
        keys.sort((a, b) => {
          try {
            const itemA = Utils.safeParseJSON(localStorage.getItem(a));
            const itemB = Utils.safeParseJSON(localStorage.getItem(b));
            return itemA.timestamp - itemB.timestamp;
          } catch (e) {
            localStorage.removeItem(a);
            localStorage.removeItem(b);
            return 0;
          }
        });
        for (let i = 0; i < keys.length - 100; i++) {
          localStorage.removeItem(keys[i]);
        }
      }
    }

    async retrieve(url, format) {
      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readonly');
              const store = transaction.objectStore(this.storeName);
              const index = store.index('url');
              const request = index.getAll(url);

              request.onsuccess = () => {
                const results = request.result;
                if (results && results.length > 0) {
                  results.sort((a, b) => b.timestamp - a.timestamp);
                  resolve(results[0].data);
                } else {
                  resolve(null);
                }
              };
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          const keys = [];
          for (let i = 0; i < localStorage.length; i++) {
            const key = localStorage.key(i);
            if (key && key.startsWith(`scraper-${url}-`)) {
              keys.push(key);
            }
          }
          keys.sort((a, b) => {
            const itemA = Utils.safeParseJSON(localStorage.getItem(a));
            const itemB = Utils.safeParseJSON(localStorage.getItem(b));
            return (itemB?.timestamp || 0) - (itemA?.timestamp || 0);
          });
          if (keys.length > 0) {
            const item = Utils.safeParseJSON(localStorage.getItem(keys[0]));
            return item ? item.data : null;
          }
          return null;

        case 'memory':
          const memoryItems = Array.from(this.memoryStorage.entries())
            .filter(([key]) => key.startsWith(`${url}-`))
            .sort((a, b) => b[1].timestamp - a[1].timestamp);
          if (memoryItems.length > 0) {
            return memoryItems[0][1].data;
          }
          return null;

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }
  }

  // Attach ScraperLib to the global object
  global.ScraperLib = ScraperLib;
})(typeof window !== 'undefined' ? window : this);