Review and analyse this whole Scraperlib library's codes, after this make the Scraperlib the best of the best possible practically feasible client-side only library you can craft or can think of to make this Scraperlib library more powerful, fast, reliable, bug free, use full, effective, efficient, bottleneck free, error free, memory leaks free, resources efficient, practical better than before.

Think really hard and deeply before coming to the conclusion.

Don't use any external libraries or anything tha's need any backend or server to work.

Always consider and handle all of the possible or potential consequences of your modification so that it's won't introduce any new errors or issues or bugs. 

You know everything whatever should be done so don't ask me just do it all.

Keep it for client side use only.

Upgrade it even further, focus solely on even more 'Quality' than ever before. Don't add anything that's need any backend or server to work. 

Send the fully working full even more significantly improved yet fully functional, practical, useable and useful codes than ever before. 

I just care about absolute and overwhelming quality over quantity.

Think harder, deeper and step by step.

/*!
 * ScraperLib.js v14.0.0
 * An exceptionally optimized, secure, and robust client-side web scraping library ready for production use
 * License: MIT
 */

(function (global) {
  'use strict';

  /**
   * Utility functions for validation and processing
   */
  const Utils = (() => {
    // Private variables and functions can be defined here

    // Validates if a string is a valid URL
    const isValidURL = url => {
      if (typeof url !== 'string' || url.length > 2048) return false;
      try {
        new URL(url);
        return true;
      } catch {
        return false;
      }
    };

    // Sanitizes a string to prevent XSS
    const sanitize = str => {
      if (typeof str !== 'string') return '';
      const temp = document.createElement('div');
      temp.textContent = str;
      return temp.innerHTML;
    };

    // Initializes the cache if available
    const initCache = async () => {
      if ('caches' in global) {
        try {
          return await caches.open('scraper-lib-cache');
        } catch (e) {
          console.error('Cache initialization failed:', e);
          return null;
        }
      }
      return null;
    };

    // Generates a cache key based on URL and options
    const generateCacheKey = async (url, options = {}) => {
      const cacheOptions = {
        useProxy: options.useProxy,
        proxyUrl: options.useProxy ? options.proxyUrl : undefined,
        selectorsHash: await hashSelectors(options.selectors),
        cacheTime: options.cacheTime,
        method: options.method,
        postDataHash:
          options.method === 'POST'
            ? await hashString(JSON.stringify(options.postData, jsonReplacer))
            : undefined,
      };
      // Exclude functions from JSON.stringify
      const cacheKeyString = `${url}-${JSON.stringify(cacheOptions, jsonReplacer)}`;
      const hash = await hashString(cacheKeyString);
      return hash;
    };

    // Custom JSON replacer to exclude functions and handle special types
    const jsonReplacer = (key, value) => {
      if (typeof value === 'function') {
        return undefined;
      }
      if (value instanceof RegExp) {
        return `__REGEXP ${value.toString()}`;
      }
      if (value instanceof Date) {
        return `__DATE ${value.toISOString()}`;
      }
      return value;
    };

    // Reviver for JSON parsing to reconstruct special types
    const jsonReviver = (key, value) => {
      if (typeof value === 'string') {
        if (value.startsWith('__REGEXP ')) {
          const m = value.substring(9).match(/\/(.*)\/([gimuy]*)/);
          if (m) {
            return new RegExp(m[1], m[2]);
          }
        }
        if (value.startsWith('__DATE ')) {
          return new Date(value.substring(7));
        }
      }
      return value;
    };

    // Generates a hash for selectors to include in cache key
    const hashSelectors = async selectors => {
      if (!selectors) return '';
      try {
        const selectorsString = JSON.stringify(selectors, jsonReplacer);
        return await hashString(selectorsString);
      } catch (e) {
        console.error('Error hashing selectors:', e);
        return '';
      }
    };

    // Generates a cryptographic hash from a string
    const hashString = async str => {
      if (typeof crypto !== 'undefined' && crypto.subtle) {
        const encoder = new TextEncoder();
        const data = encoder.encode(str);
        const hashBuffer = await crypto.subtle.digest('SHA-256', data);
        const hashArray = Array.from(new Uint8Array(hashBuffer));
        const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
        return hashHex;
      } else {
        // Fallback to SHA-256 implementation if crypto.subtle is not available
        console.warn('crypto.subtle not available, using fallback hash function');
        return simpleHash(str);
      }
    };

    // Simple hash function for fallback (non-cryptographic)
    const simpleHash = str => {
      let hash = 0;
      if (str.length === 0) return hash.toString();
      for (let i = 0; i < str.length; i++) {
        const chr = str.charCodeAt(i);
        hash = (hash << 5) - hash + chr;
        hash |= 0; // Convert to 32bit integer
        hash = hash >>> 0; // Convert to unsigned
      }
      return hash.toString(16);
    };

    // Validates and normalizes CSS selectors
    const normalizeSelector = selector => {
      if (typeof selector !== 'string') throw new Error('Selector must be a string');
      try {
        document.createDocumentFragment().querySelector(selector);
        return selector;
      } catch {
        throw new Error(`Invalid CSS selector: "${selector}"`);
      }
    };

    // Deep merge of objects with proper array handling
    const deepMerge = (target, ...sources) => {
      target = target || {};
      sources.forEach(src => {
        if (src && typeof src === 'object') {
          Object.keys(src).forEach(key => {
            const value = src[key];
            if (Array.isArray(value)) {
              target[key] = Array.isArray(target[key]) ? target[key].concat(value) : value.slice();
            } else if (value && typeof value === 'object' && !(value instanceof RegExp) && !(value instanceof Date)) {
              target[key] = deepMerge(target[key], value);
            } else {
              target[key] = value;
            }
          });
        }
      });
      return target;
    };

    // Extracts text from an element, optionally preserving formatting
    const extractText = (element, options = {}) => {
      if (!element) return '';
      const clone = element.cloneNode(true);
      // Remove scripts, styles, and noscript elements
      clone.querySelectorAll('script, style, noscript').forEach(el => el.remove());
      if (options.preserveFormatting) {
        // Replace <br> with newline characters
        clone.querySelectorAll('br').forEach(br => {
          const textNode = document.createTextNode('\n');
          br.parentNode.replaceChild(textNode, br);
        });
        // Add newline after </p> elements
        clone.querySelectorAll('p').forEach(p => {
          const textNode = document.createTextNode('\n');
          p.appendChild(textNode);
        });
      }
      return clone.textContent.trim();
    };

    // Removes scripts and styles from HTML string
    const removeScriptsAndStyles = html => {
      try {
        const doc = new DOMParser().parseFromString(html, 'text/html');
        doc.querySelectorAll('script, style, noscript').forEach(el => el.remove());
        return doc.documentElement.outerHTML;
      } catch (e) {
        console.error('Error parsing HTML for script/style removal:', e);
        return html
          .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
          .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
          .replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, '');
      }
    };

    // Chunks an array into smaller arrays of specified size
    const chunkArray = (array, size) => {
      if (!Array.isArray(array)) throw new Error('Input must be an array');
      if (!Number.isInteger(size) || size <= 0) throw new Error('Chunk size must be a positive integer');
      const result = [];
      for (let i = 0; i < array.length; i += size) {
        result.push(array.slice(i, i + size));
      }
      return result;
    };

    // Parses a string to JSON safely
    const safeParseJSON = str => {
      try {
        return JSON.parse(str, jsonReviver);
      } catch {
        return null;
      }
    };

    // Converts a NodeList to an Array
    const nodeListToArray = nodeList => Array.from(nodeList);

    // Function to delay execution
    const delay = ms => new Promise(resolve => setTimeout(resolve, ms));

    // Check for anti-scraping mechanisms (e.g., CAPTCHA)
    const detectAntiScraping = html => {
      // Simple keyword detection for common anti-scraping messages
      const antiScrapingKeywords = [
        'captcha',
        'verify you are human',
        'access denied',
        'bot detection',
        'unusual traffic',
        'request blocked',
        'please enable cookies',
        'firewall',
        'cloudflare',
        'access forbidden',
        'browser check',
        'denied',
        'not allowed',
        'error 403',
        'forbidden',
        'please turn javascript on',
        'page blocked',
      ];
      const lowerHtml = html.toLowerCase();
      return antiScrapingKeywords.some(keyword => lowerHtml.includes(keyword));
    };

    // Built-in filters for common operations
    const builtInFilters = {
      trim: value => (typeof value === 'string' ? value.trim() : value),
      toUpperCase: value => (typeof value === 'string' ? value.toUpperCase() : value),
      toLowerCase: value => (typeof value === 'string' ? value.toLowerCase() : value),
      parseInt: value => parseInt(value, 10),
      parseFloat: value => parseFloat(value),
      replace: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 2) return value;
        const [pattern, replacement] = args;
        let regex;
        if (pattern instanceof RegExp) {
          regex = pattern;
        } else if (typeof pattern === 'string') {
          regex = new RegExp(pattern, 'g');
        } else {
          return value;
        }
        return value.replace(regex, replacement);
      },
      substring: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args)) return value;
        return value.substring(args[0], args[1]);
      },
      split: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args)) return value;
        return value.split(args[0]);
      },
      join: (value, args) => {
        if (!Array.isArray(value)) return value;
        return value.join(args[0]);
      },
      regexMatch: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 1) return value;
        const [pattern, group = 0] = args;
        let regex;
        if (pattern instanceof RegExp) {
          regex = pattern;
        } else if (typeof pattern === 'string') {
          regex = new RegExp(pattern);
        } else {
          return value;
        }
        const match = value.match(regex);
        return match ? match[group] : null;
      },
      trimChars: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 1) return value;
        const chars = args[0];
        const regex = new RegExp(`^[${chars}]+|[${chars}]+$`, 'g');
        return value.replace(regex, '');
      },
      defaultValue: (value, args) => {
        return value == null || value === '' ? args[0] : value;
      },
      toDate: (value, args) => {
        const date = new Date(value);
        return isNaN(date.getTime()) ? null : date;
      },
      formatDate: (value, args) => {
        if (!(value instanceof Date)) return value;
        const options = args[0] || {};
        return value.toLocaleDateString(undefined, options);
      },
      toBoolean: value => {
        if (typeof value === 'string') {
          const val = value.toLowerCase();
          return val === 'true' || val === 'yes' || val === '1';
        }
        return Boolean(value);
      },
      encodeURI: value => (typeof value === 'string' ? encodeURI(value) : value),
      decodeURI: value => (typeof value === 'string' ? decodeURI(value) : value),
      encodeURIComponent: value => (typeof value === 'string' ? encodeURIComponent(value) : value),
      decodeURIComponent: value => (typeof value === 'string' ? decodeURIComponent(value) : value),
      toJSON: value => {
        try {
          return JSON.parse(value);
        } catch {
          return null;
        }
      },
      add: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand)) return value;
        return num + operand;
      },
      subtract: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand)) return value;
        return num - operand;
      },
      multiply: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand)) return value;
        return num * operand;
      },
      divide: (value, args) => {
        const num = parseFloat(value);
        const operand = parseFloat(args[0]);
        if (isNaN(num) || isNaN(operand) || operand === 0) return value;
        return num / operand;
      },
      unique: value => {
        if (Array.isArray(value)) {
          return Array.from(new Set(value));
        }
        return value;
      },
      flatten: value => {
        if (Array.isArray(value)) {
          return value.flat(Infinity);
        }
        return value;
      },
      camelCase: value => {
        if (typeof value !== 'string') return value;
        return value
          .replace(/(?:^\w|[A-Z]|\b\w)/g, (word, index) =>
            index === 0 ? word.toLowerCase() : word.toUpperCase()
          )
          .replace(/\s+/g, '');
      },
      snakeCase: value => {
        if (typeof value !== 'string') return value;
        return value
          .replace(/\W+/g, ' ')
          .split(/ |\B(?=[A-Z])/)
          .map(word => word.toLowerCase())
          .join('_');
      },
      chain: (value, args) => {
        if (!Array.isArray(args) || args.length === 0) return value;
        return args.reduce((val, filterStr) => {
          const filterFunc = builtInFilters[filterStr];
          if (typeof filterFunc === 'function') {
            return filterFunc(val);
          }
          return val;
        }, value);
      },
      escapeHTML: value => {
        if (typeof value !== 'string') return value;
        const div = document.createElement('div');
        div.appendChild(document.createTextNode(value));
        return div.innerHTML;
      },
      unescapeHTML: value => {
        if (typeof value !== 'string') return value;
        const div = document.createElement('div');
        div.innerHTML = value;
        return div.textContent || div.innerText || '';
      },
      removeWhitespace: value => {
        if (typeof value !== 'string') return value;
        return value.replace(/\s+/g, '');
      },
      capitalize: value => {
        if (typeof value !== 'string') return value;
        return value.charAt(0).toUpperCase() + value.slice(1);
      },
      // New filter: truncate string to specified length
      truncate: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args)) return value;
        const length = args[0];
        if (typeof length !== 'number' || length <= 0) return value;
        return value.length > length ? value.substring(0, length) + '...' : value;
      },
      // New filter: Replace HTML entities with characters
      decodeHTMLEntities: value => {
        if (typeof value !== 'string') return value;
        const txt = document.createElement('textarea');
        txt.innerHTML = value;
        return txt.value;
      },
      // New filter: Remove HTML tags
      stripHTML: value => {
        if (typeof value !== 'string') return value;
        return value.replace(/<[^>]*>?/gm, '');
      },
    };

    // Function to create a DOM element from an HTML string
    const createElementFromHTML = htmlString => {
      const template = document.createElement('template');
      template.innerHTML = htmlString.trim();
      return template.content.firstChild;
    };

    // Expose public methods
    return {
      isValidURL,
      sanitize,
      initCache,
      generateCacheKey,
      jsonReplacer,
      jsonReviver,
      hashSelectors,
      hashString,
      simpleHash,
      normalizeSelector,
      deepMerge,
      extractText,
      removeScriptsAndStyles,
      chunkArray,
      safeParseJSON,
      nodeListToArray,
      delay,
      detectAntiScraping,
      builtInFilters,
      createElementFromHTML,
    };
  })();

  /**
   * Plugin manager for extending functionality
   */
  class PluginManager {
    constructor() {
      this.plugins = new Map();
    }

    registerPlugin(name, plugin) {
      if (typeof plugin !== 'function') {
        throw new Error('Plugin must be a function');
      }
      this.plugins.set(name, plugin);
    }

    getPlugin(name) {
      return this.plugins.get(name);
    }
  }

  /**
   * Handles rate limiting to respect website policies
   */
  class RateLimiter {
    constructor(requestsPerSecond = 2) {
      this.requestsPerSecond = Math.max(1, requestsPerSecond);
      this.interval = 1000 / this.requestsPerSecond;
      this.queue = [];
      this.lastRequestTime = 0;
      this.isProcessing = false;
      this.timeoutId = null;
    }

    async acquireToken() {
      return new Promise(resolve => {
        this.queue.push(resolve);
        this._processQueue();
      });
    }

    _processQueue() {
      if (this.isProcessing || this.queue.length === 0) return;
      this.isProcessing = true;

      const now = Date.now();
      const elapsed = now - this.lastRequestTime;
      const delay = Math.max(this.interval - elapsed, 0);

      this.timeoutId = setTimeout(() => {
        this.lastRequestTime = Date.now();
        const resolve = this.queue.shift();
        if (resolve) resolve();
        this.isProcessing = false;
        if (this.queue.length > 0) {
          this._processQueue();
        }
      }, delay);
    }

    // Cleanup to prevent memory leaks
    clear() {
      clearTimeout(this.timeoutId);
      this.queue = [];
    }
  }

  /**
   * Main ScraperLib class
   */
  class ScraperLib {
    constructor(options = {}) {
      this.defaultSettings = {
        rateLimitRequests: 2,
        maxConcurrent: 5,
        timeout: 30000,
        retries: 3,
        retryDelay: 1000,
        maxRetryDelay: 30000,
        cacheTime: 3600000,
        outputFormat: 'memory',
        useCache: false,
        headers: {},
        preserveFormatting: false,
        validateSelectors: true,
        maxResponseSize: 10 * 1024 * 1024,
        useProxy: false,
        proxyUrl: '',
        filters: {}, // Global filters
        concurrencyStrategy: 'auto', // 'auto', 'parallel', 'sequential'
        logger: null, // Custom logger function
        customFetch: null, // Custom fetch function
        politenessDelay: 0, // Delay between requests to the same domain
        maxRetriesOnAntiScraping: 2, // Max retries if anti-scraping measures detected
        enableLogging: true, // Enable or disable logging
        defaultOutputPath: '', // Default output path for non-JSON outputs
        pagination: null, // Pagination config
        retryOnNetworkFailure: true, // Retry on network failures
        method: 'GET', // HTTP method
        postData: null, // Data for POST requests
        customResponseParser: null, // Custom response parser
        plugins: [], // Plugins to use
        userAgent: `ScraperLib/${ScraperLib.version}`, // Custom User-Agent
        language: 'en-US,en;q=0.9', // Accept-Language header
        maxContentLength: 0, // Maximum content length (0 means no limit)
        parseBySelector: false, // Option to parse only specified selectors
      };

      // Forbidden headers in browsers (lowercase)
      this.forbiddenHeaders = new Set([
        'accept-charset',
        'accept-encoding',
        'access-control-request-headers',
        'access-control-request-method',
        'connection',
        'content-length',
        'cookie',
        'cookie2',
        'date',
        'dnt',
        'expect',
        'host',
        'keep-alive',
        'origin',
        'referer',
        'te',
        'trailer',
        'transfer-encoding',
        'upgrade',
        'via',
        'proxy-',
        'sec-',
        'user-agent',
        'accept-language',
      ]);

      // Merge and validate default settings with user options
      this.settings = Utils.deepMerge({}, this.defaultSettings, options);
      this._validateSettings(this.settings);

      // Initialize per-URL settings
      this.urlSettings = new Map();

      this.storage = new StorageHandler();
      this.rateLimiter = new RateLimiter(this.settings.rateLimitRequests);
      this.cache = null;
      this.errorHandlers = new Map();

      // Initialize cache
      if (this.settings.useCache) {
        this.cachePromise = Utils.initCache().then(cache => {
          this.cache = cache;
        });
      } else {
        this.cachePromise = Promise.resolve();
      }

      // Initialize logger
      this.logger = this.settings.logger || console;

      // Hostname to last request time mapping for politeness
      this.hostLastRequestTime = new Map();

      // Initialize plugins
      this.pluginManager = new PluginManager();
      this._initializePlugins(this.settings.plugins);

      // Ensure that customFetch adheres to the Fetch API
      if (this.settings.customFetch && typeof this.settings.customFetch !== 'function') {
        throw new Error('customFetch must be a function adhering to the Fetch API');
      }
    }

    // Static version property
    static get version() {
      return '14.0.0';
    }

    _initializePlugins(plugins) {
      if (!Array.isArray(plugins)) return;
      plugins.forEach(plugin => {
        if (typeof plugin.name !== 'string' || typeof plugin.plugin !== 'function') {
          throw new Error('Each plugin must have a name and a plugin function');
        }
        this.pluginManager.registerPlugin(plugin.name, plugin.plugin);
      });
    }

    _validateSettings(settings) {
      const positiveNumberProps = [
        'rateLimitRequests',
        'maxConcurrent',
        'retries',
        'retryDelay',
        'maxRetryDelay',
        'cacheTime',
        'timeout',
        'maxResponseSize',
        'politenessDelay',
        'maxRetriesOnAntiScraping',
        'maxContentLength',
      ];
      positiveNumberProps.forEach(prop => {
        if (typeof settings[prop] !== 'number' || settings[prop] < 0) {
          throw new Error(`Invalid setting "${prop}": must be a positive number`);
        }
      });

      const allowedStrategies = ['auto', 'parallel', 'sequential'];
      if (!allowedStrategies.includes(settings.concurrencyStrategy)) {
        throw new Error(
          `Invalid concurrency strategy: "${settings.concurrencyStrategy}". Allowed values are ${allowedStrategies.join(
            ', '
          )}`
        );
      }

      if (settings.customFetch && typeof settings.customFetch !== 'function') {
        throw new Error('customFetch must be a function');
      }

      if (settings.pagination && typeof settings.pagination !== 'object') {
        throw new Error('pagination must be an object');
      }

      if (settings.method && !['GET', 'POST'].includes(settings.method.toUpperCase())) {
        throw new Error('method must be either GET or POST');
      }

      if (settings.method.toUpperCase() === 'POST' && !settings.postData) {
        throw new Error('postData must be provided for POST requests');
      }

      if (settings.headers && typeof settings.headers !== 'object') {
        throw new Error('headers must be an object');
      }
    }

    /**
     * Main scraping method
     */
    async scrape(configs, globalOptions = {}) {
      await this.cachePromise;

      if (typeof configs !== 'object' || configs === null) {
        throw new Error('Configs must be an object with URL keys and configuration values');
      }

      // Merge global options into settings
      const globalSettings = Utils.deepMerge({}, this.settings, globalOptions);
      this._validateSettings(globalSettings);

      const results = {};
      const urls = Object.keys(configs);
      if (urls.length === 0) {
        return results;
      }

      await this._log('Starting scraping process', { urls });

      const scrapeTasks = urls.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results));

      switch (globalSettings.concurrencyStrategy) {
        case 'parallel':
          await Promise.all(scrapeTasks);
          break;

        case 'sequential':
          for (const task of scrapeTasks) {
            await task;
          }
          break;

        case 'auto':
        default:
          const chunks = Utils.chunkArray(scrapeTasks, globalSettings.maxConcurrent);
          for (const chunk of chunks) {
            await Promise.all(chunk);
          }
          break;
      }

      await this._log('Completed scraping process', { results });

      return results;
    }

    /**
     * Wrapper for processing a URL, handles logging and error handling
     */
    async _scrapeUrlWrapper(url, config, globalSettings, results) {
      try {
        const data = await this._processUrl(url, config, globalSettings);
        results[url] = data;
      } catch (error) {
        const handler = this.errorHandlers.get(url) || this.errorHandlers.get('default');
        if (handler) {
          try {
            const data = await handler(error, url);
            results[url] = data;
          } catch (e) {
            results[url] = { error: e.message };
          }
        } else {
          results[url] = { error: error.message };
        }
        await this._log(`Error scraping ${url}: ${error.message}`, { error });
      }
    }

    /**
     * Process a single URL with retries and error handling
     */
    async _processUrl(url, config, globalSettings) {
      if (!Utils.isValidURL(url)) {
        throw new Error(`Invalid URL: ${url}`);
      }

      // Merge global settings with per-URL settings using deep merge
      const settings = Utils.deepMerge({}, globalSettings, config.options);
      this._validateSettings(settings);
      this.urlSettings.set(url, settings);

      if (settings.validateSelectors) {
        this._validateSelectors(config.selectors);
      }

      // Wait for cache initialization if needed
      if (settings.useCache) {
        await this.cachePromise;
        if (this.cache) {
          const cachedData = await this._getFromCache(url, settings);
          if (cachedData) {
            await this._log(`Using cached data for ${url}`);
            return cachedData;
          }
        }
      }

      let attempts = 0;
      let antiScrapingAttempts = 0;
      const maxAttempts = settings.retries + 1; // Including initial attempt
      while (attempts < maxAttempts) {
        attempts += 1;
        await this._respectPoliteness(url, settings);

        await this.rateLimiter.acquireToken();
        const options = this._prepareFetchOptions(config, settings);
        try {
          const response = await this._fetchWithTimeout(url, options, settings);

          if (!response.ok) {
            throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
          }

          const contentType = response.headers.get('Content-Type') || '';
          if (!contentType.includes('text/html') && !settings.customResponseParser) {
            throw new Error(`Invalid Content-Type: ${contentType}`);
          }

          let data;
          if (settings.customResponseParser) {
            data = await settings.customResponseParser(response);
          } else {
            data = await this._extractDataFromResponse(response, config.selectors, settings);
          }

          // Apply plugins
          data = await this._applyPlugins(data, settings);

          // Pagination handling
          if (settings.pagination) {
            data = await this._handlePagination(url, config, settings, data);
          }

          if (settings.outputFormat !== 'json' && settings.outputFormat !== 'memory') {
            await this.storage.store(url, data, settings.outputFormat, { customStore: settings.customStore });
          }

          if (settings.useCache && this.cache) {
            await this._storeInCache(url, data, settings);
          }

          return data;
        } catch (error) {
          if (
            Utils.detectAntiScraping(error.message) &&
            antiScrapingAttempts < settings.maxRetriesOnAntiScraping
          ) {
            antiScrapingAttempts += 1;
            await this._log(
              `Detected anti-scraping measures on ${url}. Attempt ${antiScrapingAttempts}`,
              { error }
            );
            // Delay before retrying
            await Utils.delay(settings.retryDelay * antiScrapingAttempts);
            continue;
          }

          if (attempts >= maxAttempts) {
            throw error;
          }
          // Exponential backoff with jitter
          const delay = Math.min(
            settings.retryDelay * Math.pow(2, attempts - 1) * (0.5 + Math.random() * 0.5),
            settings.maxRetryDelay
          );
          await this._log(`Retrying ${url} after delay of ${delay}ms`, { attempt: attempts, error });
          await Utils.delay(delay);
        }
      }
    }

    /**
     * Respects politeness policies by delaying requests to the same domain
     */
    async _respectPoliteness(url, settings) {
      const urlObj = new URL(url);
      const hostname = urlObj.hostname;
      const now = Date.now();
      const lastRequestTime = this.hostLastRequestTime.get(hostname) || 0;
      const elapsed = now - lastRequestTime;

      if (elapsed < settings.politenessDelay) {
        const delay = settings.politenessDelay - elapsed;
        await this._log(`Delaying request to ${url} by ${delay}ms for politeness`);
        await Utils.delay(delay);
      }

      this.hostLastRequestTime.set(hostname, Date.now());
    }

    /**
     * Validates the CSS selectors
     */
    _validateSelectors(selectors) {
      if (typeof selectors !== 'object' || selectors === null) {
        throw new Error('Selectors must be an object');
      }
      for (const key of Object.keys(selectors)) {
        const sel = selectors[key];
        if (typeof sel === 'string') {
          Utils.normalizeSelector(sel);
        } else if (sel && typeof sel.selector === 'string') {
          Utils.normalizeSelector(sel.selector);
        } else if (sel && typeof sel.customSelector === 'function') {
          // Custom selector function, assume valid
        } else if (sel && typeof sel.xpath === 'string') {
          // Could validate XPath here if needed
        } else {
          throw new Error(`Invalid selector configuration for key "${key}"`);
        }
      }
    }

    /**
     * Prepares fetch options including headers
     */
    _prepareFetchOptions(config, settings) {
      const headersInit = Utils.deepMerge({}, settings.headers, config.headers);

      // Remove forbidden headers
      Object.keys(headersInit).forEach(header => {
        const headerLower = header.toLowerCase();
        if (
          this.forbiddenHeaders.has(headerLower) ||
          Array.from(this.forbiddenHeaders).some(
            fh => fh.endsWith('-') && headerLower.startsWith(fh)
          )
        ) {
          delete headersInit[header];
        }
      });

      // Set default 'Accept' header if not present
      if (!headersInit['Accept']) {
        headersInit['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
      }

      // Set custom User-Agent
      if (!headersInit['User-Agent']) {
        headersInit['User-Agent'] = settings.userAgent;
      }

      // Set Accept-Language header
      if (!headersInit['Accept-Language']) {
        headersInit['Accept-Language'] = settings.language;
      }

      const headers = new Headers(headersInit);

      let body = null;
      if (settings.method.toUpperCase() === 'POST') {
        body = settings.postData;
        if (!(body instanceof FormData) && typeof body === 'object') {
          headers.set('Content-Type', 'application/json');
          body = JSON.stringify(body);
        } else if (typeof body === 'string') {
          headers.set('Content-Type', 'application/x-www-form-urlencoded');
        }
      }

      const options = {
        method: settings.method.toUpperCase(),
        headers: headers,
        mode: 'cors',
        credentials: 'omit',
        redirect: 'follow',
        body: body,
        cache: 'no-cache',
      };

      return options;
    }

    /**
     * Performs a fetch request with a timeout, using proxy if configured
     */
    async _fetchWithTimeout(url, options, settings) {
      const controller = new AbortController();
      options.signal = controller.signal;

      const timeoutId = setTimeout(() => {
        controller.abort();
      }, settings.timeout);

      try {
        let fetchUrl = url;
        if (settings.useProxy) {
          if (!settings.proxyUrl) {
            throw new Error('Proxy URL is not set');
          }
          // Correctly construct the fetch URL with the proxy
          if (settings.proxyUrl.includes('${url}')) {
            fetchUrl = settings.proxyUrl.replace('${url}', encodeURIComponent(url));
          } else {
            // Append the encoded URL appropriately
            const proxyUrl = new URL(settings.proxyUrl);
            proxyUrl.searchParams.append('url', url);
            fetchUrl = proxyUrl.toString();
          }
        }

        const fetchFunction = settings.customFetch || fetch;

        const response = await fetchFunction(fetchUrl, options);
        clearTimeout(timeoutId);

        // Check if response size exceeds maxResponseSize
        const reader = response.body.getReader();
        let receivedLength = 0;
        const chunks = [];
        const maxContentLength = settings.maxContentLength || Infinity;

        while (true) {
          const { done, value } = await reader.read();
          if (done) break;

          receivedLength += value.length;
          if (receivedLength > maxContentLength) {
            controller.abort();
            throw new Error('Response size exceeds maximum allowed');
          }
          chunks.push(value);
        }

        // Concatenate chunks into a Uint8Array
        const chunksAll = new Uint8Array(receivedLength);
        let position = 0;
        for (let chunk of chunks) {
          chunksAll.set(chunk, position);
          position += chunk.length;
        }

        // Decode the result into a string
        const decoder = new TextDecoder('utf-8');
        const responseText = decoder.decode(chunksAll);

        return new Response(responseText, {
          status: response.status,
          statusText: response.statusText,
          headers: response.headers,
        });
      } catch (error) {
        clearTimeout(timeoutId);
        if (error.name === 'AbortError') {
          throw new Error('Request timed out or response too large');
        }
        if (settings.retryOnNetworkFailure && (error instanceof TypeError || error.message === 'Failed to fetch')) {
          throw new Error('Network failure, will retry');
        }
        throw error;
      }
    }

    /**
     * Enhanced data extraction with advanced filters
     */
    async _extractDataFromResponse(response, selectors, settings) {
      try {
        const text = await response.text();

        // Check for anti-scraping mechanisms
        if (Utils.detectAntiScraping(text)) {
          throw new Error('Detected anti-scraping measures in the response');
        }

        // Remove scripts and styles to prevent XSS and reduce parsing overhead
        const sanitizedHtml = Utils.removeScriptsAndStyles(text);

        const parser = new DOMParser();
        const doc = parser.parseFromString(sanitizedHtml, 'text/html');

        const data = {};

        const parseBySelector = settings.parseBySelector;

        for (const key of Object.keys(selectors)) {
          const selConfig = selectors[key];

          let elements;
          if (typeof selConfig === 'string') {
            elements = doc.querySelectorAll(selConfig);
          } else if (selConfig && typeof selConfig.selector === 'string') {
            elements = doc.querySelectorAll(selConfig.selector);
          } else if (selConfig && typeof selConfig.customSelector === 'function') {
            elements = selConfig.customSelector(doc);
            if (!Array.isArray(elements)) {
              throw new Error(`customSelector for key "${key}" must return an array of elements`);
            }
          } else if (selConfig && typeof selConfig.xpath === 'string') {
            elements = this._selectXPath(doc, selConfig.xpath);
          } else {
            throw new Error(`Invalid selector configuration for key "${key}"`);
          }

          if (!elements || elements.length === 0) {
            data[key] = selConfig.single ? null : [];
            continue;
          }

          data[key] = await Promise.all(
            Array.from(elements).map(async element => {
              let value;
              if (selConfig.attr) {
                value = element.getAttribute(selConfig.attr) || '';
              } else if (selConfig.html) {
                value = element.innerHTML;
              } else {
                value = Utils.extractText(element, { preserveFormatting: settings.preserveFormatting });
              }

              // Apply per-selector filters (support chaining)
              if (selConfig.filters) {
                value = await this._applyFilters(value, selConfig.filters, element, {
                  key,
                  settings,
                });
              }

              // Apply global filters if no per-selector filters
              else if (settings.filters && settings.filters[key]) {
                value = await this._applyFilters(value, settings.filters[key], element, {
                  key,
                  settings,
                });
              }

              if (selConfig.transform && typeof selConfig.transform === 'function') {
                value = await selConfig.transform(value, element);
              }

              return value;
            })
          );

          data[key] = data[key].filter(val => val !== undefined && val !== null);

          if (selConfig.single) {
            data[key] = data[key][0] || null;
          }
        }

        // If parseBySelector is false, extract remaining data
        if (!parseBySelector) {
          // Additional data extraction logic can be added here
        }

        return data;
      } catch (e) {
        throw new Error(`Failed to extract data: ${e.message}`);
      }
    }

    /**
     * Apply filters to a value, supporting chaining and built-in filters
     */
    async _applyFilters(value, filters, element, context) {
      if (!filters) return value;

      const filtersArray = Array.isArray(filters) ? filters : [filters];

      for (let filter of filtersArray) {
        if (typeof filter === 'string') {
          // Built-in filter by name
          const filterFunc = this._parseFilterString(filter);
          if (typeof filterFunc === 'function') {
            value = filterFunc(value, element, context);
          } else {
            console.warn(`Built-in filter "${filter}" not found`);
          }
        } else if (typeof filter === 'function') {
          try {
            const filterResult = filter(value, element, context);
            value = filterResult instanceof Promise ? await filterResult : filterResult;
          } catch (e) {
            console.error(`Error in filter for key "${context.key}":`, e);
            value = null;
            break;
          }
        } else if (filter instanceof RegExp) {
          // Apply regular expression
          const match = value.match(filter);
          value = match ? match[0] : null;
        } else {
          console.warn(`Invalid filter type for key "${context.key}"`);
        }

        // If value becomes null or undefined, stop processing filters
        if (value == null) {
          break;
        }
      }

      return value;
    }

    /**
     * Parse filter string to support filter arguments safely
     */
    _parseFilterString(filterStr) {
      // Example filter string: 'replace(/foo/g, "bar")'
      const filterPattern = /^(\w+)\((.*)\)$/;
      const match = filterStr.match(filterPattern);
      if (match) {
        const filterName = match[1];
        const argsStr = match[2];
        const args = this._parseFilterArgs(argsStr);

        const filterFunc = Utils.builtInFilters[filterName];
        if (typeof filterFunc === 'function') {
          return (value, element, context) => filterFunc(value, args, element, context);
        }
      } else {
        const filterFunc = Utils.builtInFilters[filterStr];
        if (typeof filterFunc === 'function') {
          return filterFunc;
        }
      }
      return null;
    }

    /**
     * Parse filter arguments from string safely
     */
    _parseFilterArgs(argsStr) {
      try {
        const args = [];
        let currentArg = '';
        let inQuotes = false;
        let quoteChar = '';
        let escapeNext = false;
        let bracketDepth = 0;

        for (let i = 0; i < argsStr.length; i++) {
          const char = argsStr.charAt(i);

          if (escapeNext) {
            currentArg += char;
            escapeNext = false;
            continue;
          }

          if (char === '\\') {
            escapeNext = true;
            continue;
          }

          if (inQuotes) {
            if (char === quoteChar) {
              inQuotes = false;
            }
            currentArg += char;
            continue;
          }

          if (char === '"' || char === "'") {
            inQuotes = true;
            quoteChar = char;
            currentArg += char;
            continue;
          }

          if (char === '(') {
            bracketDepth++;
          } else if (char === ')') {
            bracketDepth--;
          }

          if (char === ',' && !inQuotes && bracketDepth === 0) {
            args.push(this._parseArgument(currentArg.trim()));
            currentArg = '';
            continue;
          }

          currentArg += char;
        }

        if (currentArg) {
          args.push(this._parseArgument(currentArg.trim()));
        }

        return args;
      } catch (e) {
        console.error('Failed to parse filter arguments:', e);
        return [];
      }
    }

    _parseArgument(arg) {
      if ((arg.startsWith('"') && arg.endsWith('"')) || (arg.startsWith("'") && arg.endsWith("'"))) {
        // String argument
        return arg.slice(1, -1).replace(/\\'/g, "'").replace(/\\"/g, '"');
      }

      // Try to parse as number
      const num = Number(arg);
      if (!isNaN(num)) {
        return num;
      }

      // For regex arguments, e.g., /pattern/flags
      if (arg.startsWith('/') && arg.lastIndexOf('/') > 0) {
        const lastSlash = arg.lastIndexOf('/');
        const pattern = arg.slice(1, lastSlash);
        const flags = arg.slice(lastSlash + 1);
        try {
          return new RegExp(pattern, flags);
        } catch (e) {
          console.error('Invalid regular expression:', e);
          return null;
        }
      }

      // Boolean arguments
      if (arg === 'true') return true;
      if (arg === 'false') return false;

      // Otherwise, return the arg as string
      return arg;
    }

    /**
     * Select elements using XPath
     */
    _selectXPath(doc, xpathExpression) {
      const iterator = doc.evaluate(xpathExpression, doc, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
      const nodes = [];
      for (let i = 0; i < iterator.snapshotLength; i++) {
        nodes.push(iterator.snapshotItem(i));
      }
      return nodes;
    }

    /**
     * Handles pagination by recursively scraping next pages
     */
    async _handlePagination(url, config, settings, initialData) {
      const paginationConfig = settings.pagination;
      if (!paginationConfig.selector || !paginationConfig.pageLimit) {
        return initialData;
      }

      let pageData = initialData;
      let allData = [pageData];
      let pageCount = 1;
      let nextPageUrl = this._extractNextPageUrl(pageData, paginationConfig, url);

      while (nextPageUrl && pageCount < paginationConfig.pageLimit) {
        await this.rateLimiter.acquireToken();
        await this._respectPoliteness(nextPageUrl, settings);
        const options = this._prepareFetchOptions(config, settings);

        try {
          const response = await this._fetchWithTimeout(nextPageUrl, options, settings);

          if (!response.ok) {
            throw new Error(`HTTP error on pagination: ${response.status} ${response.statusText}`);
          }

          const contentType = response.headers.get('Content-Type') || '';
          if (!contentType.includes('text/html') && !settings.customResponseParser) {
            throw new Error(`Invalid Content-Type on pagination: ${contentType}`);
          }

          let data;
          if (settings.customResponseParser) {
            data = await settings.customResponseParser(response);
          } else {
            data = await this._extractDataFromResponse(response, config.selectors, settings);
          }

          // Apply plugins
          data = await this._applyPlugins(data, settings);

          allData.push(data);
          pageCount++;
          nextPageUrl = this._extractNextPageUrl(data, paginationConfig, nextPageUrl);
        } catch (error) {
          await this._log(`Pagination error on ${nextPageUrl}: ${error.message}`, { error });
          break;
        }
      }

      // Merge all data
      const mergedData = this._mergePaginatedData(allData);
      return mergedData;
    }

    _extractNextPageUrl(data, paginationConfig, currentUrl) {
      const nextPageLinks = data[paginationConfig.selector];
      if (!nextPageLinks || nextPageLinks.length === 0) {
        return null;
      }
      let nextPageUrl = nextPageLinks[0]; // Assuming the first link is the next page
      if (!Utils.isValidURL(nextPageUrl)) {
        // Handle relative URLs
        try {
          const baseUrl = new URL(currentUrl);
          nextPageUrl = new URL(nextPageUrl, baseUrl).href;
        } catch {
          return null;
        }
      }
      return nextPageUrl;
    }

    _mergePaginatedData(allData) {
      const mergedData = {};
      allData.forEach(data => {
        Object.keys(data).forEach(key => {
          if (!mergedData[key]) {
            mergedData[key] = [];
          }
          if (Array.isArray(data[key])) {
            mergedData[key] = mergedData[key].concat(data[key]);
          } else {
            mergedData[key].push(data[key]);
          }
        });
      });
      return mergedData;
    }

    /**
     * Gets data from the cache if available and not expired
     */
    async _getFromCache(url, settings) {
      if (!this.cache) return null;
      const cacheKey = await Utils.generateCacheKey(url, settings);
      const cachedResponse = await this.cache.match(cacheKey);

      if (cachedResponse) {
        try {
          const cachedData = await cachedResponse.json();
          const age = Date.now() - cachedData.timestamp;
          if (age < settings.cacheTime) {
            return cachedData.data;
          } else {
            // Cache expired, remove it
            await this.cache.delete(cacheKey);
          }
        } catch (e) {
          console.warn('Error reading from cache:', e);
        }
      }

      return null;
    }

    /**
     * Stores data in the cache
     */
    async _storeInCache(url, data, settings) {
      if (!this.cache) return;
      const cacheKey = await Utils.generateCacheKey(url, settings);
      const cachedResponse = new Response(
        JSON.stringify({ data, timestamp: Date.now() }),
        {
          headers: { 'Content-Type': 'application/json' },
        }
      );
      try {
        await this.cache.put(cacheKey, cachedResponse);
      } catch (e) {
        console.error('Failed to store data in cache:', e);
      }
    }

    /**
     * Apply registered plugins to the data
     */
    async _applyPlugins(data, settings) {
      if (!settings.plugins || settings.plugins.length === 0) return data;
      let processedData = data;
      for (const pluginConfig of settings.plugins) {
        const plugin = this.pluginManager.getPlugin(pluginConfig.name);
        if (typeof plugin === 'function') {
          processedData = await plugin(processedData, pluginConfig.options || {});
        }
      }
      return processedData;
    }

    /**
     * Registers a custom error handler
     */
    onError(urlOrDefault, handler) {
      if (typeof handler !== 'function') {
        throw new Error('Error handler must be a function');
      }
      this.errorHandlers.set(urlOrDefault, handler);
    }

    /**
     * Logs messages using the provided logger
     */
    async _log(message, data = {}) {
      if (!this.settings.enableLogging) return;
      if (typeof this.logger === 'function') {
        this.logger(message, data);
      } else if (this.logger && typeof this.logger.log === 'function') {
        this.logger.log(message, data);
      }
    }

    /**
     * Clean up resources to prevent memory leaks
     */
    destroy() {
      this.rateLimiter.clear();
      this.hostLastRequestTime.clear();
      this.errorHandlers.clear();
      this.urlSettings.clear();
      // Additional cleanup if necessary
    }
  }

  /**
   * Handles storing scraped data
   */
  class StorageHandler {
    constructor() {
      this.dbPromise = null;
      this.storeName = 'scrapedData';
      if ('indexedDB' in global) {
        this.storageType = 'indexeddb';
        this.dbPromise = this.initIndexedDB();
      } else if (this._isLocalStorageAccessible()) {
        this.storageType = 'localstorage';
      } else {
        this.storageType = 'memory';
        this.memoryStorage = new Map();
      }
    }

    _isLocalStorageAccessible() {
      try {
        const testKey = '__test__';
        localStorage.setItem(testKey, testKey);
        localStorage.removeItem(testKey);
        return true;
      } catch {
        console.warn('localStorage is not accessible. Using in-memory storage.');
        return false;
      }
    }

    async initIndexedDB(dbName = 'ScraperLibDB') {
      return new Promise((resolve, reject) => {
        const request = indexedDB.open(dbName, 1);

        request.onerror = () => reject(new Error('IndexedDB initialization failed'));

        request.onupgradeneeded = event => {
          const db = event.target.result;
          if (!db.objectStoreNames.contains(this.storeName)) {
            const store = db.createObjectStore(this.storeName, { keyPath: 'id', autoIncrement: true });
            store.createIndex('url', 'url', { unique: false });
            store.createIndex('timestamp', 'timestamp', { unique: false });
          }
        };

        request.onsuccess = event => {
          const db = event.target.result;
          db.onerror = event => {
            console.error('Database error:', event.target.error);
          };
          resolve(db);
        };
      });
    }

    async store(url, data, format, options = {}) {
      const timestamp = Date.now();
      const storeData = { url, data, timestamp };

      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readwrite');
              const store = transaction.objectStore(this.storeName);
              const request = store.add(storeData);

              request.onsuccess = () => resolve(true);
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          try {
            const key = `scraper-${url}-${timestamp}`;
            localStorage.setItem(key, JSON.stringify(storeData));
            return true;
          } catch (e) {
            if (e.name === 'QuotaExceededError' || e.name === 'NS_ERROR_DOM_QUOTA_REACHED') {
              this._cleanupLocalStorage();
              const key = `scraper-${url}-${timestamp}`;
              localStorage.setItem(key, JSON.stringify(storeData));
              return true;
            }
            throw e;
          }

        case 'memory':
          this.memoryStorage.set(`${url}-${timestamp}`, storeData);
          return true;

        case 'custom':
          if (options.customStore && typeof options.customStore.save === 'function') {
            await options.customStore.save(url, data);
            return true;
          }
          throw new Error('Invalid custom storage handler');

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }

    _cleanupLocalStorage() {
      const keys = [];
      for (let i = 0; i < localStorage.length; i++) {
        const key = localStorage.key(i);
        if (key && key.startsWith('scraper-')) {
          keys.push(key);
        }
      }
      if (keys.length > 100) {
        keys.sort((a, b) => {
          try {
            const itemA = Utils.safeParseJSON(localStorage.getItem(a));
            const itemB = Utils.safeParseJSON(localStorage.getItem(b));
            return itemA.timestamp - itemB.timestamp;
          } catch (e) {
            localStorage.removeItem(a);
            localStorage.removeItem(b);
            return 0;
          }
        });
        for (let i = 0; i < keys.length - 100; i++) {
          localStorage.removeItem(keys[i]);
        }
      }
    }

    async retrieve(url, format) {
      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readonly');
              const store = transaction.objectStore(this.storeName);
              const index = store.index('url');
              const request = index.getAll(url);

              request.onsuccess = () => {
                const results = request.result;
                if (results && results.length > 0) {
                  results.sort((a, b) => b.timestamp - a.timestamp);
                  resolve(results[0].data);
                } else {
                  resolve(null);
                }
              };
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          const keys = [];
          for (let i = 0; i < localStorage.length; i++) {
            const key = localStorage.key(i);
            if (key && key.startsWith(`scraper-${url}-`)) {
              keys.push(key);
            }
          }
          keys.sort((a, b) => {
            const itemA = Utils.safeParseJSON(localStorage.getItem(a));
            const itemB = Utils.safeParseJSON(localStorage.getItem(b));
            return (itemB?.timestamp || 0) - (itemA?.timestamp || 0);
          });
          if (keys.length > 0) {
            const item = Utils.safeParseJSON(localStorage.getItem(keys[0]));
            return item ? item.data : null;
          }
          return null;

        case 'memory':
          const memoryItems = Array.from(this.memoryStorage.entries())
            .filter(([key]) => key.startsWith(`${url}-`))
            .sort((a, b) => b[1].timestamp - a[1].timestamp);
          if (memoryItems.length > 0) {
            return memoryItems[0][1].data;
          }
          return null;

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }
  }

  // Attach ScraperLib to the global object
  global.ScraperLib = ScraperLib;
})(typeof window !== 'undefined' ? window : this);