Review this whole codes throughly and fix all of the potential errors, issues, edge cases, bottle necks, resources consumption, possible errors or issues, might give a error or issue, bugs you can find.

Make all of the things precise, careful, appropriate, working, practical, client side only, fast, effective, efficient, reliable.


Make this library support a feature rich advance and very customisable text and url filter because css selectors doesn't support.

Make sure user can easily filter out any text and url using this library.

Make sure user can customise as which text or url to filter out globally or individually url by url.

Make sure user change settings easily in globally or individually url by url without any issue or conflicts or error or miss match.

Send the fully and practically working complete codes.

---

/*!
 * ScraperLib.js v5.1.0
 * A comprehensive, efficient, and reliable client-side web scraping library
 * License: MIT
 */

(function (global) {
  'use strict';

  /**
   * Utility functions for validation and processing
   */
  const Utils = {

    // Validates if a string is a valid URL
    isValidURL(url) {
      if (typeof url !== 'string') return false;
      try {
        new URL(url);
        return true;
      } catch {
        return false;
      }
    },

    // Sanitizes a string to prevent XSS
    sanitize(str) {
      if (typeof str !== 'string') return '';
      const temp = document.createElement('div');
      temp.textContent = str;
      return temp.textContent; // Use textContent for safety
    },

    // Initializes the cache if available
    async initCache() {
      if ('caches' in window) {
        try {
          return await caches.open('scraper-lib-cache');
        } catch (e) {
          console.error('Cache initialization failed:', e);
          return null;
        }
      }
      return null;
    },

    // Generates a cache key based on URL and options
    generateCacheKey(url, options = {}) {
      // Serialize only necessary and serializable options
      const cacheOptions = {
        useProxy: options.useProxy,
        proxyUrl: options.useProxy ? options.proxyUrl : undefined,
        selectorsHash: Utils.hashSelectors(options.selectors),
      };
      return `${url}-${JSON.stringify(cacheOptions)}`;
    },

    // Generates a hash for selectors to include in cache key
    hashSelectors(selectors) {
      if (!selectors) return '';
      try {
        return btoa(JSON.stringify(selectors));
      } catch {
        return '';
      }
    },

    // Validates and normalizes CSS selectors
    normalizeSelector(selector) {
      if (typeof selector !== 'string') throw new Error('Selector must be a string');
      try {
        document.createDocumentFragment().querySelector(selector);
        return selector;
      } catch {
        throw new Error(`Invalid CSS selector: "${selector}"`);
      }
    },

    // Extracts text from an element, optionally preserving formatting
    extractText(element, options = {}) {
      if (!element) return '';
      const clone = element.cloneNode(true);
      // Remove scripts and styles
      clone.querySelectorAll('script, style').forEach(el => el.remove());
      if (options.preserveFormatting) {
        // Replace line breaks with newline characters
        clone.querySelectorAll('br').forEach(br => {
          const textNode = document.createTextNode('\n');
          br.parentNode.replaceChild(textNode, br);
        });
        clone.querySelectorAll('p').forEach(p => {
          const textNode = document.createTextNode('\n\n');
          p.parentNode.insertBefore(textNode, p);
        });
      }
      return clone.textContent.trim();
    },

    // Removes scripts and styles from HTML string
    removeScriptsAndStyles(html) {
      try {
        const doc = new DOMParser().parseFromString(html, 'text/html');
        doc.querySelectorAll('script, style').forEach(el => el.remove());
        return doc.documentElement.outerHTML;
      } catch (e) {
        console.error('Error parsing HTML for script/style removal:', e);
        return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
                   .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
      }
    },

    // Chunks an array into smaller arrays of specified size
    chunkArray(array, size) {
      if (!Array.isArray(array)) throw new Error('Input must be an array');
      if (!Number.isInteger(size) || size <= 0) throw new Error('Chunk size must be a positive integer');
      const result = [];
      for (let i = 0; i < array.length; i += size) {
        result.push(array.slice(i, i + size));
      }
      return result;
    },

  };

  /**
   * Handles storing scraped data
   */
  class StorageHandler {
    constructor() {
      this.dbPromise = null;
      this.storeName = 'scrapedData';
      if ('indexedDB' in window) {
        this.storageType = 'indexeddb';
        this.dbPromise = this.initIndexedDB();
      } else if ('localStorage' in window && this._isLocalStorageAccessible()) {
        this.storageType = 'localstorage';
      } else {
        this.storageType = 'memory';
        this.memoryStorage = {};
      }
    }

    _isLocalStorageAccessible() {
      try {
        const testKey = '__test__';
        localStorage.setItem(testKey, testKey);
        localStorage.removeItem(testKey);
        return true;
      } catch {
        console.warn('localStorage is not accessible. Using in-memory storage.');
        return false;
      }
    }

    async initIndexedDB(dbName = 'ScraperLibDB') {
      return new Promise((resolve, reject) => {
        const request = indexedDB.open(dbName, 1);

        request.onerror = () => reject(new Error('IndexedDB initialization failed'));

        request.onupgradeneeded = (event) => {
          const db = event.target.result;
          if (!db.objectStoreNames.contains(this.storeName)) {
            const store = db.createObjectStore(this.storeName, { keyPath: 'id', autoIncrement: true });
            store.createIndex('url', 'url', { unique: false });
            store.createIndex('timestamp', 'timestamp', { unique: false });
          }
        };

        request.onsuccess = (event) => {
          const db = event.target.result;
          db.onerror = (event) => {
            console.error('Database error:', event.target.error);
          };
          resolve(db);
        };
      });
    }

    async store(url, data, format, options = {}) {
      const timestamp = Date.now();
      const storeData = { url, data, timestamp };

      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readwrite');
              const store = transaction.objectStore(this.storeName);
              const request = store.add(storeData);

              request.onsuccess = () => resolve(true);
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          try {
            const key = `scraper-${url}-${timestamp}`;
            localStorage.setItem(key, JSON.stringify(storeData));
            return true;
          } catch (e) {
            if (e.name === 'QuotaExceededError' || e.name === 'NS_ERROR_DOM_QUOTA_REACHED') {
              this._cleanupLocalStorage();
              const key = `scraper-${url}-${timestamp}`;
              localStorage.setItem(key, JSON.stringify(storeData));
              return true;
            }
            throw e;
          }

        case 'memory':
          this.memoryStorage[url] = storeData;
          return true;

        case 'custom':
          if (options.customStore && typeof options.customStore.save === 'function') {
            await options.customStore.save(url, data);
            return true;
          }
          throw new Error('Invalid custom storage handler');

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }

    _cleanupLocalStorage() {
      const keys = Object.keys(localStorage).filter(key => key.startsWith('scraper-'));
      if (keys.length > 100) {
        // Sort keys by timestamp and remove the oldest ones
        keys.sort((a, b) => {
          try {
            const itemA = JSON.parse(localStorage.getItem(a));
            const itemB = JSON.parse(localStorage.getItem(b));
            return itemA.timestamp - itemB.timestamp;
          } catch (e) {
            // Remove invalid entries
            localStorage.removeItem(a);
            localStorage.removeItem(b);
            return 0;
          }
        });
        for (let i = 0; i < keys.length - 100; i++) {
          localStorage.removeItem(keys[i]);
        }
      }
    }

    async retrieve(url, format) {
      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readonly');
              const store = transaction.objectStore(this.storeName);
              const index = store.index('url');
              const request = index.getAll(url);

              request.onsuccess = () => {
                const results = request.result;
                if (results && results.length > 0) {
                  results.sort((a, b) => b.timestamp - a.timestamp);
                  resolve(results[0].data);
                } else {
                  resolve(null);
                }
              };
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          const keys = Object.keys(localStorage)
            .filter(k => k.startsWith(`scraper-${url}-`))
            .sort((a, b) => {
              try {
                const itemA = JSON.parse(localStorage.getItem(a));
                const itemB = JSON.parse(localStorage.getItem(b));
                return itemB.timestamp - itemA.timestamp;
              } catch (e) {
                // Remove invalid entries
                localStorage.removeItem(a);
                localStorage.removeItem(b);
                return 0;
              }
            });
          if (keys.length > 0) {
            try {
              const item = JSON.parse(localStorage.getItem(keys[0]));
              return item.data;
            } catch (e) {
              // Invalid JSON, remove the item
              localStorage.removeItem(keys[0]);
              return null;
            }
          }
          return null;

        case 'memory':
          return this.memoryStorage[url] ? this.memoryStorage[url].data : null;

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }
  }

  /**
   * Handles rate limiting to respect website policies
   */
  class RateLimiter {
    constructor(requestsPerSecond = 2) {
      this.requestsPerSecond = Math.max(1, requestsPerSecond);
      this.interval = 1000 / this.requestsPerSecond;
      this.queue = [];
      this.lastRequestTime = 0;
      this.isProcessing = false;
    }

    async acquireToken() {
      return new Promise(resolve => {
        this.queue.push(resolve);
        this._processQueue();
      });
    }

    _processQueue() {
      if (this.isProcessing || this.queue.length === 0) return;
      this.isProcessing = true;

      const now = Date.now();
      const elapsed = now - this.lastRequestTime;
      const delay = Math.max(this.interval - elapsed, 0);

      setTimeout(() => {
        this.lastRequestTime = Date.now();
        const resolve = this.queue.shift();
        if (resolve) resolve();
        this.isProcessing = false;
        if (this.queue.length > 0) {
          this._processQueue();
        }
      }, delay);
    }
  }

  /**
   * Main ScraperLib class
   */
  class ScraperLib {
    constructor(options = {}) {
      this.defaultSettings = {
        rateLimitRequests: 2,
        maxConcurrent: 5,
        timeout: 30000,             // Request timeout in milliseconds
        retries: 3,                 // Number of retries on failure
        retryDelay: 1000,           // Delay between retries
        maxRetryDelay: 30000,       // Maximum delay between retries
        cacheTime: 3600000,         // Cache time in milliseconds (1 hour)
        outputFormat: 'json',       // Output format: 'json', 'indexeddb', 'localstorage', 'memory', 'custom'
        useCache: false,            // Use cache
        headers: {},                // Custom headers
        preserveFormatting: false,  // Preserve formatting when extracting text
        validateSelectors: true,    // Validate CSS selectors
        maxResponseSize: 10 * 1024 * 1024, // Max response size in bytes (10 MB)
        useProxy: false,            // Use proxy
        proxyUrl: '',               // Proxy URL
        sinceModified: false,       // Send 'If-Modified-Since' header
      };

      // Merge and validate default settings with user options
      this.settings = { ...this.defaultSettings, ...options };
      this._validateSettings(this.settings);

      // Initialize per-URL settings
      this.urlSettings = new Map();

      this.storage = new StorageHandler();
      this.rateLimiter = new RateLimiter(this.settings.rateLimitRequests);
      this.cache = null;
      this.errorHandlers = new Map();

      // Initialize cache and wait for initialization
      if (this.settings.useCache) {
        this.cachePromise = Utils.initCache().then(cache => {
          this.cache = cache;
        });
      } else {
        this.cachePromise = Promise.resolve();
      }
    }

    _validateSettings(settings) {
      const positiveNumberProps = ['rateLimitRequests', 'maxConcurrent', 'retries', 'retryDelay', 'maxRetryDelay', 'cacheTime', 'timeout', 'maxResponseSize'];
      positiveNumberProps.forEach(prop => {
        if (typeof settings[prop] !== 'number' || settings[prop] < 0) {
          throw new Error(`Invalid setting "${prop}": must be a positive number`);
        }
      });
    }

    /**
     * Main scraping method
     */
    async scrape(configs, globalOptions = {}) {
      if (typeof configs !== 'object' || configs === null || Array.isArray(configs)) {
        throw new Error('Configs must be an object with URL keys and configuration values');
      }

      // Merge global options into settings
      const globalSettings = { ...this.settings, ...globalOptions };
      this._validateSettings(globalSettings);

      // Wait for cache initialization
      await this.cachePromise;

      const results = {};
      const urls = Object.keys(configs);
      if (urls.length === 0) {
        return results;
      }

      const chunks = Utils.chunkArray(urls, globalSettings.maxConcurrent);

      for (const chunk of chunks) {
        const promises = chunk.map(url => {
          return this._processUrl(url, configs[url], globalSettings)
            .then(data => ({ url, success: true, data }))
            .catch(error => ({ url, success: false, error: error.message }));
        });

        const chunkResults = await Promise.all(promises);

        for (const result of chunkResults) {
          if (result.success) {
            results[result.url] = result.data;
          } else {
            const handler = this.errorHandlers.get(result.url) || this.errorHandlers.get('default');
            if (handler) {
              try {
                const data = await handler(new Error(result.error), result.url);
                results[result.url] = data;
              } catch (e) {
                results[result.url] = { error: e.message };
              }
            } else {
              results[result.url] = { error: result.error };
            }
          }
        }
      }

      return results;
    }

    /**
     * Process a single URL with retries and error handling
     */
    async _processUrl(url, config, globalSettings) {
      if (!Utils.isValidURL(url)) {
        throw new Error(`Invalid URL: ${url}`);
      }

      // Merge global settings with per-URL settings
      const settings = { ...globalSettings, ...config.options };
      this.urlSettings.set(url, settings);

      if (settings.validateSelectors) {
        this._validateSelectors(config.selectors);
      }

      // Wait for cache initialization if needed
      if (settings.useCache) {
        await this.cachePromise;
        if (this.cache) {
          const cachedData = await this._getFromCache(url, settings);
          if (cachedData) {
            return cachedData;
          }
        }
      }

      let attempts = 0;
      while (attempts < settings.retries) {
        attempts += 1;
        await this.rateLimiter.acquireToken();
        const options = this._prepareFetchOptions(config, settings);
        try {
          const response = await this._fetchWithTimeout(url, options, settings);
          if (!response.ok) {
            if (response.status === 304 && settings.useCache && this.cache) {
              const cachedData = await this._getFromCache(url, settings);
              if (cachedData) {
                return cachedData;
              }
            }
            throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
          }

          const contentType = response.headers.get('Content-Type') || '';
          if (!contentType.includes('text/html')) {
            throw new Error(`Invalid Content-Type: ${contentType}`);
          }

          const data = await this._extractDataFromResponse(response, config.selectors, settings);

          if (settings.outputFormat !== 'json') {
            await this.storage.store(url, data, settings.outputFormat, { customStore: settings.customStore });
          }

          if (settings.useCache && this.cache) {
            await this._storeInCache(url, data, settings);
          }

          return data;
        } catch (error) {
          if (attempts >= settings.retries) {
            throw error;
          }
          // Exponential backoff with jitter
          const delay = Math.min(
            settings.retryDelay * Math.pow(2, attempts - 1) * (0.5 + Math.random() * 0.5),
            settings.maxRetryDelay
          );
          await new Promise(resolve => setTimeout(resolve, delay));
        }
      }
    }

    /**
     * Validates the CSS selectors
     */
    _validateSelectors(selectors) {
      if (typeof selectors !== 'object' || selectors === null) {
        throw new Error('Selectors must be an object');
      }
      for (const key of Object.keys(selectors)) {
        const sel = selectors[key];
        if (typeof sel === 'string') {
          Utils.normalizeSelector(sel);
        } else if (sel && typeof sel.selector === 'string') {
          Utils.normalizeSelector(sel.selector);
        } else {
          throw new Error(`Invalid selector configuration for key "${key}"`);
        }
      }
    }

    /**
     * Prepares fetch options including headers
     */
    _prepareFetchOptions(config, settings) {
      // Remove disallowed headers
      const disallowedHeaders = [
        'User-Agent',
        'Referer',
        'Origin',
        'Cookie',
        'Host',
        'Accept-Encoding',
        'Connection',
        'Upgrade-Insecure-Requests',
      ];

      const headersInit = {
        ...settings.headers,
        ...(config.headers || {}),
      };

      disallowedHeaders.forEach(header => {
        delete headersInit[header];
      });

      const headers = new Headers(headersInit);

      if (settings.sinceModified) {
        headers.set('If-Modified-Since', new Date().toUTCString());
      }

      const options = {
        method: 'GET',
        headers: headers,
        mode: 'cors',
        credentials: 'omit',
        redirect: 'follow',
      };

      return options;
    }

    /**
     * Performs a fetch request with a timeout, using proxy if configured
     */
    async _fetchWithTimeout(url, options, settings) {
      const controller = new AbortController();
      options.signal = controller.signal;

      const timeoutId = setTimeout(() => {
        controller.abort();
      }, settings.timeout);

      try {
        let fetchUrl = url;
        if (settings.useProxy) {
          if (!settings.proxyUrl) {
            throw new Error('Proxy URL is not set');
          }
          fetchUrl = settings.proxyUrl + encodeURIComponent(url);
        }

        const response = await fetch(fetchUrl, options);
        clearTimeout(timeoutId);
        return response;
      } catch (error) {
        clearTimeout(timeoutId);
        if (error.name === 'AbortError') {
          throw new Error('Request timed out');
        }
        throw error;
      }
    }

    /**
     * Extracts data from the response using the provided selectors
     */
    async _extractDataFromResponse(response, selectors, settings) {
      try {
        const htmlText = await response.text();

        // Remove scripts and styles to prevent XSS and reduce parsing overhead
        const sanitizedHtml = Utils.removeScriptsAndStyles(htmlText);

        const parser = new DOMParser();
        const doc = parser.parseFromString(sanitizedHtml, 'text/html');

        const data = {};

        for (const key of Object.keys(selectors)) {
          const selConfig = selectors[key];

          let elements;
          if (typeof selConfig === 'string') {
            elements = doc.querySelectorAll(selConfig);
          } else if (selConfig && typeof selConfig.selector === 'string') {
            elements = doc.querySelectorAll(selConfig.selector);
          } else {
            throw new Error(`Invalid selector configuration for key "${key}"`);
          }

          if (!elements || elements.length === 0) {
            data[key] = [];
            continue;
          }

          data[key] = Array.from(elements).map(element => {
            if (selConfig.attr) {
              return element.getAttribute(selConfig.attr) || '';
            } else if (selConfig.html) {
              return element.innerHTML;
            } else {
              return Utils.extractText(element, { preserveFormatting: settings.preserveFormatting });
            }
          });
        }

        return data;
      } catch (e) {
        throw new Error(`Failed to extract data: ${e.message}`);
      }
    }

    /**
     * Gets data from the cache if available and not expired
     */
    async _getFromCache(url, settings) {
      if (!this.cache) return null;
      const cacheKey = Utils.generateCacheKey(url, settings);
      const cachedResponse = await this.cache.match(cacheKey);

      if (cachedResponse) {
        try {
          const cachedData = await cachedResponse.json();
          const age = Date.now() - cachedData.timestamp;
          if (age < settings.cacheTime) {
            return cachedData.data;
          }
        } catch (e) {
          console.warn('Error reading from cache:', e);
        }
      }

      return null;
    }

    /**
     * Stores data in the cache
     */
    async _storeInCache(url, data, settings) {
      if (!this.cache) return;
      const cacheKey = Utils.generateCacheKey(url, settings);
      const cachedResponse = new Response(
        JSON.stringify({ data, timestamp: Date.now() }), {
          headers: { 'Content-Type': 'application/json' }
        }
      );
      try {
        await this.cache.put(cacheKey, cachedResponse);
      } catch (e) {
        console.error('Failed to store data in cache:', e);
      }
    }

    /**
     * Registers a custom error handler
     */
    onError(urlOrDefault, handler) {
      if (typeof handler !== 'function') {
        throw new Error('Error handler must be a function');
      }
      this.errorHandlers.set(urlOrDefault, handler);
    }
  }

  // Attach ScraperLib to the global object
  global.ScraperLib = ScraperLib;

})(typeof window !== 'undefined' ? window : this);


---