Uncaught (in promise) TypeError: Failed to execute 'cancel' on 'ReadableStream': Cannot cancel a locked stream. Fix this entirely and send the fully working full codes. /*!
 * ScraperLib.js v5.0.0
 * A comprehensive, efficient, and reliable client-side web scraping library
 * License: MIT
 */

(function (global) {
  'use strict';

  /**
   * Utility functions for validation and processing
   */
  const Utils = {

    // Validates if a string is a valid URL
    isValidURL(url) {
      if (typeof url !== 'string') return false;
      try {
        new URL(url);
        return true;
      } catch {
        return false;
      }
    },

    // Sanitizes a string to prevent XSS
    sanitize(str) {
      if (typeof str !== 'string') return '';
      const temp = document.createElement('div');
      temp.textContent = str;
      return temp.innerHTML;
    },

    // Initializes the cache if available
    async initCache() {
      if ('caches' in window) {
        try {
          return await caches.open('scraper-lib-cache');
        } catch {
          return null;
        }
      }
      return null;
    },

    // Generates a cache key based on URL and options
    generateCacheKey(url, options = {}) {
      return `${url}-${JSON.stringify(options)}`;
    },

    // Validates and normalizes CSS selectors
    normalizeSelector(selector) {
      if (typeof selector !== 'string') throw new Error('Invalid selector');
      try {
        document.createDocumentFragment().querySelector(selector);
        return selector;
      } catch {
        throw new Error(`Invalid CSS selector: ${selector}`);
      }
    },

    // Extracts text from an element, optionally preserving formatting
    extractText(element, options = {}) {
      if (!element) return '';
      if (options.preserveFormatting) {
        // Clone the element to avoid modifying the original
        const clone = element.cloneNode(true);
        // Replace line breaks with newline characters
        clone.querySelectorAll('br').forEach(br => {
          const textNode = document.createTextNode('\n');
          br.parentNode.replaceChild(textNode, br);
        });
        clone.querySelectorAll('p').forEach(p => {
          const textNode = document.createTextNode('\n\n');
          p.parentNode.insertBefore(textNode, p);
        });
        // Remove scripts and styles
        clone.querySelectorAll('script, style').forEach(el => el.remove());

        // Get the text content
        return clone.textContent.trim();
      } else {
        // Remove scripts and styles
        element.querySelectorAll('script, style').forEach(el => el.remove());
        return element.textContent.trim();
      }
    },

    // Removes scripts and styles from HTML string
    removeScriptsAndStyles(html) {
      return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
                 .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
    },

    // Releases memory, if possible
    releaseMemory() {
      if (global.gc) {
        global.gc();
      }
    },

    // Chunks an array into smaller arrays of specified size
    chunkArray(array, size) {
      const result = [];
      for (let i = 0; i < array.length; i += size) {
        result.push(array.slice(i, i + size));
      }
      return result;
    },
  };

  /**
   * Handles storing scraped data
   */
  class StorageHandler {
    constructor() {
      this.dbPromise = null;
      if ('indexedDB' in window) {
        this.storageType = 'indexeddb';
        this.dbPromise = this.initIndexedDB();
      } else if ('localStorage' in window) {
        this.storageType = 'localstorage';
      } else {
        this.storageType = 'memory';
        this.memoryStorage = {};
      }
    }

    async initIndexedDB(dbName = 'ScraperLibDB', storeName = 'scrapedData') {
      return new Promise((resolve, reject) => {
        const request = indexedDB.open(dbName, 1);

        request.onerror = () => reject(new Error('IndexedDB initialization failed'));

        request.onupgradeneeded = (event) => {
          const db = event.target.result;
          if (!db.objectStoreNames.contains(storeName)) {
            const store = db.createObjectStore(storeName, { keyPath: 'id', autoIncrement: true });
            store.createIndex('url', 'url', { unique: false });
            store.createIndex('timestamp', 'timestamp', { unique: false });
          }
        };

        request.onsuccess = (event) => {
          const db = event.target.result;
          db.onerror = (event) => {
            console.error('Database error:', event.target.error);
          };
          resolve(db);
        };
      });
    }

    async store(url, data, format, options = {}) {
      const timestamp = Date.now();
      const storeData = {
        url,
        data,
        timestamp,
      };

      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction('scrapedData', 'readwrite');
              const store = transaction.objectStore('scrapedData');
              const request = store.add(storeData);

              request.onsuccess = () => resolve(true);
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported');

        case 'localstorage':
          try {
            const key = `scraper-${url}-${timestamp}`;
            localStorage.setItem(key, JSON.stringify(storeData));
            return true;
          } catch (e) {
            if (e.name === 'QuotaExceededError') {
              this._cleanupLocalStorage();
              const key = `scraper-${url}-${timestamp}`;
              localStorage.setItem(key, JSON.stringify(storeData));
              return true;
            }
            throw e;
          }

        case 'memory':
          this.memoryStorage[url] = storeData;
          return true;

        case 'custom':
          if (options.customStore && typeof options.customStore.save === 'function') {
            await options.customStore.save(url, data);
            return true;
          }
          throw new Error('Invalid custom storage handler');

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }

    _cleanupLocalStorage() {
      const keys = Object.keys(localStorage).filter(key => key.startsWith('scraper-'));
      if (keys.length > 100) {
        // Sort keys by timestamp and remove the oldest ones
        keys.sort((a, b) => {
          const itemA = JSON.parse(localStorage.getItem(a));
          const itemB = JSON.parse(localStorage.getItem(b));
          return itemA.timestamp - itemB.timestamp;
        });
        for (let i = 0; i < keys.length - 100; i++) {
          localStorage.removeItem(keys[i]);
        }
      }
    }

    async retrieve(url, format) {
      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction('scrapedData', 'readonly');
              const store = transaction.objectStore('scrapedData');
              const index = store.index('url');
              const request = index.get(url);

              request.onsuccess = () => resolve(request.result ? request.result.data : null);
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported');

        case 'localstorage':
          const key = Object.keys(localStorage).find(k => k.startsWith(`scraper-${url}-`));
          if (key) {
            const item = JSON.parse(localStorage.getItem(key));
            return item.data;
          }
          return null;

        case 'memory':
          return this.memoryStorage[url] ? this.memoryStorage[url].data : null;

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }
  }

  /**
   * Handles rate limiting to respect website policies
   */
  class RateLimiter {
    constructor(requestsPerSecond = 2) {
      this.requestsPerSecond = requestsPerSecond;
      this.tokens = requestsPerSecond;
      this.lastRefill = Date.now();
      this.tokenInterval = 1000 / requestsPerSecond;
      this.queue = [];
    }

    async acquireToken() {
      const now = Date.now();
      const elapsed = now - this.lastRefill;

      if (elapsed > this.tokenInterval) {
        const tokensToAdd = Math.floor(elapsed / this.tokenInterval);
        this.tokens = Math.min(this.tokens + tokensToAdd, this.requestsPerSecond);
        this.lastRefill = now;
      }

      if (this.tokens > 0) {
        this.tokens -= 1;
        return;
      } else {
        await new Promise(resolve => {
          this.queue.push(resolve);
        });
        return this.acquireToken();
      }
    }

    releaseToken() {
      this.tokens += 1;
      if (this.queue.length > 0) {
        const resolve = this.queue.shift();
        resolve();
      }
    }
  }

  /**
   * Main ScraperLib class
   */
  class ScraperLib {
    constructor(options = {}) {
      this.defaultSettings = {
        rateLimitRequests: 2,
        maxConcurrent: 5,
        timeout: 30000,             // Request timeout
        retries: 3,                 // Number of retries on failure
        retryDelay: 1000,           // Delay between retries
        maxRetryDelay: 30000,       // Maximum delay between retries
        cacheTime: 3600000,         // Cache time in milliseconds (1 hour)
        outputFormat: 'json',       // Output format: 'json', 'indexeddb', 'localstorage', 'memory', 'custom'
        useCache: false,            // Use cache
        headers: {},                // Custom headers
        preserveFormatting: false,  // Preserve formatting when extracting text
        validateSelectors: true,    // Validate CSS selectors
        maxResponseSize: 10 * 1024 * 1024, // Max response size in bytes
        userAgent: navigator.userAgent,    // User agent string
        useProxy: false,            // Use proxy
        proxyUrl: '',               // Proxy URL
        sinceModified: false,       // Send 'If-Modified-Since' header
        discardResponseAfterProcessing: true, // Discard fetched content after processing
        streaming: false,           // Support streaming of extracted data
      };

      // Merge default settings with user options
      this.settings = { ...this.defaultSettings, ...options };

      // Initialize per-URL settings
      this.urlSettings = new Map();

      this.storage = new StorageHandler();
      this.rateLimiter = new RateLimiter(this.settings.rateLimitRequests);
      this.activeRequests = new Set();
      this.cache = null;
      this.errorHandlers = new Map();

      if (this.settings.useCache) {
        Utils.initCache().then(cache => {
          this.cache = cache;
        });
      }

      // Bind methods
      this.scrape = this.scrape.bind(this);
      this.onError = this.onError.bind(this);
      this.destroy = this.destroy.bind(this);
    }

    /**
     * Main scraping method
     */
    async scrape(configs, globalOptions = {}) {
      if (typeof configs !== 'object' || configs === null) {
        throw new Error('Configs must be an object');
      }

      // Merge global options into settings
      const globalSettings = { ...this.settings, ...globalOptions };

      const results = {};
      const urls = Object.keys(configs);
      const chunks = Utils.chunkArray(urls, globalSettings.maxConcurrent);

      for (const chunk of chunks) {
        const promises = chunk.map(url => this._processUrl(url, configs[url], globalSettings).then(
          data => ({ url, success: true, data }),
          error => ({ url, success: false, error: error.message })
        ));

        const chunkResults = await Promise.all(promises);

        for (const result of chunkResults) {
          if (result.success) {
            results[result.url] = result.data;
          } else {
            const handler = this.errorHandlers.get(result.url) || this.errorHandlers.get('default');
            if (handler) {
              try {
                const data = await handler(new Error(result.error), result.url);
                results[result.url] = data;
              } catch (e) {
                results[result.url] = { error: e.message };
              }
            } else {
              results[result.url] = { error: result.error };
            }
          }
        }

        Utils.releaseMemory();
      }

      return results;
    }

    /**
     * Process a single URL with retries and error handling
     */
    async _processUrl(url, config, globalSettings) {
      if (!Utils.isValidURL(url)) {
        throw new Error(`Invalid URL: ${url}`);
      }

      // Merge global settings with per-URL settings
      const settings = { ...globalSettings, ...config.options };
      this.urlSettings.set(url, settings);

      if (settings.validateSelectors) {
        this._validateSelectors(config.selectors);
      }

      if (settings.useCache && this.cache) {
        const cachedData = await this._getFromCache(url, settings);
        if (cachedData) {
          return cachedData;
        }
      }

      await this.rateLimiter.acquireToken();

      const options = this._prepareFetchOptions(config, settings);

      let attempts = 0;
      while (attempts < settings.retries) {
        attempts += 1;
        try {
          const response = await this._fetchWithTimeout(url, options, settings);
          if (!response.ok) {
            if (response.status === 304 && settings.useCache && this.cache) {
              const cachedData = await this._getFromCache(url, settings);
              if (cachedData) {
                return cachedData;
              }
              throw new Error(`HTTP error: ${response.status} - Not Modified`);
            } else {
              throw new Error(`HTTP error: ${response.status}`);
            }
          }

          const contentLength = response.headers.get('content-length');
          if (contentLength && parseInt(contentLength) > settings.maxResponseSize) {
            throw new Error('Response too large');
          }

          const data = await this._extractDataFromResponse(response, config.selectors, settings);

          if (settings.outputFormat !== 'json') {
            await this.storage.store(url, data, settings.outputFormat, { customStore: settings.customStore });
          }

          if (settings.useCache && this.cache) {
            await this._storeInCache(url, data, settings);
          }

          // Discard the response if set
          if (settings.discardResponseAfterProcessing) {
            response.body?.cancel();
          }

          // Release the token
          this.rateLimiter.releaseToken();

          return data;
        } catch (error) {
          // Release the token in case of an error
          this.rateLimiter.releaseToken();

          if (attempts >= settings.retries) {
            throw error;
          }
          // Exponential backoff for retries
          const delay = Math.min(settings.retryDelay * Math.pow(2, attempts - 1), settings.maxRetryDelay);
          await new Promise(resolve => setTimeout(resolve, delay));
        }
      }
    }

    /**
     * Validates the CSS selectors
     */
    _validateSelectors(selectors) {
      for (const key in selectors) {
        const sel = selectors[key];
        if (typeof sel === 'string') {
          Utils.normalizeSelector(sel);
        } else if (sel && typeof sel.selector === 'string') {
          Utils.normalizeSelector(sel.selector);
        } else {
          throw new Error(`Invalid selector configuration for key "${key}"`);
        }
      }
    }

    /**
     * Prepares fetch options including headers
     */
    _prepareFetchOptions(config, settings) {
      const headers = new Headers({
        'User-Agent': settings.userAgent,
        ...settings.headers,
        ...(config.headers || {}),
      });

      if (settings.sinceModified) {
        headers.set('If-Modified-Since', new Date().toUTCString());
      }

      const options = {
        method: 'GET',
        headers: headers,
        mode: 'cors',
        credentials: 'omit',
        redirect: 'follow',
      };

      return options;
    }

    /**
     * Performs a fetch request with a timeout, using proxy if configured
     */
    async _fetchWithTimeout(url, options, settings) {
      const controller = new AbortController();
      options.signal = controller.signal;
      this.activeRequests.add(controller);

      const timeoutId = setTimeout(() => {
        controller.abort();
      }, settings.timeout);

      try {
        const fetchUrl = settings.useProxy
          ? (settings.proxyUrlPerUrl && settings.proxyUrlPerUrl[url]
            ? settings.proxyUrlPerUrl[url] + encodeURIComponent(url)
            : settings.proxyUrl + encodeURIComponent(url))
          : url;
        const response = await fetch(fetchUrl, options);
        clearTimeout(timeoutId);
        this.activeRequests.delete(controller);
        return response;
      } catch (error) {
        clearTimeout(timeoutId);
        this.activeRequests.delete(controller);
        throw error;
      }
    }

    /**
     * Extracts data from the response using the provided selectors
     */
    async _extractDataFromResponse(response, selectors, settings) {
      const htmlText = await response.text();

      // Remove scripts and styles to prevent XSS and reduce parsing overhead
      const sanitizedHtml = Utils.removeScriptsAndStyles(htmlText);

      const parser = new DOMParser();
      const doc = parser.parseFromString(sanitizedHtml, 'text/html');

      const data = {};

      for (const key in selectors) {
        const selConfig = selectors[key];

        let elements;
        if (typeof selConfig === 'string') {
          elements = doc.querySelectorAll(selConfig);
        } else if (selConfig && typeof selConfig.selector === 'string') {
          elements = doc.querySelectorAll(selConfig.selector);
        } else {
          throw new Error(`Invalid selector configuration for key "${key}"`);
        }

        if (elements.length === 0) {
          data[key] = [];
          continue;
        }

        data[key] = Array.from(elements).map(element => {
          if (selConfig.attr) {
            return element.getAttribute(selConfig.attr) || '';
          } else if (selConfig.html) {
            return element.innerHTML;
          } else {
            return Utils.extractText(element, { preserveFormatting: settings.preserveFormatting });
          }
        });
      }

      return data;
    }

    /**
     * Gets data from the cache if available and not expired
     */
    async _getFromCache(url, settings) {
      if (!this.cache) return null;
      const cacheKey = Utils.generateCacheKey(url, settings);
      const cachedResponse = await this.cache.match(cacheKey);

      if (cachedResponse) {
        const cachedData = await cachedResponse.json();
        const age = Date.now() - cachedData.timestamp;
        if (age < settings.cacheTime) {
          return cachedData.data;
        }
      }

      return null;
    }

    /**
     * Stores data in the cache
     */
    async _storeInCache(url, data, settings) {
      if (!this.cache) return;
      const cacheKey = Utils.generateCacheKey(url, settings);
      const cachedResponse = new Response(JSON.stringify({ data, timestamp: Date.now() }));
      await this.cache.put(cacheKey, cachedResponse);
    }

    /**
     * Registers a custom error handler
     */
    onError(urlOrDefault, handler) {
      if (typeof handler !== 'function') {
        throw new Error('Error handler must be a function');
      }
      this.errorHandlers.set(urlOrDefault, handler);
    }

    /**
     * Cleans up active requests and resources
     */
    destroy() {
      this.activeRequests.forEach(controller => {
        if (controller && typeof controller.abort === 'function') {
          controller.abort();
        }
      });
      this.activeRequests.clear();
      this.cache = null;
      this.storage = null;
      this.errorHandlers.clear();
      Utils.releaseMemory();
    }

  }

  // Attach ScraperLib to the global object
  global.ScraperLib = ScraperLib;

})(typeof window !== 'undefined' ? window : this);