Repository: mem0ai/mem0
File: embedchain/loaders/sitemap.py (Python)

Code before:
from embedchain.helper.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.loaders.web_page import WebPageLoader
from embedchain.utils import is_readable


@register_deserializable
class SitemapLoader(BaseLoader):
    """
    This method takes a sitemap URL or local file path as input and retrieves
    all the URLs to use the WebPageLoader to load content
    of each page.
    """

    def load_data(self, sitemap_source):
        output = []
        web_page_loader = WebPageLoader()

        if urlparse(sitemap_source).scheme in ("http", "https"):
            try:
                response = requests.get(sitemap_source)
                response.raise_for_status()
                soup = BeautifulSoup(response.text, "xml")
            except requests.RequestException as e:
                logging.error(f"Error fetching sitemap from URL: {e}")
                return {"doc_id": "", "data": []}
        elif os.path.isfile(sitemap_source):
            with open(sitemap_source, "r") as file:
                soup = BeautifulSoup(file, "xml")
        else:
            raise ValueError("Invalid sitemap source. Please provide a valid URL or local file path.")

        links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"]
        if len(links) == 0:
            links = [link.text for link in soup.find_all("loc")]

        doc_id = hashlib.sha256((" ".join(links) + sitemap_source).encode()).hexdigest()

        def load_link(link):
            try:
                each_load_data = web_page_loader.load_data(link)
                if is_readable(each_load_data.get("data")[0].get("content")):
                    return each_load_data.get("data")
                else:
                    logging.warning(f"Page is not readable (too many invalid characters): {link}")
            except ParserRejectedMarkup as e:
                logging.error(f"Failed to parse {link}: {e}")
            return None

        with concurrent.futures.ThreadPoolExecutor() as executor:
            future_to_link = {executor.submit(load_link, link): link for link in links}

Review comment (suggestion):
let's just return here, we don't need to pass any result
```suggestion
                return
```

Code after:

from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.loaders.web_page import WebPageLoader


@register_deserializable
class SitemapLoader(BaseLoader):
    """
    This method takes a sitemap URL or local file path as input and retrieves
    all the URLs to use the WebPageLoader to load content
    of each page.
    """

    def load_data(self, sitemap_source):
        output = []
        web_page_loader = WebPageLoader()

        if urlparse(sitemap_source).scheme in ("http", "https"):
            try:
                response = requests.get(sitemap_source)
                response.raise_for_status()
                soup = BeautifulSoup(response.text, "xml")
            except requests.RequestException as e:
                logging.error(f"Error fetching sitemap from URL: {e}")
                return
        elif os.path.isfile(sitemap_source):
            with open(sitemap_source, "r") as file:
                soup = BeautifulSoup(file, "xml")
        else:
            raise ValueError("Invalid sitemap source. Please provide a valid URL or local file path.")

        links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"]
        if len(links) == 0:
            links = [link.text for link in soup.find_all("loc")]

        doc_id = hashlib.sha256((" ".join(links) + sitemap_source).encode()).hexdigest()

        def load_web_page(link):
            try:
                loader_data = web_page_loader.load_data(link)
                return loader_data.get("data")
            except ParserRejectedMarkup as e:
                logging.error(f"Failed to parse {link}: {e}")
            return None

        with concurrent.futures.ThreadPoolExecutor() as executor:
            future_to_link = {executor.submit(load_web_page, link): link for link in links}
            for future in tqdm(concurrent.futures.as_completed(future_to_link), total=len(links), desc="Loading pages"):
                link = future_to_link[future]
                try: