Repository: VinciGit00/Scrapegraph-ai
File: scrapegraphai/nodes/fetch_node.py (Python)

Code before:

                parsed_content = response
   
                if not self.cut:
                    parsed_content = cleanup_html(response, source)

                if  (isinstance(self.llm_model, ChatOpenAI) and not self.script_creator) or (self.force and not self.script_creator):
                    parsed_content = convert_to_md(source, input_data[0])
                compressed_document = [Document(page_content=parsed_content)]
            else:
                self.logger.warning(
                    f"Failed to retrieve contents from the webpage at url: {source}"
                )

        else:
            self.logger.info(f"--- (Fetching HTML from: {source}) ---")
            loader_kwargs = {}

            if self.node_config is not None:
                loader_kwargs = self.node_config.get("loader_kwargs", {})

            if self.browser_base is not None:
                data =  browser_base_fetch(self.browser_base.get("api_key"),
                                                          self.browser_base.get("project_id"), source)

                document = [Document(page_content= data, metadata={"source": "html file"})]
            else:
                loader = ChromiumLoader([source], headless=self.headless, **loader_kwargs)
                document = loader.load()

            if not document or not document[0].page_content.strip():
                raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.")
            parsed_content = document[0].page_content

            if  isinstance(self.llm_model, ChatOpenAI) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled:

                parsed_content = convert_to_md(document[0].page_content, input_data[0])


            compressed_document = [
                Document(page_content=parsed_content, metadata={"source": "html file"})
            ]

        state.update(
            {
                self.output[0]: compressed_document,
            }
        )

        return state


Review comment (suggestion):
```python
 if self.browser_base is not None:
    data =  browser_base_fetch(self.browser_base.get("api_key"),
                                                  self.browser_base.get("project_id"), [source])

    document = [Document(page_content=content, metadata={"source": source}) for content in data]
```

Code after:

                parsed_content = response
   
                if not self.cut:
                    parsed_content = cleanup_html(response, source)

                if  (isinstance(self.llm_model, ChatOpenAI) and not self.script_creator) or (self.force and not self.script_creator):
                    parsed_content = convert_to_md(source, input_data[0])
                compressed_document = [Document(page_content=parsed_content)]
            else:
                self.logger.warning(
                    f"Failed to retrieve contents from the webpage at url: {source}"
                )

        else:
            self.logger.info(f"--- (Fetching HTML from: {source}) ---")
            loader_kwargs = {}

            if self.node_config is not None:
                loader_kwargs = self.node_config.get("loader_kwargs", {})

            if self.browser_base is not None:
                data =  browser_base_fetch(self.browser_base.get("api_key"),
                                            self.browser_base.get("project_id"), [source])

                document = [Document(page_content=content, metadata={"source": source}) for content in data]
            else:
                loader = ChromiumLoader([source], headless=self.headless, **loader_kwargs)
                document = loader.load()

            if not document or not document[0].page_content.strip():
                raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.")
            parsed_content = document[0].page_content

            if  isinstance(self.llm_model, ChatOpenAI) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled:

                parsed_content = convert_to_md(document[0].page_content, input_data[0])


            compressed_document = [
                Document(page_content=parsed_content, metadata={"source": "html file"})
            ]

        state.update(
            {
                self.output[0]: compressed_document,
            }
        )

        return state