ronantakizawa/codereview-bench huggingface dataset:
Repository: mem0ai/mem0
File: embedchain/loaders/json.py (Python)

Code before:
import hashlib

from langchain.document_loaders.json_loader import JSONLoader

from embedchain.loaders.base_loader import BaseLoader

langchain_json_jq_schema = 'to_entries | map("\(.key): \(.value|tostring)") | .[]'


class JsonLoader(BaseLoader):
    @staticmethod
    def load_data(content):
        """Load a json file. Each data point is a key value pair."""
        data = []
        data_content = []
        loader = JSONLoader(content, text_content=False, jq_schema=langchain_json_jq_schema)
        docs = loader.load()
        for doc in docs:
            meta_data = doc.metadata
            data.append({"content": doc.page_content, "meta_data": {"url": content, "row": meta_data["seq_num"]}})
            data_content.append(doc.page_content)
        doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
        return {"doc_id": doc_id, "data": data}


Review comment (suggestion):
```suggestion
from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader
```

Code after:
import hashlib

from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader

from embedchain.loaders.base_loader import BaseLoader

langchain_json_jq_schema = 'to_entries | map("\(.key): \(.value|tostring)") | .[]'


class JSONLoader(BaseLoader):
    @staticmethod
    def load_data(content):
        """Load a json file. Each data point is a key value pair."""
        data = []
        data_content = []
        loader = LcJSONLoader(content, text_content=False, jq_schema=langchain_json_jq_schema)
        docs = loader.load()
        for doc in docs:
            meta_data = doc.metadata
            data.append({"content": doc.page_content, "meta_data": {"url": content, "row": meta_data["seq_num"]}})
            data_content.append(doc.page_content)
        doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
        return {"doc_id": doc_id, "data": data}
Repository: mem0ai/mem0
File: embedchain/loaders/json.py (Python)

Code before:
import hashlib

from langchain.document_loaders.json_loader import JSONLoader

from embedchain.loaders.base_loader import BaseLoader

langchain_json_jq_schema = 'to_entries | map("\(.key): \(.value|tostring)") | .[]'


class JsonLoader(BaseLoader):
    @staticmethod
    def load_data(content):
        """Load a json file. Each data point is a key value pair."""
        data = []
        data_content = []
        loader = JSONLoader(content, text_content=False, jq_schema=langchain_json_jq_schema)
        docs = loader.load()
        for doc in docs:
            meta_data = doc.metadata
            data.append({"content": doc.page_content, "meta_data": {"url": content, "row": meta_data["seq_num"]}})
            data_content.append(doc.page_content)
        doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
        return {"doc_id": doc_id, "data": data}


Review comment (suggestion):
```suggestion
        loader = LcJSONLoader(content, text_content=False, jq_schema=langchain_json_jq_schema)
```

Code after:
import hashlib

from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader

from embedchain.loaders.base_loader import BaseLoader

langchain_json_jq_schema = 'to_entries | map("\(.key): \(.value|tostring)") | .[]'


class JSONLoader(BaseLoader):
    @staticmethod
    def load_data(content):
        """Load a json file. Each data point is a key value pair."""
        data = []
        data_content = []
        loader = LcJSONLoader(content, text_content=False, jq_schema=langchain_json_jq_schema)
        docs = loader.load()
        for doc in docs:
            meta_data = doc.metadata
            data.append({"content": doc.page_content, "meta_data": {"url": content, "row": meta_data["seq_num"]}})
            data_content.append(doc.page_content)
        doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
        return {"doc_id": doc_id, "data": data}
Repository: mem0ai/mem0
File: tests/loaders/test_json.py (Python)

Code before:
import hashlib
from unittest.mock import patch

from langchain.docstore.document import Document
from langchain.document_loaders.json_loader import JSONLoader

from embedchain.loaders.json import JsonLoader


def test_load_data():
    mock_document = [
        Document(page_content="content1", metadata={"seq_num": 1}),
        Document(page_content="content2", metadata={"seq_num": 2}),
    ]
    with patch.object(JSONLoader, "load", return_value=mock_document):
        content = "temp.json"

        result = JsonLoader.load_data(content)

        assert "doc_id" in result
        assert "data" in result

        expected_data = [
            {"content": "content1", "meta_data": {"url": content, "row": 1}},
            {"content": "content2", "meta_data": {"url": content, "row": 2}},
        ]

        assert result["data"] == expected_data

        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
        assert result["doc_id"] == expected_doc_id


Review comment (suggestion):
```suggestion
from embedchain.loaders.json import JSONLoader
```

Code after:
import hashlib
from unittest.mock import patch

from langchain.docstore.document import Document
from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader

from embedchain.loaders.json import JSONLoader


def test_load_data():
    mock_document = [
        Document(page_content="content1", metadata={"seq_num": 1}),
        Document(page_content="content2", metadata={"seq_num": 2}),
    ]
    with patch.object(LcJSONLoader, "load", return_value=mock_document):
        content = "temp.json"

        result = JsonLoader.load_data(content)

        assert "doc_id" in result
        assert "data" in result

        expected_data = [
            {"content": "content1", "meta_data": {"url": content, "row": 1}},
            {"content": "content2", "meta_data": {"url": content, "row": 2}},
        ]

        assert result["data"] == expected_data

        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
        assert result["doc_id"] == expected_doc_id
Repository: mem0ai/mem0
File: tests/loaders/test_json.py (Python)

Code before:
import hashlib
from unittest.mock import patch

from langchain.docstore.document import Document
from langchain.document_loaders.json_loader import JSONLoader

from embedchain.loaders.json import JsonLoader


def test_load_data():
    mock_document = [
        Document(page_content="content1", metadata={"seq_num": 1}),
        Document(page_content="content2", metadata={"seq_num": 2}),
    ]
    with patch.object(JSONLoader, "load", return_value=mock_document):
        content = "temp.json"

        result = JsonLoader.load_data(content)

        assert "doc_id" in result
        assert "data" in result

        expected_data = [
            {"content": "content1", "meta_data": {"url": content, "row": 1}},
            {"content": "content2", "meta_data": {"url": content, "row": 2}},
        ]

        assert result["data"] == expected_data

        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()

Review comment (suggestion):
```suggestion
from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader
```

Code after:
import hashlib
from unittest.mock import patch

from langchain.docstore.document import Document
from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader

from embedchain.loaders.json import JSONLoader


def test_load_data():
    mock_document = [
        Document(page_content="content1", metadata={"seq_num": 1}),
        Document(page_content="content2", metadata={"seq_num": 2}),
    ]
    with patch.object(LcJSONLoader, "load", return_value=mock_document):
        content = "temp.json"

        result = JsonLoader.load_data(content)

        assert "doc_id" in result
        assert "data" in result

        expected_data = [
            {"content": "content1", "meta_data": {"url": content, "row": 1}},
            {"content": "content2", "meta_data": {"url": content, "row": 2}},
        ]

        assert result["data"] == expected_data

        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
Repository: mem0ai/mem0
File: tests/loaders/test_json.py (Python)

Code before:
import hashlib
from unittest.mock import patch

from langchain.docstore.document import Document
from langchain.document_loaders.json_loader import JSONLoader

from embedchain.loaders.json import JsonLoader


def test_load_data():
    mock_document = [
        Document(page_content="content1", metadata={"seq_num": 1}),
        Document(page_content="content2", metadata={"seq_num": 2}),
    ]
    with patch.object(JSONLoader, "load", return_value=mock_document):
        content = "temp.json"

        result = JsonLoader.load_data(content)

        assert "doc_id" in result
        assert "data" in result

        expected_data = [
            {"content": "content1", "meta_data": {"url": content, "row": 1}},
            {"content": "content2", "meta_data": {"url": content, "row": 2}},
        ]

        assert result["data"] == expected_data

        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
        assert result["doc_id"] == expected_doc_id


Review comment (suggestion):
```suggestion
    with patch.object(LcJSONLoader, "load", return_value=mock_document):
```

Code after:
import hashlib
from unittest.mock import patch

from langchain.docstore.document import Document
from langchain.document_loaders.json_loader import JSONLoader as LcJSONLoader

from embedchain.loaders.json import JSONLoader


def test_load_data():
    mock_document = [
        Document(page_content="content1", metadata={"seq_num": 1}),
        Document(page_content="content2", metadata={"seq_num": 2}),
    ]
    with patch.object(LcJSONLoader, "load", return_value=mock_document):
        content = "temp.json"

        result = JsonLoader.load_data(content)

        assert "doc_id" in result
        assert "data" in result

        expected_data = [
            {"content": "content1", "meta_data": {"url": content, "row": 1}},
            {"content": "content2", "meta_data": {"url": content, "row": 2}},
        ]

        assert result["data"] == expected_data

        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
        assert result["doc_id"] == expected_doc_id
Repository: mem0ai/mem0
File: embedchain/chunkers/gmail.py (Python)

Code before:
from typing import Optional

from langchain.text_splitter import RecursiveCharacterTextSplitter

from embedchain.chunkers.base_chunker import BaseChunker
from embedchain.config.add_config import ChunkerConfig
from embedchain.helper.json_serializable import register_deserializable


@register_deserializable
class GMAILChunker(BaseChunker):
    """Chunker for gmail."""

    def __init__(self, config: Optional[ChunkerConfig] = None):
        if config is None:
            config = ChunkerConfig(chunk_size=1000, chunk_overlap=0, length_function=len)
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=config.chunk_size,
            chunk_overlap=config.chunk_overlap,
            length_function=config.length_function,
        )
        super().__init__(text_splitter)


Review comment (suggestion):
```suggestion
class GmailChunker(BaseChunker):
```

Code after:
from typing import Optional

from langchain.text_splitter import RecursiveCharacterTextSplitter

from embedchain.chunkers.base_chunker import BaseChunker
from embedchain.config.add_config import ChunkerConfig
from embedchain.helper.json_serializable import register_deserializable


@register_deserializable
class GmailChunker(BaseChunker):
    """Chunker for gmail."""

    def __init__(self, config: Optional[ChunkerConfig] = None):
        if config is None:
            config = ChunkerConfig(chunk_size=1000, chunk_overlap=0, length_function=len)
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=config.chunk_size,
            chunk_overlap=config.chunk_overlap,
            length_function=config.length_function,
        )
        super().__init__(text_splitter)
Repository: mem0ai/mem0
File: embedchain/embedchain.py (Python)

Code before:
        self.db.set_collection_name(name)
        # Create the collection if it does not exist
        self.db._get_or_create_collection(name)
        # TODO: Check whether it is necessary to assign to the `self.collection` attribute,
        # since the main purpose is the creation.

    def count(self) -> int:
        """
        Count the number of embeddings.

        DEPRECATED IN FAVOR OF `db.count()`

        :return: The number of embeddings.
        :rtype: int
        """
        logging.warning("DEPRECATION WARNING: Please use `app.db.count()` instead of `app.count()`.")
        return self.db.count()

    def reset(self):
        """
        Resets the database. Deletes all embeddings irreversibly.
        `App` does not have to be reinitialized after using this method.
        """
        self.db.reset()
        self.cursor.execute("DELETE FROM data_sources WHERE pipeline_id = ?", (self.config.id,))
        self.llm.memory.delete_chat_history(app_id=self.config.id)
        self.connection.commit()
        # Send anonymous telemetry
        self.telemetry.capture(event_name="reset", properties=self._telemetry_props)


Review comment (refactor):
Lets move this to a new function

```python
def clear_history(self):
        self.llm.memory.delete_chat_history(app_id=self.config.id)
```

Code after:
    def count(self) -> int:
        """
        Count the number of embeddings.

        DEPRECATED IN FAVOR OF `db.count()`

        :return: The number of embeddings.
        :rtype: int
        """
        logging.warning("DEPRECATION WARNING: Please use `app.db.count()` instead of `app.count()`.")
        return self.db.count()

    def reset(self):
        """
        Resets the database. Deletes all embeddings irreversibly.
        `App` does not have to be reinitialized after using this method.
        """
        self.db.reset()
        self.cursor.execute("DELETE FROM data_sources WHERE pipeline_id = ?", (self.config.id,))
        self.connection.commit()
        self.clear_history()
        # Send anonymous telemetry
        self.telemetry.capture(event_name="reset", properties=self._telemetry_props)

    def clear_history(self):
        self.llm.memory.delete_chat_history(app_id=self.config.id)
Repository: mem0ai/mem0
File: embedchain/memory/message.py (Python)

Code before:
import logging
from typing import Any, Dict, Optional

from embedchain.helper.json_serializable import JSONSerializable


class BaseMessage(JSONSerializable):
    """
    The base abstract message class.

    Messages are the inputs and outputs of Models.
    """

    content: str
    """The string content of the message."""

    by: str
    """The creator of the message. AI, Human, Bot etc."""

    metadata: Dict[str, Any]
    """Any additional info."""

    def __init__(self, content: str, by: str, metadata: Optional[Dict[str, Any]] = None) -> None:
        super().__init__()
        self.content = content
        self.by = by
        self.metadata = metadata

    @property
    def type(self) -> str:
        """Type of the Message, used for serialization."""

    @classmethod
    def is_lc_serializable(cls) -> bool:
        """Return whether this class is serializable."""
        return True

    def __str__(self) -> str:
        return f"{self.by}: {self.content}"


class ChatMessage(JSONSerializable):
    """
    The base abstract chat message class.

    Chat messages are the pair of (question, answer) conversation
    between human and model.
    """

Review comment (suggestion):
```suggestion
    def __init__(self, content: str, creator: str, metadata: Optional[Dict[str, Any]] = None) -> None:
```

Code after:
import logging
from typing import Any, Dict, Optional

from embedchain.helper.json_serializable import JSONSerializable


class BaseMessage(JSONSerializable):
    """
    The base abstract message class.

    Messages are the inputs and outputs of Models.
    """

    # The string content of the message.
    content: str

    # The creator of the message. AI, Human, Bot etc.
    by: str

    # Any additional info.
    metadata: Dict[str, Any]

    def __init__(self, content: str, creator: str, metadata: Optional[Dict[str, Any]] = None) -> None:
        super().__init__()
        self.content = content
        self.creator = creator
        self.metadata = metadata

    @property
    def type(self) -> str:
        """Type of the Message, used for serialization."""

    @classmethod
    def is_lc_serializable(cls) -> bool:
        """Return whether this class is serializable."""
        return True

    def __str__(self) -> str:
        return f"{self.creator}: {self.content}"


class ChatMessage(JSONSerializable):
    """
    The base abstract chat message class.

    Chat messages are the pair of (question, answer) conversation
    between human and model.
    """
Repository: mem0ai/mem0
File: embedchain/memory/message.py (Python)

Code before:
    The base abstract chat message class.

    Chat messages are the pair of (question, answer) conversation
    between human and model.
    """

    human_message: Optional[BaseMessage] = None
    ai_message: Optional[BaseMessage] = None

    def add_user_message(self, message: str, metadata: Optional[dict] = None):
        if self.human_message:
            logging.info(
                "Human message already exists in the chat message,\
                overwritting it with new message."
            )

        self.human_message = BaseMessage(content=message, by="human", metadata=metadata)

    def add_ai_message(self, message: str, metadata: Optional[dict] = None):
        if self.ai_message:
            logging.info(
                "AI message already exists in the chat message,\
                overwritting it with new message."
            )

        self.ai_message = BaseMessage(content=message, by="ai", metadata=metadata)

    def __str__(self) -> str:
        return f"{self.human_message} | {self.ai_message}"


Review comment (suggestion):
```suggestion
        self.ai_message = BaseMessage(content=message, creator="ai", metadata=metadata)
```

Code after:
    The base abstract chat message class.

    Chat messages are the pair of (question, answer) conversation
    between human and model.
    """

    human_message: Optional[BaseMessage] = None
    ai_message: Optional[BaseMessage] = None

    def add_user_message(self, message: str, metadata: Optional[dict] = None):
        if self.human_message:
            logging.info(
                "Human message already exists in the chat message,\
                overwritting it with new message."
            )

        self.human_message = BaseMessage(content=message, creator="human", metadata=metadata)

    def add_ai_message(self, message: str, metadata: Optional[dict] = None):
        if self.ai_message:
            logging.info(
                "AI message already exists in the chat message,\
                overwritting it with new message."
            )

        self.ai_message = BaseMessage(content=message, creator="ai", metadata=metadata)

    def __str__(self) -> str:
        return f"{self.human_message} | {self.ai_message}"
Repository: mem0ai/mem0
File: embedchain/chunkers/rss_feed.py (Python)

Code before:
from typing import Optional

from langchain.text_splitter import RecursiveCharacterTextSplitter

from embedchain.chunkers.base_chunker import BaseChunker
from embedchain.config.add_config import ChunkerConfig
from embedchain.helper.json_serializable import register_deserializable


@register_deserializable
class RSSFeedChunker(BaseChunker):
    """Chunker for RSS Feed."""

    def __init__(self, config: Optional[ChunkerConfig] = None):
        if config is None:
            config = ChunkerConfig(chunk_size=1000, chunk_overlap=0, length_function=len)
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=config.chunk_size,
            chunk_overlap=config.chunk_overlap,
            length_function=config.length_function,
        )
        super().__init__(text_splitter)


Review comment (suggestion):
```suggestion
            config = ChunkerConfig(chunk_size=2000, chunk_overlap=0, length_function=len)
```

Code after:
from typing import Optional

from langchain.text_splitter import RecursiveCharacterTextSplitter

from embedchain.chunkers.base_chunker import BaseChunker
from embedchain.config.add_config import ChunkerConfig
from embedchain.helper.json_serializable import register_deserializable


@register_deserializable
class RSSFeedChunker(BaseChunker):
    """Chunker for RSS Feed."""

    def __init__(self, config: Optional[ChunkerConfig] = None):
        if config is None:
            config = ChunkerConfig(chunk_size=2000, chunk_overlap=0, length_function=len)
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=config.chunk_size,
            chunk_overlap=config.chunk_overlap,
            length_function=config.length_function,
        )
        super().__init__(text_splitter)
Repository: mem0ai/mem0
File: embedchain/loaders/substack.py (Python)

Code before:
import concurrent.futures
import hashlib
import logging

import requests

try:
    from bs4 import BeautifulSoup
    from bs4.builder import ParserRejectedMarkup
except ImportError:
    raise ImportError(
        'Sitemap requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
    ) from None

from embedchain.helper.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.utils import is_readable


@register_deserializable
class SubstackLoader(BaseLoader):
    """
    This method takes a sitemap URL as input and retrieves
    all the URLs to use the WebPageLoader to load content
    of each page.
    """

    def load_data(self, url: str):
        output = []
        response = requests.get(url)
        response.raise_for_status()

        soup = BeautifulSoup(response.text, "xml")
        links = [link.text for link in soup.find_all("loc") if link.parent.name == "url" and "/p/" in link.text]
        if len(links) == 0:
            links = [link.text for link in soup.find_all("loc") if "/p/" in link.text]


Review comment (suggestion):
```suggestion
        'Substack requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
```

Code after:
import time
import hashlib
import logging
import requests

from embedchain.helper.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.utils import is_readable


@register_deserializable
class SubstackLoader(BaseLoader):
    """
    This method takes a sitemap URL as input and retrieves
    all the URLs to use the WebPageLoader to load content
    of each page.
    """

    def load_data(self, url: str):
        try:
            from bs4 import BeautifulSoup
            from bs4.builder import ParserRejectedMarkup
        except ImportError:
            raise ImportError(
                'Substack requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
            ) from None

        output = []
        response = requests.get(url)
        response.raise_for_status()

        soup = BeautifulSoup(response.text, "xml")
        links = [link.text for link in soup.find_all("loc") if link.parent.name == "url" and "/p/" in link.text]
        if len(links) == 0:
            links = [link.text for link in soup.find_all("loc") if "/p/" in link.text]

        doc_id = hashlib.sha256((" ".join(links) + url).encode()).hexdigest()