Repository: mem0ai/mem0
File: embedchain/embedchain.py (Python)

Code before:
            # Add hashed source
            m["hash"] = source_hash

            # Note: Metadata is the function argument
            if metadata:
                # Spread whatever is in metadata into the new object.
                m.update(metadata)

            new_metadatas.append(m)
        metadatas = new_metadatas

        if dry_run:
            return list(documents), metadatas, ids, 0

        # Count before, to calculate a delta in the end.
        chunks_before_addition = self.db.count()

        self.db.add(documents=documents, metadatas=metadatas, ids=ids, **kwargs)
        count_new_chunks = self.db.count() - chunks_before_addition

        print(f"Successfully saved {src} ({chunker.data_type}). New chunks count: {count_new_chunks}")
        return list(documents), metadatas, ids, count_new_chunks

    @staticmethod
    def _format_result(results):
        return [
            (Document(page_content=result[0], metadata=result[1] or {}), result[2])
            for result in zip(
                results["documents"][0],
                results["metadatas"][0],
                results["distances"][0],
            )
        ]

    def _retrieve_from_database(
        self,
        input_query: str,
        config: Optional[BaseLlmConfig] = None,
        where=None,
        citations: bool = False,
        **kwargs: Optional[dict[str, Any]],
    ) -> Union[list[tuple[str, str, str]], list[str]]:
        """
        Queries the vector database based on the given input query.
        Gets relevant doc based on the query

        :param input_query: The query to use.
        :type input_query: str
        :param config: The query configuration, defaults to None
        :type config: Optional[BaseLlmConfig], optional
        :param where: A dictionary of key-value pairs to filter the database results, defaults to None

Review comment (performance):
Can we pass the `batch_size` as kwarg and set its default value to 2048? That way, if someone is using some different embedding model service, they can configure it easily. 

Code after:
            # Add hashed source
            m["hash"] = source_hash

            # Note: Metadata is the function argument
            if metadata:
                # Spread whatever is in metadata into the new object.
                m.update(metadata)

            new_metadatas.append(m)
        metadatas = new_metadatas

        if dry_run:
            return list(documents), metadatas, ids, 0
        
        # Count before, to calculate a delta in the end.
        chunks_before_addition = self.db.count()

        
        # Filter out empty documents and ensure they meet the API requirements
        valid_documents = [doc for doc in documents if doc and isinstance(doc, str)]

        documents = valid_documents

        # Chunk documents into batches of 2048 and handle each batch
        # helps wigth large loads of embeddings  that hit OpenAI limits
        document_batches = [documents[i:i+2048] for i in range(0, len(documents), 2048)]
        for batch in document_batches:
            try:
                # Add only valid batches
                if batch:
                    self.db.add(documents=batch, metadatas=metadatas, ids=ids, **kwargs)
            except Exception as e:
                print(f"Failed to add batch due to a bad request: {e}")
                # Handle the error, e.g., by logging, retrying, or skipping
                pass


        count_new_chunks = self.db.count() - chunks_before_addition
        print(f"Successfully saved {src} ({chunker.data_type}). New chunks count: {count_new_chunks}")
        
        return list(documents), metadatas, ids, count_new_chunks


    @staticmethod
    def _format_result(results):
        return [
            (Document(page_content=result[0], metadata=result[1] or {}), result[2])
            for result in zip(
                results["documents"][0],
                results["metadatas"][0],
                results["distances"][0],