Repository: mem0ai/mem0
File: embedchain/embedchain.py (Python)

Code before:
            existing_doc_id = None
        embeddings_data = chunker.create_chunks(loader, src)

        # spread chunking results
        documents = embeddings_data["documents"]
        metadatas = embeddings_data["metadatas"]
        ids = embeddings_data["ids"]
        new_doc_id = embeddings_data["doc_id"]

        if existing_doc_id and existing_doc_id == new_doc_id:
            print("Doc content has not changed. Skipping creating chunks and embeddings")
            return [], [], [], 0

        # this means that doc content has changed.
        if existing_doc_id and existing_doc_id != new_doc_id:
            print("Doc content has changed. Recomputing chunks and embeddings intelligently.")
            self.db.delete({"doc_id": existing_doc_id})

        # get existing ids, and discard doc if any common id exist.
        where = {"app_id": self.config.id} if self.config.id is not None else {}
        # where={"url": src}
        db_result = self.db.get(
            ids=ids,
            where=where,  # optional filter
        )
        existing_ids = set(db_result["ids"])

        if len(existing_ids):
            data_dict = {id: (doc, meta) for id, doc, meta in zip(ids, documents, metadatas)}
            data_dict = {id: value for id, value in data_dict.items() if id not in existing_ids}

            if not data_dict:
                print(f"All data from {src} already exists in the database.")
                # Make sure to return a matching return type
                return [], [], [], 0

            ids = list(data_dict.keys())
            documents, metadatas = zip(*data_dict.values())

        # Loop though all metadatas and add extras.
        new_metadatas = []
        for m in metadatas:
            # Add app id in metadatas so that they can be queried on later
            if self.config.id:
                m["app_id"] = self.config.id

            # Add hashed source
            m["hash"] = source_id

            # Note: Metadata is the function argument
            if metadata:

Review comment (suggestion):
This is going to throw index out of range error when `existing_embeddings_data` is empty since you are trying to fetch `0th` index from an empty array. :) 

Code after:
        elif chunker.data_type.value in [item.value for item in IndirectDataType]:
            # These types have a indirect source reference
            # As long as the reference is the same, they can be updated.
            existing_embeddings_data = self.db.get(
                where={
                    "url": src,
                },
                limit=1,
            )
            try:
                existing_doc_id = existing_embeddings_data.get("metadatas", [])[0]["doc_id"]
            except Exception:
                existing_doc_id = None
        elif chunker.data_type.value in [item.value for item in SpecialDataType]:
            # These types don't contain indirect references.
            # Through custom logic, they can be attributed to a source and be updated.
            if chunker.data_type == DataType.QNA_PAIR:
                # QNA_PAIRs update the answer if the question already exists.
                existing_embeddings_data = self.db.get(
                    where={
                        "question": src[0],
                    },
                    limit=1,
                )
                try:
                    existing_doc_id = existing_embeddings_data.get("metadatas", [])[0]["doc_id"]
                except Exception:
                    existing_doc_id = None
            else:
                raise NotImplementedError(
                    f"SpecialDataType {chunker.data_type} must have a custom logic to check for existing data"
                )
        else:
            raise TypeError(
                f"{chunker.data_type} is type {type(chunker.data_type)}. "
                "When it should be  DirectDataType, IndirectDataType or SpecialDataType."
            )

        # Create chunks
        embeddings_data = chunker.create_chunks(loader, src)

        # spread chunking results
        documents = embeddings_data["documents"]
        metadatas = embeddings_data["metadatas"]
        ids = embeddings_data["ids"]
        new_doc_id = embeddings_data["doc_id"]

        if existing_doc_id and existing_doc_id == new_doc_id:
            print("Doc content has not changed. Skipping creating chunks and embeddings")
            return [], [], [], 0