Repository: neo4j/neo4j-graphrag-python

Issue:
"not in" filtering is not working.
Hi, I'd like to use the pre-filtering feature. I'm using `VectorRetriever`.
[pre-filters](https://neo4j.com/docs/neo4j-graphrag-python/current/user_guide_rag.html#pre-filters)

But following error occurred.
```
neo4j.exceptions.CypherSyntaxError: {code: Neo.ClientError.Statement.SyntaxError} {message: Invalid input 'NOT': expected an expression or ')' (line 1, column 131 (offset: 130))
"MATCH (node:`Chunk`) WHERE node.`embedding` IS NOT NULL AND size(node.`embedding`) = toInteger($embedding_dimension) AND (node.id NOT IN $param_0) WITH node, vector.similarity.cosine(node.`embedding`, $query_vector) AS score ORDER BY score DESC LIMIT $top_k RETURN node {.id, .text, .content_id} AS node, labels(node) AS nodeLabels, elementId(node) AS elementId, elementId(node) AS id, score"
```

My implementation is:
```python
filters ={
    "id": {
        "$nin": docIds # list of strings
    }
retriever_result = retriever.search(query_text=query_text, filters=filters)
```

I think `WHERE NOT thing IN list` not `WHERE thing NOT IN list` is the correct syntax.
It seems that the current version of `_single_condition_cypher()` function in `filters.py` can't handle this.
```python
def _single_condition_cypher(
    field: str,
    native_operator_class: Type[Operator],
    value: Any,
    param_store: ParameterStore,
    node_alias: str = DEFAULT_NODE_ALIAS,
) -> str:
    native_op = native_operator_class(node_alias=node_alias)
    param_name = param_store.add(native_op.cleaned_value(value))
    query_snippet = f"{native_op.lhs(field)} {native_op.CYPHER_OPERATOR} ${param_name}"
    return query_snippet
```

Discussion:
@stellasia: Hi @bldev2473 ,

Thank you for raising this issue. A fix has just been merged and will be part of the next release.
@stellasia: 1.7.0 has just been released, with the fix, so closing this issue now.

Solution patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2abf200..fcec454 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,11 @@
 
 - Improved log output readability in Retrievers and GraphRAG and added embedded vector to retriever result metadata for debugging.
 
+### Fixed
+
+- Fixed a bug where the `$nin` operator for metadata pre-filtering in retrievers would create an invalid Cypher query.
+
+
 ## 1.6.1
 
 ### Added
diff --git a/examples/customize/retrievers/use_pre_filters.py b/examples/customize/retrievers/use_pre_filters.py
index cc8c5e2..6e381cf 100644
--- a/examples/customize/retrievers/use_pre_filters.py
+++ b/examples/customize/retrievers/use_pre_filters.py
@@ -4,26 +4,29 @@ import neo4j
 from neo4j_graphrag.embeddings import OpenAIEmbeddings
 from neo4j_graphrag.retrievers import VectorRetriever
 
-URI = "neo4j://localhost:7687"
-AUTH = ("neo4j", "password")
-
-INDEX_NAME = "embedding-name"
+URI = "neo4j+s://demo.neo4jlabs.com"
+AUTH = ("recommendations", "recommendations")
+DATABASE = "recommendations"
+INDEX_NAME = "moviePlotsEmbedding"
 DIMENSION = 1536
 
-# Connect to Neo4j database
-driver = neo4j.GraphDatabase.driver(URI, auth=AUTH)
-
 
-# Initialize the retriever
-retriever = VectorRetriever(driver, INDEX_NAME, embedder=OpenAIEmbeddings())
+# Connect to Neo4j database
+with neo4j.GraphDatabase.driver(URI, auth=AUTH) as driver:
+    # Initialize the retriever
+    retriever = VectorRetriever(driver, INDEX_NAME, embedder=OpenAIEmbeddings())
 
-# Perform the search
-query_text = "Find me a book about Fremen"
-pre_filters = {"int_property": {"$gt": 100}}
-print(
-    retriever.search(
+    # Perform the search
+    query_text = "Find me a movie about love"
+    pre_filters = {"int_property": {"$gt": 100}}
+    # pre_filters = {
+    #     "year": {
+    #         "$nin": ["1999", "2000"]
+    #     }
+    # }
+    retriever_result = retriever.search(
         query_text=query_text,
         top_k=1,
         filters=pre_filters,
     )
-)
+    print(retriever_result)
diff --git a/src/neo4j_graphrag/filters.py b/src/neo4j_graphrag/filters.py
index 2431a12..ed51863 100644
--- a/src/neo4j_graphrag/filters.py
+++ b/src/neo4j_graphrag/filters.py
@@ -100,7 +100,9 @@ class InOperator(Operator):
 
 
 class NinOperator(InOperator):
-    CYPHER_OPERATOR = "NOT IN"
+    def lhs(self, field: str) -> str:
+        lhs = super().lhs(field)
+        return f"NOT {lhs}"
 
 
 class LikeOperator(Operator):
@@ -114,8 +116,8 @@ class LikeOperator(Operator):
 
 class ILikeOperator(LikeOperator):
     def lhs(self, field: str) -> str:
-        safe_field_cypher = self.safe_field_cypher(field)
-        return f"toLower({self.node_alias}.{safe_field_cypher})"
+        lhs = super().lhs(field)
+        return f"toLower({lhs})"
 
     def cleaned_value(self, value: str) -> str:
         value = super().cleaned_value(value)
@@ -368,6 +370,5 @@ def get_metadata_filter(
         contains the query parameters
     """
     param_store = ParameterStore()
-    return _construct_metadata_filter(
-        filter, param_store, node_alias=node_alias
-    ), param_store.params
+    query = _construct_metadata_filter(filter, param_store, node_alias=node_alias)
+    return query, param_store.params


Test patch:
diff --git a/tests/unit/test_filters.py b/tests/unit/test_filters.py
index 14065cc..a62168d 100644
--- a/tests/unit/test_filters.py
+++ b/tests/unit/test_filters.py
@@ -165,7 +165,7 @@ def test_single_condition_cypher_nin(param_store_empty: ParameterStore) -> None:
     generated = _single_condition_cypher(
         "field", NinOperator, ["a", "b", "c"], param_store=param_store_empty
     )
-    assert generated == "node.field NOT IN $param_0"
+    assert generated == "NOT node.field IN $param_0"
     assert param_store_empty.params == {"param_0": ["a", "b", "c"]}
 
 
@@ -552,7 +552,7 @@ def test_get_metadata_filter_in_operator() -> None:
 def test_get_metadata_filter_not_in_operator() -> None:
     filters = {"field": {"$nin": ["a", "b"]}}
     query, params = get_metadata_filter(filters)
-    assert query == "node.field NOT IN $param_0"
+    assert query == "NOT node.field IN $param_0"
     assert params == {"param_0": ["a", "b"]}