Repository: milvus-io/milvus
Pull request #41096: test: add JsonQueryChecker in test
Author: login: zhuwenxing
Labels: name: kind/improvement
description: Changes related to something improve, likes ut and code refactor
name: sig/testing
name: size/M
description: Denotes a PR that changes 30-99 lines.
name: approved
name: lgtm
name: area/test
name: ci-passed
name: dco-passed
description: DCO check passed.
name: kind/test
Created: 2025-04-03T10:00:26Z | Merged: 2025-04-10T02:28:30Z


/kind improvement


Diff:
diff --git a/tests/python_client/chaos/checker.py b/tests/python_client/chaos/checker.py
index 34b5f34851950..d8923f6f07ffa 100644
--- a/tests/python_client/chaos/checker.py
+++ b/tests/python_client/chaos/checker.py
@@ -23,6 +23,7 @@
 from common import common_type as ct
 from common.milvus_sys import MilvusSys
 from chaos import constants
+from faker import Faker
 
 from common.common_type import CheckTasks
 from utils.util_log import test_log as log
@@ -228,6 +229,7 @@ class Op(Enum):
     query = 'query'
     text_match = 'text_match'
     phrase_match = 'phrase_match'
+    json_query = 'json_query'
     delete = 'delete'
     delete_freshness = 'delete_freshness'
     compact = 'compact'
@@ -364,6 +366,7 @@ def __init__(self, collection_name=None, partition_name=None, shards_num=2, dim=
                                     timeout=timeout,
                                     enable_traceback=enable_traceback)
         self.scalar_field_names = cf.get_scalar_field_name_list(schema=schema)
+        self.json_field_names = cf.get_json_field_name_list(schema=schema)
         self.float_vector_field_names = cf.get_float_vec_field_name_list(schema=schema)
         self.binary_vector_field_names = cf.get_binary_vec_field_name_list(schema=schema)
         self.bm25_sparse_field_names = cf.get_bm25_vec_field_name_list(schema=schema)
@@ -379,6 +382,28 @@ def __init__(self, collection_name=None, partition_name=None, shards_num=2, dim=
                                      timeout=timeout,
                                      enable_traceback=enable_traceback,
                                      check_task=CheckTasks.check_nothing)
+        # create index for json fields
+        for f in self.json_field_names:
+            if f in indexed_fields:
+                continue
+            self.c_wrap.create_index(f,
+                                    {"index_type": "INVERTED",
+                                    "params": {"json_path": f"{f}['name']", "json_cast_type": "varchar"}},
+                                    timeout=timeout,
+                                    enable_traceback=enable_traceback,
+                                    check_task=CheckTasks.check_nothing)
+            self.c_wrap.create_index(f,
+                                    {"index_type": "INVERTED",
+                                    "params": {"json_path": f"{f}['address']", "json_cast_type": "varchar"}},
+                                    timeout=timeout,
+                                    enable_traceback=enable_traceback,
+                                    check_task=CheckTasks.check_nothing)
+            self.c_wrap.create_index(f,
+                                    {"index_type": "INVERTED",
+                                    "params": {"json_path": f"{f}['count']", "json_cast_type": "double"}},
+                                    timeout=timeout,
+                                    enable_traceback=enable_traceback,
+                                    check_task=CheckTasks.check_nothing)   
         # create index for float vector fields
         for f in self.float_vector_field_names:
             if f in indexed_fields:
@@ -1459,6 +1484,55 @@ def keep_running(self):
             sleep(constants.WAIT_PER_OP / 10)
 
 
+class JsonQueryChecker(Checker):
+    """check json query operations in a dependent thread"""
+
+    def __init__(self, collection_name=None, shards_num=2, replica_number=1, schema=None):
+        if collection_name is None:
+            collection_name = cf.gen_unique_str("JsonQueryChecker_")
+        super().__init__(collection_name=collection_name, shards_num=shards_num, schema=schema)
+        res, result = self.c_wrap.create_index(self.float_vector_field_name,
+                                               constants.DEFAULT_INDEX_PARAM,
+                                               timeout=timeout,
+                                               enable_traceback=enable_traceback,
+                                               check_task=CheckTasks.check_nothing)
+        self.c_wrap.load(replica_number=replica_number)  # do load before query
+        self.insert_data()
+        self.term_expr = self.get_term_expr()
+
+    def get_term_expr(self):
+        json_field_name = random.choice(self.json_field_names)
+        fake = Faker()
+        address_list = [fake.address() for _ in range(10)]
+        name_list = [fake.name() for _ in range(10)]
+        number_list = [random.randint(0, 100) for _ in range(10)]
+        path = random.choice([ "name", "count"])
+        path_value = {
+            "address": address_list, # TODO not used in json query because of issue
+            "name": name_list,
+            "count": number_list
+        }
+        return f"{json_field_name}['{path}'] <= '{path_value[path][random.randint(0, len(path_value[path]) - 1)]}'"
+        
+
+    @trace()
+    def json_query(self):
+        res, result = self.c_wrap.query(self.term_expr, timeout=query_timeout,
+                                        check_task=CheckTasks.check_query_not_empty)
+        return res, result
+
+    @exception_handler()
+    def run_task(self):
+        self.term_expr = self.get_term_expr()
+        res, result = self.json_query()
+        return res, result
+
+    def keep_running(self):
+        while self._keep_running:
+            self.run_task()
+            sleep(constants.WAIT_PER_OP / 10)
+
+
 class DeleteChecker(Checker):
     """check delete operations in a dependent thread"""
 
diff --git a/tests/python_client/chaos/testcases/test_concurrent_operation.py b/tests/python_client/chaos/testcases/test_concurrent_operation.py
index 0415d09d15149..ac411c0ac3d02 100644
--- a/tests/python_client/chaos/testcases/test_concurrent_operation.py
+++ b/tests/python_client/chaos/testcases/test_concurrent_operation.py
@@ -12,6 +12,7 @@
                            QueryChecker,
                            TextMatchChecker,
                            PhraseMatchChecker,
+                           JsonQueryChecker,
                            DeleteChecker,
                            Op,
                            ResultAnalyzer
@@ -83,6 +84,7 @@ def init_health_checkers(self, collection_name=None):
             Op.query: QueryChecker(collection_name=c_name),
             Op.text_match: TextMatchChecker(collection_name=c_name),
             Op.phrase_match: PhraseMatchChecker(collection_name=c_name),
+            Op.json_query: JsonQueryChecker(collection_name=c_name),
             Op.delete: DeleteChecker(collection_name=c_name),
         }
         log.info(f"init_health_checkers: {checkers}")
diff --git a/tests/python_client/chaos/testcases/test_single_request_operation.py b/tests/python_client/chaos/testcases/test_single_request_operation.py
index 34a286d2fc619..c20630c9a3e18 100644
--- a/tests/python_client/chaos/testcases/test_single_request_operation.py
+++ b/tests/python_client/chaos/testcases/test_single_request_operation.py
@@ -14,6 +14,7 @@
                            QueryChecker,
                            TextMatchChecker,
                            PhraseMatchChecker,
+                           JsonQueryChecker,
                            IndexCreateChecker,
                            DeleteChecker,
                            CollectionDropChecker,
@@ -82,6 +83,7 @@ def init_health_checkers(self, collection_name=None):
             Op.query: QueryChecker(collection_name=c_name),
             Op.text_match: TextMatchChecker(collection_name=c_name),
             Op.phrase_match: PhraseMatchChecker(collection_name=c_name),
+            Op.json_query: JsonQueryChecker(collection_name=c_name),
             Op.delete: DeleteChecker(collection_name=c_name),
             Op.drop: CollectionDropChecker(collection_name=c_name)
         }
diff --git a/tests/python_client/common/common_func.py b/tests/python_client/common/common_func.py
index f6a9c2e55b0e6..fa7efb5c568f3 100644
--- a/tests/python_client/common/common_func.py
+++ b/tests/python_client/common/common_func.py
@@ -796,7 +796,7 @@ def gen_all_datatype_collection_schema(description=ct.default_desc, primary_fiel
         gen_string_field(name="document", max_length=2000, enable_analyzer=True, enable_match=True, nullable=nullable),
         gen_string_field(name="text", max_length=2000, enable_analyzer=True, enable_match=True,
                          analyzer_params=analyzer_params),
-        gen_json_field(),
+        gen_json_field(nullable=nullable),
         gen_array_field(name="array_int", element_type=DataType.INT64),
         gen_array_field(name="array_float", element_type=DataType.FLOAT),
         gen_array_field(name="array_varchar", element_type=DataType.VARCHAR, max_length=200),
@@ -1886,6 +1886,16 @@ def get_scalar_field_name_list(schema=None):
             vec_fields.append(field.name)
     return vec_fields
 
+def get_json_field_name_list(schema=None):
+    json_fields = []
+    if schema is None:
+        schema = gen_default_collection_schema()
+    fields = schema.fields
+    for field in fields:
+        if field.dtype == DataType.JSON:
+            json_fields.append(field.name)
+    return json_fields
+
 
 def get_binary_vec_field_name(schema=None):
     if schema is None:
@@ -1940,6 +1950,10 @@ def gen_varchar_data(length: int, nb: int, text_mode=False):
 
 def gen_data_by_collection_field(field, nb=None, start=None):
     # if nb is None, return one data, else return a list of data
+    nullable = field.nullable
+    if nullable is True:
+        if random.random() < 0.1:
+            return None
     data_type = field.dtype
     enable_analyzer = field.params.get("enable_analyzer", False)
     if data_type == DataType.BOOL:
@@ -1981,8 +1995,8 @@ def gen_data_by_collection_field(field, nb=None, start=None):
         return gen_varchar_data(length=length, nb=nb, text_mode=enable_analyzer)
     if data_type == DataType.JSON:
         if nb is None:
-            return {"name": fake.name(), "address": fake.address()}
-        data = [{"name": str(i), "address": i} for i in range(nb)]
+            return {"name": fake.name(), "address": fake.address(), "count": random.randint(0, 100)}
+        data = [{"name": str(i), "address": i, "count": random.randint(0, 100)} for i in range(nb)]
         return data
     if data_type == DataType.FLOAT_VECTOR:
         dim = field.params['dim']


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR introduces support for JSON queries in the test suite by adding a new JsonQueryChecker and updating the JSON field handling in the client’s test functions. Key changes include:
- Passing the nullable parameter to gen_json_field and adding a helper function to extract JSON field names.
- Integrating JsonQueryChecker into single and concurrent operation tests.
- Extending the checker module to create indexes for JSON fields and implementing the JsonQueryChecker class.

### Reviewed Changes

Copilot reviewed 4 out of 4 changed files in this pull request and generated 2 comments.

| File                                                       | Description                                                    |
| ---------------------------------------------------------- | -------------------------------------------------------------- |
| tests/python_client/common/common_func.py                  | Updated JSON field generation and added get_json_field_name_list.|
| tests/python_client/chaos/testcases/test_single_request_operation.py | Added JsonQueryChecker import and usage in health checkers. |
| tests/python_client/chaos/testcases/test_concurrent_operation.py      | Added JsonQueryChecker import and usage in health checkers. |
| tests/python_client/chaos/checker.py                        | Introduced new enum member json_query, JSON indexing, and the JsonQueryChecker class. |


<details>
<summary>Comments suppressed due to low confidence (1)</summary>

**tests/python_client/chaos/checker.py:1496**
* The variable 'timeout' is referenced but not defined in this scope; ensure it is properly passed or defined for the create_index call in the JsonQueryChecker __init__.
```
timeout=timeout,
```
</details>


state: COMMENTED
author_association: NONE
submitted_at: 2025-04-03T10:01:56Z
user: login: zhuwenxing
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T10:08:55Z
user: login: zhuwenxing
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T10:09:32Z

Review comments:
diff_hunk: @@ -1459,6 +1484,55 @@ def keep_running(self):
             sleep(constants.WAIT_PER_OP / 10)
 
 
+class JsonQueryChecker(Checker):
+    """check json query operations in a dependent thread"""
+
+    def __init__(self, collection_name=None, shards_num=2, replica_number=1, schema=None):
+        if collection_name is None:
+            collection_name = cf.gen_unique_str("JsonQueryChecker_")
+        super().__init__(collection_name=collection_name, shards_num=shards_num, schema=schema)
+        res, result = self.c_wrap.create_index(self.float_vector_field_name,
path: tests/python_client/chaos/checker.py
user: login: Copilot
body: It appears that 'self.float_vector_field_name' is used in JsonQueryChecker, but since this checker is intended for JSON fields, consider using the appropriate JSON field attribute (e.g., selecting one from self.json_field_names).
```suggestion
        json_field_name = random.choice(self.json_field_names)
        res, result = self.c_wrap.create_index(json_field_name,
```
created_at: 2025-04-03T10:01:56Z
author_association: NONE
diff_hunk: @@ -1459,6 +1484,55 @@ def keep_running(self):
             sleep(constants.WAIT_PER_OP / 10)
 
 
+class JsonQueryChecker(Checker):
+    """check json query operations in a dependent thread"""
+
+    def __init__(self, collection_name=None, shards_num=2, replica_number=1, schema=None):
+        if collection_name is None:
+            collection_name = cf.gen_unique_str("JsonQueryChecker_")
+        super().__init__(collection_name=collection_name, shards_num=shards_num, schema=schema)
+        res, result = self.c_wrap.create_index(self.float_vector_field_name,
+                                               constants.DEFAULT_INDEX_PARAM,
+                                               timeout=timeout,
+                                               enable_traceback=enable_traceback,
+                                               check_task=CheckTasks.check_nothing)
+        self.c_wrap.load(replica_number=replica_number)  # do load before query
+        self.insert_data()
+        self.term_expr = self.get_term_expr()
+
+    def get_term_expr(self):
+        json_field_name = random.choice(self.json_field_names)
+        fake = Faker()
+        address_list = [fake.address() for _ in range(10)]
+        name_list = [fake.name() for _ in range(10)]
+        number_list = [random.randint(0, 100) for _ in range(10)]
+        path = random.choice([ "name", "count"])
+        path_value = {
+            "address": address_list, # TODO not used in json query because of issue
+            "name": name_list,
+            "count": number_list
+        }
+        return f"{json_field_name}['{path}'] <= '{path_value[path][random.randint(0, len(path_value[path]) - 1)]}'"
+        
+
+    @trace()
+    def json_query(self):
+        res, result = self.c_wrap.query(self.term_expr, timeout=query_timeout,
path: tests/python_client/chaos/checker.py
user: login: Copilot
body: The variable 'query_timeout' is used in json_query but is not defined; consider passing it as an argument or using a default timeout value.
```suggestion
        res, result = self.c_wrap.query(self.term_expr, timeout=self.query_timeout,
```
created_at: 2025-04-03T10:01:56Z
author_association: NONE
diff_hunk: @@ -1459,6 +1484,55 @@ def keep_running(self):
             sleep(constants.WAIT_PER_OP / 10)
 
 
+class JsonQueryChecker(Checker):
+    """check json query operations in a dependent thread"""
+
+    def __init__(self, collection_name=None, shards_num=2, replica_number=1, schema=None):
+        if collection_name is None:
+            collection_name = cf.gen_unique_str("JsonQueryChecker_")
+        super().__init__(collection_name=collection_name, shards_num=shards_num, schema=schema)
+        res, result = self.c_wrap.create_index(self.float_vector_field_name,
+                                               constants.DEFAULT_INDEX_PARAM,
+                                               timeout=timeout,
+                                               enable_traceback=enable_traceback,
+                                               check_task=CheckTasks.check_nothing)
+        self.c_wrap.load(replica_number=replica_number)  # do load before query
+        self.insert_data()
+        self.term_expr = self.get_term_expr()
+
+    def get_term_expr(self):
+        json_field_name = random.choice(self.json_field_names)
+        fake = Faker()
+        address_list = [fake.address() for _ in range(10)]
+        name_list = [fake.name() for _ in range(10)]
+        number_list = [random.randint(0, 100) for _ in range(10)]
+        path = random.choice([ "name", "count"])
+        path_value = {
+            "address": address_list, # TODO not used in json query because of issue
+            "name": name_list,
+            "count": number_list
+        }
+        return f"{json_field_name}['{path}'] <= '{path_value[path][random.randint(0, len(path_value[path]) - 1)]}'"
+        
+
+    @trace()
+    def json_query(self):
+        res, result = self.c_wrap.query(self.term_expr, timeout=query_timeout,
path: tests/python_client/chaos/checker.py
user: login: zhuwenxing
body: it is a global variable
created_at: 2025-04-03T10:08:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1459,6 +1484,55 @@ def keep_running(self):
             sleep(constants.WAIT_PER_OP / 10)
 
 
+class JsonQueryChecker(Checker):
+    """check json query operations in a dependent thread"""
+
+    def __init__(self, collection_name=None, shards_num=2, replica_number=1, schema=None):
+        if collection_name is None:
+            collection_name = cf.gen_unique_str("JsonQueryChecker_")
+        super().__init__(collection_name=collection_name, shards_num=shards_num, schema=schema)
+        res, result = self.c_wrap.create_index(self.float_vector_field_name,
path: tests/python_client/chaos/checker.py
user: login: zhuwenxing
body: creating index for float_vector_field_name is to make sure collection can be loaded
created_at: 2025-04-03T10:09:32Z
author_association: CONTRIBUTOR

PR comments:
user: login: mergify[bot]
created_at: 2025-04-03T11:38:43Z
body: @zhuwenxing E2e jenkins job failed, comment `/run-cpu-e2e` can trigger the job again.
author_association: CONTRIBUTOR
user: login: zhuwenxing
created_at: 2025-04-04T05:45:14Z
body: /run-cpu-e2e
author_association: CONTRIBUTOR
user: login: mergify[bot]
created_at: 2025-04-04T06:38:00Z
body: @zhuwenxing E2e jenkins job failed, comment `/run-cpu-e2e` can trigger the job again.
author_association: CONTRIBUTOR
user: login: mergify[bot]
created_at: 2025-04-08T05:00:37Z
body: @zhuwenxing E2e jenkins job failed, comment `/run-cpu-e2e` can trigger the job again.
author_association: CONTRIBUTOR
user: login: mergify[bot]
created_at: 2025-04-08T14:11:55Z
body: @zhuwenxing E2e jenkins job failed, comment `/run-cpu-e2e` can trigger the job again.
author_association: CONTRIBUTOR
user: login: yanliang567
created_at: 2025-04-10T02:26:42Z
body: /lgtm
/approve
author_association: CONTRIBUTOR
user: login: sre-ci-robot
created_at: 2025-04-10T02:26:56Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/milvus-io/milvus/pull/41096#issuecomment-2791391874" title="Approved">yanliang567</a>*, *<a href="https://github.com/milvus-io/milvus/pull/41096#" title="Author self-approved">zhuwenxing</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=milvus-io%2Fmilvus).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[tests/OWNERS](https://github.com/milvus-io/milvus/blob/master/tests/OWNERS)~~ [yanliang567]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR