Repository: alibaba/GraphScope (language: python)

Issue:
Support `union` step for Interactive gremlin service
Currently `union` step is not supported in gremlin service provided by interactive.
[BUG] Query failure on interactive
Codegen failure on interactive
```cypher
Match(n) return labels(n) as type, count(*) as cnt;
```


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml
index a2cea64d4bfd..9b7e34ea3ed1 100644
--- a/.github/workflows/hqps-db-ci.yml
+++ b/.github/workflows/hqps-db-ci.yml
@@ -101,6 +101,8 @@ jobs:
         mkdir -p ${INTERACTIVE_WORKSPACE}/data/graph_algo
         cp ${GITHUB_WORKSPACE}/flex/tests/interactive/graph_algo_test.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml
         cp ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml
+        mkdir -p ${INTERACTIVE_WORKSPACE}/data/modern_graph
+        cp ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
 
         # load graph
         cd ${GITHUB_WORKSPACE}/flex/build
@@ -110,6 +112,8 @@ jobs:
         GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/movies/indices/
         export FLEX_DATA_DIR=../interactive/examples/graph_algo
         GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/graph_algo/indices/
+        export FLEX_DATA_DIR=../interactive/examples/modern_graph
+        GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml -l ../interactive/examples/modern_graph/bulk_load.yaml -d ${INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
 
     - name: Test HQPS admin http service
       env: 
@@ -156,7 +160,7 @@ jobs:
           cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
-          eval ${cmd}
+          eval ${cmd} || exit 1
         done
 
         for i in 1 2 3 4 5 6 7 8 9 10 11 12;
@@ -165,7 +169,7 @@ jobs:
           cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
-          eval ${cmd}
+          eval ${cmd} || exit 1
         done
 
         # test movie graph, 8,9,10 are not supported now
@@ -177,7 +181,7 @@ jobs:
           cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
           echo $cmd
-          eval ${cmd}
+          eval ${cmd} || exit 1
         done
 
     - name: Test cypher procedure generation
@@ -202,7 +206,7 @@ jobs:
         export ENGINE_TYPE=hiactor
         # change the default_graph config in ./engine_config_test.yaml to ldbc
         sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml
-        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
+        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
               ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
 
     - name: Run End-to-End cypher adhoc movie query test
@@ -216,7 +220,7 @@ jobs:
         export ENGINE_TYPE=hiactor
         # change the default_graph config in ./engine_config_test.yaml to movies
         sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml
-        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies \
+        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies \
           ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
 
     - name: Run End-to-End cypher adhoc graph_algo query test
@@ -230,7 +234,19 @@ jobs:
         export ENGINE_TYPE=hiactor
         # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to graph_algo
         sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test.yaml
-        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
+        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
           ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
 
+    - name: Run Gremlin test on modern graph
+      env:
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
+        LD_LIBRARY_PATH: /usr/local/lib
+      run: |
+        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
+        export ENGINE_TYPE=hiactor
+        sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ./engine_config_test.yaml
+        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph \
+          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml gremlin
+
 
diff --git a/docs/flex/interactive/configuration.md b/docs/flex/interactive/configuration.md
index ac4dd0ec3561..e3ae888eb40a 100644
--- a/docs/flex/interactive/configuration.md
+++ b/docs/flex/interactive/configuration.md
@@ -44,6 +44,14 @@ compiler:
     - FilterIntoJoinRule
     - NotExistToAntiJoinRule
   query_timeout: 20000  # query timeout in milliseconds, default 20000
+  endpoint:
+    default_listen_address: localhost
+    bolt_connector: # cypher query endpoint 
+      disabled: false # disable cypher endpoint or not.
+      port: 7687
+    gremlin_connector: # gremlin query endpoint 
+      disabled: false # disable gremlin endpoint or not.
+      port: 8182
 http_service:
   default_listen_address: localhost
   admin_port: 7777
@@ -69,6 +77,11 @@ In this following table, we use the `.` notation to represent the hierarchy with
 | compiler.planner.rules.FilterMatchRule | N/A | An optimization rule that pushes filter (`Where`) conditions into the `Match` clause | 0.0.1 |
 | compiler.planner.rules.FilterIntoJoinRule | N/A | A native Calcite optimization rule that pushes filter conditions to the Join participants before performing the join | 0.0.1 |
 | compiler.planner.rules.NotMatchToAntiJoinRule | N/A | An optimization rule that transforms a "not exist" pattern into an anti-join operation  | 0.0.1 |
+| compiler.endpoint.default_listen_address | localhost | The address for compiler endpoint to bind | 0.0.3 |
+| compiler.endpoint.bolt_connector.disabled | false | Whether to disable the cypher endpoint| 0.0.3 |
+| compiler.endpoint.bolt_connector.port | 7687 | The port for compiler's cypher endpoint.| 0.0.3 |
+| compiler.endpoint.gremlin_connector.disabled | true | Whether to disable the gremlin endpoint| 0.0.3 |
+| compiler.endpoint.gremlin_connector.port | 8182 | The port for compiler's cypher endpoint.| 0.0.3 |
 | http_service.default_listen_address | localhost | The address for http service to bind | 0.0.2 |
 | http_service.admin_port | 7777 | The port for admin service to listen on | 0.0.2 |
 | http_service.query_port | 10000 | The port for query service to listen on, for stored procedure queries, user can directory submit queries to query_port without compiler involved | 0.0.2 |
diff --git a/docs/flex/interactive/getting_started.md b/docs/flex/interactive/getting_started.md
index c41e8c202986..ef594e3c346c 100644
--- a/docs/flex/interactive/getting_started.md
+++ b/docs/flex/interactive/getting_started.md
@@ -34,7 +34,7 @@ To start the Interactive service, provide the raw graph data and meta info. For
 ```bash
 bin/gs_interactive service start
 ```
-You should be able to see the port (by default: 7687) for accepting Cypher queries.  
+You should be able to see a message telling you what the `Gremlin/Cypher` endpoint that you should connect to.
 
 ### Stop the Service
 To stop the Interactive service, simple type in the following command:
@@ -65,34 +65,15 @@ Execute the command below to terminate the service. Please be aware that this ac
 bin/gs_interactive destroy
 ```
 
-## Running Queries
+## Running Cypher Queries
 GraphScope Interactive seamlessly integrates with the Neo4j ecosystem. You can establish a connection to the Interactive service using Neo4j's Bolt connector and execute Cypher queries. Our implementation of Cypher queries aligns with the standards set by the [openCypher](http://www.opencypher.org/) project. For a detailed overview of the supported Cypher queries, please visit [supported_cypher](https://graphscope.io/docs/latest/interactive_engine/neo4j/supported_cypher).
 
-### Connect to the Service
-Connect to the GraphScope Interactive Service using the cypher-shell from Neo4j:
+Follow the instructions in [Connect-to-cypher-service](../../interactive_engine/neo4j/cypher_sdk) to connect to the Cypher service using either the Python client or cypher-shell.
 
-```bash
-# Get Cypher-shell from: https://dist.neo4j.org/cypher-shell/cypher-shell-4.4.22.zip
-wget -O cypher-shell-4.4.22.zip https://dist.neo4j.org/cypher-shell/cypher-shell-4.4.22.zip
-unzip cypher-shell-4.4.22.zip
-./cypher-shell/cypher-shell -a neo4j://localhost:7687
-```
-
-User name and password are not needed for now, while such authentication will be provided in a future version. 
 
-### Submit Queries
-Once connected, you can start submitting your queries:
+Note: Cypher queries submitted to GraphScope Interactive are compiled into a dynamic library for execution. While the initial compilation might take some time, the execution for subsequent uses (of the **same** query) will be much faster since it is cached by Interactive.
 
-```cypher
-@neo4j> MATCH(v:person { name: "peter"}) RETURN v.age;
-```
-Then you can get the age of `peter`.
-```txt
-+-----+
-| age |
-+-----+
-| 35  |
-+-----+
-```
+## Running Gremlin Queries
 
-Note: Cypher queries submitted to GraphScope Interactive are compiled into a dynamic library for execution. While the initial compilation might take some time, a future version will provide caching the library to ensure faster execution for subsequent uses (of the **same** query).
\ No newline at end of file
+GraphScope Interactive supports the property graph model and Gremlin traversal language defined by Apache TinkerPop,
+Please refer to the following link to connect to the Tinkerpop Gremlin service provided by GraphScope Interactive: [Connect-to-gremlin-service](../../interactive_engine/tinkerpop/tinkerpop_gremlin)
\ No newline at end of file
diff --git a/docs/flex/interactive/glossary.md b/docs/flex/interactive/glossary.md
index 3c4ad511841a..230b5a9e7d58 100644
--- a/docs/flex/interactive/glossary.md
+++ b/docs/flex/interactive/glossary.md
@@ -10,6 +10,10 @@ In GraphScope Interactive, a cypher query with parameters (denote with $ prefix)
 
 Cypher is a popular graph query language, used by [neo4j](https://neo4j.com/developer/cypher/).
 
+### Gremlin
+
+[Gremlin](https://tinkerpop.apache.org/gremlin.html) is the graph traversal language of Apache TinkerPop. Gremlin is a functional, data-flow language that enables users to succinctly express complex traversals on (or queries of) their application's property graph. Every Gremlin traversal is composed of a sequence of (potentially nested) steps.
+
 ### Compile.
 
 Compile refers to the process of compiling a stored procedure into a dynamic lib that the compute engine can load execute.
\ No newline at end of file
diff --git a/docs/flex/interactive/overview.md b/docs/flex/interactive/overview.md
index 4ab5cc0eb1fb..f13bf309631c 100644
--- a/docs/flex/interactive/overview.md
+++ b/docs/flex/interactive/overview.md
@@ -23,9 +23,9 @@ GraphScope Interactive stands on the shoulders of two pivotal pillars:
 GraphScope Interactive boasts several key features:
 
 1. **Exceptional Query Throughput**: As highlighted, GraphScope Interactive's foundation is laid on benchmarking triumphs, enabling it to process tens of thousands of queries swiftly.
-2. **Versatility in Language Support**: GraphScope Interactive supports the use of Neo4j's declarative query language, Cypher, for crafting stored procedures. For those who prefer a more comprehensive approach, C++ programming is also supported for stored procedure development.
+2. **Versatility in Language Support**: GraphScope Interactive supports two graph query languages: TinkerPop Gremlin and Neo4j Cypher. Cypher can also be used for crafting stored procedures. For those who prefer a more comprehensive approach, C++ programming is also supported for stored procedure development.
 3. **Future-Ready Expansion Capabilities**: Drawing from the prowess of GraphScope Flex, GraphScope Interactive is primed for adaptability:
-    * Support for Multiple Query Languages: In the near future, GraphScope Interactive will extend its language support to include [Gremlin](https://tinkerpop.apache.org/gremlin.html), and [GQL](https://www.gqlstandards.org/), further enhancing its versatility.
+    * Support for Multiple Query Languages: In the near future, GraphScope Interactive will extend its language support to [GQL](https://www.gqlstandards.org/), further enhancing its versatility.
     * Scalability: GraphScope Interactive possesses the potential for distributed processing. This means it can be expanded with few effort to handle larger-scale graphs, ensuring it remains effective as your data grows.
 4. **Massive Graph Support**: To enhance system throughput and query performance, we store all graph data in memory by default. However, when encountering graph data too large to fit entirely in memory, we offload excess data to disk storage. This approach, while effectively handling massive-scale graph data, may result in reduced throughput. Moving forward, we aim to adopt a master-slave architecture designed to enhance concurrent querying performance. This will be achieved by distributing the workload across multiple machines, thereby optimizing resource utilization to improve throughput. 
 
@@ -44,9 +44,20 @@ The application of Cypher in GraphScope Interactive can be categorized into two
 1. **On-the-Fly Read Queries**: Users can submit Cypher read queries directly to the engine. These queries are then automatically compiled into a dynamic library for execution, providing immediate results.
 2. **Stored Procedures**: Users can create a Cypher file with specified parameters denoted as `${param_name}`. Using the admin tool, this code can be compiled into a stored procedure and registered within the system. This stored procedure can then be invoked for future use via Cypher's CALL clause.
 
+The Cypher endpoint is enabled by default and listens on port 7687. You can also turn of the bolt connector or customize the port it by setting properties under `compiler.endpoint.bolt_connector`. For more detail explanation, please refer to [Configuration](./configuration) guide. For instructions on how to connect to the Cypher endpoint, please see [GettingStarted](./getting_started#running-cypher-queries).
+
+## The Gremlin Query Language
+
+GraphScope Interactive also support [Gremlin]((https://tinkerpop.apache.org/gremlin.html)), a graph traversal language of Apache TinkerPop, and is widely adopted by many other graph databases. Although the syntax of Gremlin is definitely different from cypher, thanks to the unified Intermediate Representation (IR) used by GraphScope Interactive, we can achieve good unification at the execution level(physical plan).
+
+You can use Gremlin Console/Driver to connect to GraphScope Interactive' Gremlin endpoint to submit gremlin queries. Since Gremlin doesn't support stored procedure, you can only run **On-the-Fly** Gremlin queries.
+
+The Gremlin endpoint is disabled by default. To enable the gremlin endpoint, you need to set `compiler.endpoint.gremlin_connector.disabled` to `false` in the configuration.
+For more configurations about the Gremlin connector, please consult the [Configuration](./configuration) guide. To learn how to connect to the Gremlin endpoint, refer to [GettingStarted](./getting_started#running-gremlin-queries).
 
 ## Limitations
 
-1. GraphScope Interactive now only supports a subset of Cypher read clause, as indicated in [supported_cypher](https://graphscope.io/docs/latest/interactive_engine/neo4j/supported_cypher). Write and DDL (for defining and modifying schema) functionalities are not currently provided. 
-2. Writes are designed to be fulfilled via built-in stored procedures. Currently writes are not supported.
-3. ACID is currently not supported, but it is planned to be supported in the near future.
\ No newline at end of file
+1. GraphScope Interactive now only supports a subset of Cypher read clause, as indicated in [supported_cypher](../../interactive_engine/neo4j/supported_cypher). Write and DDL (for defining and modifying schema) functionalities are not currently provided. 
+2. GraphScope Interactive also does not fully support all traversal steps of Gremlin, please check [supported_gremlin_step](../../interactive_engine/tinkerpop/supported_gremlin_steps). It's important to note that the `g.E()` step is currently not supported in Interactive.
+3. Writes are designed to be fulfilled via built-in stored procedures. Currently writes are not supported.
+4. ACID is currently not supported, but it is planned to be supported in the near future.
\ No newline at end of file
diff --git a/docs/flex/interactive/tools/cypher_shell.md b/docs/flex/interactive/tools/cypher_shell.md
deleted file mode 100644
index 5bda11a6ef31..000000000000
--- a/docs/flex/interactive/tools/cypher_shell.md
+++ /dev/null
@@ -1,20 +0,0 @@
-# Cypher Shell
-
-GraphScope interactive is compatible with the existing neo4j cypher ecosystem. Users can use Cypher Shell as a client to connect to the database service and submit queries for execution.
-
-## Downloading
-We currently only support Cypher Shell version 4.4.22. You can download from the [official release](https://dist.neo4j.org/cypher-shell/cypher-shell-4.4.22.zip).
-
-## Usage
-
-You can connect to the started GraphScope Interactive Database though the bolt connector port. The default port is 7687. 
-
-```bash
-cypher-shell -a neo4j://{ip}:7687.
-```
-
-{ip} refers to the IP address of the machine where the database service is running. If you are running cypher shell and database service on the same host, then use `localhost` is ok.
-
-## Limitation
-
-Currently write operations are not supported in GraphScope Interactive, so you cannot use `create` clause in cypher shell.
\ No newline at end of file
diff --git a/flex/bin/load_plan_and_gen.sh b/flex/bin/load_plan_and_gen.sh
index 6f4df31706f6..9cf71a27ff6c 100755
--- a/flex/bin/load_plan_and_gen.sh
+++ b/flex/bin/load_plan_and_gen.sh
@@ -234,7 +234,7 @@ compile_hqps_so() {
 
   #only do codegen when receives a .pb file.
   if [[ $last_file_name == *.pb ]]; then
-    cmd="${CODEGEN_RUNNER} -e hqps -i ${input_path} -o ${output_cc_path}"
+    cmd="${CODEGEN_RUNNER} -e hqps -i ${input_path} -o ${output_cc_path} -g ${graph_schema_path}"
     info "Codegen command = ${cmd}"
     eval ${cmd}
     info "----------------------------"
@@ -250,7 +250,7 @@ compile_hqps_so() {
     info "----------------------------"
     info "Codegen from cypher query done."
     info "----------------------------"
-    cmd="${CODEGEN_RUNNER} -e hqps -i ${output_pb_path} -o ${output_cc_path}"
+    cmd="${CODEGEN_RUNNER} -e hqps -i ${output_pb_path} -o ${output_cc_path} -g ${graph_schema_path}"
     info "Codegen command = ${cmd}"
     eval ${cmd}
     # then. do .pb to .cc
diff --git a/flex/codegen/CMakeLists.txt b/flex/codegen/CMakeLists.txt
index 87f6834e656c..875e0cca5906 100644
--- a/flex/codegen/CMakeLists.txt
+++ b/flex/codegen/CMakeLists.txt
@@ -5,6 +5,6 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
 
 if (BUILD_HQPS)
     add_executable(gen_code_from_plan gen_code_from_plan.cc)
-    target_link_libraries(gen_code_from_plan hqps_plan_proto ${GLOG_LIBRARIES}  ${Boost_LIBRARIES})
+    target_link_libraries(gen_code_from_plan flex_rt_mutable_graph flex_utils hqps_plan_proto ${GLOG_LIBRARIES}  ${Boost_LIBRARIES})
     install_flex_target(gen_code_from_plan)
 endif()
diff --git a/flex/codegen/gen_code_from_plan.cc b/flex/codegen/gen_code_from_plan.cc
index 0bbfc5bbb881..3d02b5976162 100644
--- a/flex/codegen/gen_code_from_plan.cc
+++ b/flex/codegen/gen_code_from_plan.cc
@@ -26,6 +26,7 @@ limitations under the License.
 
 #include "flex/codegen/src/hqps_generator.h"
 #include "flex/codegen/src/pegasus_generator.h"
+#include "flex/storages/rt_mutable_graph/schema.h"
 
 namespace bpo = boost::program_options;
 
@@ -71,7 +72,8 @@ void deserialize_plan_and_gen_pegasus(const std::string& input_file_path,
 }
 
 void deserialize_plan_and_gen_hqps(const std::string& input_file_path,
-                                   const std::string& output_file_path) {
+                                   const std::string& output_file_path,
+                                   const std::string& graph_schema_path) {
   LOG(INFO) << "Start deserializing from: " << input_file_path;
   std::string content_str = read_binary_str_from_path(input_file_path);
   LOG(INFO) << "Deserialized plan size : " << content_str.size() << ", from "
@@ -82,8 +84,29 @@ void deserialize_plan_and_gen_hqps(const std::string& input_file_path,
   LOG(INFO) << "deserialized plan size : " << plan_pb.ByteSizeLong();
   VLOG(1) << "deserialized plan : " << plan_pb.DebugString();
   BuildingContext context;
-  QueryGenerator<uint8_t> query_generator(context, plan_pb);
-  auto res = query_generator.GenerateQuery();
+  std::shared_ptr<QueryGenerator<uint8_t>> query_generator;
+  // load schema
+  if (!graph_schema_path.empty()) {
+    auto schema_res = Schema::LoadFromYaml(graph_schema_path);
+    if (schema_res.ok()) {
+      LOG(INFO) << "Generate code with schema info: " << graph_schema_path;
+      query_generator = std::make_shared<QueryGenerator<uint8_t>>(
+          context, plan_pb, schema_res.value());
+    } else {
+      LOG(ERROR) << "Fail to load schema from path: " << graph_schema_path
+                 << ", " << schema_res.status().error_message()
+                 << ",generate code without schema info.";
+      query_generator =
+          std::make_shared<QueryGenerator<uint8_t>>(context, plan_pb);
+    }
+  } else {
+    LOG(INFO)
+        << "graph schema path is empty, generate code without schema info";
+    query_generator =
+        std::make_shared<QueryGenerator<uint8_t>>(context, plan_pb);
+  }
+
+  auto res = query_generator->GenerateQuery();
   LOG(INFO) << "Start writing to: " << output_file_path;
   output_code_to_file(res, output_file_path);
 }
@@ -94,7 +117,9 @@ int main(int argc, char** argv) {
   desc.add_options()("help", "Display help message")(
       "engine,e", bpo::value<std::string>(), "engine type")(
       "input,i", bpo::value<std::string>(), "input plan path")(
-      "output,o", bpo::value<std::string>(), "output file path");
+      "output,o", bpo::value<std::string>(), "output file path")(
+      "graph,g", bpo::value<std::string>()->default_value(""),
+      "graph schema path");
 
   bpo::variables_map vm;
   bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
@@ -108,6 +133,7 @@ int main(int argc, char** argv) {
   std::string input_path = vm["input"].as<std::string>();
   std::string output_path = vm["output"].as<std::string>();
   std::string engine_type = vm["engine"].as<std::string>();
+  std::string graph_schema_path = vm["graph"].as<std::string>();
 
   if (!std::filesystem::exists(input_path)) {
     LOG(ERROR) << "input file: [" << input_path << "] not found";
@@ -123,7 +149,8 @@ int main(int argc, char** argv) {
     gs::deserialize_plan_and_gen_pegasus(input_path, output_path);
   } else if (engine_type == "hqps") {
     LOG(INFO) << "Start generating hqps code";
-    gs::deserialize_plan_and_gen_hqps(input_path, output_path);
+    gs::deserialize_plan_and_gen_hqps(input_path, output_path,
+                                      graph_schema_path);
   } else {
     LOG(ERROR) << "Unknown engine type: " << engine_type
                << ", valid engine types: "
diff --git a/flex/codegen/src/building_context.h b/flex/codegen/src/building_context.h
index ee25d4bd5e1b..b4135943c86f 100644
--- a/flex/codegen/src/building_context.h
+++ b/flex/codegen/src/building_context.h
@@ -64,6 +64,10 @@ struct TagIndMapping {
       return -1;
     }
     print_debug_info();
+    if (tag_id >= (int) tag_id_2_tag_inds_.size()) {
+      // If tag_id is not found, we assume it is the head node.
+      return -1;
+    }
     CHECK(tag_id < (int) tag_id_2_tag_inds_.size())
         << "tag id: " << tag_id << " not found";
     return tag_id_2_tag_inds_[tag_id];
@@ -117,6 +121,15 @@ struct TagIndMapping {
     return tag_id_2_tag_inds_;
   }
 
+  bool HasTagId(int32_t tag_id) const {
+    return tag_id < (int32_t) tag_id_2_tag_inds_.size() &&
+           tag_id_2_tag_inds_[tag_id] != -1;
+  }
+
+  int32_t GetMaxTagId() const {
+    return tag_id_2_tag_inds_.size() - 1;
+  }
+
   // convert tag_ind (us) to tag ids
   std::vector<int32_t> tag_ind_2_tag_ids_;
   // convert tag ids(pb) to tag_inds
diff --git a/flex/codegen/src/codegen_utils.h b/flex/codegen/src/codegen_utils.h
index db527880c475..841549cfb4ea 100644
--- a/flex/codegen/src/codegen_utils.h
+++ b/flex/codegen/src/codegen_utils.h
@@ -206,10 +206,13 @@ codegen::ParamConst variable_to_param_const(const common::Variable& var,
       param_const.var_name = var.property().key().name();
       param_const.type =
           common_data_type_pb_2_data_type(var.node_type().data_type());
+    } else if (var_property.has_id()) {
+      param_const.var_name = ctx.GetNextVarName();
+      param_const.type = codegen::DataType::kGlobalVertexId;
     } else {
-      LOG(FATAL) << "Unexpected property type";
+      LOG(FATAL) << "Unexpected property type: " << var_property.DebugString();
     }
-  } else if (var.has_tag()) {
+  } else {  // var.tag() could be null or not
     // check is vertex or is edge from node_type
     if (var.has_node_type()) {
       auto node_type = var.node_type();
@@ -222,7 +225,7 @@ codegen::ParamConst variable_to_param_const(const common::Variable& var,
         if (graph_type.element_opt() ==
             common::GraphDataType::GraphElementOpt::
                 GraphDataType_GraphElementOpt_VERTEX) {
-          param_const.type = codegen::DataType::kVertexId;
+          param_const.type = codegen::DataType::kGlobalVertexId;
         } else if (graph_type.element_opt() ==
                    common::GraphDataType::GraphElementOpt::
                        GraphDataType_GraphElementOpt_EDGE) {
@@ -260,6 +263,18 @@ std::string interval_to_str(const common::Extract::Interval& interval) {
   }
 }
 
+std::string join_string(const std::vector<std::string>& strs,
+                        const std::string& split) {
+  std::stringstream ss;
+  for (size_t i = 0; i < strs.size(); ++i) {
+    ss << strs[i];
+    if (i + 1 < strs.size()) {
+      ss << split;
+    }
+  }
+  return ss.str();
+}
+
 }  // namespace gs
 
 #endif  // CODEGEN_SRC_CODEGEN_UTILS_H_
\ No newline at end of file
diff --git a/flex/codegen/src/graph_types.h b/flex/codegen/src/graph_types.h
index 78d0f601a02b..e38c5d3697f8 100644
--- a/flex/codegen/src/graph_types.h
+++ b/flex/codegen/src/graph_types.h
@@ -37,13 +37,14 @@ enum class DataType {
   kInt64Array = 5,
   kInt32Array = 6,
   kBoolean = 7,
-  kVertexId = 8,
+  kGlobalVertexId = 8,
   kEdgeId = 9,
   kLength = 10,
   kDate = 11,
   kTime = 12,
   kTimeStamp = 13,
-  kLabelId = 14
+  kLabelId = 14,
+  kEmpty = 15,
 };
 
 // a parameter const, the real data will be feed at runtime.
@@ -189,8 +190,8 @@ static std::string data_type_2_string(const codegen::DataType& data_type) {
     return "std::vector<int32_t>";
   case codegen::DataType::kBoolean:
     return "bool";
-  case codegen::DataType::kVertexId:
-    return VERTEX_ID_T;
+  case codegen::DataType::kGlobalVertexId:
+    return GLOBAL_VERTEX_ID_T;
   case codegen::DataType::kLength:
     return LENGTH_KEY_T;
   case codegen::DataType::kEdgeId:
@@ -199,6 +200,8 @@ static std::string data_type_2_string(const codegen::DataType& data_type) {
     return "Date";
   case codegen::DataType::kLabelId:
     return "LabelKey";
+  case codegen::DataType::kEmpty:
+    return GRAPE_EMPTY_TYPE;
   default:
     // LOG(FATAL) << "unknown data type" << static_cast<int>(data_type);
     throw std::runtime_error(
@@ -243,7 +246,7 @@ static std::string data_type_2_rust_string(const codegen::DataType& data_type) {
     return "Vector<i32>";
   case codegen::DataType::kBoolean:
     return "bool";
-  case codegen::DataType::kVertexId:
+  case codegen::DataType::kGlobalVertexId:
     return "ID";
   default:
     LOG(FATAL) << "unknown data type" << static_cast<int>(data_type);
diff --git a/flex/codegen/src/hqps/hqps_case_when_builder.h b/flex/codegen/src/hqps/hqps_case_when_builder.h
index 4c97c288930f..f3ea57a16b21 100644
--- a/flex/codegen/src/hqps/hqps_case_when_builder.h
+++ b/flex/codegen/src/hqps/hqps_case_when_builder.h
@@ -113,7 +113,8 @@ class CaseWhenBuilder : public ExprBuilder {
     auto ret_type_str = common_data_type_pb_2_str(res_data_type_);
     formater % class_name_ % ret_type_str % constructor_param_str %
         field_init_code_str % func_call_template_typename_str % "auto" %
-        func_call_params_str % func_call_impl_str % private_filed_str;
+        func_call_params_str % func_call_impl_str % private_filed_str %
+        get_filter_null_str();
 
     std::string str = formater.str();
 
diff --git a/flex/codegen/src/hqps/hqps_dedup_builder.h b/flex/codegen/src/hqps/hqps_dedup_builder.h
index 6f11dc66bbe8..118a9c1c7b1f 100644
--- a/flex/codegen/src/hqps/hqps_dedup_builder.h
+++ b/flex/codegen/src/hqps/hqps_dedup_builder.h
@@ -35,7 +35,12 @@ enum DedupProp {
 };
 
 static constexpr const char* DEDUP_OP_TEMPLATE_STR =
-    "auto %1%= Engine::template Dedup<%2%>(std::move(%3%));";
+    "auto %1%= Engine::template Dedup<%2%>(%3%, std::move(%4%), "
+    "std::tuple{%5%});";
+
+// we dedup on innerId, we should use global id.
+static constexpr const char* GLOBAL_ID_SELECTOR = "GlobalIdSelector()";
+static constexpr const char* Label_ID_SELECTOR = "LabelIdSelector()";
 
 class DedupOpBuilder {
  public:
@@ -43,10 +48,17 @@ class DedupOpBuilder {
 
   // dedup on kId
   DedupOpBuilder& dedup_on_inner_id(int32_t tag_id) {
-    // dedup_prop_ = "none";
-    // dedup_prop_type_ = DedupProp::kInnerId;
     int32_t real_tag_ind = ctx_.GetTagInd(tag_id);
     dedup_tag_ids_.emplace_back(real_tag_ind);
+    dedup_props_.emplace_back(GLOBAL_ID_SELECTOR);
+    return *this;
+  }
+
+  // dedup on kLabel
+  DedupOpBuilder& dedup_on_label(int32_t tag_id) {
+    int32_t real_tag_ind = ctx_.GetTagInd(tag_id);
+    dedup_tag_ids_.emplace_back(real_tag_ind);
+    dedup_props_.emplace_back(Label_ID_SELECTOR);
     return *this;
   }
 
@@ -66,13 +78,16 @@ class DedupOpBuilder {
       dedup_tag_ids_str = ss.str();
     }
     boost::format formater(DEDUP_OP_TEMPLATE_STR);
-    formater % next_ctx_name % dedup_tag_ids_str % prev_ctx_name;
+    auto dedup_prop_str = join_string(dedup_props_, ",");
+    formater % next_ctx_name % dedup_tag_ids_str % ctx_.GraphVar() %
+        prev_ctx_name % dedup_prop_str;
     return formater.str();
   }
 
  private:
   BuildingContext& ctx_;
   std::vector<int32_t> dedup_tag_ids_;
+  std::vector<std::string> dedup_props_;
 };
 
 static std::string BuildDedupOp(
@@ -83,13 +98,20 @@ static std::string BuildDedupOp(
   CHECK(keys.size() > 0) << "Dedup keys size should be gt 0";
 
   for (auto& key : keys) {
+    int32_t tag = -1;
+    if (key.has_tag()) {
+      tag = key.tag().id();
+    }
     if (key.has_property()) {
-      LOG(FATAL) << "dedup on property" << key.property().DebugString()
-                 << "not supported";
-      // dedup_builder.dedup_prop(key.property());
+      VLOG(10) << "dedup on property";
+      if (key.property().has_label()) {
+        dedup_builder.dedup_on_label(tag);
+      } else {
+        LOG(FATAL) << "Not support dedup on property: " << key.DebugString();
+      }
     } else {
-      VLOG(10) << "dedup on innerid";
-      dedup_builder.dedup_on_inner_id(key.tag().id());
+      VLOG(10) << "dedup on innerid: " << tag;
+      dedup_builder.dedup_on_inner_id(tag);
     }
   }
 
diff --git a/flex/codegen/src/hqps/hqps_edge_expand_builder.h b/flex/codegen/src/hqps/hqps_edge_expand_builder.h
index f0e63226c1ac..9a2b678d7730 100644
--- a/flex/codegen/src/hqps/hqps_edge_expand_builder.h
+++ b/flex/codegen/src/hqps/hqps_edge_expand_builder.h
@@ -42,7 +42,13 @@ static constexpr const char*
     EDGE_EXPAND_E_OPT_MULTI_EDGE_NO_FILTER_TEMPLATE_STR =
         "auto %1% = gs::make_edge_expand_multie_opt<%2%>(%3%, %4%, %5%);\n";
 
-// This opt can only be used by both edge expandv, with multiple edge triplet,
+// Edge expande, with multiple edge triplet. with filter.
+static constexpr const char* EDGE_EXPAND_E_OPT_MULTI_EDGE_FILTER_TEMPLATE_STR =
+    "auto %1% = gs::make_filter(%2%(%3%) %4%);\n"
+    "auto %5% = gs::make_edge_expand_multie_opt<%6%>(%7%, %8%, %9%, "
+    "std::move(%1%));\n";
+
+//  This opt can only be used by both edge expandv, with multiple edge triplet,
 static constexpr const char*
     EDGE_EXPAND_V_OPT_MULTI_EDGE_NO_FILTER_TEMPLATE_STR =
         "auto %1% = gs::make_edge_expand_multiv_opt(%2%, %3%);\n";
@@ -212,6 +218,49 @@ std::string make_prop_tuple_array_tuple(
   return ss.str();
 }
 
+static void BuildExprFromPredicate(BuildingContext& ctx,
+                                   const common::Expression& expr,
+                                   std::string& expr_func_name,
+                                   std::string& func_construct_params_str,
+                                   std::string& property_selectors_str) {
+  auto expr_builder = ExprBuilder(ctx);
+  expr_builder.set_return_type(common::DataType::BOOLEAN);
+  expr_builder.AddAllExprOpr(expr.operators());
+  std::string expr_code;
+  std::vector<codegen::ParamConst> func_call_param_const;
+  std::vector<std::pair<int32_t, std::string>> expr_tag_props;
+  std::vector<common::DataType> unused_expr_ret_type;
+  std::tie(expr_func_name, func_call_param_const, expr_tag_props, expr_code,
+           unused_expr_ret_type) = expr_builder.Build();
+  VLOG(10) << "Found expr in edge_expand_opt:  " << expr_func_name;
+  // generate code.
+  ctx.AddExprCode(expr_code);
+
+  {
+    std::stringstream ss;
+    for (size_t i = 0; i < func_call_param_const.size(); ++i) {
+      ss << func_call_param_const[i].var_name;
+      if (i != func_call_param_const.size() - 1) {
+        ss << ", ";
+      }
+    }
+    func_construct_params_str = ss.str();
+  }
+  {
+    std::stringstream ss;
+    if (expr_tag_props.size() > 0) {
+      ss << ",";
+    }
+    for (size_t i = 0; i < expr_tag_props.size(); ++i) {
+      ss << expr_tag_props[i].second;
+      if (i != expr_tag_props.size() - 1) {
+        ss << ", ";
+      }
+    }
+    property_selectors_str = ss.str();
+  }
+}
+
 template <typename LabelT>
 static std::pair<std::string, std::string> BuildOneLabelEdgeExpandOpt(
     BuildingContext& ctx, const internal::Direction& direction,
@@ -284,43 +333,8 @@ static std::pair<std::string, std::string> BuildOneLabelEdgeExpandOpt(
   // first check whether expand_opt contains expression.
   if (params.has_predicate()) {
     VLOG(10) << "Found expr in edge expand";
-    auto& expr = params.predicate();
-    auto expr_builder = ExprBuilder(ctx);
-    expr_builder.set_return_type(common::DataType::BOOLEAN);
-    expr_builder.AddAllExprOpr(expr.operators());
-    std::string expr_code;
-    std::vector<codegen::ParamConst> func_call_param_const;
-    std::vector<std::pair<int32_t, std::string>> expr_tag_props;
-    std::vector<common::DataType> unused_expr_ret_type;
-    std::tie(expr_func_name, func_call_param_const, expr_tag_props, expr_code,
-             unused_expr_ret_type) = expr_builder.Build();
-    VLOG(10) << "Found expr in edge_expand_opt:  " << expr_func_name;
-    // generate code.
-    ctx.AddExprCode(expr_code);
-
-    {
-      std::stringstream ss;
-      for (size_t i = 0; i < func_call_param_const.size(); ++i) {
-        ss << func_call_param_const[i].var_name;
-        if (i != func_call_param_const.size() - 1) {
-          ss << ", ";
-        }
-      }
-      func_construct_params_str = ss.str();
-    }
-    {
-      std::stringstream ss;
-      if (expr_tag_props.size() > 0) {
-        ss << ",";
-      }
-      for (size_t i = 0; i < expr_tag_props.size(); ++i) {
-        ss << expr_tag_props[i].second;
-        if (i != expr_tag_props.size() - 1) {
-          ss << ", ";
-        }
-      }
-      property_selectors_str = ss.str();
-    }
+    BuildExprFromPredicate(ctx, params.predicate(), expr_func_name,
+                           func_construct_params_str, property_selectors_str);
   }
 
   {
@@ -380,8 +394,9 @@ static std::pair<std::string, std::string> BuildMultiLabelEdgeExpandOpt(
     const algebra::QueryParams& params,
     const physical::EdgeExpand::ExpandOpt& expand_opt,
     const physical::PhysicalOpr::MetaData& meta_data) {
+  std::string expr_var_name = ctx.GetNextExprVarName();
   std::string opt_var_name = ctx.GetNextEdgeOptName();
-  std::string func_construct_params_str, property_selectors_str,
+  std::string expr_func_name, func_construct_params_str, property_selectors_str,
       edge_label_id_str, dst_label_ids_str, edge_prop_selectors_str,
       edge_expand_e_types_str;
 
@@ -400,28 +415,46 @@ static std::pair<std::string, std::string> BuildMultiLabelEdgeExpandOpt(
   auto edge_named_prop_array =
       make_prop_tuple_array_tuple(prop_names, prop_types);
 
+  if (params.has_predicate()) {
+    VLOG(10) << "Found expr in edge expand with multiple label triplet";
+    BuildExprFromPredicate(ctx, params.predicate(), expr_func_name,
+                           func_construct_params_str, property_selectors_str);
+  }
+
   boost::format formater;
   if (expand_opt ==
       physical::EdgeExpand::ExpandOpt::EdgeExpand_ExpandOpt_EDGE) {
     auto edge_triplet_2d_array =
         edge_label_triplet_to_array_str(edge_label_triplet);
-    formater =
-        boost::format(EDGE_EXPAND_E_OPT_MULTI_EDGE_NO_FILTER_TEMPLATE_STR);
-    formater % opt_var_name % func_template_str %
-        gs::direction_pb_to_str(direction) % edge_triplet_2d_array %
-        edge_named_prop_array;
+    if (params.has_predicate()) {
+      formater =
+          boost::format(EDGE_EXPAND_E_OPT_MULTI_EDGE_FILTER_TEMPLATE_STR);
+      formater % expr_var_name % expr_func_name % func_construct_params_str %
+          property_selectors_str % opt_var_name % func_template_str %
+          gs::direction_pb_to_str(direction) % edge_triplet_2d_array %
+          edge_named_prop_array;
+    } else {
+      formater =
+          boost::format(EDGE_EXPAND_E_OPT_MULTI_EDGE_NO_FILTER_TEMPLATE_STR);
+      formater % opt_var_name % func_template_str %
+          gs::direction_pb_to_str(direction) % edge_triplet_2d_array %
+          edge_named_prop_array;
+    }
   } else if (expand_opt ==
              physical::EdgeExpand::ExpandOpt::EdgeExpand_ExpandOpt_VERTEX) {
-    auto edge_triplet_2d_vector =
-        edge_label_triplet_to_vector_str(edge_label_triplet);
-    formater =
-        boost::format(EDGE_EXPAND_V_OPT_MULTI_EDGE_NO_FILTER_TEMPLATE_STR);
-    formater % opt_var_name % gs::direction_pb_to_str(direction) %
-        edge_triplet_2d_vector;
+    if (params.has_predicate()) {
+      LOG(FATAL) << "Currently not supported";
+    } else {
+      auto edge_triplet_2d_vector =
+          edge_label_triplet_to_vector_str(edge_label_triplet);
+      formater =
+          boost::format(EDGE_EXPAND_V_OPT_MULTI_EDGE_NO_FILTER_TEMPLATE_STR);
+      formater % opt_var_name % gs::direction_pb_to_str(direction) %
+          edge_triplet_2d_vector;
+    }
   } else {
     throw std::runtime_error("Unknown expand opt");
   }
-
   return std::make_pair(opt_var_name, formater.str());
 }
 
@@ -509,8 +542,8 @@ class EdgeExpandOpBuilder {
         } else if (direction_ == internal::Direction::kIn) {
           dst_vertex_labels_.emplace_back(triplet.src_label().value());
         } else {  // kBoth
-          auto src = triplet.src_label().value();
-          dst_vertex_labels_.emplace_back(src);
+          dst_vertex_labels_.emplace_back(triplet.src_label().value());
+          dst_vertex_labels_.emplace_back(dst_label.value());
         }
       }
       VLOG(10) << "before join: " << gs::to_string(dst_vertex_labels_);
@@ -537,7 +570,7 @@ class EdgeExpandOpBuilder {
     // EdgeExpandOpt.
     std::unordered_set<LabelT> edge_label_set(edge_labels_.begin(),
                                               edge_labels_.end());
-    if (edge_label_set.size() == 1) {
+    if (edge_label_set.size() == 1 && dst_vertex_labels_.size() == 1) {
       LOG(INFO) << "Building simple edge expand opt, with only one edge label";
       std::tie(opt_name, opt_code) = BuildOneLabelEdgeExpandOpt(
           ctx_, direction_, query_params_, dst_vertex_labels_, expand_opt_,
diff --git a/flex/codegen/src/hqps/hqps_expr_builder.h b/flex/codegen/src/hqps/hqps_expr_builder.h
index 8755482799cf..febfcdb93a1e 100644
--- a/flex/codegen/src/hqps/hqps_expr_builder.h
+++ b/flex/codegen/src/hqps/hqps_expr_builder.h
@@ -40,6 +40,7 @@ static constexpr const char* EXPR_BUILDER_TEMPLATE_STR =
     "struct %1% {\n"
     "  public: \n"
     "   using result_t = %2%;\n"
+    "   static constexpr bool filter_null = %10%;\n"
     "   %1%(%3%) %4% {}\n"
     "   %5%\n"
     "   inline %6% operator()(%7%) const {\n"
@@ -71,6 +72,12 @@ static std::pair<int32_t, std::string> variable_to_tag_id_property_selector(
       prop_name = var.property().key().name();
       prop_type = data_type_2_string(
           common_data_type_pb_2_data_type(var.node_type().data_type()));
+    } else if (var_property.has_id()) {
+      // prop_name = "id";
+      // prop_type = data_type_2_string(
+      //     common_data_type_pb_2_data_type(var.node_type().data_type()));
+      prop_name = "global_id";
+      prop_type = data_type_2_string(codegen::DataType::kGlobalVertexId);
     } else {
       LOG(FATAL) << "Unexpected property type: " << var.DebugString();
     }
@@ -88,6 +95,18 @@ static std::pair<int32_t, std::string> variable_to_tag_id_property_selector(
     if (var.node_type().type_case() == common::IrDataType::kDataType) {
       prop_type = data_type_2_string(
           common_data_type_pb_2_data_type(var.node_type().data_type()));
+    } else if (var.node_type().type_case() == common::IrDataType::kGraphType) {
+      if (var.node_type().graph_type().element_opt() ==
+          common::GraphDataType::GraphElementOpt::
+              GraphDataType_GraphElementOpt_VERTEX) {
+        prop_type = data_type_2_string(codegen::DataType::kGlobalVertexId);
+      } else if (var.node_type().graph_type().element_opt() ==
+                 common::GraphDataType::GraphElementOpt::
+                     GraphDataType_GraphElementOpt_EDGE) {
+        prop_type = data_type_2_string(codegen::DataType::kEdgeId);
+      } else {
+        LOG(FATAL) << "Unexpect graph type: " << var.node_type().DebugString();
+      }
     } else {
       prop_type = GRAPE_EMPTY_TYPE;
     }
@@ -122,6 +141,12 @@ static std::string logical_to_str(const common::Logical& logical) {
     return "< WithIn > ";
   case common::Logical::ISNULL:
     return "NONE ==";  // Convert
+  case common::Logical::ENDSWITH:
+    return "< EndWith > ";
+  case common::Logical::STARTSWITH:
+    return "< StartWith > ";
+  case common::Logical::REGEX:
+    return "< Regex > ";
   default:
     throw std::runtime_error("unknown logical");
   }
@@ -181,7 +206,7 @@ static std::string value_pb_to_str(const common::Value& value) {
 
 bool contains_vertex_id(const std::vector<codegen::ParamConst>& params) {
   for (auto& param : params) {
-    if (param.type == codegen::DataType::kVertexId ||
+    if (param.type == codegen::DataType::kGlobalVertexId ||
         param.type == codegen::DataType::kEdgeId) {
       return true;
     }
@@ -252,7 +277,7 @@ static common::DataType eval_expr_return_type(const common::Expression& expr) {
 class ExprBuilder {
  public:
   ExprBuilder(BuildingContext& ctx, int var_id = 0, bool no_build = false)
-      : ctx_(ctx) {
+      : ctx_(ctx), null_presents_(false) {
     if (!no_build) {
       // no build indicates whether we will use this builder as a helper.
       // If set to true, we will not let queryClassName and next_expr_name
@@ -265,6 +290,10 @@ class ExprBuilder {
     res_data_type_.emplace_back(data_type);
   }
 
+  const std::vector<common::DataType>& get_return_type() const {
+    return res_data_type_;
+  }
+
   void add_return_type(common::DataType data_type) {
     res_data_type_.emplace_back(data_type);
   }
@@ -343,6 +372,9 @@ class ExprBuilder {
     case common::ExprOpr::kLogical: {
       auto logical = opr.logical();
       auto str = logical_to_str(logical);
+      if (logical == common::Logical::ISNULL) {
+        null_presents_ = true;
+      }
       VLOG(10) << "Got expr opt logical: " << str;
       expr_nodes_.emplace_back(std::move(str));
       break;
@@ -408,8 +440,8 @@ class ExprBuilder {
     }
   }
 
-  // Add extract operator with var. Currently not support extract on a complicated
-  // expression.
+  // Add extract operator with var. Currently not support extract on a
+  // complicated expression.
   void AddExtractOpr(const common::Extract& extract_opr,
                      const common::ExprOpr& expr_opr) {
     CHECK(expr_opr.item_case() == common::ExprOpr::kVar)
@@ -477,7 +509,7 @@ class ExprBuilder {
     formater % class_name_ % common_data_type_pb_2_str(res_data_type_) %
         constructor_param_str % field_init_code_str %
         func_call_template_typename_str % "auto" % func_call_params_str %
-        func_call_impl_str % private_filed_str;
+        func_call_impl_str % private_filed_str % get_filter_null_str();
 
     std::string str = formater.str();
 
@@ -488,6 +520,15 @@ class ExprBuilder {
   bool empty() const { return expr_nodes_.empty(); }
 
  protected:
+  // If null presents in our expression, we need to get null records. Otherwise,
+  // we can feed expression with non-null records.
+  std::string get_filter_null_str() const {
+    if (null_presents_) {
+      return "false";
+    } else {
+      return "true";
+    }
+  }
   // return the concatenated string of constructor's input params
   std::string get_constructor_params_str() const {
     std::stringstream ss;
@@ -518,12 +559,21 @@ class ExprBuilder {
 
   std::string get_func_call_typename_str() const {
     std::string typename_template = "";
-    if (contains_vertex_id(func_call_vars_)) {
+    if (contains_edge_id(func_call_vars_)) {
       typename_template = "template <typename vertex_id_t>";
     }
     return typename_template;
   }
 
+  bool contains_edge_id(const std::vector<codegen::ParamConst>& params) const {
+    for (auto& param : params) {
+      if (param.type == codegen::DataType::kEdgeId) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   std::string get_func_call_params_str() const {
     std::stringstream ss;
     for (size_t i = 0; i < func_call_vars_.size(); ++i) {
@@ -591,6 +641,7 @@ class ExprBuilder {
   std::vector<common::DataType> res_data_type_;
 
   std::string class_name_;
+  bool null_presents_;
 };
 }  // namespace gs
 
diff --git a/flex/codegen/src/hqps/hqps_fold_builder.h b/flex/codegen/src/hqps/hqps_fold_builder.h
index 1dc7ae6d6b5c..742b8a164d5e 100644
--- a/flex/codegen/src/hqps/hqps_fold_builder.h
+++ b/flex/codegen/src/hqps/hqps_fold_builder.h
@@ -51,8 +51,23 @@ std::pair<std::string, std::string> gen_agg_var_and_code_for_fold(
   auto& vars = agg_func.vars();
   for (int32_t i = 0; i < vars.size(); ++i) {
     auto& var = vars[i];
-    auto raw_tag_id = var.tag().id();
-    in_tags.push_back(ctx.GetTagInd(raw_tag_id));
+    VLOG(10) << "var " << i << " " << var.DebugString();
+    int32_t raw_tag_id;
+    if (var.has_tag()) {
+      raw_tag_id = var.tag().id();
+    } else {
+      raw_tag_id = -1;
+    }
+
+    if (raw_tag_id == -1) {
+      in_tags.push_back(-1);
+    } else if (ctx.GetTagIdAndIndMapping().HasTagId(raw_tag_id)) {
+      in_tags.push_back(ctx.GetTagInd(raw_tag_id));
+    } else if (raw_tag_id == ctx.GetTagIdAndIndMapping().GetMaxTagId() + 1) {
+      in_tags.push_back(-1);
+    } else {
+      LOG(WARNING) << "tag id " << raw_tag_id << " not found in tag id mapping";
+    }
     VLOG(10) << "var " << i << " tag id " << raw_tag_id << " real tag id "
              << in_tags[i];
     if (var.has_property()) {
diff --git a/flex/codegen/src/hqps/hqps_group_by_builder.h b/flex/codegen/src/hqps/hqps_group_by_builder.h
index 1aae290bd792..34a04a31825d 100644
--- a/flex/codegen/src/hqps/hqps_group_by_builder.h
+++ b/flex/codegen/src/hqps/hqps_group_by_builder.h
@@ -83,8 +83,22 @@ std::pair<std::string, std::string> gen_agg_var_and_code(
   auto& vars = agg_func.vars();
   for (int32_t i = 0; i < vars.size(); ++i) {
     auto& var = vars[i];
-    auto raw_tag_id = var.tag().id();
-    in_tags.push_back(ctx.GetTagInd(raw_tag_id));
+    int32_t raw_tag_id = -1;
+    if (var.has_tag()) {
+      raw_tag_id = var.tag().id();
+    }
+    if (raw_tag_id == -1) {
+      in_tags.push_back(-1);
+    } else if (ctx.GetTagIdAndIndMapping().HasTagId(raw_tag_id)) {
+      in_tags.push_back(ctx.GetTagInd(raw_tag_id));
+    } else if (raw_tag_id == ctx.GetTagIdAndIndMapping().GetMaxTagId() + 1) {
+      in_tags.push_back(-1);
+    } else {
+      LOG(WARNING) << "tag id " << raw_tag_id << " not found in tag id mapping";
+      in_tags.push_back(-1);
+    }
+
+    // in_tags.push_back(ctx.GetTagInd(raw_tag_id));
     if (var.has_property()) {
       auto var_prop = var.property();
       if (var_prop.item_case() == common::Property::kId) {
@@ -143,13 +157,16 @@ class GroupByOpBuilder {
   // counting of tag_ind
   GroupByOpBuilder& AddKeyAlias(const physical::GroupBy::KeyAlias& key_alias) {
     CHECK(key_alias.has_alias());
-    CHECK(key_alias.key().has_tag());
+    // CHECK(key_alias.key().has_tag());
     std::string prop_name = "None";
     std::string prop_type;
     auto group_key_var_name = ctx_.GetNextGroupKeyName();
 
     // first in tag id, then alias id
-    int32_t input_tag_id = ctx_.GetTagInd(key_alias.key().tag().id());
+    int32_t input_tag_id = -1;
+    if (key_alias.key().has_tag()) {
+      input_tag_id = ctx_.GetTagInd(key_alias.key().tag().id());
+    }
     int32_t output_tag_id =
         new_tag_id_mapping.CreateOrGetTagInd(key_alias.alias().value());
     // output_col_id should equal to current key's length
diff --git a/flex/codegen/src/hqps/hqps_project_builder.h b/flex/codegen/src/hqps/hqps_project_builder.h
index 18e82454f355..e4027090925e 100644
--- a/flex/codegen/src/hqps/hqps_project_builder.h
+++ b/flex/codegen/src/hqps/hqps_project_builder.h
@@ -37,6 +37,13 @@ static constexpr const char* PROJECT_MAPPER_VAR_TEMPLATE_STR =
 static constexpr const char* PROJECT_MAPPER_EXPR_TEMPLATE_STR =
     "gs::make_mapper_with_expr<%1%>(%2%(%3%) %4%)";
 
+static constexpr const char* PROJECT_MAPPER_KEY_VALUE_TEMPLATE_STR =
+    "gs::make_key_value_mapper<%1%>(\"%2%\", %3%)";
+
+// multiple key-value pairs
+static constexpr const char* PROJECT_KEY_VALUES_TEMPLATE_STR =
+    "gs::make_key_value_mappers(%1%)";
+
 static constexpr const char* PROJECT_OP_TEMPLATE_STR =
     "auto %1% = Engine::Project<%2%>(%3%, std::move(%4%), std::tuple{%5%});\n";
 
@@ -98,7 +105,7 @@ std::tuple<std::string, std::string, std::string> concatenate_expr_built_result(
 // same(excluding null)
 std::string project_case_when_from_project_mapping(
     BuildingContext& ctx, const common::Case& expr_case,
-    common::DataType data_type, int32_t out_alias_tag) {
+    common::DataType data_type) {
   // the common::case contains three expression, input-expr, when-expr,
   // then-expr the input-expr can be null. if it is null, it means we just
   // evaluate when-expr, then-expr like case when.
@@ -145,8 +152,7 @@ std::string project_case_when_from_project_mapping(
 }
 
 std::string project_expression_from_project_mapping(
-    BuildingContext& ctx, const common::Expression& expr,
-    int32_t out_alias_tag) {
+    BuildingContext& ctx, const common::Expression& expr) {
   auto expr_builder = ExprBuilder(ctx);
   VLOG(10) << "Projecting expression: " << expr.DebugString();
   auto ret_data_type = eval_expr_return_type(expr);
@@ -181,47 +187,117 @@ std::string project_expression_from_project_mapping(
   return formater.str();
 }
 
+std::pair<std::string, codegen::DataType> get_prop_name_type_from_variable(
+    const common::Variable& var) {
+  if (!var.has_property()) {
+    return std::make_pair("", codegen::DataType::kEmpty);
+  }
+  auto& property = var.property();
+  if (property.item_case() == common::Property::kId) {
+    return std::make_pair("", codegen::DataType::kGlobalVertexId);
+  } else if (property.item_case() == common::Property::kKey) {
+    return std::make_pair(
+        property.key().name(),
+        common_data_type_pb_2_data_type(var.node_type().data_type()));
+  } else if (property.item_case() == common::Property::kLen) {
+    return std::make_pair("length", codegen::DataType::kLength);
+  } else if (property.item_case() == common::Property::kLabel) {
+    // return the label id.
+    return std::make_pair("label", codegen::DataType::kLabelId);
+  } else {
+    LOG(FATAL) << "Unknown property type" << property.DebugString();
+  }
+}
+
+std::string project_key_value_to_string(BuildingContext& ctx,
+                                        std::string key_str,
+                                        const common::Variable& variable) {
+  int32_t real_in_col_id = -1;
+  if (variable.has_tag()) {
+    auto tag_id = variable.tag().id();
+    if (tag_id != -1) {
+      if (ctx.GetTagIdAndIndMapping().HasTagId(tag_id)) {
+        real_in_col_id = ctx.GetTagIdAndIndMapping().GetTagInd(tag_id);
+      } else if (ctx.GetTagIdAndIndMapping().GetMaxTagId() + 1 == tag_id) {
+        real_in_col_id = -1;
+      } else {
+        LOG(WARNING) << "Tag id: " << tag_id << " not found in tag id mapping";
+        return "";
+      }
+    }
+  }
+
+  auto [prop_name, data_type] = get_prop_name_type_from_variable(variable);
+  // when project key_value to string with empty type, we should project to
+  // globalId.
+  if (data_type == codegen::DataType::kEmpty) {
+    data_type = codegen::DataType::kGlobalVertexId;
+  }
+
+  boost::format select_formater(PROPERTY_SELECTOR);
+  select_formater % data_type_2_string(data_type) % prop_name;
+  auto selector_str = select_formater.str();
+
+  boost::format formater(PROJECT_MAPPER_KEY_VALUE_TEMPLATE_STR);
+  formater % real_in_col_id % key_str % selector_str;
+  return formater.str();
+}
+
 std::string project_variable_mapping_to_string(BuildingContext& ctx,
-                                               const common::ExprOpr& expr_op,
-                                               int32_t real_res_col_id) {
-  int32_t in_tag_id = -2;
+                                               const common::ExprOpr& expr_op);
+
+std::string project_key_values_to_string(
+    BuildingContext& ctx, const common::VariableKeyValues& key_values) {
+  std::vector<std::string> key_value_strs;
+  auto& mappings = key_values.key_vals();
+  for (int32_t i = 0; i < mappings.size(); ++i) {
+    auto& key_value = mappings[i];
+    auto& key = key_value.key();
+    CHECK(key.item_case() == common::Value::kStr);
+    auto& value = key_value.value();
+    auto key_value_str = project_key_value_to_string(ctx, key.str(), value);
+    if (!key_value_str.empty()) {
+      key_value_strs.emplace_back(key_value_str);
+    }
+  }
+
+  auto key_value_str = join_string(key_value_strs, ", ");
+  boost::format formater(PROJECT_KEY_VALUES_TEMPLATE_STR);
+  formater % key_value_str;
+  return formater.str();
+}
+
+std::string project_variable_mapping_to_string(BuildingContext& ctx,
+                                               const common::ExprOpr& expr_op) {
+  int32_t in_tag_id =
+      -2;  // in_tag_id used for expr_opr except for map and case_when
   std::vector<std::string> prop_names;
   std::vector<codegen::DataType> data_types;
-  bool project_self = false;
   switch (expr_op.item_case()) {
   case common::ExprOpr::kCase: {
     VLOG(10) << "Got case when in projecting";
     auto case_when = expr_op.case_();
     VLOG(10) << case_when.DebugString();
     return project_case_when_from_project_mapping(
-        ctx, case_when, expr_op.node_type().data_type(), real_res_col_id);
+        ctx, case_when, expr_op.node_type().data_type());
+  }
+  case common::ExprOpr::kMap: {
+    VLOG(10) << "Got map in projecting";
+    auto& map = expr_op.map();
+    return project_key_values_to_string(ctx, map);
   }
   case common::ExprOpr::kVar: {
     VLOG(10) << "Got var in projecting";
     auto& var = expr_op.var();
-    in_tag_id = var.tag().id();
-    if (var.has_property()) {
-      auto& prop = var.property();
-      if (prop.item_case() == common::Property::kId) {
-        project_self = true;
-      } else if (prop.item_case() == common::Property::kKey) {
-        prop_names.push_back(prop.key().name());
-        data_types.push_back(
-            common_data_type_pb_2_data_type(var.node_type().data_type()));
-      } else if (prop.item_case() == common::Property::kLen) {
-        prop_names.push_back("length");
-        data_types.push_back(codegen::DataType::kLength);
-      } else if (prop.item_case() == common::Property::kLabel) {
-        // return the label id.
-        prop_names.push_back("label");
-        data_types.push_back(codegen::DataType::kLabelId);
-      } else {
-        LOG(FATAL) << "Unknown property type" << prop.DebugString();
-      }
+    if (var.has_tag()) {
+      in_tag_id = var.tag().id();
     } else {
-      VLOG(10) << "receives no property, project itself";
-      project_self = true;
+      in_tag_id = -1;
     }
+    // auto& prop = var.property();
+    auto [prop_name, data_type] = get_prop_name_type_from_variable(var);
+    prop_names.push_back(prop_name);
+    data_types.push_back(data_type);
     break;
   }
   case common::ExprOpr::kVarMap: {
@@ -241,77 +317,43 @@ std::string project_variable_mapping_to_string(BuildingContext& ctx,
       } else {
         CHECK(in_tag_id == var.tag().id()) << "can only support one tag";
       }
-
-      auto& prop = var.property();
-      // if (prop.has_id()) {
-      if (prop.item_case() == common::Property::kId) {
-        LOG(FATAL) << "Not support project id in projecting with vars";
-        // } else if (prop.has_key()) {
-      } else if (prop.item_case() == common::Property::kKey) {
-        prop_names.push_back(prop.key().name());
-        data_types.push_back(
-            common_data_type_pb_2_data_type(var.node_type().data_type()));
-      } else {
-        LOG(FATAL) << "Unknown property type" << prop.DebugString();
-      }
+      auto [prop_name, data_type] = get_prop_name_type_from_variable(var);
+      prop_names.push_back(prop_name);
+      data_types.push_back(data_type);
     }
     break;
   }
+
   default:
-    LOG(FATAL) << "Unknown variable type";
+    LOG(FATAL) << "Unknown variable type: " << expr_op.DebugString();
   }
-  auto real_in_col_id = ctx.GetTagInd(in_tag_id);
-  VLOG(10) << "real_in_tag_id: " << real_in_col_id
-           << " in_tag_id: " << in_tag_id;
-
-  if (project_self) {
-    VLOG(10) << "Projecting self";
-    CHECK(prop_names.size() == 0 && data_types.size() == 0);
-    std::string selector_str;
-    {
-      boost::format select_formater(PROPERTY_SELECTOR);
-      select_formater % EMPTY_TYPE % "";
-      selector_str = select_formater.str();
-    }
-    boost::format formater(PROJECT_MAPPER_VAR_TEMPLATE_STR);
 
-    formater % real_in_col_id % selector_str;
-    return formater.str();
-  } else {
-    VLOG(10) << "Projecting properties" << gs::to_string(prop_names);
-    CHECK(prop_names.size() == data_types.size());
-    CHECK(prop_names.size() == 1);
-    std::string selector_str;
-    {
-      boost::format select_formater(PROPERTY_SELECTOR);
-      select_formater % data_type_2_string(data_types[0]) % prop_names[0];
-      selector_str = select_formater.str();
+  int32_t real_in_col_id = -1;
+  if (in_tag_id != -1) {
+    if (ctx.GetTagIdAndIndMapping().HasTagId(in_tag_id)) {
+      real_in_col_id = ctx.GetTagIdAndIndMapping().GetTagInd(in_tag_id);
+    } else if (ctx.GetTagIdAndIndMapping().GetMaxTagId() + 1 == in_tag_id) {
+      real_in_col_id = -1;
+    } else {
+      LOG(WARNING) << "Tag id: " << in_tag_id << " not found in tag id mapping";
+      return "";
     }
-    boost::format formater(PROJECT_MAPPER_VAR_TEMPLATE_STR);
-    formater % real_in_col_id % selector_str;
-    return formater.str();
   }
-}
 
-std::string project_mapping_to_string(
-    BuildingContext& ctx, const physical::Project::ExprAlias& mapping,
-    TagIndMapping& new_tag_ind_map) {
-  int32_t res_alias = mapping.alias().value();
-  // TODO: Currently we assume each expr_alias contains only property for that
-  // input tag
-
-  auto real_res_alias = new_tag_ind_map.CreateOrGetTagInd(res_alias);
-  auto& expr = mapping.expr();
-  // CHECK(expr.operators_size() == 1) << "can only support one variable";
-  if (expr.operators_size() > 1) {
-    return project_expression_from_project_mapping(ctx, expr, real_res_alias);
-  } else if (expr.operators_size() == 1) {
-    auto& expr_op = expr.operators(0);
-    return project_variable_mapping_to_string(ctx, expr_op, real_res_alias);
-  } else {
-    LOG(FATAL) << "expect at least one expr opr";
-    return "";
-  }
+  VLOG(10) << "real_in_tag_id: " << real_in_col_id
+           << " in_tag_id: " << in_tag_id;
+
+  CHECK(prop_names.size() == data_types.size());
+  std::string selector_str;
+  VLOG(10) << "Projecting properties" << gs::to_string(prop_names);
+  CHECK(prop_names.size() == 1);
+  boost::format select_formater(PROPERTY_SELECTOR);
+  select_formater % data_type_2_string(data_types[0]) % prop_names[0];
+  selector_str = select_formater.str();
+
+  boost::format formater(PROJECT_MAPPER_VAR_TEMPLATE_STR);
+  formater % real_in_col_id % selector_str;
+  return formater.str();
 }
 
 /**
@@ -329,6 +371,11 @@ class ProjectOpBuilder {
     return *this;
   }
 
+  ProjectOpBuilder& res_alias(int32_t res_alias) {
+    res_alias_ = res_alias;
+    return *this;
+  }
+
   ProjectOpBuilder& add_mapping(const physical::Project::ExprAlias& mapping) {
     mappings_.push_back(mapping);
     return *this;
@@ -337,20 +384,35 @@ class ProjectOpBuilder {
   // return make_project code and call project code.
   std::string Build() const {
     TagIndMapping new_tag_id_mapping;
+    if (is_append_) {
+      new_tag_id_mapping = ctx_.GetTagIdAndIndMapping();
+    }
     std::string project_cols_code;
     std::string prev_ctx_name, next_ctx_name;
     std::tie(prev_ctx_name, next_ctx_name) = ctx_.GetPrevAndNextCtxName();
     std::stringstream ss;
     for (size_t i = 0; i < mappings_.size(); ++i) {
-      ss << project_mapping_to_string(ctx_, mappings_[i], new_tag_id_mapping);
-      if (i != mappings_.size() - 1) {
-        ss << ", ";
+      auto res = project_mapping_to_string(mappings_[i]);
+      if (!res.empty()) {
+        ss << res;
+        if (i != mappings_.size() - 1) {
+          ss << ", ";
+        }
+        // Only insert res_alias to new_tag_ind_map when res is not empty
+        int32_t res_alias = -1;
+        if (mappings_[i].has_alias()) {
+          res_alias = mappings_[i].alias().value();
+        }
+        new_tag_id_mapping.CreateOrGetTagInd(res_alias);
       }
     }
     project_cols_code = ss.str();
+    LOG(INFO) << "res alias: " << res_alias_.has_value()
+              << " res_alias: " << res_alias_.value();
+    bool is_temp = res_alias_.has_value() && res_alias_.value() == -1;
 
     boost::format formater(PROJECT_OP_TEMPLATE_STR);
-    formater % next_ctx_name % project_is_append_str(is_append_) %
+    formater % next_ctx_name % project_is_append_str(is_append_, is_temp) %
         ctx_.GraphVar() % prev_ctx_name % project_cols_code;
     ctx_.UpdateTagIdAndIndMapping(new_tag_id_mapping);
 
@@ -358,8 +420,24 @@ class ProjectOpBuilder {
   }
 
  private:
+  std::string project_mapping_to_string(
+      const physical::Project::ExprAlias& mapping) const {
+    auto& expr = mapping.expr();
+    // CHECK(expr.operators_size() == 1) << "can only support one variable";
+    if (expr.operators_size() > 1) {
+      return project_expression_from_project_mapping(ctx_, expr);
+    } else if (expr.operators_size() == 1) {
+      auto& expr_op = expr.operators(0);
+      return project_variable_mapping_to_string(ctx_, expr_op);
+    } else {
+      LOG(FATAL) << "expect at least one expr opr";
+      return "";
+    }
+  }
+
   BuildingContext& ctx_;
   bool is_append_;
+  std::optional<int32_t> res_alias_;
   std::vector<physical::Project::ExprAlias> mappings_;
 };
 
@@ -368,6 +446,7 @@ static std::string BuildProjectOp(
     const physical::PhysicalOpr::MetaData& meta_data) {
   ProjectOpBuilder builder(ctx);
   builder.is_append(project_pb.is_append());
+  builder.res_alias(meta_data.alias());
   auto& mappings = project_pb.mappings();
   for (int32_t i = 0; i < mappings.size(); ++i) {
     builder.add_mapping(mappings[i]);
diff --git a/flex/codegen/src/hqps/hqps_scan_builder.h b/flex/codegen/src/hqps/hqps_scan_builder.h
index 80a440ce8cc9..153d553c8eb0 100644
--- a/flex/codegen/src/hqps/hqps_scan_builder.h
+++ b/flex/codegen/src/hqps/hqps_scan_builder.h
@@ -15,6 +15,7 @@ limitations under the License.
 #ifndef CODEGEN_SRC_HQPS_HQPS_SCAN_BUILDER_H_
 #define CODEGEN_SRC_HQPS_HQPS_SCAN_BUILDER_H_
 
+#include <optional>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -23,10 +24,12 @@ limitations under the License.
 #include "flex/codegen/src/building_context.h"
 #include "flex/codegen/src/codegen_utils.h"
 #include "flex/codegen/src/graph_types.h"
+#include "flex/codegen/src/hqps/hqps_expr_builder.h"
 #include "flex/codegen/src/pb_parser/query_params_parser.h"
 #include "flex/proto_generated_gie/algebra.pb.h"
 #include "flex/proto_generated_gie/common.pb.h"
 #include "flex/proto_generated_gie/physical.pb.h"
+#include "flex/storages/rt_mutable_graph/schema.h"
 
 // #define FAST_SCAN
 
@@ -48,6 +51,16 @@ static constexpr const char* SCAN_OP_TEMPLATE_NO_EXPR_STR =
     "auto %1% = Engine::template ScanVertex<%2%>(%3%, %4%, "
     "Filter<TruePredicate>());\n";
 
+static constexpr const char* SCAN_OP_BOTH_OID_EXPR_TEMPLATE_STR =
+    "auto %1% = gs::make_filter(%2%(%3%) %4%);\n"
+    "auto %5% = Engine::template ScanVertexWithOidExpr<%6%, %7%>(%8%, %9%, "
+    "%10%, std::move(%1%));\n";
+
+static constexpr const char* SCAN_OP_BOTH_GID_EXPR_TEMPLATE_STR =
+    "auto %1% = gs::make_filter(%2%(%3%) %4%);\n"
+    "auto %5% = Engine::template ScanVertexWithGidExpr<%6%, %7%>(%8%, %9%, "
+    "%10%, std::move(%1%));\n";
+
 /// Args
 /// 1. res_ctx_name
 /// 2. AppendOpt,
@@ -57,6 +70,15 @@ static constexpr const char* SCAN_OP_TEMPLATE_NO_EXPR_STR =
 static constexpr const char* SCAN_OP_WITH_OID_ONE_LABEL_TEMPLATE_STR =
     "auto %1% = Engine::template ScanVertexWithOid<%2%,%3%>(%4%, %5%, %6%);\n";
 
+/// Args
+/// 1. res_ctx_name
+/// 2. AppendOpt,
+/// 3. graph name
+/// 4. vertex label
+/// 5. gid
+static constexpr const char* SCAN_OP_WITH_GID_ONE_LABEL_TEMPLATE_STR =
+    "auto %1% = Engine::template ScanVertexWithGid<%2%,%3%>(%4%, %5%, %6%);\n";
+
 /// Args
 /// 1. res_ctx_name
 /// 2. AppendOpt,
@@ -64,16 +86,27 @@ static constexpr const char* SCAN_OP_WITH_OID_ONE_LABEL_TEMPLATE_STR =
 /// 4. vertex label
 /// 5. oid
 static constexpr const char* SCAN_OP_WITH_OID_MUL_LABEL_TEMPLATE_STR =
-    "auto %1% = Engine::template ScanVertexWithOid<%2%>(%3%, "
-    "std::array<label_id_t, %4%>{%5%}, %6%);\n";
+    "auto %1% = Engine::template ScanVertexWithOid<%2%, %3%>(%4%, "
+    "std::array<label_id_t, %5%>{%6%}, %7%);\n";
+
+/// Args
+/// 1. res_ctx_name
+/// 2. AppendOpt,
+/// 3. graph name
+/// 4. vertex label
+/// 5. gid
+static constexpr const char* SCAN_OP_WITH_GID_MUL_LABEL_TEMPLATE_STR =
+    "auto %1% = Engine::template ScanVertexWithGid<%2%, %3%>(%4%, "
+    "std::array<label_id_t, %5%>{%6%}, %7%);\n";
 
 /**
  * @brief When building scanOp, we ignore the data type provided in the pb.
- *
+ * The expression can from index predicate or query params, or both.
  */
 class ScanOpBuilder {
  public:
-  ScanOpBuilder(BuildingContext& ctx) : ctx_(ctx) {}
+  ScanOpBuilder(BuildingContext& ctx, const std::optional<Schema>& schema)
+      : ctx_(ctx), schema_(schema), expr_builder_(ctx_) {}
 
   ScanOpBuilder& scanOpt(const physical::Scan::ScanOpt& opt) {
     if (opt != physical::Scan::ScanOpt::Scan_ScanOpt_VERTEX) {
@@ -114,41 +147,17 @@ class ScanOpBuilder {
         intersection(labels_ids_, expr_label_ids);
       }
 
-      auto expr_builder = ExprBuilder(ctx_);
-      expr_builder.set_return_type(common::DataType::BOOLEAN);
-      expr_builder.AddAllExprOpr(query_params.predicate().operators());
-
-      std::string expr_code;
-      std::vector<codegen::ParamConst> func_call_param_const;
-      std::vector<std::pair<int32_t, std::string>> expr_tag_props;
-      std::vector<common::DataType> unused_expr_ret_type;
-      std::tie(expr_func_name_, func_call_param_const, expr_tag_props,
-               expr_code, unused_expr_ret_type) = expr_builder.Build();
-      VLOG(10) << "Found expr in scan:  " << expr_func_name_;
-      // generate code.
-      ctx_.AddExprCode(expr_code);
-      expr_var_name_ = ctx_.GetNextExprVarName();
-      {
-        std::stringstream ss;
-        for (size_t i = 0; i < func_call_param_const.size(); ++i) {
-          ss << func_call_param_const[i].var_name;
-          if (i != func_call_param_const.size() - 1) {
-            ss << ",";
-          }
-        }
-        expr_construct_params_ = ss.str();
-      }
-      {
-        std::stringstream ss;
-        if (expr_tag_props.size() > 0) {
-          ss << ",";
-          for (size_t i = 0; i + 1 < expr_tag_props.size(); ++i) {
-            ss << expr_tag_props[i].second << ", ";
-          }
-          ss << expr_tag_props[expr_tag_props.size() - 1].second;
-        }
-        selectors_str_ = ss.str();
-      }
+      // TODO: make expr_builder a member of ScanOpBuilder
+      // auto expr_builder = ExprBuilder(ctx_);
+      expr_builder_.set_return_type(common::DataType::BOOLEAN);
+      // Add extra (, ) to wrap the code, since we may append index_predicate
+      // afterwards.
+      common::ExprOpr left_brace, right_brace;
+      left_brace.set_brace(common::ExprOpr_Brace_LEFT_BRACE);
+      right_brace.set_brace(common::ExprOpr_Brace_RIGHT_BRACE);
+      expr_builder_.AddExprOpr(left_brace);
+      expr_builder_.AddAllExprOpr(query_params.predicate().operators());
+      expr_builder_.AddExprOpr(right_brace);
     }
     return *this;
   }
@@ -165,33 +174,69 @@ class ScanOpBuilder {
       throw std::runtime_error(
           std::string("Currently only support one predicate"));
     }
-    CHECK(expr_func_name_.empty()) << "Predicate is already given by expr";
+
     auto or_predicate = predicate.or_predicates(0);
     if (or_predicate.predicates_size() != 1) {
       throw std::runtime_error(
           std::string("Currently only support one and predicate"));
     }
     auto triplet = or_predicate.predicates(0);
+    if (!triplet.has_key()) {
+      LOG(FATAL) << "Expect key in index predicate.";
+    }
+    auto& key = triplet.key();
+    if (key.has_id()) {
+      // scan with global id.
+      scan_with_oid_gid_ = false;
+    } else if (key.has_key()) {
+      // scan with primary key.We only support one primary key.
+      scan_with_oid_gid_ = true;
+    } else {
+      LOG(FATAL) << "Expect id or key in index predicate.";
+    }
+
     if (triplet.value_case() == algebra::IndexPredicate::Triplet::kConst) {
       // FUTURE: check property is really the primary key.
       auto const_value = triplet.const_();
       switch (const_value.item_case()) {
       case common::Value::kI32:
-        oid_ = std::to_string(const_value.i32());
-        oid_type_name_ = "int32_t";
+        oid_or_gid_ = std::to_string(const_value.i32());
+        oid_or_gid_type_name_ = "int32_t";
         break;
       case common::Value::kI64:
-        oid_ = std::to_string(const_value.i64());
-        oid_type_name_ = "int64_t";
+        oid_or_gid_ = std::to_string(const_value.i64());
+        oid_or_gid_type_name_ = "int64_t";
         break;
       case common::Value::kStr:
-        oid_ = const_value.str();
-        oid_type_name_ = "std::string_view";
+        oid_or_gid_ = const_value.str();
+        oid_or_gid_type_name_ = "std::string_view";
+      case common::Value::kI32Array:
+        oid_or_gid_ = "";
+        for (int32_t i = 0; i < const_value.i32_array().item_size(); ++i) {
+          oid_or_gid_ += std::to_string(const_value.i32_array().item(i));
+          if (i + 1 != const_value.i32_array().item_size()) {
+            oid_or_gid_ += ", ";
+          }
+        }
+        oid_or_gid_type_name_ = "int32_t";
+        break;
+      case common::Value::kI64Array:
+        oid_or_gid_ = "";
+        for (int32_t i = 0; i < const_value.i64_array().item_size(); ++i) {
+          oid_or_gid_ += std::to_string(const_value.i64_array().item(i));
+          if (i + 1 != const_value.i64_array().item_size()) {
+            oid_or_gid_ += ", ";
+          }
+        }
+        oid_or_gid_type_name_ = "int64_t";
+        break;
+
       default:
-        LOG(FATAL) << "Currently only support int, long as primary key";
+        LOG(FATAL) << "Currently only support int, long as primary key: "
+                   << const_value.DebugString();
       }
-      VLOG(1) << "Found oid: " << oid_
-              << " in index scan, type: " << oid_type_name_;
+      VLOG(1) << "Found oid/gid: " << oid_or_gid_
+              << " in index scan, type: " << oid_or_gid_type_name_;
     } else {
       // dynamic param
       auto dyn_param_pb = triplet.param();
@@ -199,72 +244,131 @@ class ScanOpBuilder {
       VLOG(10) << "receive param const in index predicate: "
                << dyn_param_pb.DebugString();
       ctx_.AddParameterVar(param_const);
-      // set to oid_ and oid_type_name_
-      oid_ = param_const.var_name;
-      oid_type_name_ = data_type_2_string(param_const.type);
+      // set to oid_or_gid_ and oid_or_gid_type_name_
+      oid_or_gid_ = param_const.var_name;
+      oid_or_gid_type_name_ = data_type_2_string(param_const.type);
     }
 
+    get_real_oid_gid_type_name();
+    // add std::vector{ %} to oid_or_gid_
+    oid_or_gid_ =
+        "std::vector<" + oid_or_gid_type_name_ + ">{" + oid_or_gid_ + "}";
     return *this;
   }
 
+  void build_expr(std::string& expr_var_name, std::string& expr_func_name,
+                  std::string& expr_construct_params,
+                  std::string& selectors_str) const {
+    std::string expr_code;
+    std::vector<codegen::ParamConst> func_call_param_const;
+    std::vector<std::pair<int32_t, std::string>> expr_tag_props;
+    std::vector<common::DataType> unused_expr_ret_type;
+    std::tie(expr_func_name, func_call_param_const, expr_tag_props, expr_code,
+             unused_expr_ret_type) = expr_builder_.Build();
+    VLOG(10) << "Found expr in scan:  " << expr_func_name;
+    // generate code.
+    ctx_.AddExprCode(expr_code);
+    expr_var_name = ctx_.GetNextExprVarName();
+    {
+      std::stringstream ss;
+      for (size_t i = 0; i < func_call_param_const.size(); ++i) {
+        ss << func_call_param_const[i].var_name;
+        if (i != func_call_param_const.size() - 1) {
+          ss << ",";
+        }
+      }
+      expr_construct_params = ss.str();
+    }
+    {
+      std::stringstream ss;
+      if (expr_tag_props.size() > 0) {
+        ss << ",";
+        for (size_t i = 0; i + 1 < expr_tag_props.size(); ++i) {
+          ss << expr_tag_props[i].second << ", ";
+        }
+        ss << expr_tag_props[expr_tag_props.size() - 1].second;
+      }
+      selectors_str = ss.str();
+    }
+  }
+
   std::string Build() const {
     // 1. If common expression predicate presents, scan with expression
-    if (!expr_func_name_.empty()) {
+    if (!expr_builder_.get_return_type().empty()) {
       VLOG(1) << "Scan with expression";
-      return scan_with_expr(labels_ids_, expr_var_name_, expr_func_name_,
-                            expr_construct_params_, selectors_str_);
+
+      std::string expr_var_name, expr_func_name, expr_construct_params,
+          selectors_str;  // The expression decode from params.
+
+      build_expr(expr_var_name, expr_func_name, expr_construct_params,
+                 selectors_str);
+      if (oid_or_gid_.empty()) {
+        return scan_with_expr(expr_var_name, expr_func_name,
+                              expr_construct_params, selectors_str);
+      } else {
+        return scan_with_expr_and_oid_gid(expr_var_name, expr_func_name,
+                                          expr_construct_params, selectors_str);
+      }
     } else {
-      // If oid_ not empty, scan with oid
-      if (!oid_.empty()) {
-        VLOG(1) << "Scan with oid: " << oid_;
-        return scan_with_oid(labels_ids_, oid_, oid_type_name_);
+      // If oid_or_gid_ not empty, scan with oid
+      if (!oid_or_gid_.empty()) {
+        VLOG(1) << "Scan with oid: " << oid_or_gid_;
+        return scan_with_oid_gid();
       } else {
         // If no oid, scan without expression
         VLOG(1) << "Scan without expression";
-        return scan_without_expr(labels_ids_);
+        return scan_without_expr();
       }
     }
   }
 
  private:
-  std::string scan_with_oid(const std::vector<int32_t>& label_ids,
-                            const std::string& oid,
-                            const std::string& oid_type_name) const {
-    VLOG(10) << "Scan with oid: " << oid;
+  std::string scan_with_oid_gid() const {
+    VLOG(10) << "Scan with oid/gid: " << oid_or_gid_;
+
     std::string next_ctx_name = ctx_.GetCurCtxName();
     auto append_opt = res_alias_to_append_opt(res_alias_);
 
-    if (label_ids.size() == 1) {
-      boost::format formater(SCAN_OP_WITH_OID_ONE_LABEL_TEMPLATE_STR);
-      formater % next_ctx_name % append_opt % oid_type_name % ctx_.GraphVar() %
-          label_ids[0] % oid;
+    boost::format formater;
+    if (labels_ids_.size() == 1) {
+      if (scan_with_oid_gid_) {
+        formater = boost::format(SCAN_OP_WITH_OID_ONE_LABEL_TEMPLATE_STR);
+      } else {
+        formater = boost::format(SCAN_OP_WITH_GID_ONE_LABEL_TEMPLATE_STR);
+      }
+      formater % next_ctx_name % append_opt % oid_or_gid_type_name_ %
+          ctx_.GraphVar() % labels_ids_[0] % oid_or_gid_;
       return formater.str();
     } else {
-      boost::format formater(SCAN_OP_WITH_OID_MUL_LABEL_TEMPLATE_STR);
+      if (scan_with_oid_gid_) {
+        formater = boost::format(SCAN_OP_WITH_OID_MUL_LABEL_TEMPLATE_STR);
+      } else {
+        formater = boost::format(SCAN_OP_WITH_GID_MUL_LABEL_TEMPLATE_STR);
+      }
       std::stringstream ss;
-      for (size_t i = 0; i + 1 < label_ids.size(); ++i) {
-        ss << std::to_string(label_ids[i]) << ", ";
+      for (size_t i = 0; i + 1 < labels_ids_.size(); ++i) {
+        ss << std::to_string(labels_ids_[i]) << ", ";
       }
-      ss << std::to_string(label_ids[label_ids.size() - 1]);
-      formater % next_ctx_name % append_opt % ctx_.GraphVar() %
-          label_ids.size() % ss.str() % oid;
+      ss << std::to_string(labels_ids_[labels_ids_.size() - 1]);
+      formater % next_ctx_name % append_opt % oid_or_gid_type_name_ %
+          ctx_.GraphVar() % labels_ids_.size() % ss.str() % oid_or_gid_;
       return formater.str();
     }
   }
 
-  std::string scan_without_expr(const std::vector<int32_t>& label_ids) const {
+  std::string scan_without_expr() const {
     std::string label_ids_str;
     {
       std::stringstream ss;
-      CHECK(label_ids.size() > 0);
-      if (label_ids.size() == 1) {
-        ss << label_ids[0];
+      CHECK(labels_ids_.size() > 0);
+      if (labels_ids_.size() == 1) {
+        ss << labels_ids_[0];
       } else {
-        ss << "std::array<label_id_t, " << label_ids.size() << "> {";
-        for (size_t i = 0; i + 1 < label_ids.size(); ++i) {
-          ss << std::to_string(label_ids[i]) << ", ";
+        ss << "std::array<label_id_t, " << labels_ids_.size() << "> {";
+        for (size_t i = 0; i + 1 < labels_ids_.size(); ++i) {
+          ss << std::to_string(labels_ids_[i]) << ", ";
         }
-        ss << std::to_string(label_ids[label_ids.size() - 1]);
+        ss << std::to_string(labels_ids_[labels_ids_.size() - 1]);
         ss << "}";
       }
       label_ids_str = ss.str();
@@ -275,8 +379,7 @@ class ScanOpBuilder {
     return formater.str();
   }
 
-  std::string scan_with_expr(const std::vector<int32_t>& label_ids,
-                             const std::string& expr_var_name,
+  std::string scan_with_expr(const std::string& expr_var_name,
                              const std::string& expr_func_name,
                              const std::string& expr_construct_params,
                              const std::string& selectors_str) const {
@@ -284,15 +387,15 @@ class ScanOpBuilder {
     std::string label_ids_str;
     {
       std::stringstream ss;
-      CHECK(label_ids.size() > 0);
-      if (label_ids.size() == 1) {
-        ss << label_ids[0];
+      CHECK(labels_ids_.size() > 0);
+      if (labels_ids_.size() == 1) {
+        ss << labels_ids_[0];
       } else {
-        ss << "std::array<label_id_t, " << label_ids.size() << "> {";
-        for (size_t i = 0; i + 1 < label_ids.size(); ++i) {
-          ss << std::to_string(label_ids[i]) << ", ";
+        ss << "std::array<label_id_t, " << labels_ids_.size() << "> {";
+        for (size_t i = 0; i + 1 < labels_ids_.size(); ++i) {
+          ss << std::to_string(labels_ids_[i]) << ", ";
         }
-        ss << std::to_string(label_ids[label_ids.size() - 1]);
+        ss << std::to_string(labels_ids_[labels_ids_.size() - 1]);
         ss << "}";
       }
       label_ids_str = ss.str();
@@ -305,23 +408,114 @@ class ScanOpBuilder {
     return formater.str();
   }
 
+  std::string scan_with_expr_and_oid_gid(
+      const std::string& expr_var_name, const std::string& expr_func_name,
+      const std::string& expr_construct_params,
+      const std::string& selectors_str) const {
+    std::string next_ctx_name = ctx_.GetCurCtxName();
+    std::string label_ids_str;
+    {
+      std::stringstream ss;
+      CHECK(labels_ids_.size() > 0);
+      if (labels_ids_.size() == 1) {
+        ss << labels_ids_[0];
+      } else {
+        ss << "std::array<label_id_t, " << labels_ids_.size() << "> {";
+        for (size_t i = 0; i + 1 < labels_ids_.size(); ++i) {
+          ss << std::to_string(labels_ids_[i]) << ", ";
+        }
+        ss << std::to_string(labels_ids_[labels_ids_.size() - 1]);
+        ss << "}";
+      }
+      label_ids_str = ss.str();
+    }
+
+    boost::format formater;
+    if (scan_with_oid_gid_) {
+      formater = boost::format(SCAN_OP_BOTH_OID_EXPR_TEMPLATE_STR);
+    } else {
+      formater = boost::format(SCAN_OP_BOTH_GID_EXPR_TEMPLATE_STR);
+    }
+    formater % expr_var_name % expr_func_name % expr_construct_params %
+        selectors_str % next_ctx_name % res_alias_to_append_opt(res_alias_) %
+        oid_or_gid_type_name_ % ctx_.GraphVar() % label_ids_str % oid_or_gid_;
+    return formater.str();
+  }
+
+  void get_real_oid_gid_type_name() {
+    if (scan_with_oid_gid_) {
+      if (!schema_.has_value()) {
+        LOG(INFO) << "No schema found";
+        return;
+      }
+      auto& real_schema = schema_.value();
+      if (labels_ids_.size() < 1) {
+        LOG(FATAL) << "No label id found";
+      }
+      std::unordered_set<std::string> oid_types;
+      for (auto label_id : labels_ids_) {
+        if (label_id >= real_schema.vertex_label_num()) {
+          LOG(FATAL) << "label id " << label_id << " is not a valid label id";
+        }
+        auto pk_types = real_schema.get_vertex_primary_key(label_id);
+        CHECK(pk_types.size() == 1) << "Currently only support one primary key";
+        oid_types.insert(pk_type_to_string(std::get<0>(pk_types[0])));
+      }
+      if (oid_types.size() > 1) {
+        LOG(FATAL) << "Find different oid types in different labels: "
+                   << oid_types.size()
+                   << " types, currently can only support 1.";
+      }
+      if (oid_types.size() < 1) {
+        LOG(FATAL) << "No oid type found in schema";
+      }
+      LOG(INFO) << "parsed oid type name: " << oid_or_gid_type_name_
+                << ", real oid type "
+                   "name: "
+                << *oid_types.begin();
+      oid_or_gid_type_name_ = *oid_types.begin();
+    } else {
+      // We have builtin gid type. We don't need to check the schema.
+      oid_or_gid_type_name_ = "gid_t";
+    }
+  }
+
+  std::string pk_type_to_string(const gs::PropertyType type) const {
+    if (type == gs::PropertyType::Int32()) {
+      return "int32_t";
+    } else if (type == gs::PropertyType::Int64()) {
+      return "int64_t";
+    } else if (type == gs::PropertyType::UInt32()) {
+      return "uint32_t";
+    } else if (type == gs::PropertyType::UInt64()) {
+      return "uint64_t";
+    } else if (type == gs::PropertyType::String()) {
+      return "std::string_view";
+    } else {
+      LOG(FATAL) << "Currently only support int, long, string as primary key";
+    }
+  }
+
   BuildingContext& ctx_;
+  const std::optional<Schema>& schema_;
+  ExprBuilder expr_builder_;
   physical::Scan::ScanOpt scan_opt_;
   std::vector<int32_t> labels_ids_;
-  std::string expr_var_name_, expr_func_name_, expr_construct_params_,
-      selectors_str_;  // The expression decode from params.
-  std::string oid_;    // the oid decode from idx predicate, or param name.
-  std::string oid_type_name_;
+  bool scan_with_oid_gid_;  // true if oid , false if gid
+  std::string oid_or_gid_;  // the oid decode from idx predicate, or param name,
+                            // or the global vertex id.
+  std::string oid_or_gid_type_name_;  // the type of oid_or_gid_
   int res_alias_;
 };
 
-static std::string BuildScanOp(
-    BuildingContext& ctx, const physical::Scan& scan_pb,
-    const physical::PhysicalOpr::MetaData& meta_data) {
+static std::string BuildScanOp(BuildingContext& ctx,
+                               const physical::Scan& scan_pb,
+                               const physical::PhysicalOpr::MetaData& meta_data,
+                               const std::optional<Schema>& schema) {
   if (!scan_pb.has_params()) {
     throw std::runtime_error(std::string("expect scan pb has params"));
   }
-  auto builder = ScanOpBuilder(ctx).scanOpt(scan_pb.scan_opt());
+  auto builder = ScanOpBuilder(ctx, schema).scanOpt(scan_pb.scan_opt());
   if (scan_pb.has_alias()) {
     VLOG(10) << "scan pb has alias" << scan_pb.alias().value();
     builder.resAlias(scan_pb.alias().value());
diff --git a/flex/codegen/src/hqps/hqps_sort_builder.h b/flex/codegen/src/hqps/hqps_sort_builder.h
index 1e04c5898443..97ecb1c93a5e 100644
--- a/flex/codegen/src/hqps/hqps_sort_builder.h
+++ b/flex/codegen/src/hqps/hqps_sort_builder.h
@@ -67,12 +67,10 @@ std::string sort_pair_pb_to_order_pair(
       throw std::runtime_error("Unknown sort property type" +
                                sort_property.DebugString());
     }
-  } else if (pair.key().has_tag()) {
+  } else {
     sort_prop_name = "";
     sort_prop_type =
         single_common_data_type_pb_2_str(pair.key().node_type().data_type());
-  } else {
-    throw std::runtime_error("Unknown sort property type" + pair.DebugString());
   }
 
   boost::format formater(ORDERING_PAIR_TEMPLATE_STR);
diff --git a/flex/codegen/src/hqps_generator.h b/flex/codegen/src/hqps_generator.h
index 8b1eb78cf391..4bd72e725763 100644
--- a/flex/codegen/src/hqps_generator.h
+++ b/flex/codegen/src/hqps_generator.h
@@ -34,6 +34,7 @@ limitations under the License.
 #include "flex/codegen/src/hqps/hqps_sink_builder.h"
 #include "flex/codegen/src/hqps/hqps_sort_builder.h"
 #include "flex/proto_generated_gie/physical.pb.h"
+#include "flex/storages/rt_mutable_graph/schema.h"
 
 namespace gs {
 
@@ -57,6 +58,7 @@ static constexpr const char* QUERY_TEMPLATE_STR =
     "  using Engine = SyncEngine<%4%>;\n"
     "  using label_id_t = typename %4%::label_id_t;\n"
     "  using vertex_id_t = typename %4%::vertex_id_t;\n"
+    "  using gid_t = typename %4%::gid_t;\n"
     " // constructor\n"
     "  %3%(const GraphDBSession& session) : %6%(session) {}\n"
     "// Query function for query class\n"
@@ -116,10 +118,40 @@ static std::array<std::string, 4> BuildIntersectOp(
 // get_v can contains labels and filters.
 // what ever it takes, we will always fuse label info into edge_expand,
 // but if get_v contains expression, we will not fuse it into edge_expand
-bool simple_get_v(const physical::GetV& get_v_op) {
+bool simple_get_v(const physical::GetV& get_v_op,
+                  const physical::EdgeExpand& edge_expand_op) {
   if (get_v_op.params().has_predicate()) {
     return false;
   }
+  // There are 5 possible combinations of get_v and edge_expand
+  // Direction::Out & GetVOpt::End
+  // Direction::Out & GetVOpt::Start
+  // Direction::In & GetVOpt::End
+  // Direction::In & GetVOpt::Start
+  // Whatever Direction & GetVOpt::Other
+  //
+  if (get_v_op.opt() == physical::GetV::OTHER) {
+    return true;
+  }
+  if (get_v_op.opt() == physical::GetV::BOTH) {
+    return false;
+  }
+  if (get_v_op.opt() == physical::GetV::END) {
+    if (edge_expand_op.direction() == physical::EdgeExpand::OUT) {
+      return true;
+    }
+    if (edge_expand_op.direction() == physical::EdgeExpand::IN) {
+      return false;
+    }
+  }
+  if (get_v_op.opt() == physical::GetV::START) {
+    if (edge_expand_op.direction() == physical::EdgeExpand::OUT) {
+      return false;
+    }
+    if (edge_expand_op.direction() == physical::EdgeExpand::IN) {
+      return true;
+    }
+  }
   return true;
 }
 
@@ -178,7 +210,11 @@ class QueryGenerator {
   static constexpr bool FUSE_EDGE_GET_V = true;
   static constexpr bool FUSE_PATH_EXPAND_V = true;
   QueryGenerator(BuildingContext& ctx, const physical::PhysicalPlan& plan)
-      : ctx_(ctx), plan_(plan) {}
+      : ctx_(ctx), plan_(plan), schema_() {}
+
+  QueryGenerator(BuildingContext& ctx, const physical::PhysicalPlan& plan,
+                 const Schema& schema)
+      : ctx_(ctx), plan_(plan), schema_(schema) {}
 
   std::string GenerateQuery() {
     // During generate query body, we will track the parameters
@@ -314,7 +350,7 @@ class QueryGenerator {
         LOG(INFO) << "Found a scan operator";
         auto& scan_op = opr.scan();
 
-        ss << BuildScanOp(ctx_, scan_op, meta_data) << std::endl;
+        ss << BuildScanOp(ctx_, scan_op, meta_data, schema_) << std::endl;
         break;
       }
 
@@ -330,7 +366,7 @@ class QueryGenerator {
             extract_vertex_labels(get_v_op, dst_vertex_labels);
 
             if (FUSE_EDGE_GET_V) {
-              if (simple_get_v(get_v_op) &&
+              if (simple_get_v(get_v_op, real_edge_expand) &&
                   intermediate_edge_op(real_edge_expand)) {
                 CHECK(dst_vertex_labels.size() > 0);
                 VLOG(10) << "When fusing edge+get_v, get_v has labels: "
@@ -378,9 +414,12 @@ class QueryGenerator {
       case physical::PhysicalOpr::Operator::kProject: {  // project
         // project op can result into multiple meta data
         // auto& meta_data = meta_datas[0];
-        physical::PhysicalOpr::MetaData meta_data;
         LOG(INFO) << "Found a project operator";
         auto& project_op = opr.project();
+        physical::PhysicalOpr_MetaData meta_data;
+        if (meta_datas.size() > 0) {
+          meta_data = meta_datas[0];
+        }
         std::string call_project_code;
         call_project_code = BuildProjectOp(ctx_, project_op, meta_data);
         ss << call_project_code;
@@ -535,6 +574,7 @@ class QueryGenerator {
 
   BuildingContext& ctx_;
   const physical::PhysicalPlan& plan_;
+  std::optional<Schema> schema_;
 };
 
 // When building a join op, we need to consider the following cases:
diff --git a/flex/codegen/src/pegasus/pegasus_expr_builder.h b/flex/codegen/src/pegasus/pegasus_expr_builder.h
index 81bda1d48789..bc0ef59fc381 100644
--- a/flex/codegen/src/pegasus/pegasus_expr_builder.h
+++ b/flex/codegen/src/pegasus/pegasus_expr_builder.h
@@ -110,7 +110,7 @@ static std::string value_pb_to_str(const common::Value& value) {
 
 bool contains_vertex_id(const std::vector<codegen::ParamConst>& params) {
   for (auto& param : params) {
-    if (param.type == codegen::DataType::kVertexId) {
+    if (param.type == codegen::DataType::kGlobalVertexId) {
       return true;
     }
   }
diff --git a/flex/codegen/src/string_utils.h b/flex/codegen/src/string_utils.h
index d62e17db6893..36410650a535 100644
--- a/flex/codegen/src/string_utils.h
+++ b/flex/codegen/src/string_utils.h
@@ -42,6 +42,7 @@ static constexpr const char* LABEL_ID_T_CASTER = "(label_id_t)";
 static constexpr const char* EMPTY_TYPE = "grape::EmptyType";
 static constexpr const char* INNER_ID_PROPERTY_NAME = "InnerIdProperty";
 static constexpr const char* VERTEX_ID_T = "vertex_id_t";
+static constexpr const char* GLOBAL_VERTEX_ID_T = "GlobalId";
 static constexpr const char* EDGE_ID_T = "const DefaultEdge<vertex_id_t>&";
 static constexpr const char* LENGTH_KEY_T = "LengthKey";
 static constexpr const char* MAKE_PROJECT_EXPR = "make_project_expr";
@@ -55,9 +56,13 @@ static constexpr const char* PROPERTY_SELECTOR =
     "gs::PropertySelector<%1%>(\"%2%\")";
 static constexpr const char* PROP_NAME_ARRAY = "gs::PropNameArray<%1%>{%2%}";
 
-std::string project_is_append_str(bool is_append) {
+std::string project_is_append_str(bool is_append, bool is_temp) {
   if (is_append) {
-    return "PROJ_TO_APPEND";
+    if (is_temp) {
+      return "PROJ_TO_APPEND_TEMP";
+    } else {
+      return "PROJ_TO_APPEND_PERSIST";
+    }
   } else {
     return "PROJ_TO_NEW";
   }
diff --git a/flex/engines/hqps_db/core/base_engine.h b/flex/engines/hqps_db/core/base_engine.h
index a755da9ca1ff..1786617663cd 100644
--- a/flex/engines/hqps_db/core/base_engine.h
+++ b/flex/engines/hqps_db/core/base_engine.h
@@ -71,78 +71,6 @@ class BaseEngine {
     return LimitOp::Limit(std::move(ctx), lower, upper);
   }
 
-  //////////////////////////////////////Dedup/////////////////////////
-  // Only can dedup head node.
-  template <
-      int alias_to_use, typename CTX_HEAD_T, int cur_alias, int base_tag,
-      typename... CTX_PREV,
-      typename RES_T = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>>
-  static RES_T Dedup(
-      Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx) {
-    if constexpr (alias_to_use != cur_alias) {
-      // When we dedup a intermediate node, we need to
-      // 1) first dedup current node, no duplicate in us.
-      // 2) then iterate whole context, for later nodes, we only preserve the
-      // first element met.
-      //
-      // the result context type should be same with previous.
-      // 1 -> (2, 3)
-      // 2 -> (4, 5), 3 -> (6, 7);
-      //
-      // dedup on col 2, then we 1 -> (2, 3), 2 -> 4, 3 -> 6;
-
-      // first remove all possible duplication introduced by later csr.
-      ctx.template Dedup<alias_to_use>();
-    }
-    auto& select_node = gs::Get<alias_to_use>(ctx);
-    // dedup inplace, and return the offset_array to old node.
-    auto offset_to_old_node = select_node.Dedup();
-    // The offset need to be changed.
-    ctx.template UpdateChildNode<alias_to_use>(std::move(offset_to_old_node));
-    return ctx;
-  }
-
-  /// @brief /////////////Dedup on multiple keys////////////////
-  /// @param ctx
-  /// @return
-  template <
-      int... alias_to_use, typename CTX_HEAD_T, int cur_alias, int base_tag,
-      typename... CTX_PREV,
-      typename RES_T = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>,
-      typename std::enable_if<(sizeof...(alias_to_use) > 1)>::type* = nullptr>
-  static RES_T Dedup(
-      Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx) {
-    // get all ele_t of context
-    using CTX_T = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>;
-    using ctx_iter_t = typename CTX_T::iterator;
-    using ctx_all_ele_t = std::remove_reference_t<decltype(
-        std::declval<ctx_iter_t>().GetAllElement())>;
-    using dedup_tuple_t =
-        std::tuple<std::tuple_element_t<alias_to_use, ctx_all_ele_t>...>;
-    std::unordered_set<dedup_tuple_t, boost::hash<dedup_tuple_t>> dedup_set;
-    std::vector<size_t> active_indices;
-    std::vector<size_t> new_offset;
-    auto& cur_ = ctx.GetMutableHead();
-    new_offset.reserve(cur_.Size());
-    new_offset.emplace_back(0);
-    size_t cnt = 0;
-    for (auto iter : ctx) {
-      auto eles = iter.GetAllElement();
-      dedup_tuple_t dedup_tuple =
-          std::make_tuple(std::get<alias_to_use>(eles)...);
-      if (dedup_set.find(dedup_tuple) == dedup_set.end()) {
-        dedup_set.insert(dedup_tuple);
-        active_indices.emplace_back(cnt);
-      }
-      cnt += 1;
-      new_offset.emplace_back(active_indices.size());
-    }
-
-    cur_.SubSetWithIndices(active_indices);
-    ctx.merge_offset_with_back(new_offset);
-    return ctx;
-  }
-
   /////////////////////////Apply///////////////////////////////
   /// With a apply function, we get the result, and join with current node.
   //  append the result data to current traversal.
diff --git a/flex/engines/hqps_db/core/context.h b/flex/engines/hqps_db/core/context.h
index 41d48f57ee1d..3bad0e308961 100644
--- a/flex/engines/hqps_db/core/context.h
+++ b/flex/engines/hqps_db/core/context.h
@@ -1184,7 +1184,9 @@ class Context<HEAD_T, cur_alias, base_tag, grape::EmptyType> {
 
   // must return 1,1,1,.
   std::vector<offset_t> ObtainOffsetFromTag(int dst_tag) const {
-    CHECK(dst_tag == cur_alias);
+    CHECK(dst_tag == cur_alias || (dst_tag == 0 && cur_alias == -1) ||
+          dst_tag == -1)
+        << "dst tag: " << dst_tag << ", cur_alias: " << cur_alias;
     auto size = cur_.Size();
     std::vector<offset_t> res;
     res.reserve(size + 1);
diff --git a/flex/engines/hqps_db/core/null_record.h b/flex/engines/hqps_db/core/null_record.h
index 291a5885d12b..ba2b9ccb9648 100644
--- a/flex/engines/hqps_db/core/null_record.h
+++ b/flex/engines/hqps_db/core/null_record.h
@@ -29,7 +29,12 @@ static constexpr const None NONE;
 template <typename T>
 struct NullRecordCreator {
   static inline T GetNull() {
-    static T null_value = std::numeric_limits<T>::max();
+    static_assert(
+        std::numeric_limits<
+            std::remove_const_t<std::remove_reference_t<T>>>::is_specialized,
+        "NullRecordCreator only support numeric type");
+    using type = std::remove_const_t<std::remove_reference_t<T>>;
+    static type null_value = std::numeric_limits<type>::max();
     return null_value;
   }
 };
@@ -39,6 +44,39 @@ struct NullRecordCreator<std::string_view> {
   static inline std::string_view GetNull() { return ""; }
 };
 
+template <>
+struct NullRecordCreator<Date> {
+  static inline Date GetNull() {
+    return Date(std::numeric_limits<int64_t>::max());
+  }
+};
+
+template <>
+struct NullRecordCreator<Any> {
+  static inline Any GetNull() { return Any(); }
+};
+
+template <>
+struct NullRecordCreator<LabelKey> {
+  static inline LabelKey GetNull() {
+    return LabelKey(std::numeric_limits<LabelKey::label_data_type>::max());
+  }
+};
+
+template <>
+struct NullRecordCreator<grape::EmptyType> {
+  static inline grape::EmptyType GetNull() { return grape::EmptyType(); }
+};
+
+// bool doesn't have a null value
+
+template <>
+struct NullRecordCreator<GlobalId> {
+  static inline GlobalId GetNull() {
+    return GlobalId(std::numeric_limits<GlobalId::gid_t>::max());
+  }
+};
+
 template <typename... T>
 struct NullRecordCreator<std::tuple<T...>> {
   static inline std::tuple<T...> GetNull() {
@@ -53,6 +91,13 @@ struct NullRecordCreator<Path<VID_T, LabelT>> {
   }
 };
 
+static inline bool IsNull(const bool& b) { return false; }
+
+template <typename A, typename B>
+static inline bool IsNull(const std::pair<A, B>& pair) {
+  return IsNull(pair.first) && IsNull(pair.second);
+}
+
 template <size_t Ind = 0, typename... T>
 static inline bool IsNull(const std::tuple<T...>& tuple) {
   if constexpr (Ind + 1 < sizeof...(T)) {
@@ -110,6 +155,31 @@ template <typename T>
 bool operator!=(const None& lhs, const T& rhs) {
   return !IsNull(rhs);
 }
+
+template <size_t Is, typename T>
+bool has_null(const T& t) {
+  if constexpr (Is < std::tuple_size<T>::value) {
+    if (IsNull(std::get<Is>(t))) {
+      return true;
+    } else {
+      return has_null<Is + 1>(t);
+    }
+  } else {
+    return false;
+  }
+}
+
+template <typename... T>
+bool HasNull(const std::tuple<T...>& t) {
+  // at least one element in the tuple is null
+  return has_null<0>(t);
+}
+
+template <typename T>
+bool HasNull(const T& t) {
+  return IsNull(t);
+}
+
 }  // namespace gs
 
 #endif  // ENGINES_HQPS_ENGINE_NULL_RECORD_H_
\ No newline at end of file
diff --git a/flex/engines/hqps_db/core/operator/edge_expand.h b/flex/engines/hqps_db/core/operator/edge_expand.h
index 218cccdd02ed..0527b1b1b13d 100644
--- a/flex/engines/hqps_db/core/operator/edge_expand.h
+++ b/flex/engines/hqps_db/core/operator/edge_expand.h
@@ -84,34 +84,6 @@ struct EdgeExpandEState {
         limit_(limit) {}
 };
 
-template <typename GRAPH_INTERFACE, typename VERTEX_SET_T, size_t num_labels,
-          typename EDGE_FILTER_T, typename... T>
-struct EdgeExpandEMultiDstState {
-  const GRAPH_INTERFACE& graph_;
-  VERTEX_SET_T& cur_vertex_set_;
-  Direction direction_;
-  typename GRAPH_INTERFACE::label_id_t edge_label_;
-  std::array<typename GRAPH_INTERFACE::label_id_t, num_labels> other_label_;
-  const PropNameArray<T...>& prop_names_;
-  const EDGE_FILTER_T& edge_filter_;
-  size_t limit_;
-
-  EdgeExpandEMultiDstState(
-      const GRAPH_INTERFACE& frag, VERTEX_SET_T& v_set, Direction direction,
-      typename GRAPH_INTERFACE::label_id_t edge_label,
-      std::array<typename GRAPH_INTERFACE::label_id_t, num_labels> other_label,
-      const PropNameArray<T...>& prop_names, const EDGE_FILTER_T& edge_filter,
-      size_t limit)
-      : graph_(frag),
-        cur_vertex_set_(v_set),
-        direction_(direction),
-        edge_label_(edge_label),
-        other_label_(other_label),
-        limit_(limit),
-        prop_names_(prop_names),
-        edge_filter_(edge_filter) {}
-};
-
 template <typename GRAPH_INTERFACE>
 class EdgeExpand {
   using label_id_t = typename GRAPH_INTERFACE::label_id_t;
@@ -169,66 +141,96 @@ class EdgeExpand {
   /// @param edge_expand_opt
   /// @return
   template <
-      typename VERTEX_SET_T, typename... SELECTOR,
-      typename RES_T = std::pair<vertex_set_t, std::vector<offset_t>>,
+      typename VERTEX_SET_T,
+      typename RES_T =
+          std::pair<RowVertexSet<label_id_t, vertex_id_t, grape::EmptyType>,
+                    std::vector<offset_t>>,
       typename std::enable_if<VERTEX_SET_T::is_two_label_set>::type* = nullptr>
   static RES_T EdgeExpandV(const GRAPH_INTERFACE& graph,
                            const VERTEX_SET_T& cur_vertex_set,
                            Direction direction, label_id_t edge_label,
                            label_id_t other_label,
-                           Filter<TruePredicate, SELECTOR...>&& edge_filter,
+                           const Filter<TruePredicate>& edge_filter,
                            size_t limit = INT_MAX) {
     VLOG(10) << "[EdgeExpandV] for two label vertex set size: "
              << cur_vertex_set.Size();
     auto state = EdgeExpandVState(graph, cur_vertex_set, direction, edge_label,
                                   other_label, std::move(edge_filter), limit);
 
-    std::vector<vertex_id_t> vids;
-    std::vector<offset_t> offset;
     static constexpr size_t num_src_labels = VERTEX_SET_T::num_labels;
-    using nbr_list_array_t = typename GRAPH_INTERFACE::nbr_list_array_t;
-    nbr_list_array_t nbr_list_array;
-    nbr_list_array.resize(state.cur_vertex_set_.Size());
+    // using nbr_list_array_t = typename GRAPH_INTERFACE::nbr_list_array_t;
+    // nbr_list_array_t nbr_list_array;
+    // nbr_list_array.resize(state.cur_vertex_set_.Size());
+    std::vector<std::vector<vertex_id_t>> tmp_vids;
+    tmp_vids.resize(state.cur_vertex_set_.Size());
 
     for (size_t i = 0; i < num_src_labels; ++i) {
       std::vector<vertex_id_t> cur_vids;
       std::vector<int32_t> active_inds;
       std::tie(cur_vids, active_inds) = state.cur_vertex_set_.GetVertices(i);
-      label_id_t cur_label = state.cur_vertex_set_.GetLabel(i);
-      label_id_t src_label, dst_label;
-      std::tie(src_label, dst_label) =
-          get_graph_label_pair(direction, cur_label, state.other_label_);
+      label_id_t src_label = state.cur_vertex_set_.GetLabel(i);
+      label_id_t dst_label = state.other_label_;
+      // label_id_t src_label, dst_label;
+      // std::tie(src_label, dst_label) =
+      //     get_graph_label_pair(direction, cur_label, state.other_label_);
 
       VLOG(10) << "[EdgeExpandV-TwoLabelSet]: from label: "
-               << ",edge label: " << state.edge_label_ << "src: " << src_label
-               << ",dst: " << dst_label << ",dire: " << state.direction_;
-      auto tmp_nbr_list_array = state.graph_.GetOtherVertices(
-          src_label, dst_label, state.edge_label_, cur_vids,
-          gs::to_string(state.direction_), state.limit_);
-      // nbr_lists.emplace_back(std::move(nbr_list_array));
-
-      CHECK(tmp_nbr_list_array.size() == active_inds.size());
-      for (size_t i = 0; i < active_inds.size(); ++i) {
-        auto dst_ind = active_inds[i];
-        CHECK(nbr_list_array.get(dst_ind).size() == 0);
-        nbr_list_array.get_vector(dst_ind).swap(
-            tmp_nbr_list_array.get_vector(i));
+               << ",edge label: " << gs::to_string(state.edge_label_)
+               << "src: " << gs::to_string(src_label)
+               << ",dst: " << gs::to_string(dst_label)
+               << ",dire: " << state.direction_;
+      if (state.direction_ == Direction::In ||
+          state.direction_ == Direction::Both) {
+        auto tmp_nbr_list_array = state.graph_.GetOtherVertices(
+            src_label, dst_label, state.edge_label_, cur_vids, "In",
+            state.limit_);
+
+        CHECK(tmp_nbr_list_array.size() == active_inds.size());
+        for (size_t j = 0; j < active_inds.size(); ++j) {
+          auto dst_ind = active_inds[j];
+          for (auto nbr : tmp_nbr_list_array.get(j)) {
+            tmp_vids[dst_ind].emplace_back(nbr.neighbor());
+          }
+        }
+      }
+      if (state.direction_ == Direction::Out ||
+          state.direction_ == Direction::Both) {
+        auto tmp_nbr_list_array = state.graph_.GetOtherVertices(
+            src_label, dst_label, state.edge_label_, cur_vids, "Out",
+            state.limit_);
+
+        CHECK(tmp_nbr_list_array.size() == active_inds.size());
+        for (size_t j = 0; j < active_inds.size(); ++j) {
+          auto dst_ind = active_inds[j];
+          for (auto nbr : tmp_nbr_list_array.get(j)) {
+            tmp_vids[dst_ind].emplace_back(nbr.neighbor());
+          }
+        }
       }
     }
-    CHECK(nbr_list_array.size() == state.cur_vertex_set_.Size());
 
+    std::vector<vertex_id_t> vids;
+    std::vector<offset_t> offset;
     offset.reserve(state.cur_vertex_set_.Size() + 1);
-    // first gather size.
     offset.emplace_back(vids.size());
-    for (size_t i = 0; i < nbr_list_array.size(); ++i) {
-      for (auto nbr : nbr_list_array.get(i)) {
-        // TODO: use edge_filter to filter.
-        vids.emplace_back(nbr.neighbor());
+    {
+      // gather tmp_vids total size
+      size_t total_size = 0;
+      for (size_t i = 0; i < tmp_vids.size(); ++i) {
+        total_size += tmp_vids[i].size();
+      }
+      vids.reserve(total_size);
+    }
+    for (size_t i = 0; i < tmp_vids.size(); ++i) {
+      for (size_t j = 0; j < tmp_vids[i].size(); ++j) {
+        vids.emplace_back(tmp_vids[i][j]);
       }
       offset.emplace_back(vids.size());
     }
-    vertex_set_t result_set(std::move(vids), state.other_label_);
-    auto pair = std::make_pair(std::move(result_set), std::move(offset));
+
+    auto pair = std::make_pair(
+        make_default_row_vertex_set(std::move(vids), state.other_label_),
+        std::move(offset));
     return pair;
   }
 
@@ -247,7 +249,7 @@ class EdgeExpand {
       const GRAPH_INTERFACE& graph,
       const GeneralVertexSet<vertex_id_t, label_id_t, SET_T...>& cur_vertex_set,
       Direction direction, label_id_t edge_label, label_id_t other_label,
-      Filter<TruePredicate, SELECTOR...>&& edge_filter,
+      const Filter<TruePredicate, SELECTOR...>& edge_filter,
       size_t limit = INT_MAX) {
     VLOG(10) << "[EdgeExpandV] for general vertex set size: "
              << cur_vertex_set.Size();
@@ -264,16 +266,17 @@ class EdgeExpand {
     for (size_t i = 0; i < src_labels.size(); ++i) {
       std::vector<vertex_id_t> cur_vids;
       std::vector<int32_t> active_inds;
-      std::tie(cur_vids, active_inds) =
-          state.cur_vertex_set_.GetVerticesWithIndex(i);
-      label_id_t cur_label = state.cur_vertex_set_.GetLabel(i);
-      label_id_t src_label, dst_label;
-      std::tie(src_label, dst_label) =
-          get_graph_label_pair(direction, cur_label, state.other_label_);
-
-      VLOG(10) << "[EdgeExpandV]: from label: "
-               << ",edge label: " << state.edge_label_ << "src: " << src_label
-               << ",dst: " << dst_label << ",dire: " << state.direction_;
+      std::tie(cur_vids, active_inds) = state.cur_vertex_set_.GetVertices(i);
+      label_id_t src_label = state.cur_vertex_set_.GetLabel(i);
+      label_id_t dst_label = state.other_label_;
+      // std::tie(src_label, dst_label) =
+      //     get_graph_label_pair(direction, cur_label, state.other_label_);
+
+      VLOG(10) << "[EdgeExpandV]: "
+               << "edge label: " << std::to_string(state.edge_label_)
+               << "src: " << std::to_string(src_label)
+               << ",dst: " << std::to_string(dst_label)
+               << ",dire: " << state.direction_;
       auto tmp_nbr_list_array = state.graph_.GetOtherVertices(
           src_label, dst_label, state.edge_label_, cur_vids,
           gs::to_string(state.direction_), state.limit_);
@@ -444,20 +447,37 @@ class EdgeExpand {
   /// @param v_sets
   /// @param edge_expand_opt
   /// @return
-  template <typename VERTEX_SET_T, typename EDGE_FILTER_T, typename... SET_T>
+  template <typename VERTEX_SET_T, typename EDGE_FILTER_T, typename... SET_T,
+            typename std::enable_if_t<VERTEX_SET_T::is_row_vertex_set ||
+                                      VERTEX_SET_T::is_two_label_set ||
+                                      VERTEX_SET_T::is_general_set>* = nullptr>
   static auto EdgeExpandV(
       const GRAPH_INTERFACE& graph, const VERTEX_SET_T& cur_vertex_set,
       Direction direction,
       const std::vector<std::array<label_id_t, 3>>& edge_triplets,
       const EDGE_FILTER_T& edge_filter) {
     CHECK(edge_triplets.size() > 0);
-    using result_pair_t = std::pair<vertex_set_t, std::vector<offset_t>>;
+    // result_set_t is the type of calling EdgeExpandV with cur_vertex_set
+    // and edge_triplets[i]
+    using result_set_t = decltype(
+        EdgeExpandV(graph, cur_vertex_set, direction, edge_triplets[0][2],
+                    edge_triplets[0][1], std::move(edge_filter))
+            .first);
+    using result_pair_t = std::pair<result_set_t, std::vector<offset_t>>;
     std::vector<result_pair_t> result_pairs;
-    for (auto i = 0; i < edge_triplets.size(); ++i) {
-      auto copied_filter = edge_filter;
-      result_pairs.emplace_back(
-          EdgeExpandV(graph, cur_vertex_set, direction, edge_triplets[i][2],
-                      edge_triplets[i][1], std::move(copied_filter)));
+    for (size_t i = 0; i < edge_triplets.size(); ++i) {
+      if (direction == Direction::In || direction == Direction::Both) {
+        auto copied_filter = edge_filter;
+        result_pairs.emplace_back(EdgeExpandV(
+            graph, cur_vertex_set, Direction::In, edge_triplets[i][2],
+            edge_triplets[i][0], std::move(copied_filter)));
+      }
+      if (direction == Direction::Out || direction == Direction::Both) {
+        auto copied_filter = edge_filter;
+        result_pairs.emplace_back(EdgeExpandV(
+            graph, cur_vertex_set, Direction::Out, edge_triplets[i][2],
+            edge_triplets[i][1], std::move(copied_filter)));
+      }
     }
 
     size_t offset_array_size = result_pairs[0].second.size();
@@ -469,21 +489,24 @@ class EdgeExpand {
 
     std::vector<vertex_id_t> res_vids;
     std::vector<grape::Bitset> res_bitset;
-    std::unordered_map<label_id_t, int32_t> label_to_ind;
+    std::map<label_id_t, int32_t> label_to_ind;
+    std::vector<label_id_t> copied_labels;
     std::vector<offset_t> res_offset;
 
     size_t num_labels = 0;
     {
-      for (auto i = 0; i < edge_triplets.size(); ++i) {
+      for (size_t i = 0; i < edge_triplets.size(); ++i) {
         auto& triplet = edge_triplets[i];
         if (direction == Direction::In || direction == Direction::Both) {
           if (label_to_ind.find(triplet[0]) == label_to_ind.end()) {
             label_to_ind[triplet[0]] = num_labels++;
+            copied_labels.emplace_back(triplet[0]);
           }
         }
         if (direction == Direction::Out || direction == Direction::Both) {
           if (label_to_ind.find(triplet[1]) == label_to_ind.end()) {
             label_to_ind[triplet[1]] = num_labels++;
+            copied_labels.emplace_back(triplet[1]);
           }
         }
       }
@@ -507,28 +530,44 @@ class EdgeExpand {
     for (size_t i = 0; i < prev_set_size; ++i) {
       for (size_t j = 0; j < result_pairs.size(); ++j) {
         auto& vertex_set = result_pairs[j].first;
-        auto& vertex_set_label = vertex_set.GetLabel();
-        CHECK(label_to_ind.find(vertex_set_label) != label_to_ind.end())
-            << "label " << vertex_set_label << " not found";
-        auto res_label_ind = label_to_ind[vertex_set_label];
-        auto& offset_array = result_pairs[j].second;
-        auto& vec = vertex_set.GetVertices();
-        auto start_off = offset_array[i];
-        auto end_off = offset_array[i + 1];
-        for (auto k = start_off; k < end_off; ++k) {
-          res_vids.emplace_back(vec[k]);
-          res_bitset[res_label_ind].set_bit(cur_ind);
-          // res_bitset[j].set_bit(cur_ind);
-          cur_ind += 1;
+        using type = typename std::remove_reference<decltype(vertex_set)>::type;
+        if constexpr (type::is_row_vertex_set) {
+          auto& vertex_set_label = vertex_set.GetLabel();
+          CHECK(label_to_ind.find(vertex_set_label) != label_to_ind.end())
+              << "label " << vertex_set_label << " not found";
+          auto res_label_ind = label_to_ind[vertex_set_label];
+          auto& offset_array = result_pairs[j].second;
+          auto& vec = vertex_set.GetVertices();
+          auto start_off = offset_array[i];
+          auto end_off = offset_array[i + 1];
+          for (auto k = start_off; k < end_off; ++k) {
+            res_vids.emplace_back(vec[k]);
+            res_bitset[res_label_ind].set_bit(cur_ind);
+            cur_ind += 1;
+          }
+        } else if constexpr (type::is_two_label_set || type::is_general_set) {
+          auto label_vec = vertex_set.GetLabelVec();
+          auto& vec = vertex_set.GetVertices();
+          auto& offset_array = result_pairs[j].second;
+          auto start_off = offset_array[i];
+          auto end_off = offset_array[i + 1];
+          for (auto k = start_off; k < end_off; ++k) {
+            auto vertex_set_label = label_vec[k].label_id;
+            CHECK(label_to_ind.find(vertex_set_label) != label_to_ind.end())
+                << "label " << vertex_set_label << " not found";
+            auto res_label_ind = label_to_ind[vertex_set_label];
+            res_vids.emplace_back(vec[k]);
+            res_bitset[res_label_ind].set_bit(cur_ind);
+            cur_ind += 1;
+          }
+        } else {
+          LOG(FATAL) << "unsupported vertex set type";
         }
       }
       res_offset.emplace_back(cur_ind);
     }
     CHECK(cur_ind == total_size);
-    std::vector<label_id_t> copied_labels;
-    for (auto pair : label_to_ind) {
-      copied_labels.emplace_back(pair.first);
-    }
+
     CHECK(copied_labels.size() == num_labels &&
           res_bitset.size() == num_labels);
     if constexpr (sizeof...(SET_T) > 0) {
@@ -551,80 +590,134 @@ class EdgeExpand {
   /// @param v_sets
   /// @param edge_expand_opt
   /// @return
-  template <typename... SET_T, size_t num_labels, size_t... Is,
-            typename std::enable_if<(num_labels == 2)>::type* = nullptr,
+  template <typename VERTEX_SET_T, size_t num_labels, size_t... Is,
+            typename std::enable_if<(num_labels == 2 &&
+                                     (VERTEX_SET_T::is_row_vertex_set ||
+                                      VERTEX_SET_T::is_two_label_set ||
+                                      VERTEX_SET_T::is_general_set))>::type* =
+                nullptr,
             typename RES_T = std::pair<
                 TwoLabelVertexSet<vertex_id_t, label_id_t, grape::EmptyType>,
                 std::vector<offset_t>>>
   static RES_T EdgeExpandV(
-      const GRAPH_INTERFACE& graph,
-      const RowVertexSet<label_id_t, vertex_id_t, SET_T...>& cur_vertex_set,
+      const GRAPH_INTERFACE& graph, const VERTEX_SET_T& cur_vertex_set,
       Direction direction, label_id_t edge_label,
       std::array<label_id_t, num_labels>& other_labels,
       std::array<Filter<TruePredicate>, num_labels>&& edge_filter,
       std::index_sequence<Is...>) {
-    label_id_t src_label, dst_label;
+    using vid_and_offset_t =
+        std::pair<std::vector<vertex_id_t>, std::vector<offset_t>>;
+    std::array<std::vector<vid_and_offset_t>, num_labels> vid_and_offset;
 
-    std::tie(src_label, dst_label) = get_graph_label_pair(
-        direction, cur_vertex_set.GetLabel(), other_labels[0]);
-    LOG(INFO) << "EdgeExpandV: with two dst labels"
-              << gs::to_string(other_labels);
-
-    auto vid_and_offset1 = graph.GetOtherVerticesV2(
-        src_label, dst_label, edge_label, cur_vertex_set.GetVertices(),
-        gs::to_string(direction), INT_MAX);
-
-    if (direction == Direction::In) {
-      src_label = other_labels[1];
-    } else {
-      dst_label = other_labels[1];
+    for (size_t i = 0; i < num_labels; ++i) {
+      auto raw_dst_label = other_labels[i];
+      if constexpr (VERTEX_SET_T::is_row_vertex_set) {
+        auto raw_src_label = cur_vertex_set.GetLabel();
+        VLOG(10) << "expand from row vertex set: " << cur_vertex_set.Size()
+                 << ", edge label: " << gs::to_string(edge_label)
+                 << ", src label: " << gs::to_string(raw_src_label)
+                 << ", dst label: " << gs::to_string(raw_dst_label)
+                 << ", direction: " << gs::to_string(direction);
+        auto vid_and_offset1 = graph.GetOtherVerticesV2(
+            raw_src_label, raw_dst_label, edge_label,
+            cur_vertex_set.GetVertices(), gs::to_string(direction), INT_MAX);
+        VLOG(10) << ", vid_and_offset1 size: " << vid_and_offset1.first.size()
+                 << ", vid_and_offset1 offset size: "
+                 << vid_and_offset1.second.size();
+        vid_and_offset[i].emplace_back(std::move(vid_and_offset1));
+      } else if constexpr (VERTEX_SET_T::is_two_label_set ||
+                           VERTEX_SET_T::is_general_set) {
+        auto raw_src_label = cur_vertex_set.GetLabel(i);
+        auto labels = cur_vertex_set.GetLabels();
+        using nbr_list_array_t = typename GRAPH_INTERFACE::nbr_list_array_t;
+        nbr_list_array_t nbr_list_array;
+        nbr_list_array.resize(cur_vertex_set.Size());
+        size_t cnt = 0;
+        for (auto j = 0; j < labels.size(); ++j) {
+          auto raw_src_label = labels[j];
+          // std::tie(src_label, dst_label) =
+          //     get_graph_label_pair(direction, raw_src_label, raw_dst_label);
+          auto [vids, active_inds] = cur_vertex_set.GetVertices(j);
+          auto tmp_nbr_list_array =
+              graph.GetOtherVertices(raw_src_label, raw_dst_label, edge_label,
+                                     vids, gs::to_string(direction), INT_MAX);
+          for (size_t k = 0; k < active_inds.size(); ++k) {
+            auto dst_ind = active_inds[k];
+            CHECK(nbr_list_array.get(dst_ind).size() == 0);
+            cnt += tmp_nbr_list_array.get_vector(k).size();
+            nbr_list_array.get_vector(dst_ind).swap(
+                tmp_nbr_list_array.get_vector(k));
+          }
+        }
+        std::vector<vertex_id_t> cur_vids;
+        std::vector<offset_t> cur_offset;
+        cur_vids.reserve(cnt);
+        cur_offset.reserve(cur_vertex_set.Size() + 1);
+        cur_offset.emplace_back(0);
+        for (size_t j = 0; j < nbr_list_array.size(); ++j) {
+          for (auto nbr : nbr_list_array.get(j)) {
+            cur_vids.emplace_back(nbr.neighbor());
+          }
+          cur_offset.emplace_back(cur_vids.size());
+        }
+        VLOG(10) << "expand with edge label: " << gs::to_string(edge_label)
+                 << ", src label: " << gs::to_string(raw_src_label)
+                 << ", dst label: " << gs::to_string(raw_dst_label)
+                 << ", direction: " << gs::to_string(direction)
+                 << ", cur_vids size: " << cur_vids.size()
+                 << ", cur_offset:" << gs::to_string(cur_offset);
+        vid_and_offset[i].emplace_back(
+            std::make_pair(std::move(cur_vids), std::move(cur_offset)));
+      } else {
+        LOG(FATAL) << "not possible";
+      }
     }
 
-    auto vid_and_offset2 = graph.GetOtherVerticesV2(
-        src_label, dst_label, edge_label, cur_vertex_set.GetVertices(),
-        gs::to_string(direction), INT_MAX);
-
-    auto& vids1 = vid_and_offset1.first;
-    auto& off1 = vid_and_offset1.second;
-    auto& vids2 = vid_and_offset2.first;
-    auto& off2 = vid_and_offset2.second;
     size_t prev_set_size = cur_vertex_set.Size();
-    CHECK(off1.size() == prev_set_size + 1);
-    CHECK(off2.size() == prev_set_size + 1);
+    for (size_t i = 0; i < vid_and_offset.size(); ++i) {
+      for (size_t j = 0; j < vid_and_offset[i].size(); ++j) {
+        CHECK(vid_and_offset[i][j].second.size() == prev_set_size + 1);
+      }
+    }
 
     std::vector<vertex_id_t> res_vids;
+    std::vector<offset_t> offset_vec;
     grape::Bitset res_bitset;
 
-    size_t total_size = vids1.size() + vids2.size();
+    size_t total_size = 0;
+    {
+      for (size_t i = 0; i < vid_and_offset.size(); ++i) {
+        for (size_t j = 0; j < vid_and_offset[i].size(); ++j) {
+          total_size += vid_and_offset[i][j].first.size();
+        }
+      }
+    }
     VLOG(10) << "total size: " << total_size;
     res_vids.reserve(total_size);
     res_bitset.init(total_size);
+    offset_vec.reserve(prev_set_size + 1);
+    offset_vec.emplace_back(0);
 
-    size_t cur = 0;
     for (size_t i = 0; i < prev_set_size; ++i) {
-      auto start_off = off1[i];
-      auto end_off = off1[i + 1];
-      for (auto k = start_off; k < end_off; ++k) {
-        res_vids.emplace_back(vids1[k]);
-        res_bitset.set_bit(cur++);
-      }
-      start_off = off2[i];
-      end_off = off2[i + 1];
-      for (auto k = start_off; k < end_off; ++k) {
-        res_vids.emplace_back(vids2[k]);
-        cur++;
-      }
-    }
-    {
-      for (size_t i = 0; i < off1.size(); ++i) {
-        off1[i] += off2[i];
+      for (size_t j = 0; j < 2; ++j) {  // num_labels == 2
+        auto& vid_and_offsets = vid_and_offset[j];
+        for (auto& vid_and_offset : vid_and_offsets) {
+          auto start_off = vid_and_offset.second[i];
+          auto end_off = vid_and_offset.second[i + 1];
+          for (auto k = start_off; k < end_off; ++k) {
+            if (j == 0) {
+              res_bitset.set_bit(res_vids.size());
+            }
+            res_vids.emplace_back(vid_and_offset.first[k]);
+          }
+        }
       }
+      offset_vec.emplace_back(res_vids.size());
     }
-    CHECK(cur == total_size);
     auto copied_labels(other_labels);
     TwoLabelVertexSet<vertex_id_t, label_id_t, grape::EmptyType> res_set(
         std::move(res_vids), std::move(copied_labels), std::move(res_bitset));
-    return std::make_pair(std::move(res_set), std::move(off1));
+    return std::make_pair(std::move(res_set), std::move(offset_vec));
   }
 
   /// @brief Expand from multi label vertices and though multi edge labels,
@@ -871,10 +964,12 @@ class EdgeExpand {
   /// @param edge_filter
   /// @param limit
   /// @return /
-  template <size_t num_pairs, typename... PropTuple, typename... SET_T>
+  template <size_t num_pairs, typename... PropTuple, typename VERTEX_SET_T,
+            typename std::enable_if<(VERTEX_SET_T::is_general_set ||
+                                     VERTEX_SET_T::is_two_label_set)>::type* =
+                nullptr>
   static auto EdgeExpandEMultiTriplet(
-      const GRAPH_INTERFACE& graph,
-      const GeneralVertexSet<vertex_id_t, label_id_t, SET_T...>& cur_vertex_set,
+      const GRAPH_INTERFACE& graph, const VERTEX_SET_T& cur_vertex_set,
       Direction& direction,
       std::array<std::array<label_id_t, 3>, num_pairs>& edge_labels,
       std::tuple<PropTupleArrayT<PropTuple>...>& prop_names,
@@ -890,9 +985,6 @@ class EdgeExpand {
       }
     }
 
-    LOG(INFO) << "[EdgeExpandEMultiTriplet] real labels: "
-              << gs::to_string(edge_labels);
-
     // for each triplet, returns a vector of edge iters.
     auto& vertices = cur_vertex_set.GetVertices();
     using sub_graph_t = typename GRAPH_INTERFACE::sub_graph_t;
@@ -990,6 +1082,71 @@ class EdgeExpand {
     return std::make_pair(std::move(set), std::move(offsets));
   }
 
+  /// @brief Expand from multi label vertices and though multi edge labels, With
+  /// Expression.
+  /// @tparam ...PropTuple
+  /// @tparam ...SET_T
+  /// @tparam num_pairs
+  /// @param graph
+  /// @param cur_vertex_set
+  /// @param direction
+  /// @param edge_labels
+  /// @param prop_names
+  /// @param edge_filter
+  /// @param limit
+  /// @return /
+  template <
+      size_t num_pairs, typename... PropTuple, typename VERTEX_SET_T,
+      typename EXPR, typename... SELECTOR,
+      typename std::enable_if<((VERTEX_SET_T::is_general_set ||
+                                VERTEX_SET_T::is_two_label_set) &&
+                               !IsTruePredicate<EXPR>::value)>::type* = nullptr>
+  static auto EdgeExpandEMultiTriplet(
+      const GRAPH_INTERFACE& graph, const VERTEX_SET_T& cur_vertex_set,
+      Direction& direction,
+      std::array<std::array<label_id_t, 3>, num_pairs>& edge_labels,
+      std::tuple<PropTupleArrayT<PropTuple>...>& prop_names,
+      Filter<EXPR, SELECTOR...>&& edge_filter, size_t limit) {
+    // Get the unfilted UnTypedEdgeSet and offset.
+    auto pair = EdgeExpandEMultiTriplet<num_pairs, PropTuple...>(
+        graph, cur_vertex_set, direction, edge_labels, prop_names,
+        Filter<TruePredicate>(), limit);
+    auto& set = pair.first;
+    auto& old_offset = pair.second;
+    auto& cur_prop_names = std::get<0>(prop_names);
+    CHECK(cur_prop_names.size() == 1);
+    using prop_t = typename std::tuple_element_t<
+        0, std::tuple_element_t<0, std::tuple<PropTuple...>>>;
+    std::vector<size_t> repeat_array;
+    repeat_array.resize(set.Size(), 0);
+    std::vector<offset_t> new_offset;
+    new_offset.emplace_back(0);
+    size_t cur_ind = 0;
+    size_t active_cnt = 0;
+    for (auto iter : set) {
+      auto prop = iter.GetData();
+      if (iter.GetPropNames().size() == 1 &&
+          iter.GetPropNames()[0] == cur_prop_names[0]) {
+        if (prop.type == AnyConverter<prop_t>::type()) {
+          if (edge_filter.expr_(AnyConverter<prop_t>::from_any(prop))) {
+            CHECK(cur_ind < repeat_array.size());
+            repeat_array[cur_ind] = 1;
+          }
+        }
+      }
+      cur_ind += 1;
+      new_offset.emplace_back(active_cnt);
+    }
+    LOG(INFO) << "Selected edge count: " << active_cnt;
+    // merge the new_offset with the old offset
+    for (size_t i = 0; i < old_offset.size(); ++i) {
+      old_offset[i] = new_offset[old_offset[i]];
+    }
+    KeyAlias<-1, -1> tmp_alias;
+    auto new_set = set.ProjectWithRepeatArray(repeat_array, tmp_alias);
+    return std::make_pair(std::move(new_set), std::move(old_offset));
+  }
+
   // for input vertex set with only one label.
   template <typename... T, typename... SET_T, typename EDGE_FILTER_T,
             typename std::enable_if<sizeof...(T) == 0>::type* = nullptr,
@@ -1071,7 +1228,7 @@ class EdgeExpand {
                          Filter<EDGE_FILTER_T, SELECTOR...>, T...>(
             graph, cur_vertex_set, direction, edge_label, other_label, props,
             edge_filter, limit);
-    return EdgeExpandETwoLabelSetImplWithExpr(state);
+    return EdgeExpandEImplWithExpr(state);
   }
 
   // specialization for two label vertex set, with no filter.
@@ -1091,7 +1248,47 @@ class EdgeExpand {
                          Filter<EDGE_FILTER_T, SELECTOR...>, T...>(
             graph, cur_vertex_set, direction, edge_label, other_label, props,
             edge_filter, limit);
-    return EdgeExpandETwoLabelSetImplNoExpr(state);
+    return EdgeExpandETwoLabelImplNoExpr(state);
+  }
+
+  // EdgeExpand E for general vertex set, with expression.
+  template <typename... T, typename... SET_T, typename EDGE_FILTER_T,
+            typename... SELECTOR,
+            typename std::enable_if<
+                !IsTruePredicate<EDGE_FILTER_T>::value>::type* = nullptr>
+  static auto EdgeExpandE(
+      const GRAPH_INTERFACE& graph,
+      GeneralVertexSet<vertex_id_t, label_id_t, SET_T...>& cur_vertex_set,
+      Direction direction, label_id_t edge_label, label_id_t other_label,
+      Filter<EDGE_FILTER_T, SELECTOR...>& edge_filter,
+      PropNameArray<T...>& props, size_t limit = INT_MAX) {
+    auto state =
+        EdgeExpandEState<GRAPH_INTERFACE,
+                         GeneralVertexSet<vertex_id_t, label_id_t, SET_T...>,
+                         Filter<EDGE_FILTER_T, SELECTOR...>, T...>(
+            graph, cur_vertex_set, direction, edge_label, other_label, props,
+            edge_filter, limit);
+    return EdgeExpandEImplWithExpr(state);
+  }
+
+  // specialization for general vertex set, with no filter.
+  template <typename... T, typename... SET_T, typename EDGE_FILTER_T,
+            typename... SELECTOR,
+            typename std::enable_if<
+                IsTruePredicate<EDGE_FILTER_T>::value>::type* = nullptr>
+  static auto EdgeExpandE(
+      const GRAPH_INTERFACE& graph,
+      GeneralVertexSet<vertex_id_t, label_id_t, SET_T...>& cur_vertex_set,
+      Direction direction, label_id_t edge_label, label_id_t other_label,
+      Filter<EDGE_FILTER_T, SELECTOR...>& edge_filter,
+      PropNameArray<T...>& props, size_t limit = INT_MAX) {
+    auto state =
+        EdgeExpandEState<GRAPH_INTERFACE,
+                         GeneralVertexSet<vertex_id_t, label_id_t, SET_T...>,
+                         Filter<EDGE_FILTER_T, SELECTOR...>, T...>(
+            graph, cur_vertex_set, direction, edge_label, other_label, props,
+            edge_filter, limit);
+    return EdgeExpandEImplWithExpr(state);
   }
 
  private:
@@ -1099,9 +1296,8 @@ class EdgeExpand {
   static auto EdgeExpandVFromSingleLabel(
       EdgeExpandVState<GRAPH_INTERFACE, VERTEX_SET_T,
                        Filter<TruePredicate, SELECTOR...>>& state) {
-    label_id_t src_label, dst_label;
-    std::tie(src_label, dst_label) = get_graph_label_pair(
-        state.direction_, state.cur_vertex_set_.GetLabel(), state.other_label_);
+    label_id_t src_label = state.cur_vertex_set_.GetLabel();
+    label_id_t dst_label = state.other_label_;
 
     VLOG(10) << "[EdgeExpandV]: from label: "
              << std::to_string(state.cur_vertex_set_.GetLabel())
@@ -1134,44 +1330,30 @@ class EdgeExpand {
     return pair;
   }
 
-  // the input set is two label set, and the result set is one label set,
-  //  when true predicate is passed, we just return general edge set.
-  template <typename... T, typename VERTEX_SET_T>
-  static auto EdgeExpandETwoLabelSetImplNoExpr(
-      EdgeExpandEState<GRAPH_INTERFACE, VERTEX_SET_T, Filter<TruePredicate>,
-                       T...>& state) {
-    auto prop_names = state.prop_names_;
-    static constexpr size_t num_labels = VERTEX_SET_T::num_labels;
-    auto& two_label_set = state.cur_vertex_set_;
-    auto total_vertices_num = two_label_set.Size();
-    VLOG(10) << "[EdgeExpandETwoLabelSetImpl]" << prop_names.size()
-             << ", total vnum: " << total_vertices_num;
-
-    using adj_list_array_t =
-        typename GRAPH_INTERFACE::template adj_list_array_t<T...>;
-    adj_list_array_t res_adj_list_arrays;
-    res_adj_list_arrays.resize(total_vertices_num);
-    // overall vid array.
-    std::vector<vertex_id_t> vids_arrays(two_label_set.GetVertices());
-    std::array<std::vector<offset_t>, num_labels> offset_arrays;
-
+  template <typename VERTEX_SET_T, typename... EDATA_T>
+  static void fetch_adj_list_array_from_graph(
+      const GRAPH_INTERFACE& graph, const VERTEX_SET_T& vertex_set,
+      Direction direction, label_id_t edge_label, label_id_t other_label,
+      PropNameArray<EDATA_T...>& prop_names, int32_t limit,
+      typename GRAPH_INTERFACE::template adj_list_array_t<EDATA_T...>&
+          res_adj_list_arrays) {
+    auto num_labels = vertex_set.GetLabels().size();
+    auto direction_str = gs::to_string(direction);
     label_id_t src_label, dst_label;
-
-    auto direction_str = gs::to_string(state.direction_);
     for (size_t i = 0; i < num_labels; ++i) {
-      if (state.direction_ == Direction::In) {
-        src_label = state.other_label_;
-        dst_label = two_label_set.GetLabel(i);
-      } else if (state.direction_ == Direction::Out) {
-        src_label = two_label_set.GetLabel(i);
-        dst_label = state.other_label_;
+      if (direction == Direction::In) {
+        src_label = other_label;
+        dst_label = vertex_set.GetLabel(i);
+      } else if (direction == Direction::Out) {
+        src_label = vertex_set.GetLabel(i);
+        dst_label = other_label;
       } else {
         // If direction is both, we need to make sure what is src and what is
         // dst.
-        src_label = two_label_set.GetLabel(i);
-        dst_label = state.other_label_;
-        auto& schema = state.graph_.schema();
-        if (!schema.exist(src_label, dst_label, state.edge_label_)) {
+        src_label = vertex_set.GetLabel(i);
+        dst_label = other_label;
+        auto& schema = graph.schema();
+        if (!schema.exist(src_label, dst_label, edge_label)) {
           std::swap(src_label, dst_label);
         }
       }
@@ -1179,10 +1361,10 @@ class EdgeExpand {
               << ", dst label: " << (int) dst_label;
       std::vector<vertex_id_t> cur_vids;
       std::vector<int32_t> cur_active_inds;
-      std::tie(cur_vids, cur_active_inds) = two_label_set.GetVertices(i);
-      auto tmp = state.graph_.template GetEdges<T...>(
-          src_label, dst_label, state.edge_label_, cur_vids, direction_str,
-          state.limit_, prop_names);
+      std::tie(cur_vids, cur_active_inds) = vertex_set.GetVertices(i);
+      auto tmp = graph.template GetEdges<EDATA_T...>(
+          src_label, dst_label, edge_label, cur_vids, direction_str, limit,
+          prop_names);
       CHECK(tmp.size() == cur_active_inds.size());
       if (i == 0) {
         // first time, update flag field.
@@ -1193,6 +1375,31 @@ class EdgeExpand {
         res_adj_list_arrays.set(cur_active_inds[j], tmp.get(j));
       }
     }
+  }
+
+  // the input set is two label set or general set.
+  template <typename... T, typename VERTEX_SET_T>
+  static auto EdgeExpandETwoLabelImplNoExpr(
+      EdgeExpandEState<GRAPH_INTERFACE, VERTEX_SET_T, Filter<TruePredicate>,
+                       T...>& state) {
+    auto prop_names = state.prop_names_;
+    static constexpr size_t num_labels = VERTEX_SET_T::num_labels;
+    auto& two_label_set = state.cur_vertex_set_;
+    auto total_vertices_num = two_label_set.Size();
+    VLOG(10) << "[EdgeExpandETwoLabelImplNoExpr]" << prop_names.size()
+             << ", total vnum: " << total_vertices_num;
+
+    using adj_list_array_t =
+        typename GRAPH_INTERFACE::template adj_list_array_t<T...>;
+    adj_list_array_t res_adj_list_arrays;
+    res_adj_list_arrays.resize(total_vertices_num);
+    // overall vid array.
+    std::vector<vertex_id_t> vids_arrays(two_label_set.GetVertices());
+    std::array<std::vector<offset_t>, num_labels> offset_arrays;
+
+    fetch_adj_list_array_from_graph(
+        state.graph_, two_label_set, state.direction_, state.edge_label_,
+        state.other_label_, prop_names, state.limit_, res_adj_list_arrays);
 
     std::vector<size_t> offset;
     offset.reserve(two_label_set.Size() + 1);
@@ -1226,19 +1433,21 @@ class EdgeExpand {
     return std::make_pair(std::move(edge_set), std::move(offset));
   }
 
-  // the input set is two label set, and the result set is one label set,
-  //  Evaluate the filter expression, producing a flat edge set.
-  template <typename... T, typename VERTEX_SET_T, typename FUNC_T,
-            typename... SELECTOR>
-  static auto EdgeExpandETwoLabelSetImplWithExpr(
+  // EdgeExpand with single edge label, the input vertex_set can be two label
+  // set or general set. True predicate or real expression.
+  template <
+      typename... T, typename VERTEX_SET_T, typename FUNC_T,
+      typename... SELECTOR,
+      typename std::enable_if<(VERTEX_SET_T::is_two_label_set ||
+                               VERTEX_SET_T::is_general_set)>::type* = nullptr>
+  static auto EdgeExpandEImplWithExpr(
       EdgeExpandEState<GRAPH_INTERFACE, VERTEX_SET_T,
                        Filter<FUNC_T, SELECTOR...>, T...>& state) {
     auto prop_names = state.prop_names_;
-    static constexpr size_t num_labels = VERTEX_SET_T::num_labels;
-    static_assert(num_labels == 2, "num_labels should be 2");
-    auto& two_label_set = state.cur_vertex_set_;
-    auto total_vertices_num = two_label_set.Size();
-    VLOG(10) << "[EdgeExpandETwoLabelSetImplWithExpr]" << prop_names.size()
+    // static constexpr size_t num_labels = VERTEX_SET_T::num_labels;
+    auto& vertex_set = state.cur_vertex_set_;
+    auto total_vertices_num = vertex_set.Size();
+    VLOG(10) << "[EdgeExpandEImplWithExpr]" << prop_names.size()
              << ", total vnum: " << total_vertices_num;
 
     using adj_list_array_t =
@@ -1246,53 +1455,12 @@ class EdgeExpand {
     adj_list_array_t res_adj_list_arrays;
     res_adj_list_arrays.resize(total_vertices_num);
     // overall vid array.
-    std::vector<vertex_id_t> vids_arrays(two_label_set.GetVertices());
-    std::array<std::vector<offset_t>, num_labels> offset_arrays;
+    std::vector<vertex_id_t> vids_arrays(vertex_set.GetVertices());
 
-    label_id_t src_label, dst_label;
-
-    auto direction_str = gs::to_string(state.direction_);
-    for (size_t i = 0; i < num_labels; ++i) {
-      if (state.direction_ == Direction::In) {
-        src_label = state.other_label_;
-        dst_label = two_label_set.GetLabel(i);
-      } else if (state.direction_ == Direction::Out) {
-        src_label = two_label_set.GetLabel(i);
-        dst_label = state.other_label_;
-      } else {
-        // If direction is both, we need to make sure what is src and what is
-        // dst.
-        src_label = two_label_set.GetLabel(i);
-        dst_label = state.other_label_;
-        auto& schema = state.graph_.schema();
-        if (!schema.exist(src_label, dst_label, state.edge_label_)) {
-          std::swap(src_label, dst_label);
-        }
-      }
-      std::vector<vertex_id_t> cur_vids;
-      std::vector<int32_t> cur_active_inds;
-      std::tie(cur_vids, cur_active_inds) = two_label_set.GetVertices(i);
-      auto tmp = state.graph_.template GetEdges<T...>(
-          src_label, dst_label, state.edge_label_, cur_vids, direction_str,
-          state.limit_, prop_names);
-      CHECK(tmp.size() == cur_active_inds.size());
-      if (i == 0) {
-        // first time, update flag field.
-        res_adj_list_arrays.set_flag(tmp.get_flag());
-      }
-      for (size_t j = 0; j < cur_active_inds.size(); ++j) {
-        // res_adj_list_arrays[cur_active_inds[j]] = tmp.get(j);
-        res_adj_list_arrays.set(cur_active_inds[j], tmp.get(j));
-      }
-    }
-    using edge_tuple_t = std::tuple<vertex_id_t, vertex_id_t, std::tuple<T...>>;
-    std::vector<edge_tuple_t> edge_tuples;
-    std::vector<size_t> offset;
-    std::vector<label_t> label_inds;
-    std::vector<std::array<label_id_t, 3>> label_triplets;
+    fetch_adj_list_array_from_graph(
+        state.graph_, vertex_set, state.direction_, state.edge_label_,
+        state.other_label_, prop_names, state.limit_, res_adj_list_arrays);
 
-    offset.reserve(two_label_set.Size() + 1);
-    offset.emplace_back(0);
     size_t num_pre_edges = 0;
     {
       // Construct offset from adj_list.
@@ -1302,40 +1470,50 @@ class EdgeExpand {
       }
     }
     VLOG(10) << "num edges, before filtering: " << num_pre_edges;
+    using edge_tuple_t = std::tuple<vertex_id_t, vertex_id_t, std::tuple<T...>>;
+    std::vector<edge_tuple_t> edge_tuples;
+    std::vector<size_t> offset;
+    std::vector<label_t> label_inds;
     edge_tuples.reserve(num_pre_edges);
     label_inds.reserve(num_pre_edges);
+    std::unordered_map<label_id_t, size_t> label_to_index;
+    std::vector<std::vector<std::string>> vec_vec_prop_names;
+    std::vector<std::array<label_id_t, 3>> label_triplets;
 
-    auto expr_filter = state.edge_filter_.expr_;
-    auto& old_bitset = two_label_set.GetBitset();
-    auto labels_vec = two_label_set.GetLabels();
-    label_triplets.emplace_back(std::array<label_id_t, 3>{
-        labels_vec[0], state.other_label_, state.edge_label_});
-    label_triplets.emplace_back(std::array<label_id_t, 3>{
-        labels_vec[1], state.other_label_, state.edge_label_});
+    auto labels_vec = vertex_set.GetLabels();
+    auto vertex_labels_vec = vertex_set.GetLabelVec();
+    for (auto label : labels_vec) {
+      CHECK(label_to_index.find(label) == label_to_index.end());
+      label_to_index[label] = label_triplets.size();
+      label_triplets.emplace_back(std::array<label_id_t, 3>{
+          label, state.other_label_, state.edge_label_});
+      vec_vec_prop_names.emplace_back(array_to_vec(prop_names));
+    }
+
+    offset.reserve(vertex_set.Size() + 1);
+    offset.emplace_back(0);
 
+    auto expr_filter = state.edge_filter_.expr_;
     for (size_t i = 0; i < res_adj_list_arrays.size(); ++i) {
       auto edges = res_adj_list_arrays.get(i);
       auto src = vids_arrays[i];
+      CHECK(label_to_index.find(vertex_labels_vec[i].label_id) !=
+            label_to_index.end());
+      auto cur_src_label_ind = label_to_index[vertex_labels_vec[i].label_id];
       for (auto edge : edges) {
         auto& props = edge.properties();
         // current hack impl for edge property
         if (expr_filter(std::get<0>(props))) {
           edge_tuples.emplace_back(
               std::make_tuple(src, edge.neighbor(), props));
-          if (old_bitset.get_bit(i)) {
-            label_inds.emplace_back(0);
-          } else {
-            label_inds.emplace_back(1);
-          }
+          label_inds.emplace_back(cur_src_label_ind);
         }
       }
       offset.emplace_back(edge_tuples.size());
     }
     LOG(INFO) << "Got edge tuples: " << edge_tuples.size() << " from "
               << num_pre_edges << " edges";
-    std::vector<std::vector<std::string>> vec_vec_prop_names;
-    vec_vec_prop_names.emplace_back(array_to_vec(prop_names));
-    vec_vec_prop_names.emplace_back(array_to_vec(prop_names));
+
     auto edge_set = FlatEdgeSet<vertex_id_t, label_id_t, std::tuple<T...>>(
         std::move(edge_tuples), std::move(label_triplets), vec_vec_prop_names,
         std::move(label_inds), state.direction_);
@@ -1374,46 +1552,71 @@ class EdgeExpand {
              << ", dst label: " << (int) dst_label
              << ", edge label: " << (int) state.edge_label_;
 
-    auto adj_list_array = state.graph_.template GetEdges<T>(
-        src_label, dst_label, state.edge_label_, cur_set.GetVertices(),
-        gs::to_string(state.direction_), state.limit_, prop_names);
-
     std::vector<size_t> offset;
     offset.reserve(cur_set.Size() + 1);
     size_t size = 0;
     offset.emplace_back(size);
-    CHECK(cur_set.Size() == adj_list_array.size())
-        << "cur_set.Size(): " << cur_set.Size()
-        << ", adj_list_array.size():" << adj_list_array.size();
     std::vector<std::tuple<vertex_id_t, vertex_id_t, std::tuple<T>>>
         prop_tuples;
     prop_tuples.reserve(cur_set.Size() + 1);
-    // Construct offset from adj_list.
+    using adj_list_array_t =
+        typename GRAPH_INTERFACE::template adj_list_array_t<T>;
+    std::vector<std::pair<adj_list_array_t, Direction>> adj_list_array_vec;
+
+    if (state.direction_ == Direction::Out ||
+        state.direction_ == Direction::Both) {
+      auto adj_list_array = state.graph_.template GetEdges<T>(
+          src_label, dst_label, state.edge_label_,
+          state.cur_vertex_set_.GetVertices(), gs::to_string(Direction::Out),
+          state.limit_, prop_names);
+      adj_list_array_vec.emplace_back(
+          std::make_pair(std::move(adj_list_array), Direction::Out));
+    }
+    if (state.direction_ == Direction::In ||
+        state.direction_ == Direction::Both) {
+      auto adj_list_array = state.graph_.template GetEdges<T>(
+          src_label, dst_label, state.edge_label_,
+          state.cur_vertex_set_.GetVertices(), gs::to_string(Direction::In),
+          state.limit_, prop_names);
+      adj_list_array_vec.emplace_back(
+          std::make_pair(std::move(adj_list_array), Direction::In));
+    }
+    std::vector<Direction> directions;
+
     auto cur_set_iter = cur_set.begin();
     auto end_iter = cur_set.end();
-    for (size_t i = 0; i < adj_list_array.size(); ++i) {
-      auto edges = adj_list_array.get(i);
+    for (size_t i = 0; i < cur_set.Size(); ++i) {
       CHECK(cur_set_iter != end_iter);
       auto src = cur_set_iter.GetVertex();
-      for (auto edge : edges) {
-        auto& props = edge.properties();
-        // current hack impl for edge property
-        // TODO: better performance
-        if (run_expr_filter(state.edge_filter_.expr_, props)) {
-          prop_tuples.emplace_back(
-              std::make_tuple(src, edge.neighbor(), props));
+      for (auto& adj_list_pair : adj_list_array_vec) {
+        auto& adj_list_array = adj_list_pair.first;
+        CHECK(cur_set.Size() == adj_list_array.size())
+            << "cur_set.Size(): " << cur_set.Size()
+            << ", adj_list_array.size():" << adj_list_array.size();
+        auto& direction = adj_list_pair.second;
+        auto edges = adj_list_array.get(i);
+        for (auto edge : edges) {
+          auto& props = edge.properties();
+          // current hack impl for edge property
+          // TODO: better performance
+          if (run_expr_filter(state.edge_filter_.expr_, props)) {
+            prop_tuples.emplace_back(
+                std::make_tuple(src, edge.neighbor(), props));
+            directions.emplace_back(direction);
+          }
         }
       }
       ++cur_set_iter;
       offset.emplace_back(prop_tuples.size());
     }
+
     VLOG(10) << "num edges: " << prop_tuples.size();
 
     std::array<label_id_t, 3> label_triplet{src_label, dst_label,
                                             state.edge_label_};
     SingleLabelEdgeSet<vertex_id_t, label_id_t, std::tuple<T>> edge_set(
         std::move(prop_tuples), std::move(label_triplet),
-        std::vector{array_to_vec(prop_names)}, state.direction_);
+        std::vector{array_to_vec(prop_names)}, std::move(directions));
 
     CHECK(offset.back() == edge_set.Size())
         << "offset: " << offset.back() << ", " << edge_set.Size();
@@ -1504,20 +1707,6 @@ class EdgeExpand {
     return adj_list_array;
   }
 
-  static std::tuple<label_id_t, label_id_t> get_graph_label_pair(
-      Direction& direction, label_id_t query_src_label,
-      label_id_t query_dst_label) {
-    label_id_t src_label, dst_label;
-    if (direction == Direction::In) {
-      src_label = query_dst_label;
-      dst_label = query_src_label;
-    } else {
-      src_label = query_src_label;
-      dst_label = query_dst_label;
-    }
-    return std::tuple{src_label, dst_label};
-  }
-
   static void expand_other_vertices_and_put_back(
       const GRAPH_INTERFACE& graph,
       std::vector<std::vector<vertex_id_t>>& ret_nbr_vertices,
diff --git a/flex/engines/hqps_db/core/operator/get_v.h b/flex/engines/hqps_db/core/operator/get_v.h
index f590bf1ba97a..e396a20e608b 100644
--- a/flex/engines/hqps_db/core/operator/get_v.h
+++ b/flex/engines/hqps_db/core/operator/get_v.h
@@ -289,8 +289,8 @@ class GetVertex {
     auto property_getters_array = get_prop_getters_from_selectors(
         graph, set.GetLabels(), filter.selectors_);
     t0 += grape::GetCurrentTime();
-    LOG(INFO) << "Get property tuple for general set of size: " << set.Size()
-              << " cost: " << t0;
+    LOG(INFO) << "Get property tuple for set" << demangle(set)
+              << " of size: " << set.Size() << " cost: " << t0;
     return set.project_vertices(labels, filter.expr_, property_getters_array);
   }
 
diff --git a/flex/engines/hqps_db/core/operator/group_by.h b/flex/engines/hqps_db/core/operator/group_by.h
index 30bed8bbfa8c..508d6c0ee7e2 100644
--- a/flex/engines/hqps_db/core/operator/group_by.h
+++ b/flex/engines/hqps_db/core/operator/group_by.h
@@ -101,25 +101,25 @@ struct GroupValueResT<CTX_T,
 template <typename SET_T>
 struct GroupValueResTImpl<SET_T, AggFunc::COUNT,
                           std::tuple<PropertySelector<grape::EmptyType>>> {
-  using result_t = Collection<size_t>;
+  using result_t = Collection<int64_t>;
 };
 
 template <typename SET_T>
 struct GroupValueResTImpl<SET_T, AggFunc::COUNT_DISTINCT,
                           std::tuple<PropertySelector<grape::EmptyType>>> {
-  using result_t = Collection<size_t>;
+  using result_t = Collection<int64_t>;
 };
 
 // PropSelectorTuple doesn't effect the result type.
 template <typename SET_TUPLE_T, typename PropSelectorTuple>
 struct GroupValueResTImpl<SET_TUPLE_T, AggFunc::COUNT_DISTINCT,
                           PropSelectorTuple> {
-  using result_t = Collection<size_t>;
+  using result_t = Collection<int64_t>;
 };
 
 template <typename SET_TUPLE_T, typename PropSelectorTuple>
 struct GroupValueResTImpl<SET_TUPLE_T, AggFunc::COUNT, PropSelectorTuple> {
-  using result_t = Collection<size_t>;
+  using result_t = Collection<int64_t>;
 };
 
 template <typename T>
@@ -128,6 +128,12 @@ struct GroupValueResTImpl<Collection<T>, AggFunc::SUM,
   using result_t = Collection<T>;
 };
 
+template <typename VID_T, typename LabelT, typename... SET_T, typename PropT>
+struct GroupValueResTImpl<TwoLabelVertexSetImpl<VID_T, LabelT, SET_T...>,
+                          AggFunc::SUM, std::tuple<PropertySelector<PropT>>> {
+  using result_t = Collection<PropT>;
+};
+
 // specialization for to_set
 // TODO: to set for pairs.
 template <typename T>
@@ -151,6 +157,27 @@ struct GroupValueResTImpl<Collection<T>, AggFunc::TO_LIST,
   using result_t = Collection<std::vector<T>>;
 };
 
+// When group vertices to list, we should return a vector of global ids.
+template <typename VID_T, typename LabelT, typename... T>
+struct GroupValueResTImpl<TwoLabelVertexSet<VID_T, LabelT, T...>,
+                          AggFunc::TO_LIST,
+                          std::tuple<PropertySelector<grape::EmptyType>>> {
+  using result_t = Collection<std::vector<GlobalId>>;
+};
+
+template <typename VID_T, typename LabelT, typename... T>
+struct GroupValueResTImpl<RowVertexSet<LabelT, VID_T, T...>, AggFunc::TO_LIST,
+                          std::tuple<PropertySelector<grape::EmptyType>>> {
+  using result_t = Collection<std::vector<GlobalId>>;
+};
+
+template <typename VID_T, typename LabelT, typename... T>
+struct GroupValueResTImpl<GeneralVertexSet<VID_T, LabelT, T...>,
+                          AggFunc::TO_LIST,
+                          std::tuple<PropertySelector<grape::EmptyType>>> {
+  using result_t = Collection<std::vector<GlobalId>>;
+};
+
 // get the vertex's certain properties as list
 template <typename LabelT, typename VID_T, typename... SET_T, typename PropT>
 struct GroupValueResTImpl<RowVertexSet<LabelT, VID_T, SET_T...>,
@@ -168,6 +195,36 @@ struct GroupValueResTImpl<Collection<T>, AggFunc::MIN,
   using result_t = Collection<T>;
 };
 
+template <typename VID_T, typename LabelT, typename... T, typename PropT>
+struct GroupValueResTImpl<RowVertexSet<LabelT, VID_T, T...>, AggFunc::MIN,
+                          std::tuple<PropertySelector<PropT>>> {
+  using result_t = Collection<PropT>;
+};
+
+template <typename VID_T, typename LabelT, typename... T, typename PropT>
+struct GroupValueResTImpl<TwoLabelVertexSet<VID_T, LabelT, T...>, AggFunc::MIN,
+                          std::tuple<PropertySelector<PropT>>> {
+  using result_t = Collection<PropT>;
+};
+
+template <typename VID_T, typename LabelT, typename... T, typename PropT>
+struct GroupValueResTImpl<RowVertexSet<LabelT, VID_T, T...>, AggFunc::AVG,
+                          std::tuple<PropertySelector<PropT>>> {
+  using result_t = Collection<PropT>;
+};
+
+template <typename VID_T, typename LabelT, typename... T, typename PropT>
+struct GroupValueResTImpl<TwoLabelVertexSet<VID_T, LabelT, T...>, AggFunc::AVG,
+                          std::tuple<PropertySelector<PropT>>> {
+  using result_t = Collection<PropT>;
+};
+
+template <typename VID_T, typename LabelT, typename... T, typename PropT>
+struct GroupValueResTImpl<GeneralVertexSet<VID_T, LabelT, T...>, AggFunc::AVG,
+                          std::tuple<PropertySelector<PropT>>> {
+  using result_t = Collection<PropT>;
+};
+
 // support get max of vertexset's id
 template <typename LabelT, typename VID_T, typename... SET_T, typename T>
 struct GroupValueResTImpl<RowVertexSet<LabelT, VID_T, SET_T...>, AggFunc::MAX,
@@ -175,6 +232,12 @@ struct GroupValueResTImpl<RowVertexSet<LabelT, VID_T, SET_T...>, AggFunc::MAX,
   using result_t = Collection<T>;
 };
 
+template <typename LabelT, typename VID_T, typename... SET_T, typename T>
+struct GroupValueResTImpl<TwoLabelVertexSet<VID_T, LabelT, SET_T...>,
+                          AggFunc::MAX, std::tuple<PropertySelector<T>>> {
+  using result_t = Collection<T>;
+};
+
 // support get first from vertexset
 template <typename LabelT, typename VID_T, typename... SET_T, typename T>
 struct GroupValueResTImpl<RowVertexSet<LabelT, VID_T, SET_T...>, AggFunc::FIRST,
@@ -194,6 +257,15 @@ struct GroupValueResTImpl<TwoLabelVertexSet<VID_T, LabelT, SET_T...>,
       typename AggFirst<TwoLabelVertexSet<VID_T, LabelT, SET_T...>>::result_t;
 };
 
+template <typename VID_T, typename LabelT, typename... SET_T>
+struct GroupValueResTImpl<GeneralVertexSet<VID_T, LabelT, SET_T...>,
+                          AggFunc::FIRST,
+                          std::tuple<PropertySelector<grape::EmptyType>>> {
+  // the old_set_t is vertex_set or collection
+  using result_t =
+      typename AggFirst<GeneralVertexSet<VID_T, LabelT, SET_T...>>::result_t;
+};
+
 // get first from collection
 template <typename T>
 struct GroupValueResTImpl<Collection<T>, AggFunc::FIRST,
@@ -356,6 +428,7 @@ class GroupByOp {
   }
 
   // group by only one key_alias
+  // TODO: Filter out null values.
   template <typename CTX_HEAD_T, int cur_alias, int base_tag,
             typename... CTX_PREV, typename GROUP_KEY, typename... AGG_T,
             typename RES_T = typename GroupResT<
@@ -384,13 +457,23 @@ class GroupByOp {
         KeyedT<old_key_set_t, key_alias_t>::create_keyed_builder(
             old_key_set, std::get<0>(group_keys).selector_);
 
-    // VLOG(10) << "Create keyed set builder";
+    // TODO(zhanglei): Refactor the Builders for each agg function, avoid
+    // specialization for each ds.
     auto value_set_builder_tuple = create_keyed_value_set_builder_tuple(
         graph, ctx, agg_tuple, std::make_index_sequence<grouped_value_num>());
 
-    // if group_key use property, we need property getter
-    // else we just insert into key_set
-    if constexpr (group_key_on_property<key_alias_t>::value) {
+    // using key_res_ele_t = typename CommonBuilderT<
+    //     Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>,
+    //     GROUP_KEY>::result_ele_t;
+    // std::unordered_map<key_res_ele_t, size_t, boost::hash<key_res_ele_t>>
+    //     key_set;
+    // size_t cur_ind = 0;
+    // Create tuple of prop getters from aggregating tuples.
+    // auto value_prop_getter_tuple = create_value_prop_getter_tuple(
+    //     graph, ctx, agg_tuple,
+    //     std::make_index_sequence<grouped_value_num>());
+
+    if constexpr (group_key_on_property<GROUP_KEY>::value) {
       auto named_property = create_prop_desc_from_selector<GROUP_KEY::col_id>(
           std::get<0>(group_keys).selector_);
       auto prop_getter =
@@ -399,23 +482,36 @@ class GroupByOp {
         auto ele_tuple = iter.GetAllIndexElement();
         auto data_tuple = iter.GetAllData();
 
-        auto key_ele = gs::get_from_tuple<GROUP_KEY::col_id>(ele_tuple);
-        size_t ind = insert_to_keyed_set_with_prop_getter(keyed_set_builder,
-                                                          prop_getter, key_ele);
-
-        insert_to_value_set_builder(value_set_builder_tuple, ele_tuple,
-                                    data_tuple, ind);
+        auto key_ind_ele = gs::get_from_tuple<GROUP_KEY::col_id>(ele_tuple);
+        auto key_prop = prop_getter.get_view(key_ind_ele);
+        // apply get_view on key_ind_ele with value_prop_getter_tuple
+        if (IsNull(key_prop)) {
+          VLOG(10) << "Skip null values";
+          continue;
+        }
+        int32_t ind = insert_to_keyed_set_with_prop_getter(
+            keyed_set_builder, prop_getter, key_ind_ele);
+        if (ind != -1) {
+          insert_to_value_set_builder(value_set_builder_tuple, ele_tuple,
+                                      data_tuple, ind);
+        }
       }
     } else {
       for (auto iter : ctx) {
         auto ele_tuple = iter.GetAllIndexElement();
         auto data_tuple = iter.GetAllData();
 
-        auto key_ele = gs::get_from_tuple<key_alias_t::tag_id>(ele_tuple);
-        auto data_ele = gs::get_from_tuple<key_alias_t::tag_id>(data_tuple);
-        size_t ind = insert_to_keyed_set(keyed_set_builder, key_ele, data_ele);
-        insert_to_value_set_builder(value_set_builder_tuple, ele_tuple,
-                                    data_tuple, ind);
+        auto key_ele = gs::get_from_tuple<GROUP_KEY::col_id>(ele_tuple);
+        if (IsNull(key_ele)) {
+          continue;
+        }
+        auto data_ele = gs::get_from_tuple<GROUP_KEY::col_id>(data_tuple);
+        int32_t ind = insert_to_keyed_set(keyed_set_builder, key_ele, data_ele);
+        // CHECK(ind != -1);
+        if (ind != -1) {
+          insert_to_value_set_builder(value_set_builder_tuple, ele_tuple,
+                                      data_tuple, ind);
+        }
       }
     }
 
@@ -457,9 +553,6 @@ class GroupByOp {
     static constexpr size_t grouped_value_num = std::tuple_size_v<agg_tuple_t>;
     static constexpr size_t group_key_num = std::tuple_size_v<alias_tuple_t>;
 
-    // the result context must be one-to-one mapping.
-    auto key_set_ref_tuple = std::tie(gs::Get<KEY_ALIAS::col_id>(ctx)...);
-
     auto value_set_builder_tuple = create_keyed_value_set_builder_tuple(
         graph, ctx, aggs, std::make_index_sequence<grouped_value_num>());
     VLOG(10) << "Create value set builders";
@@ -476,9 +569,12 @@ class GroupByOp {
     std::unordered_map<con_key_ele_t, int, boost::hash<con_key_ele_t>>
         key_tuple_set;
 
-    auto named_properties = create_prop_descs_from_group_keys(group_keys);
+    auto group_key_prop_desc = create_prop_descs_from_group_keys(group_keys);
     auto prop_getters =
-        create_prop_getters_from_prop_desc(graph, ctx, named_properties);
+        create_prop_getters_from_prop_desc(graph, ctx, group_key_prop_desc);
+    // auto value_prop_getter_tuple = create_value_prop_getter_tuple(
+    // graph, ctx, aggs, std::make_index_sequence<grouped_value_num>());
+
     size_t cur_ind = 0;
     for (auto iter : ctx) {
       auto ele_tuple = iter.GetAllElement();
@@ -487,18 +583,25 @@ class GroupByOp {
       auto key_data_tuple =
           std::make_tuple(gs::get_from_tuple<KEY_ALIAS::col_id>(data_tuple)...);
       auto key_tuple = create_key_tuple_ele(ele_tuple, prop_getters);
+      // auto value_prop =
+      // apply_get_view(value_prop_getter_tuple, ind_ele_tuple, data_tuple,
+      //  std::make_index_sequence<grouped_value_num>());
+      // TODO: determine what to do if the key or value has null value
+      if (HasNull(key_tuple)) {
+        VLOG(10) << "Skip null values: " << gs::to_string(key_tuple);
+        continue;
+      }
       size_t ind = 0;
       if (key_tuple_set.find(key_tuple) != key_tuple_set.end()) {
         // already exist
         ind = key_tuple_set[key_tuple];
       } else {
         // not exist
-        ind = cur_ind++;
         insert_into_comment_builder_tuple<0>(
             keyed_set_builder_tuple, group_keys, key_tuple, key_data_tuple);
+        ind = cur_ind++;
         key_tuple_set[key_tuple] = ind;
       }
-      // CHECK insert key.
       insert_to_value_set_builder(value_set_builder_tuple, ind_ele_tuple,
                                   data_tuple, ind);
     }
@@ -530,14 +633,17 @@ class GroupByOp {
   // ind is the index of the key in the key set
   template <size_t Is = 0, typename ele_tuple_t, typename data_tuple_t,
             typename... SET_T>
-  static void insert_to_value_set_builder(
+  static bool insert_to_value_set_builder(
       std::tuple<SET_T...>& value_set_builder, const ele_tuple_t& ele_tuple,
       const data_tuple_t& data_tuple, size_t ind) {
-    std::get<Is>(value_set_builder).insert(ind, ele_tuple, data_tuple);
+    if (!std::get<Is>(value_set_builder).insert(ind, ele_tuple, data_tuple)) {
+      return false;
+    }
     if constexpr (Is + 1 < sizeof...(SET_T)) {
-      insert_to_value_set_builder<Is + 1>(value_set_builder, ele_tuple,
-                                          data_tuple, ind);
+      return insert_to_value_set_builder<Is + 1>(value_set_builder, ele_tuple,
+                                                 data_tuple, ind);
     }
+    return true;
   }
 
   template <typename... BUILDER_T, size_t... Is>
@@ -557,6 +663,29 @@ class GroupByOp {
         graph, ctx.GetPrevCols(), ctx.GetHead(), std::get<Is>(agg_tuple))...);
   }
 
+  // create tuple of prop getters for aggregation functions
+  template <typename... CTX_PREV, typename HEAD_T, int cur_alias, int base_tag,
+            typename... AGG_T, size_t... Is>
+  static auto create_value_prop_getter_tuple(
+      const GRAPH_INTERFACE& graph,
+      Context<HEAD_T, cur_alias, base_tag, CTX_PREV...>& ctx,
+      std::tuple<AGG_T...>& agg_tuple, std::index_sequence<Is...>) {
+    return std::make_tuple(create_prop_getter_from_prop_desc(
+        graph, ctx,
+        create_prop_desc_from_aggregate_prop(std::get<Is>(agg_tuple)))...);
+  }
+
+  // Apply get_view on each element in ele_tuple with prop_getter_tuple
+  template <typename... PROP_GETTER_T, typename... ELE_T, typename... DATA_T,
+            size_t... Is>
+  static auto apply_get_view(std::tuple<PROP_GETTER_T...>& prop_getter_tuple,
+                             const std::tuple<ELE_T...>& ele_tuple,
+                             const std::tuple<DATA_T...>& data_tuple,
+                             std::index_sequence<Is...>) {
+    return std::make_tuple(
+        std::get<Is>(prop_getter_tuple).get_from_all_element(ele_tuple)...);
+  }
+
   // create for ctx with only one column
   template <typename HEAD_T, int cur_alias, int base_tag, typename... AGG_T,
             size_t... Is>
@@ -586,7 +715,7 @@ class GroupByOp {
       const HEAD_T& head,
       AggregateProp<_agg_func, std::tuple<PropertySelector<T>>,
                     std::integer_sequence<int32_t, tag_id>>& agg) {
-    if constexpr (tag_id < sizeof...(SET_T)) {
+    if constexpr (tag_id >= 0 && tag_id < (int32_t) sizeof...(SET_T)) {
       auto old_set = gs::get_from_tuple<tag_id>(tuple);
       using old_set_t = typename std::remove_const_t<
           std::remove_reference_t<decltype(old_set)>>;
@@ -638,6 +767,18 @@ class GroupByOp {
     }
   }
 
+  template <typename... SET_T, typename HEAD_T, AggFunc _agg_func, typename T,
+            int tag_id>
+  static auto create_keyed_value_set_builder(
+      const GRAPH_INTERFACE& graph, const HEAD_T& head,
+      AggregateProp<_agg_func, std::tuple<PropertySelector<T>>,
+                    std::integer_sequence<int32_t, tag_id>>& agg) {
+    static_assert(tag_id == 0 || tag_id == -1);
+    return KeyedAggT<GRAPH_INTERFACE, HEAD_T, _agg_func, std::tuple<T>,
+                     std::integer_sequence<int32_t, tag_id>>::
+        create_agg_builder(head, graph, agg.selectors_);
+  }
+
   // create builder for single key_alias
   template <typename... SET_T, typename HEAD_T, int col_id, typename KEY_PROP>
   static auto create_unkeyed_set_builder(
@@ -656,21 +797,9 @@ class GroupByOp {
     }
   }
 
-  template <typename... SET_T, typename HEAD_T, AggFunc _agg_func, typename T,
-            int tag_id>
-  static auto create_keyed_value_set_builder(
-      const GRAPH_INTERFACE& graph, const HEAD_T& head,
-      AggregateProp<_agg_func, std::tuple<PropertySelector<T>>,
-                    std::integer_sequence<int32_t, tag_id>>& agg) {
-    static_assert(tag_id == 0 || tag_id == -1);
-    return KeyedAggT<GRAPH_INTERFACE, HEAD_T, _agg_func, std::tuple<T>,
-                     std::integer_sequence<int32_t, tag_id>>::
-        create_agg_builder(head, graph, agg.selectors_);
-  }
-
   // insert_to_key_set with respect to property type
   template <typename BuilderT, typename PROP_GETTER, typename ELE>
-  static inline auto insert_to_keyed_set_with_prop_getter(
+  static inline int32_t insert_to_keyed_set_with_prop_getter(
       BuilderT& builder, const PROP_GETTER& prop_getter, const ELE& ele) {
     return builder.insert(prop_getter.get_view(ele));
   }
@@ -719,7 +848,6 @@ class GroupByOp {
     auto& ele = std::get<Ind>(eles);
     auto& d = std::get<Ind>(data);
     insert_to_keyed_set_with_group_key(builder, group_key, ele, d);
-
     if constexpr (Ind + 1 < sizeof...(BuilderT)) {
       insert_into_comment_builder_tuple<Ind + 1>(builders, keys, eles, data);
     }
@@ -730,15 +858,20 @@ class GroupByOp {
   static inline void insert_to_keyed_set_with_group_key(
       BuilderT& builder, const GroupKey<col_id, T>& group_key, const ELE& ele,
       const DATA& data) {
-    builder.Insert(ele);
+    if constexpr (std::is_same_v<DATA, grape::EmptyType> ||
+                  std::is_same_v<DATA, std::tuple<grape::EmptyType>>) {
+      builder.Insert(ele);
+    } else {
+      builder.Insert(ele, data);
+    }
   }
 
-  template <typename BuilderT, int col_id, typename ELE, typename DATA>
-  static inline void insert_to_keyed_set_with_group_key(
-      BuilderT& builder, const GroupKey<col_id, grape::EmptyType>& group_key,
-      const ELE& ele, const DATA& data) {
-    insert_into_builder_v2_impl(builder, ele, data);
-  }
+  // template <typename BuilderT, int col_id, typename ELE, typename DATA>
+  // static inline bool insert_to_keyed_set_with_group_key(
+  //     BuilderT& builder, const GroupKey<col_id, grape::EmptyType>& group_key,
+  //     const ELE& ele, const DATA& data) {
+  //   return insert_into_builder_v2_impl(builder, ele, data);
+  // }
 };
 }  // namespace gs
 
diff --git a/flex/engines/hqps_db/core/operator/path_expand.h b/flex/engines/hqps_db/core/operator/path_expand.h
index 000309c22701..d0869c70f056 100644
--- a/flex/engines/hqps_db/core/operator/path_expand.h
+++ b/flex/engines/hqps_db/core/operator/path_expand.h
@@ -138,7 +138,7 @@ class PathExpand {
       CHECK(cur_path.size() == vertices_vec.size());
       next_path.resize(vertices_vec.size());
 
-      for (auto i = 0; i < vertices_vec.size(); ++i) {
+      for (size_t i = 0; i < vertices_vec.size(); ++i) {
         auto& tmp_path = cur_path[i];
         auto start = cur_offset_vec[i];
         auto end = cur_offset_vec[i + 1];
@@ -159,7 +159,7 @@ class PathExpand {
     }
 
     std::vector<Path<vid_t, LabelT>> res_path;
-    for (auto i = 0; i < vertices_vec.size(); ++i) {
+    for (size_t i = 0; i < vertices_vec.size(); ++i) {
       auto& tmp_path = prev_path[i];
       res_path.insert(res_path.end(), tmp_path.begin(), tmp_path.end());
       res_offsets.emplace_back(res_path.size());
@@ -205,7 +205,7 @@ class PathExpand {
 
   // PathExpandV for two_label_vertex set as input.
   template <typename... V_SET_T, typename VERTEX_FILTER_T, typename LabelT,
-            typename EDGE_FILTER_T, typename RES_SET_T = vertex_set_t<int32_t>,
+            typename EDGE_FILTER_T, typename RES_SET_T = vertex_set_t<Dist>,
             typename RES_T = std::pair<RES_SET_T, std::vector<offset_t>>>
   static RES_T PathExpandV(
       const GRAPH_INTERFACE& graph,
@@ -223,7 +223,7 @@ class PathExpand {
     std::tie(input_v_1, active_ind1) = vertex_set.GetVertices(1);
 
     std::vector<vertex_id_t> vids_vec0, vids_vec1;
-    std::vector<int32_t> dist_vec0, dist_vec1;
+    std::vector<Dist> dist_vec0, dist_vec1;
     std::vector<offset_t> offsets0, offsets1;
     std::tie(vids_vec0, dist_vec0, offsets0) = PathExpandRawVMultiV(
         graph, vertex_set.GetLabel(0), input_v_0, range, edge_expand_opt);
@@ -233,7 +233,7 @@ class PathExpand {
 
     // Default vertex set to vertex set.
     std::vector<vertex_id_t> res_vids;
-    std::vector<int32_t> res_dist;
+    std::vector<Dist> res_dist;
     std::vector<offset_t> res_offsets;
     res_vids.reserve(vids_vec0.size() + vids_vec1.size());
     res_dist.reserve(dist_vec0.size() + dist_vec1.size());
@@ -287,12 +287,10 @@ class PathExpand {
     auto cur_label = vertex_set.GetLabel();
     auto& range = path_expand_opt.range_;
     auto& edge_expand_opt = path_expand_opt.edge_expand_opt_;
-    auto& get_v_opt = path_expand_opt.get_v_opt_;
     auto tuple = PathExpandRawVMultiV(
         graph, cur_label, vertex_set.GetVertices(), range, edge_expand_opt);
 
     // Default vertex set to vertex set.
-    auto& vids_vec = std::get<0>(tuple);
     auto tuple_vec = single_col_vec_to_tuple_vec(std::move(std::get<1>(tuple)));
     auto row_vertex_set = make_row_vertex_set(std::move(std::get<0>(tuple)),
                                               edge_expand_opt.other_label_,
@@ -465,7 +463,6 @@ class PathExpand {
       EdgeExpandOpt<LabelT, EDGE_FILTER_T, SELECTOR...>& edge_expand_opt) {
     // auto src_label = vertex_set.GetLabel();
     // auto src_vertices_vec = vertex_set.GetVertices();
-    auto src_vertices_size = src_vertices_vec.size();
     vertex_id_t src_id = src_vertices_vec[0];
 
     std::vector<vertex_id_t> gids;
@@ -484,12 +481,16 @@ class PathExpand {
       dists.emplace_back(0);
     }
 
+    label_id_t real_src_label, dst_label;
+    std::tie(real_src_label, dst_label) = get_graph_label_pair(
+        edge_expand_opt.dir_, src_label, edge_expand_opt.other_label_);
+
     double visit_array_time = 0.0;
-    for (auto cur_hop = 1; cur_hop < range.limit_; ++cur_hop) {
+    for (size_t cur_hop = 1; cur_hop < range.limit_; ++cur_hop) {
       std::vector<size_t> unused;
       std::tie(tmp_vec, unused) = graph.GetOtherVerticesV2(
-          src_label, edge_expand_opt.other_label_, edge_expand_opt.edge_label_,
-          tmp_vec, gs::to_string(edge_expand_opt.dir_), INT_MAX);
+          real_src_label, dst_label, edge_expand_opt.edge_label_, tmp_vec,
+          gs::to_string(edge_expand_opt.dir_), INT_MAX);
       // remove duplicate
       size_t limit = 0;
       for (size_t i = 0; i < tmp_vec.size(); ++i) {
@@ -565,11 +566,15 @@ class PathExpand {
     }
     offsets[0].emplace_back(src_vertices_size);
 
+    label_id_t real_src_label, dst_label;
+    std::tie(real_src_label, dst_label) = get_graph_label_pair(
+        edge_expand_opt.dir_, src_label, edge_expand_opt.other_label_);
+
     double visit_array_time = 0.0;
-    for (auto cur_hop = 1; cur_hop < range.limit_; ++cur_hop) {
+    for (size_t cur_hop = 1; cur_hop < range.limit_; ++cur_hop) {
       double t0 = -grape::GetCurrentTime();
       auto pair = graph.GetOtherVerticesV2(
-          src_label, edge_expand_opt.other_label_, edge_expand_opt.edge_label_,
+          real_src_label, dst_label, edge_expand_opt.edge_label_,
           gids[cur_hop - 1], gs::to_string(edge_expand_opt.dir_), INT_MAX);
 
       gids[cur_hop].swap(pair.first);
@@ -609,10 +614,8 @@ class PathExpand {
           auto start = offsets[j][i];
           auto end = offsets[j][i + 1];
           for (auto k = start; k < end; ++k) {
-            auto gid = gids[j][k];
             flat_gids.emplace_back(gids[j][k]);
             dists.emplace_back(j);
-            // }
           }
         }
         flat_offsets.emplace_back(flat_gids.size());
diff --git a/flex/engines/hqps_db/core/operator/project.h b/flex/engines/hqps_db/core/operator/project.h
index 9356fd0abbd1..e1873227fb8a 100644
--- a/flex/engines/hqps_db/core/operator/project.h
+++ b/flex/engines/hqps_db/core/operator/project.h
@@ -59,6 +59,27 @@ struct ResultOfContextKeyAlias<
   using result_t = ctx_node_t;
 };
 
+template <typename CTX_HEAD_T, int cur_alias, int base_tag,
+          typename... CTX_PREV, int in_col_id>
+struct ResultOfContextKeyAlias<
+    Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>,
+    IdentityMapper<in_col_id, PropertySelector<GlobalId>>> {
+  using context_t = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>;
+  using result_t = Collection<typename GlobalId::gid_t>;
+};
+
+// Mapping a node to
+// template <typename CTX_HEAD_T, int cur_alias, int base_tag,
+//           typename... CTX_PREV, typename... Mapper>
+// struct ResultOfContextKeyAlias<
+//     Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>,
+//     KeyValueMappers<Mapper...>> {
+//   using context_t = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>;
+//   using ctx_node_t = std::remove_reference_t<decltype(
+//       std::declval<context_t>().template GetNode<in_col_id>())>;
+//   using result_t = Collection
+// };
+
 template <int new_head_alias, typename new_head_t, int cur_alias,
           typename old_head_t, int base_tag, typename tuple>
 struct ResultContextTWithPrevTuple;
@@ -73,63 +94,15 @@ struct ResultContextTWithPrevTuple<new_head_alias, new_head_t, cur_alias,
                               old_head_t, base_tag, T...>::result_t;
 };
 
-template <bool is_append, typename CTX_T, typename PROJECT_OPT>
-struct ProjectResT;
-
-template <typename CTX_HEAD_T, int cur_alias, int base_tag,
-          typename... CTX_PREV, typename... KEY_ALIAS_T>
-struct ProjectResT<true, Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>,
-                   ProjectOpt<KEY_ALIAS_T...>> {
-  using old_ctx_t = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>;
-  using project_opt_t = ProjectOpt<KEY_ALIAS_T...>;
-  static constexpr size_t num_key_alias = sizeof...(KEY_ALIAS_T);
-  using last_key_alias_t =
-      std::tuple_element_t<num_key_alias - 1, std::tuple<KEY_ALIAS_T...>>;
-
-  using new_head_t =
-      typename ResultOfContextKeyAlias<old_ctx_t, last_key_alias_t>::result_t;
-  static constexpr int new_head_alias = last_key_alias_t::res_alias;
-  using proj_res_tuple_t = std::tuple<
-      typename ResultOfContextKeyAlias<old_ctx_t, KEY_ALIAS_T>::result_t...>;
-  using first_n_of_key_alias_tuple =
-      typename first_n<sizeof...(KEY_ALIAS_T) - 1, proj_res_tuple_t>::type;
-  using result_t = typename ResultContextTWithPrevTuple<
-      new_head_alias, new_head_t, cur_alias, CTX_HEAD_T, base_tag,
-      typename TupleCatT<std::tuple<CTX_PREV...>,
-                         first_n_of_key_alias_tuple>::tuple_cat_t>::result_t;
-};
-
-template <typename CTX_HEAD_T, int cur_alias, int base_tag,
-          typename... CTX_PREV, typename... KEY_ALIAS_T>
-struct ProjectResT<false, Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>,
-                   ProjectOpt<KEY_ALIAS_T...>> {
-  using old_ctx_t = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>;
-  using project_opt_t = ProjectOpt<KEY_ALIAS_T...>;
-  static constexpr size_t num_key_alias = sizeof...(KEY_ALIAS_T);
-  using last_key_alias_t =
-      std::tuple_element_t<num_key_alias - 1, std::tuple<KEY_ALIAS_T...>>;
-
-  using new_head_t =
-      typename ResultOfContextKeyAlias<old_ctx_t, last_key_alias_t>::result_t;
-  static constexpr int new_head_alias = last_key_alias_t::res_alias;
-  using proj_res_tuple_t = std::tuple<
-      typename ResultOfContextKeyAlias<old_ctx_t, KEY_ALIAS_T>::result_t...>;
-  using first_n_of_key_alias_tuple =
-      typename first_n<sizeof...(KEY_ALIAS_T) - 1, proj_res_tuple_t>::type;
-  using result_t = typename ResultContextTWithPrevTuple<
-      new_head_alias, new_head_t, cur_alias, CTX_HEAD_T, base_tag,
-      typename TupleCatT<std::tuple<CTX_PREV...>,
-                         first_n_of_key_alias_tuple>::tuple_cat_t>::result_t;
-};
-
 template <typename GRAPH_INTERFACE>
 class ProjectOp {
  public:
   // specialized to append
   // Project a previous tag and append to traversal.
-  template <bool is_append, typename CTX_HEAD_T, int cur_alias, int base_tag,
-            typename... CTX_PREV, typename... ProjectMapper,
-            typename std::enable_if<is_append>::type* = nullptr>
+  template <
+      ProjectDesc is_append, typename CTX_HEAD_T, int cur_alias, int base_tag,
+      typename... CTX_PREV, typename... ProjectMapper,
+      typename std::enable_if<(is_append != ProjectDesc::New)>::type* = nullptr>
   static auto ProjectImpl(
       const GRAPH_INTERFACE& graph,
       Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
@@ -142,19 +115,22 @@ class ProjectOp {
       offsets[i] = i;
     }
 
-    return apply_projects_append<0>(graph, std::move(ctx), mappers, offsets);
+    return apply_projects_append<0, is_append>(graph, std::move(ctx), mappers,
+                                               offsets);
   }
 
   // implementation for project is false, only proj one column
-  template <bool is_append, typename CTX_HEAD_T, int cur_alias, int base_tag,
-            typename... CTX_PREV, typename... ProjectMapper,
-            typename std::enable_if<!is_append && (sizeof...(ProjectMapper) ==
-                                                   1)>::type* = nullptr>
+  template <
+      ProjectDesc is_append, typename CTX_HEAD_T, int cur_alias, int base_tag,
+      typename... CTX_PREV, typename... ProjectMapper,
+      typename std::enable_if<(is_append == ProjectDesc::New) &&
+                              (sizeof...(ProjectMapper) == 1)>::type* = nullptr>
   static auto ProjectImpl(
       const GRAPH_INTERFACE& graph,
       Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
       std::tuple<ProjectMapper...>&& mappers) {
     auto node_size = gs::Get<-1>(ctx).Size();
+    LOG(INFO) << "Project with: " << demangle(std::get<0>(mappers));
     VLOG(10) << "Current head size: " << node_size;
 
     auto head = apply_single_project(graph, ctx, std::get<0>(mappers));
@@ -164,10 +140,11 @@ class ProjectOp {
   }
 
   // implementation for project is false. project multiple columns
-  template <bool is_append, typename CTX_HEAD_T, int cur_alias, int base_tag,
-            typename... CTX_PREV, typename... ProjectMapper,
-            typename std::enable_if<!is_append && (sizeof...(ProjectMapper) >
-                                                   1)>::type* = nullptr>
+  template <
+      ProjectDesc is_append, typename CTX_HEAD_T, int cur_alias, int base_tag,
+      typename... CTX_PREV, typename... ProjectMapper,
+      typename std::enable_if<(is_append == ProjectDesc::New) &&
+                              (sizeof...(ProjectMapper) > 1)>::type* = nullptr>
   static auto ProjectImpl(
       const GRAPH_INTERFACE& graph,
       Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
@@ -194,7 +171,8 @@ class ProjectOp {
                                          std::move(offsets));
   }
 
-  template <size_t Is, typename CTX_T, typename... ProjectMapper,
+  template <size_t Is, ProjectDesc append_opt, typename CTX_T,
+            typename... ProjectMapper,
             typename std::enable_if<(Is < sizeof...(ProjectMapper) -
                                               1)>::type* = nullptr>
   static auto apply_projects_append(const GRAPH_INTERFACE& graph, CTX_T&& ctx,
@@ -202,14 +180,22 @@ class ProjectOp {
                                     std::vector<offset_t>& offsets) {
     auto new_node = apply_single_project(graph, ctx, std::get<Is>(key_alias));
     std::vector<offset_t> res_offsets(offsets);
-    auto res = ctx.template AddNode<AppendOpt::Persist>(std::move(new_node),
-                                                        std::move(res_offsets));
-    return apply_projects_append<Is + 1>(graph, std::move(res), key_alias,
-                                         offsets);
+    if constexpr (append_opt == ProjectDesc::AppendTemp) {
+      auto res = ctx.template AddNode<AppendOpt::Temp>(std::move(new_node),
+                                                       std::move(res_offsets));
+      return apply_projects_append<Is + 1, append_opt>(graph, std::move(res),
+                                                       key_alias, offsets);
+    } else {
+      auto res = ctx.template AddNode<AppendOpt::Persist>(
+          std::move(new_node), std::move(res_offsets));
+      return apply_projects_append<Is + 1, append_opt>(graph, std::move(res),
+                                                       key_alias, offsets);
+    }
   }
 
   // For the last element, return the result
-  template <size_t Is, typename CTX_T, typename... ProjectMapper,
+  template <size_t Is, ProjectDesc append_opt, typename CTX_T,
+            typename... ProjectMapper,
             typename std::enable_if<(Is == sizeof...(ProjectMapper) -
                                                1)>::type* = nullptr>
   static auto apply_projects_append(const GRAPH_INTERFACE& graph, CTX_T&& ctx,
@@ -217,8 +203,14 @@ class ProjectOp {
                                     std::vector<offset_t>& offsets) {
     auto new_node = apply_single_project(graph, ctx, std::get<Is>(key_alias));
     std::vector<offset_t> res_offsets(offsets);
-    return ctx.template AddNode<AppendOpt::Persist>(std::move(new_node),
-                                                    std::move(res_offsets));
+
+    if constexpr (append_opt == ProjectDesc::AppendTemp) {
+      return ctx.template AddNode<AppendOpt::Temp>(std::move(new_node),
+                                                   std::move(res_offsets));
+    } else {
+      return ctx.template AddNode<AppendOpt::Persist>(std::move(new_node),
+                                                      std::move(res_offsets));
+    }
   }
 
   // Apply single project on old context's node until the indicated index of
@@ -233,24 +225,31 @@ class ProjectOp {
   }
 
   // Apply single project with IdentityMapper.
-  template <typename CTX_T, int in_col_id, typename T>
+  template <typename CTX_T, int in_col_id, typename SelectorValueType,
+            typename std::enable_if<(
+                !std::is_same_v<grape::EmptyType, SelectorValueType> &&
+                !std::is_same_v<GlobalId, SelectorValueType>)>::type* = nullptr>
   static auto apply_single_project(
       const GRAPH_INTERFACE& graph, CTX_T& ctx,
-      IdentityMapper<in_col_id, PropertySelector<T>>& mapper) {
+      IdentityMapper<in_col_id, PropertySelector<SelectorValueType>>& mapper) {
     auto& node = ctx.template GetNode<in_col_id>();
     // Create a empty copy.
     auto offset_array = ctx.ObtainOffsetFromTag(in_col_id);
     auto repeat_array = offset_array_to_repeat_array(std::move(offset_array));
     // A col describe what content is used to project
-    return apply_single_project_impl<T>(
+    return apply_single_project_impl<SelectorValueType>(
         graph, node, mapper.selector_.prop_name_, repeat_array);
   }
 
   // Project self.
-  template <typename CTX_T, int in_col_id>
+  // Selector with GlobalId or grape::EmptyType is different
+  template <typename CTX_T, int in_col_id, typename SelectorValueType,
+            typename std::enable_if<
+                (std::is_same_v<grape::EmptyType, SelectorValueType>)>::type* =
+                nullptr>
   static auto apply_single_project(
       const GRAPH_INTERFACE& graph, CTX_T& ctx,
-      IdentityMapper<in_col_id, InternalIdSelector>& mapper) {
+      IdentityMapper<in_col_id, PropertySelector<SelectorValueType>>& mapper) {
     auto& node = ctx.template GetNode<in_col_id>();
     // Create a empty copy.
     auto offset_array = ctx.ObtainOffsetFromTag(in_col_id);
@@ -259,6 +258,37 @@ class ProjectOp {
     return node.ProjectWithRepeatArray(repeat_array, key_alias);
   }
 
+  // Project to GlobalId
+  template <typename CTX_T, int in_col_id, typename SelectorValueType,
+            typename std::enable_if<
+                (std::is_same_v<GlobalId, SelectorValueType>)>::type* = nullptr>
+  static auto apply_single_project(
+      const GRAPH_INTERFACE& graph, CTX_T& ctx,
+      IdentityMapper<in_col_id, PropertySelector<SelectorValueType>>& mapper) {
+    auto& node = ctx.template GetNode<in_col_id>();
+    static_assert(std::remove_reference_t<decltype(
+                      node)>::is_vertex_set);  // edge_set not supported
+    // Create a empty copy.
+    auto offset_array = ctx.ObtainOffsetFromTag(in_col_id);
+    auto repeat_array = offset_array_to_repeat_array(std::move(offset_array));
+    auto prop_getter = create_global_id_prop_getter_from_prop_desc(
+        graph, node, GlobalIdProperty<in_col_id>{});
+    std::vector<typename GlobalId::gid_t> res_prop_vec;
+    // iterate over node with ind_ele
+    auto iter = node.begin();
+    auto end = node.end();
+    size_t i = 0;
+    CHECK(repeat_array.size() == node.Size());
+    for (; iter != end; ++iter) {
+      auto ele_tuple = iter.GetIndexElement();
+      for (size_t j = 0; j < repeat_array[i]; ++j) {
+        res_prop_vec.push_back(prop_getter.get_view(ele_tuple).global_id);
+      }
+      ++i;
+    }
+    return Collection<typename GlobalId::gid_t>(std::move(res_prop_vec));
+  }
+
   // Project with  single mapper
   template <typename CTX_T, typename EXPR, typename... SELECTOR,
             int32_t... in_col_id>
@@ -278,7 +308,7 @@ class ProjectOp {
         create_prop_getters_from_prop_desc(graph, ctx, prop_desc);
     LOG(INFO) << "In project with expression, successfully got prop getters";
     for (auto iter : ctx) {
-      auto ele_tuple = iter.GetAllElement();
+      auto ele_tuple = iter.GetAllIndexElement();
       res_vec.emplace_back(evaluate_proj_expr(expr, ele_tuple, prop_getters));
     }
     return Collection<expr_result_t>(std::move(res_vec));
@@ -309,39 +339,12 @@ class ProjectOp {
 
   // single label vertex set.
   template <
-      typename T, typename LabelT, typename VID_T, typename... SET_T,
-      typename std::enable_if<(!std::is_same_v<T, LabelKey>)>::type* = nullptr>
-  static auto apply_single_project_impl(
-      const GRAPH_INTERFACE& graph,
-      RowVertexSetImpl<LabelT, VID_T, SET_T...>& node,
-      const std::string& prop_name, const std::vector<size_t>& repeat_array) {
-    // Get property from storage.
-    auto prop_tuple_vec = graph.template GetVertexPropsFromVid<T>(
-        node.GetLabel(), node.GetVertices(), {prop_name});
-    // VLOG(10) << "Finish fetching properties";
-    node.fillBuiltinProps(prop_tuple_vec, {prop_name}, repeat_array);
-    std::vector<T> res_prop_vec;
-    for (size_t i = 0; i < repeat_array.size(); ++i) {
-      for (size_t j = 0; j < repeat_array[i]; ++j) {
-        res_prop_vec.push_back(std::get<0>(prop_tuple_vec[i]));
-      }
-    }
-    // check builtin properties.
-    // Found if there is any builtin properties need.
-
-    return Collection<T>(std::move(res_prop_vec));
-  }
-
-  // single keyed label vertex set.
-  template <
-      typename T, typename LabelT, typename KEY_T, typename VID_T,
-      typename... SET_T,
-      typename std::enable_if<(!std::is_same_v<T, LabelKey>)>::type* = nullptr>
+      typename T, typename NODE_T,
+      typename std::enable_if<(!std::is_same_v<T, LabelKey> &&
+                               NODE_T::is_row_vertex_set)>::type* = nullptr>
   static auto apply_single_project_impl(
-      const GRAPH_INTERFACE& graph,
-      KeyedRowVertexSetImpl<LabelT, KEY_T, VID_T, SET_T...>& node,
-      const std::string& prop_name, const std::vector<size_t>& repeat_array) {
-    LOG(INFO) << "[Single project on KeyedRowVertexSet:]" << node.GetLabel();
+      const GRAPH_INTERFACE& graph, NODE_T& node, const std::string& prop_name,
+      const std::vector<size_t>& repeat_array) {
     // Get property from storage.
     auto prop_tuple_vec = graph.template GetVertexPropsFromVid<T>(
         node.GetLabel(), node.GetVertices(), {prop_name});
@@ -519,6 +522,28 @@ class ProjectOp {
         std::move(lengths_vec));
   }
 
+  ///////////////////Apply KeyValueMapper to all data structures.
+  template <typename CTX_T, typename... MAPPER>
+  static auto apply_single_project(
+      const GRAPH_INTERFACE& graph, CTX_T& ctx,
+      KeyValueMappers<MAPPER...>& key_value_mappers) {
+    LOG(INFO) << "Project KeyValueMapper: " << demangle(key_value_mappers);
+    // the result is collection<VariableKeyValue>
+    std::vector<VariableKeyValue> res_vec;
+    res_vec.reserve(ctx.GetHead().Size());
+    auto prop_desc =
+        create_prop_descs_from_mappers<MAPPER...>(key_value_mappers);
+    LOG(INFO) << "Prop Desc: " << demangle(prop_desc);
+    auto prop_getters =
+        create_prop_getters_from_prop_desc(graph, ctx, prop_desc);
+    for (auto iter : ctx) {
+      auto ele_tuple = iter.GetAllIndexElement();
+      res_vec.emplace_back(
+          evaluate_kv_mapper(ele_tuple, prop_getters, key_value_mappers));
+    }
+    return Collection<VariableKeyValue>(std::move(res_vec));
+  }
+
   // evaluate expression in project op
   template <typename EXPR, typename... ELE, typename... PROP_GETTER>
   static inline auto evaluate_proj_expr(
@@ -537,6 +562,89 @@ class ProjectOp {
       std::index_sequence<Is...>) {
     return expr(std::get<Is>(prop_getter_tuple).get_from_all_element(eles)...);
   }
+
+  // create prop desc from mappers
+  template <typename... MAPPER>
+  static inline auto create_prop_descs_from_mappers(
+      KeyValueMappers<MAPPER...>& key_value_mappers) {
+    return create_prop_descs_from_mappers_impl(
+        key_value_mappers, std::make_index_sequence<sizeof...(MAPPER)>());
+  }
+
+  template <int32_t... in_col_id, typename... T, size_t... Is>
+  static inline auto create_prop_descs_from_mappers_impl(
+      KeyValueMappers<KeyValueMapper<in_col_id, PropertySelector<T>>...>&
+          key_value_mappers,
+      std::index_sequence<Is...>) {
+    return std::make_tuple(create_prop_desc_from_mapper(
+        std::get<Is>(key_value_mappers.mappers_))...);
+  }
+
+  // create prop desc from mapper
+  template <int32_t in_col_id, typename T>
+  static inline auto create_prop_desc_from_mapper(
+      KeyValueMapper<in_col_id, PropertySelector<T>>& mapper) {
+    return create_prop_desc_from_selector<in_col_id>(mapper.value_selector_);
+  }
+
+  // evaluate_kv_mapper
+  template <typename... ELE, typename... PROP_GETTER, typename... Mapper>
+  static inline auto evaluate_kv_mapper(
+      std::tuple<ELE...>& eles, std::tuple<PROP_GETTER...>& prop_getter_tuple,
+      const KeyValueMappers<Mapper...>& key_value_mappers) {
+    return evaluate_kv_mapper_impl(
+        eles, prop_getter_tuple, key_value_mappers,
+        std::make_index_sequence<sizeof...(PROP_GETTER)>());
+  }
+
+  template <typename... ELE, typename... PROP_GETTER, typename... Mapper,
+            size_t... Is>
+  static inline auto evaluate_kv_mapper_impl(
+      std::tuple<ELE...>& eles, std::tuple<PROP_GETTER...>& prop_getter_tuple,
+      const KeyValueMappers<Mapper...>& key_value_mappers,
+      std::index_sequence<Is...>) {
+    LOG(INFO) << "Prop Getters: " << demangle(prop_getter_tuple);
+    static_assert(sizeof...(PROP_GETTER) == sizeof...(Mapper));
+    VariableKeyValue res;
+
+    emplace_key_value_mapper<0>(res, eles, prop_getter_tuple,
+                                key_value_mappers);
+    // (res.emplace(
+    //      std::get<Is>(key_value_mappers.mappers_).key_,
+    //      std::get<Is>(prop_getter_tuple)
+    //          .get_view(gs::get_from_tuple<std::tuple_element_t<
+    //                        Is, std::tuple<Mapper...>>::in_col_id>(eles))),
+    //  ...);
+    for (auto iter : res) {
+      LOG(INFO) << "Key: " << iter.first
+                << " Value: " << iter.second.to_string();
+    }
+    return res;
+  }
+
+  template <size_t Is, typename... ELE, typename... PROP_GETTER,
+            typename... Mapper>
+  static inline void emplace_key_value_mapper(
+      VariableKeyValue& res, std::tuple<ELE...>& eles,
+      std::tuple<PROP_GETTER...>& prop_getter_tuple,
+      const KeyValueMappers<Mapper...>& key_value_mappers) {
+    if constexpr (Is < sizeof...(Mapper)) {
+      auto cur_value =
+          std::get<Is>(prop_getter_tuple)
+              .get_view(gs::get_from_tuple<std::tuple_element_t<
+                            Is, std::tuple<Mapper...>>::in_col_id>(eles));
+      if (!IsNull(cur_value)) {
+        res.emplace_back(std::get<Is>(key_value_mappers.mappers_).key_,
+                         cur_value);
+      } else {
+        LOG(INFO) << "cur value is null: " << gs::to_string(cur_value);
+        res.emplace_back(std::get<Is>(key_value_mappers.mappers_).key_,
+                         Any::From(grape::EmptyType()));
+      }
+      emplace_key_value_mapper<Is + 1>(res, eles, prop_getter_tuple,
+                                       key_value_mappers);
+    }
+  }
 };
 }  // namespace gs
 
diff --git a/flex/engines/hqps_db/core/operator/scan.h b/flex/engines/hqps_db/core/operator/scan.h
index 8df1676d9c97..3135a18b9308 100644
--- a/flex/engines/hqps_db/core/operator/scan.h
+++ b/flex/engines/hqps_db/core/operator/scan.h
@@ -27,12 +27,25 @@ limitations under the License.
 
 namespace gs {
 
+// define struct FilterNull, which's value is true if FUNC::filter_null is
+// defined and value is true
+template <typename FUNC, typename Void = void>
+struct FilterNull {
+  static constexpr bool value = false;
+};
+
+template <typename FUNC>
+struct FilterNull<FUNC, typename std::enable_if<FUNC::filter_null>::type> {
+  static constexpr bool value = FUNC::filter_null;
+};
+
 // scan for a single vertex
 template <typename GRAPH_INTERFACE>
 class Scan {
  public:
   using label_id_t = typename GRAPH_INTERFACE::label_id_t;
   using vertex_id_t = typename GRAPH_INTERFACE::vertex_id_t;
+  using gid_t = typename GRAPH_INTERFACE::gid_t;
   using vertex_set_t = DefaultRowVertexSet<label_id_t, vertex_id_t>;
   using two_label_set_t =
       TwoLabelVertexSet<vertex_id_t, label_id_t, grape::EmptyType>;
@@ -99,15 +112,37 @@ class Scan {
   template <typename OID_T>
   static vertex_set_t ScanVertexWithOid(const GRAPH_INTERFACE& graph,
                                         const label_id_t& v_label_id,
-                                        OID_T oid) {
+                                        std::vector<OID_T>&& oids) {
     std::vector<vertex_id_t> gids;
     vertex_id_t vid;
-    if (graph.ScanVerticesWithOid(v_label_id, oid, vid)) {
-      gids.emplace_back(vid);
+    for (auto oid : oids) {
+      if (graph.ScanVerticesWithOid(v_label_id, oid, vid)) {
+        gids.emplace_back(vid);
+      }
     }
     return make_default_row_vertex_set(std::move(gids), v_label_id);
   }
 
+  static vertex_set_t ScanVertexWithGid(const GRAPH_INTERFACE& graph,
+                                        const label_id_t& v_label_id,
+                                        std::vector<gid_t>&& gids) {
+    std::vector<vertex_id_t> lids;
+    auto vnum = graph.vertex_num(v_label_id);
+    for (auto gid : gids) {
+      vertex_id_t vid;
+      if (GlobalId::get_label_id(gid) == v_label_id) {
+        vid = GlobalId::get_vid(gid);
+      }
+      if (vid < vnum) {
+        lids.emplace_back(vid);
+      }
+    }
+    VLOG(10) << "Scan vertex with gid, label: " << v_label_id
+             << ", valid vertices cnt: " << lids.size()
+             << ", input cnt: " << gids.size();
+    return make_default_row_vertex_set(std::move(lids), v_label_id);
+  }
+
   /// @brief Scan vertex with oid
   /// @param graph
   /// @param v_label_ids
@@ -116,17 +151,142 @@ class Scan {
   template <typename OID_T, size_t num_labels>
   static auto ScanVertexWithOid(
       const GRAPH_INTERFACE& graph,
-      const std::array<label_id_t, num_labels>& v_label_ids, OID_T oid) {
+      const std::array<label_id_t, num_labels>& v_label_ids,
+      std::vector<OID_T>&& oids) {
     std::vector<vertex_id_t> gids;
     std::vector<label_id_t> labels_vec;
     std::vector<grape::Bitset> bitsets;
     vertex_id_t vid;
     for (size_t i = 0; i < num_labels; ++i) {
-      if (graph.ScanVerticesWithOid(v_label_ids[i], oid, vid)) {
-        labels_vec.emplace_back(v_label_ids[i]);
+      for (auto oid : oids) {
+        if (graph.ScanVerticesWithOid(v_label_ids[i], oid, vid)) {
+          labels_vec.emplace_back(v_label_ids[i]);
+          gids.emplace_back(vid);
+        }
+      }
+    }
+    bitsets.resize(labels_vec.size());
+    for (size_t i = 0; i < bitsets.size(); ++i) {
+      bitsets[i].init(gids.size());
+      bitsets[i].set_bit(i);
+    }
+
+    return make_general_set(std::move(gids), std::move(labels_vec),
+                            std::move(bitsets));
+  }
+
+  template <size_t num_labels>
+  static auto ScanVertexWithGid(
+      const GRAPH_INTERFACE& graph,
+      const std::array<label_id_t, num_labels>& v_label_ids,
+      std::vector<gid_t>&& gids) {
+    std::unordered_map<label_id_t, size_t> label_to_index;
+
+    size_t valid_label_num = 0;
+    for (size_t i = 0; i < num_labels; ++i) {
+      if (v_label_ids[i] < graph.schema().vertex_label_num()) {
+        label_to_index[v_label_ids[i]] = valid_label_num++;
+      }
+    }
+
+    std::vector<vertex_id_t> lids;
+    std::vector<label_id_t> label_ind_vec;
+    for (auto gid : gids) {
+      auto label_id = GlobalId::get_label_id(gid);
+      auto vid = GlobalId::get_vid(gid);
+      if (label_to_index.find(label_id) != label_to_index.end()) {
+        label_ind_vec.emplace_back(label_to_index[label_id]);
+        lids.emplace_back(vid);
+      }
+    }
+
+    std::vector<grape::Bitset> bitsets;
+    bitsets.resize(valid_label_num);
+    for (size_t i = 0; i < bitsets.size(); ++i) {
+      bitsets[i].init(lids.size());
+    }
+    for (size_t i = 0; i < label_ind_vec.size(); ++i) {
+      bitsets[label_ind_vec[i]].set_bit(i);
+    }
+    std::vector<label_id_t> labels_vec(valid_label_num);
+    for (auto& pair : label_to_index) {
+      labels_vec[pair.second] = pair.first;
+    }
+    return make_general_set(std::move(lids), std::move(labels_vec),
+                            std::move(bitsets));
+  }
+
+  /// @brief Scan vertex with oid and Expr
+  /// @param graph
+  /// @param v_label_id
+  /// @param oid
+  /// @return
+  template <typename OID_T, typename EXPR, typename... SELECTOR>
+  static vertex_set_t ScanVertexWithOidExpr(
+      const GRAPH_INTERFACE& graph, const label_id_t& v_label_id,
+      std::vector<OID_T>&& oids, Filter<EXPR, SELECTOR...>&& filter) {
+    std::vector<vertex_id_t> gids;
+    vertex_id_t vid;
+    for (auto oid : oids) {
+      if (graph.ScanVerticesWithOid(v_label_id, oid, vid)) {
         gids.emplace_back(vid);
       }
     }
+
+    auto real_gids = filter_vertex_with_selector(
+        graph, v_label_id, filter.expr_, filter.selectors_, gids);
+    return make_default_row_vertex_set(std::move(real_gids), v_label_id);
+  }
+
+  template <typename EXPR, typename... SELECTOR>
+  static vertex_set_t ScanVertexWithGidExpr(
+      const GRAPH_INTERFACE& graph, const label_id_t& v_label_id,
+      std::vector<gid_t>&& gids, Filter<EXPR, SELECTOR...>&& filter) {
+    std::vector<vertex_id_t> lids;
+    auto vnum = graph.vertex_num(v_label_id);
+    for (auto gid : gids) {
+      if (GlobalId::get_label_id(gid) == v_label_id) {
+        auto vid = GlobalId::get_vid(gid);
+        if (vid < vnum) {
+          lids.emplace_back(vid);
+        }
+      }
+    }
+
+    auto real_lids = filter_vertex_with_selector(
+        graph, v_label_id, filter.expr_, filter.selectors_, lids);
+    return make_default_row_vertex_set(std::move(real_lids), v_label_id);
+  }
+
+  /// @brief Scan vertex with oid
+  /// @param graph
+  /// @param v_label_ids
+  /// @param oid
+  /// @return
+  template <typename OID_T, size_t num_labels, typename FilterT>
+  static auto ScanVertexWithOidExpr(
+      const GRAPH_INTERFACE& graph,
+      const std::array<label_id_t, num_labels>& v_label_ids,
+      std::vector<OID_T>&& oids, FilterT&& filter) {
+    std::vector<vertex_id_t> gids;
+    std::vector<label_id_t> labels_vec;
+    std::vector<grape::Bitset> bitsets;
+    vertex_id_t vid;
+    for (size_t i = 0; i < num_labels; ++i) {
+      std::vector<vertex_id_t> tmp_gids;
+      for (auto oid : oids) {
+        if (graph.ScanVerticesWithOid(v_label_ids[i], oid, vid)) {
+          tmp_gids.emplace_back(vid);
+        }
+      }
+      auto real_gids = filter_vertex_with_selector(
+          graph, v_label_ids[i], filter.expr_, filter.selectors_, tmp_gids);
+      for (auto gid : real_gids) {
+        labels_vec.emplace_back(v_label_ids[i]);
+        gids.emplace_back(gid);
+      }
+    }
+
     bitsets.resize(labels_vec.size());
     for (size_t i = 0; i < bitsets.size(); ++i) {
       bitsets[i].init(gids.size());
@@ -137,6 +297,48 @@ class Scan {
                             std::move(bitsets));
   }
 
+  template <size_t num_labels, typename FilterT>
+  static auto ScanVertexWithGidExpr(
+      const GRAPH_INTERFACE& graph,
+      const std::array<label_id_t, num_labels>& v_label_ids,
+      std::vector<gid_t>&& gids, FilterT&& filter) {
+    std::unordered_map<label_id_t, size_t> label_to_index;
+    size_t valid_label_num = 0;
+    for (size_t i = 0; i < num_labels; ++i) {
+      if (v_label_ids[i] < graph.schema().vertex_label_num()) {
+        label_to_index[v_label_ids[i]] = valid_label_num++;
+      }
+    }
+
+    std::vector<vertex_id_t> lids;
+    std::vector<label_id_t> label_ind_vec;
+    for (auto gid : gids) {
+      auto label_id = GlobalId::get_label_id(gid);
+      auto vid = GlobalId::get_vid(gid);
+      if (label_to_index.find(label_id) != label_to_index.end() &&
+          eval_vertex_with_expr(graph, label_id, filter.expr_,
+                                filter.selectors_, vid)) {
+        label_ind_vec.emplace_back(label_to_index[label_id]);
+        lids.emplace_back(vid);
+      }
+    }
+
+    std::vector<grape::Bitset> bitsets;
+    bitsets.resize(valid_label_num);
+    for (size_t i = 0; i < bitsets.size(); ++i) {
+      bitsets[i].init(lids.size());
+    }
+    for (size_t i = 0; i < label_ind_vec.size(); ++i) {
+      bitsets[label_ind_vec[i]].set_bit(i);
+    }
+    std::vector<label_id_t> labels_vec(valid_label_num);
+    for (auto& pair : label_to_index) {
+      labels_vec[pair.second] = pair.first;
+    }
+    return make_general_set(std::move(lids), std::move(labels_vec),
+                            std::move(bitsets));
+  }
+
  private:
   template <typename EXPR, typename... SELECTOR, size_t num_labels>
   static GeneralVertexSet<vertex_id_t, label_id_t, grape::EmptyType>
@@ -167,22 +369,6 @@ class Scan {
     return make_general_set(std::move(gids), labels_vec, std::move(bitsets));
   }
 
-  template <typename FUNC, typename... PropT>
-  static std::vector<vertex_id_t> scan_vertex1_impl(
-      const GRAPH_INTERFACE& graph, const label_id_t& v_label_id,
-      const FUNC& func, const std::tuple<PropT...>& props) {
-    std::vector<vertex_id_t> gids;
-    auto filter = [&](vertex_id_t v,
-                      const std::tuple<typename PropT::prop_t...>& real_props) {
-      if (apply_on_tuple(func, real_props)) {
-        gids.push_back(v);
-      }
-    };
-
-    graph.template ScanVertices(v_label_id, props, filter);
-    return gids;
-  }
-
   template <typename FUNC, typename... SELECTOR>
   static std::vector<vertex_id_t> scan_vertex_with_selector(
       const GRAPH_INTERFACE& graph, const label_id_t& v_label_id,
@@ -195,8 +381,64 @@ class Scan {
             gids.push_back(v);
           }
         };
+    // if FUNC has filter_null constexpr member, we can use it to filter
+    // vertices
+    if constexpr (FilterNull<FUNC>::value) {
+      graph.template ScanVertices(v_label_id, selectors, filter, true);
+    } else {
+      graph.template ScanVertices(v_label_id, selectors, filter, false);
+    }
+    return gids;
+  }
+
+  template <typename FUNC, typename... SELECTOR>
+  static inline bool eval_vertex_with_expr(
+      const GRAPH_INTERFACE& graph, const label_id_t& v_label_id,
+      const FUNC& func, const std::tuple<SELECTOR...>& selectors,
+      vertex_id_t vid) {
+    std::tuple<typename SELECTOR::prop_t...> real_props;
+    if constexpr (sizeof...(SELECTOR) == 0) {
+      return apply_on_tuple(func, real_props);
+    } else {
+      auto columns =
+          graph.GetPropertyColumnWithSelectors(v_label_id, selectors);
+      if (exists_nullptr_in_tuple(columns)) {
+        VLOG(10) << "When scanning for label " << std::to_string(v_label_id)
+                 << ", there is null column, using default NULL value";
+      }
+      get_tuple_from_column_tuple(vid, real_props, columns);
+      return apply_on_tuple(func, real_props);
+    }
+  }
 
-    graph.template ScanVertices(v_label_id, selectors, filter);
+  // Filter the vertex with selector
+  template <typename FUNC, typename... SELECTOR>
+  static std::vector<vertex_id_t> filter_vertex_with_selector(
+      const GRAPH_INTERFACE& graph, const label_id_t& v_label_id,
+      const FUNC& func, const std::tuple<SELECTOR...>& selectors,
+      std::vector<vertex_id_t>& vids) {
+    std::vector<vertex_id_t> gids;
+    std::tuple<typename SELECTOR::prop_t...> real_props;
+    if constexpr (sizeof...(SELECTOR) == 0) {
+      for (auto vid : vids) {
+        if (apply_on_tuple(func, real_props)) {
+          gids.push_back(vid);
+        }
+      }
+    } else {
+      auto columns =
+          graph.GetPropertyColumnWithSelectors(v_label_id, selectors);
+      if (exists_nullptr_in_tuple(columns)) {
+        VLOG(10) << "When scanning for label " << std::to_string(v_label_id)
+                 << ", there is null column, using default NULL value";
+      }
+      for (auto vid : vids) {
+        get_tuple_from_column_tuple(vid, real_props, columns);
+        if (apply_on_tuple(func, real_props)) {
+          gids.push_back(vid);
+        }
+      }
+    }
     return gids;
   }
 };
diff --git a/flex/engines/hqps_db/core/operator/sink.h b/flex/engines/hqps_db/core/operator/sink.h
index 5e7b3849a68c..36b6816b86d8 100644
--- a/flex/engines/hqps_db/core/operator/sink.h
+++ b/flex/engines/hqps_db/core/operator/sink.h
@@ -80,12 +80,21 @@ void SumEleSize(std::tuple<T...>& t, size_t& size) {
 template <typename T, typename std::enable_if<
                           (std::is_same_v<T, int32_t>)>::type* = nullptr>
 void template_set_value(common::Value* value, T v) {
-  value->set_i32(v);
+  if (v == NullRecordCreator<int32_t>::GetNull()) {
+    value->mutable_none();
+  } else {
+    value->set_i32(v);
+  }
 }
+
 template <typename T, typename std::enable_if<
                           (std::is_same_v<T, uint32_t>)>::type* = nullptr>
 void template_set_value(common::Value* value, T v) {
-  value->set_i32(v);
+  if (v == NullRecordCreator<int32_t>::GetNull()) {
+    value->mutable_none();
+  } else {
+    value->set_i32(v);
+  }
 }
 
 template <typename T,
@@ -164,20 +173,72 @@ template <typename T>
 void template_set_tuple_value(results::Collection* collection,
                               const std::vector<T>& t) {
   for (size_t i = 0; i < t.size(); ++i) {
-    auto cur_ele = collection->add_collection()->mutable_object();
+    auto cur_collection = collection->add_collection();
     // if is tuple
     if constexpr (gs::is_tuple<T>::value) {
       LOG(WARNING) << "PLEASE FIXME: tuple in vector is not supported "
                       "yet.";
-      template_set_value(cur_ele, gs::to_string(t[i]));
+      template_set_value(cur_collection->mutable_object(), gs::to_string(t[i]));
+    } else if constexpr (std::is_same_v<T, GlobalId>) {
+      cur_collection->mutable_vertex()->mutable_label()->set_id(
+          t[i].label_id());
+      cur_collection->mutable_vertex()->set_id(t[i].global_id);
     } else {
-      template_set_value(cur_ele, t[i]);
+      template_set_value(cur_collection->mutable_object(), t[i]);
     }
   }
 }
 
 /////Sink Any
 
+void set_any_to_element(const Any& any, results::Element* element) {
+  if (any.type == PropertyType::Bool()) {
+    element->mutable_object()->set_boolean(any.value.b);
+  } else if (any.type == PropertyType::Int32()) {
+    element->mutable_object()->set_i32(any.value.i);
+  } else if (any.type == PropertyType::UInt32()) {
+    // FIXME(zhanglei): temporarily use i32, fix this after common.proto is
+    // changed
+    element->mutable_object()->set_i32(any.value.ui);
+  } else if (any.type == PropertyType::Int64()) {
+    element->mutable_object()->set_i64(any.value.l);
+  } else if (any.type == PropertyType::UInt64()) {
+    // FIXME(zhanglei): temporarily use i64, fix this after common.proto is
+    // changed
+    element->mutable_object()->set_i64(any.value.ul);
+  } else if (any.type == PropertyType::Double()) {
+    element->mutable_object()->set_f64(any.value.db);
+  } else if (any.type == PropertyType::Float()) {
+    element->mutable_object()->set_f64(any.value.f);
+  } else if (any.type == PropertyType::Date()) {
+    element->mutable_object()->set_i64(any.value.d.milli_second);
+  } else if (any.type == PropertyType::String()) {
+    element->mutable_object()->mutable_str()->assign(any.value.s.data(),
+                                                     any.value.s.size());
+  } else if (any.type == PropertyType::VertexGlobalId()) {
+    auto vertex = element->mutable_vertex();
+    vertex->mutable_label()->set_id(any.value.vertex_gid.label_id());
+    vertex->set_id(any.value.vertex_gid.global_id);
+  } else if (any.type == PropertyType::Label()) {
+    element->mutable_object()->set_i32(any.value.label_key.label_id);
+  } else if (any.type == PropertyType::Empty()) {
+    element->mutable_object()->mutable_none();
+  } else {
+    LOG(WARNING) << "Unexpected property type: "
+                 << static_cast<int>(any.type.type_enum);
+  }
+}
+
+void template_set_key_value(results::KeyValues* map,
+                            const VariableKeyValue& key_value) {
+  for (auto& kv : key_value) {
+    auto cur_kv = map->add_key_values();
+    cur_kv->mutable_key()->set_str(kv.first);
+    auto value = cur_kv->mutable_value();
+    set_any_to_element(kv.second, value);
+  }
+}
+
 void set_any_to_common_value(const Any& any, common::Value* value) {
   if (any.type == PropertyType::Bool()) {
     value->set_boolean(any.value.b);
@@ -705,7 +766,7 @@ class SinkOp {
                             const std::vector<size_t>& repeat_offsets,
                             int32_t tag_id) {
     if (repeat_offsets.empty()) {
-      CHECK(collection.Size() == results_vec.results_size())
+      CHECK(collection.Size() == (size_t) results_vec.results_size())
           << "size neq " << collection.Size() << " "
           << results_vec.results_size();
       for (size_t i = 0; i < collection.Size(); ++i) {
@@ -717,9 +778,15 @@ class SinkOp {
         auto record = row->mutable_record();
         auto new_col = record->add_columns();
         new_col->mutable_name_or_id()->set_id(tag_id);
-        auto mutable_collection =
-            new_col->mutable_entry()->mutable_collection();
-        template_set_tuple_value(mutable_collection, collection.Get(i));
+        if constexpr (gs::is_vector<T>::value &&
+                      gs::is_pair<typename T::value_type>::value) {
+          auto mutable_map = new_col->mutable_entry()->mutable_map();
+          template_set_key_value(mutable_map, collection.Get(i));
+        } else {
+          auto mutable_collection =
+              new_col->mutable_entry()->mutable_collection();
+          template_set_tuple_value(mutable_collection, collection.Get(i));
+        }
       }
     } else {
       CHECK(repeat_offsets.size() == collection.Size());
@@ -730,9 +797,53 @@ class SinkOp {
           auto record = row->mutable_record();
           auto new_col = record->add_columns();
           new_col->mutable_name_or_id()->set_id(tag_id);
-          auto mutable_collection =
-              new_col->mutable_entry()->mutable_collection();
-          template_set_tuple_value(mutable_collection, collection.Get(i));
+          if constexpr (gs::is_vector<T>::value &&
+                        gs::is_pair<typename T::value_type>::value) {
+            auto mutable_map = new_col->mutable_entry()->mutable_map();
+            template_set_key_value(mutable_map, collection.Get(i));
+          } else {
+            auto mutable_collection =
+                new_col->mutable_entry()->mutable_collection();
+            template_set_tuple_value(mutable_collection, collection.Get(i));
+          }
+        }
+      }
+    }
+  }
+
+  // sink for collection of unordered_map.
+  template <size_t Ind, size_t act_tag_id, typename KeyT, typename ValueT>
+  static void sink_col_impl(
+      const GRAPH_INTERFACE& graph, results::CollectiveResults& results_vec,
+      const Collection<std::unordered_map<KeyT, ValueT>>& collection,
+      const std::vector<size_t>& repeat_offsets, int32_t tag_id) {
+    if (repeat_offsets.empty()) {
+      CHECK(collection.Size() == (size_t) results_vec.results_size())
+          << "size neq " << collection.Size() << " "
+          << results_vec.results_size();
+      for (size_t i = 0; i < collection.Size(); ++i) {
+        // auto& row = results_vec[i];
+        auto row = results_vec.mutable_results(i);
+        CHECK(row->record().columns_size() == Ind)
+            << "record column size: " << row->record().columns_size()
+            << ", ind: " << Ind;
+        auto record = row->mutable_record();
+        auto new_col = record->add_columns();
+        new_col->mutable_name_or_id()->set_id(tag_id);
+        auto mutable_map = new_col->mutable_entry()->mutable_map();
+        template_set_key_value(mutable_map, collection.Get(i));
+      }
+    } else {
+      CHECK(repeat_offsets.size() == collection.Size());
+      size_t cur_ind = 0;
+      for (size_t i = 0; i < collection.Size(); ++i) {
+        for (size_t j = 0; j < repeat_offsets[i]; ++j) {
+          auto row = results_vec.mutable_results(cur_ind++);
+          auto record = row->mutable_record();
+          auto new_col = record->add_columns();
+          new_col->mutable_name_or_id()->set_id(tag_id);
+          auto mutable_map = new_col->mutable_entry()->mutable_map();
+          template_set_key_value(mutable_map, collection.Get(i));
         }
       }
     }
@@ -745,6 +856,7 @@ class SinkOp {
       const GRAPH_INTERFACE& graph, results::CollectiveResults& results_vec,
       const GeneralVertexSet<VID_T, LabelT, SET_T...>& vertex_set,
       const std::vector<size_t>& repeat_offsets, int32_t tag_id) {
+    LOG(INFO) << "Sink for general vertex set: " << demangle(vertex_set);
     auto& schema = graph.schema();
     auto vertices_vec = vertex_set.GetVertices();
     auto labels_vec = vertex_set.GetLabels();
@@ -766,10 +878,10 @@ class SinkOp {
       }
     }
 
-    label_t label;
+    label_t label = std::numeric_limits<label_t>::max();
     size_t label_vec_ind;
     if (repeat_offsets.empty()) {
-      CHECK(vertex_set.Size() == results_vec.results_size())
+      CHECK(vertex_set.Size() == (size_t) results_vec.results_size())
           << "size neq " << vertex_set.Size() << " "
           << results_vec.results_size();
 
@@ -792,6 +904,10 @@ class SinkOp {
             break;
           }
         }
+        if (label == std::numeric_limits<label_t>::max()) {
+          LOG(ERROR) << "No label found for vertex: " << vertices_vec[i];
+          continue;
+        }
         mutable_vertex->mutable_label()->set_id(label);
         mutable_vertex->set_id(encode_unique_vertex_id(label, vertices_vec[i]));
         // label must be set
@@ -825,6 +941,10 @@ class SinkOp {
               break;
             }
           }
+          if (label == std::numeric_limits<label_t>::max()) {
+            LOG(ERROR) << "No label found for vertex: " << vertices_vec[i];
+            continue;
+          }
           mutable_vertex->mutable_label()->set_id(label);
           mutable_vertex->set_id(
               encode_unique_vertex_id(label, vertices_vec[i]));
@@ -843,6 +963,7 @@ class SinkOp {
         }
       }
     }
+    // LOG(INFO) << "result_vec" << results_vec.DebugString();
   }
 
   template <size_t Ind, size_t act_tag_id, typename EDGE_SET_T,
@@ -853,12 +974,12 @@ class SinkOp {
                             const std::vector<size_t>& repeat_offsets,
                             int32_t tag_id) {
     if (repeat_offsets.empty()) {
-      CHECK(edge_set.Size() == results_vec.results_size())
+      CHECK(edge_set.Size() == (size_t) results_vec.results_size())
           << "size neq " << edge_set.Size() << " "
           << results_vec.results_size();
       auto iter = edge_set.begin();
       auto end_iter = edge_set.end();
-      for (size_t i = 0; i < results_vec.results_size(); ++i) {
+      for (int32_t i = 0; i < results_vec.results_size(); ++i) {
         auto row = results_vec.mutable_results(i);
         CHECK(row->record().columns_size() == Ind)
             << "record column size: " << row->record().columns_size()
@@ -871,8 +992,10 @@ class SinkOp {
         CHECK(iter != end_iter);
         auto unique_edge_label = generate_edge_label_id(
             iter.GetSrcLabel(), iter.GetDstLabel(), iter.GetEdgeLabel());
-        mutable_edge->mutable_label()->set_id(unique_edge_label);
-        mutable_edge->set_id(encode_unique_edge_id(unique_edge_label, i));
+        mutable_edge->mutable_label()->set_id(iter.GetEdgeLabel());
+        mutable_edge->set_id(encode_unique_edge_id(
+            unique_edge_label, iter.GetSrc(),
+            iter.GetDst()));  // Currently we don't have a unique id for edge.
         mutable_edge->set_src_id(
             encode_unique_vertex_id(iter.GetSrcLabel(), iter.GetSrc()));
         mutable_edge->mutable_src_label()->set_id(iter.GetSrcLabel());
@@ -888,7 +1011,7 @@ class SinkOp {
         // todo: set properties.
       }
     } else {
-      CHECK(repeat_offsets.size() == edge_set.Size());
+      CHECK(repeat_offsets.size() == (size_t) edge_set.Size());
       size_t cur_ind = 0;
       auto iter = edge_set.begin();
       auto end_iter = edge_set.end();
@@ -906,9 +1029,10 @@ class SinkOp {
               new_col->mutable_entry()->mutable_element()->mutable_edge();
           auto unique_edge_label = generate_edge_label_id(
               iter.GetSrcLabel(), iter.GetDstLabel(), iter.GetEdgeLabel());
-          mutable_edge->mutable_label()->set_id(unique_edge_label);
-          mutable_edge->set_id(
-              encode_unique_edge_id(unique_edge_label, cur_ind - 1));
+          mutable_edge->mutable_label()->set_id(iter.GetEdgeLabel());
+          mutable_edge->set_id(encode_unique_edge_id(
+              unique_edge_label, iter.GetSrc(),
+              iter.GetDst()));  // Currently we don't have a unique id for edge.
           mutable_edge->set_src_id(
               encode_unique_vertex_id(iter.GetSrcLabel(), iter.GetSrc()));
           mutable_edge->mutable_src_label()->set_id(iter.GetSrcLabel());
@@ -1002,29 +1126,35 @@ class SinkOp {
     vertex->mutable_label()->set_id(label);
   }
 
-  static vid_t encode_unique_vertex_id(label_id_t label_id, vid_t vid) {
+  static uint64_t encode_unique_vertex_id(label_id_t label_id, vid_t vid) {
     // encode label_id and vid to a unique vid
-    vid_t unique_vid = label_id;
-    static constexpr int num_bits = sizeof(vid_t) * 8 - sizeof(label_id_t) * 8;
-    unique_vid = unique_vid << num_bits;
-    unique_vid = unique_vid | vid;
-    return unique_vid;
+    GlobalId global_id(label_id, vid);
+    return global_id.global_id;
   }
 
-  static int64_t encode_unique_edge_id(label_id_t label_id, size_t index) {
-    // encode label_id and vid to a unique vid
+  static int64_t encode_unique_edge_id(uint32_t label_id, vid_t src,
+                                       vid_t dst) {
+    // We assume label_id is only used by 24 bits.
     int64_t unique_edge_id = label_id;
-    static constexpr int num_bits =
-        sizeof(int64_t) * 8 - sizeof(label_id_t) * 8;
+    static constexpr int num_bits = sizeof(int64_t) * 8 - sizeof(uint32_t) * 8;
     unique_edge_id = unique_edge_id << num_bits;
-    unique_edge_id = unique_edge_id | index;
+    // bitmask for top 44 bits set to 1
+    int64_t bitmask = 0xFFFFFFFFFF000000;
+    // 24 bit | 20 bit | 20 bit
+    if (bitmask & (int64_t) src || bitmask & (int64_t) dst) {
+      LOG(ERROR) << "src or dst is too large to be encoded in 20 bits: " << src
+                 << " " << dst;
+    }
+    unique_edge_id = unique_edge_id | (src << 20);
+    unique_edge_id = unique_edge_id | dst;
     return unique_edge_id;
   }
 
-  static label_id_t generate_edge_label_id(label_id_t src_label_id,
-                                           label_id_t dst_label_id,
-                                           label_id_t edge_label_id) {
-    label_id_t unique_edge_label_id = src_label_id;
+  // actually produce uint24_t.
+  static uint32_t generate_edge_label_id(label_id_t src_label_id,
+                                         label_id_t dst_label_id,
+                                         label_id_t edge_label_id) {
+    uint32_t unique_edge_label_id = src_label_id;
     static constexpr int num_bits = sizeof(label_id_t) * 8;
     unique_edge_label_id = unique_edge_label_id << num_bits;
     unique_edge_label_id = unique_edge_label_id | dst_label_id;
diff --git a/flex/engines/hqps_db/core/operator/sort.h b/flex/engines/hqps_db/core/operator/sort.h
index bff03c5e6e05..7abc89921c6b 100644
--- a/flex/engines/hqps_db/core/operator/sort.h
+++ b/flex/engines/hqps_db/core/operator/sort.h
@@ -331,15 +331,6 @@ class SortOp {
     return ctx.Flat(std::move(index_eles));
   }
 
-  template <size_t Is = 0, typename... IND_ELE, typename... GETTER>
-  static inline void update_prop_getter(std::tuple<GETTER...>& getters,
-                                        std::tuple<IND_ELE...>& ind_eles) {
-    std::get<Is>(getters).set_ind_ele(ind_eles);
-    if constexpr (Is + 1 < sizeof...(GETTER)) {
-      update_prop_getter<Is + 1>(getters, ind_eles);
-    }
-  }
-
   template <typename... ORDER_PAIR, typename CTX_HEAD_T, int cur_alias,
             int base_tag, typename... CTX_PREV, size_t... Is>
   static auto create_prop_getter_tuple(
diff --git a/flex/engines/hqps_db/core/params.h b/flex/engines/hqps_db/core/params.h
index b3ad7048d6a1..291edece1e07 100644
--- a/flex/engines/hqps_db/core/params.h
+++ b/flex/engines/hqps_db/core/params.h
@@ -18,6 +18,7 @@
 #include <time.h>
 #include <climits>
 #include <iostream>
+#include <regex>
 #include <string>
 // #include "grape/grape.h"
 #include "flex/engines/hqps_db/core/utils/hqps_type.h"
@@ -31,8 +32,17 @@ namespace gs {
 #define INPUT_COL_ID(x) (x)
 #define LAST_COL -1
 
-#define PROJ_TO_NEW false
-#define PROJ_TO_APPEND true
+enum class ProjectDesc {
+  New = 0,  // Create a new column in Context.(Project To new context will
+            // persist all columns)
+  AppendTemp = 1,     // Append the result to the last column in Context.
+  AppendPersist = 2,  // Append the result to the last column in Context and
+                      // persist it.
+};
+
+#define PROJ_TO_NEW ProjectDesc::New
+#define PROJ_TO_APPEND_TEMP ProjectDesc::AppendTemp
+#define PROJ_TO_APPEND_PERSIST ProjectDesc::AppendPersist
 
 // Indicator flag for appending the result column to Context.
 enum class AppendOpt {
@@ -71,6 +81,8 @@ struct PropertySelector {
 };
 
 using InternalIdSelector = PropertySelector<grape::EmptyType>;
+using GlobalIdSelector = PropertySelector<GlobalId>;
+using LabelIdSelector = PropertySelector<LabelKey>;
 
 // @brief Mapping a vertex/edge to new data with expr& selector.
 // @tparam EXPR
@@ -91,6 +103,42 @@ struct IdentityMapper {
   IdentityMapper() = default;
 };
 
+// Create a key-value mapping record with key(string) and value selector.
+// Implement IR's VariableKeyValues.
+template <int _in_col_id, typename SELECTOR>
+struct KeyValueMapper {
+  static constexpr int in_col_id = _in_col_id;
+  std::string key_;
+  SELECTOR value_selector_;
+  KeyValueMapper(const std::string& key, SELECTOR&& value_selector)
+      : key_(key), value_selector_(std::move(value_selector)) {}
+};
+
+template <int in_col_id, typename SELECTOR>
+auto make_key_value_mapper(const std::string& key, SELECTOR&& value_selector) {
+  return KeyValueMapper<in_col_id, SELECTOR>(key, std::move(value_selector));
+}
+
+template <typename... Mapper>
+struct KeyValueMappers {};
+
+template <int... in_col_id, typename... SELECTOR>
+struct KeyValueMappers<KeyValueMapper<in_col_id, SELECTOR>...> {
+  std::tuple<KeyValueMapper<in_col_id, SELECTOR>...> mappers_;
+  KeyValueMappers(KeyValueMapper<in_col_id, SELECTOR>&&... mappers)
+      : mappers_(std::move(mappers)...) {}
+};
+
+template <int... in_col_id, typename... SELECTOR>
+auto make_key_value_mappers(KeyValueMapper<in_col_id, SELECTOR>&&... mappers) {
+  return KeyValueMappers<KeyValueMapper<in_col_id, SELECTOR>...>(
+      std::move(mappers)...);
+}
+
+// a type alias for unordered_map, which is used to store the key-value mapping
+// for VariableKeyValues
+using VariableKeyValue = std::vector<std::pair<std::string, Any>>;
+
 template <typename EXPR, typename... SELECTOR>
 struct Filter {
   using expr_t = EXPR;
@@ -151,12 +199,12 @@ struct AggFuncReturnValue {
 
 template <typename T>
 struct AggFuncReturnValue<AggFunc::COUNT, T> {
-  using return_t = size_t;
+  using return_t = int64_t;
 };
 
 template <typename T>
 struct AggFuncReturnValue<AggFunc::COUNT_DISTINCT, T> {
-  using return_t = size_t;
+  using return_t = int64_t;
 };
 
 // for grouping values, for which key, and to which alias, applying which
@@ -225,6 +273,12 @@ struct InnerIdProperty {
   InnerIdProperty() = default;
 };
 
+template <int _tag_id_ = -1>
+struct GlobalIdProperty {
+  static constexpr int tag_id = _tag_id_;
+  GlobalIdProperty() = default;
+};
+
 // Denote the length of a path
 struct LengthKey {
   using length_data_type = int32_t;
@@ -246,11 +300,6 @@ inline bool operator>(const LabelKey& lhs, const LabelKey& rhs) {
   return lhs.label_id > rhs.label_id;
 }
 
-// overload hash_value for LabelKey
-inline std::size_t hash_value(const LabelKey& key) {
-  return std::hash<int32_t>()(key.label_id);
-}
-
 // static constexpr size_t dist_col = 0;
 
 using groot_prop_label_t = std::string;
@@ -518,7 +567,7 @@ struct EdgeExpandEMultiLabelOpt<num_labels, LabelT, EDGE_FILTER_FUNC,
   Direction dir_;
   LabelT edge_label_;
   std::array<LabelT, num_labels> other_label_;
-  EDGE_FILTER_FUNC edge_filter_;
+  Filter<EDGE_FILTER_FUNC, SELECTOR...> edge_filter_;
 };
 
 // EdgeExpandE with multiple edge triplet pairs.
@@ -573,7 +622,7 @@ auto make_edge_expande_opt(Direction dir, LabelT edge_label,
 }
 
 // make edge expand with multiple labels
-template <typename LabelT, typename FILTER_T, typename... PropTuple>
+template <typename LabelT, typename... PropTuple, typename FILTER_T>
 auto make_edge_expand_multie_opt(
     Direction dir,
     std::array<std::array<LabelT, 3>, sizeof...(PropTuple)>&&
@@ -1025,6 +1074,140 @@ NamedProperty<T, _tag_id> alias_tag_prop_to_named_property(
   return NamedProperty<T, _tag_id>(alias_tag_prop.tag_prop_.prop_names_[0]);
 }
 
+// customized operator
+// 0. WithIn
+const struct WithIn_ {
+} WithIn;
+
+template <typename T>
+struct WithProxy {
+  WithProxy(const T& t) : t_(t) {}
+  const T& t_;
+};
+
+template <typename T>
+WithProxy<T> operator<(const T& lhs, const WithIn_& rhs) {
+  return WithProxy<T>(lhs);
+}
+
+template <
+    typename T1, typename T2, size_t N,
+    typename std::enable_if<std::is_pod_v<T1> && (N == 1)>::type* = nullptr>
+bool operator>(const WithProxy<T1>& lhs, const std::array<T2, N>& rhs) {
+  return lhs.t_ == rhs[0];
+}
+
+template <
+    typename T1, typename T2, size_t N,
+    typename std::enable_if<std::is_pod_v<T1> && (N > 1)>::type* = nullptr>
+bool operator>(const WithProxy<T1>& lhs, const std::array<T2, N>& rhs) {
+  return rhs.end() != std::find(rhs.begin(), rhs.end(), lhs.t_);
+}
+
+template <size_t N, typename std::enable_if<(N > 0)>::type* = nullptr>
+bool operator>(const WithProxy<LabelKey>& lhs,
+               const std::array<int64_t, N>& rhs) {
+  return rhs.end() != std::find(rhs.begin(), rhs.end(), lhs.t_.label_id);
+}
+
+template <size_t N, typename std::enable_if<(N == 0)>::type* = nullptr>
+bool operator>(const WithProxy<LabelKey>& lhs,
+               const std::array<int64_t, N>& rhs) {
+  return false;
+}
+
+template <size_t N, typename std::enable_if<(N > 0)>::type* = nullptr>
+bool operator>(const WithProxy<GlobalId>& lhs,
+               const std::array<int64_t, N>& rhs) {
+  return rhs.end() != std::find(rhs.begin(), rhs.end(), lhs.t_.global_id);
+}
+
+template <size_t N, typename std::enable_if<(N == 0)>::type* = nullptr>
+bool operator>(const WithProxy<GlobalId>& lhs,
+               const std::array<int64_t, N>& rhs) {
+  return false;
+}
+
+template <
+    typename T1, typename T2, size_t N,
+    typename std::enable_if<std::is_pod_v<T1> && (N == 0)>::type* = nullptr>
+bool operator>(const WithProxy<T1>& lhs, const std::array<T2, N>& rhs) {
+  return false;
+}
+
+const struct EndWith_ {
+} EndWith;
+
+const struct StartWith_ {
+} StartWith;
+
+template <typename T>
+struct StartWithProxy {
+  StartWithProxy(const T& t) : t_(t) {}
+  const T& t_;
+};
+
+template <typename T>
+struct EndWithProxy {
+  EndWithProxy(const T& t) : t_(t) {}
+  const T& t_;
+};
+
+template <typename T>
+inline StartWithProxy<T> operator<(const T& lhs, const StartWith_& rhs) {
+  return StartWithProxy<T>(lhs);
+}
+
+inline bool operator>(const StartWithProxy<std::string>& lhs,
+                      const std::string& rhs) {
+  return lhs.t_.find(rhs, 0) == 0;
+}
+
+// string_view
+inline bool operator>(const StartWithProxy<std::string_view>& lhs,
+                      const std::string_view& rhs) {
+  return lhs.t_.find(rhs, 0) == 0;
+}
+
+// same as startwith
+template <typename T>
+inline EndWithProxy<T> operator<(const T& lhs, const EndWith_& rhs) {
+  return EndWithProxy<T>(lhs);
+}
+
+inline bool operator>(const EndWithProxy<std::string>& lhs,
+                      const std::string& rhs) {
+  return lhs.t_.rfind(rhs) == (lhs.t_.size() - rhs.size());
+}
+
+inline bool operator>(const EndWithProxy<std::string_view>& lhs,
+                      const std::string_view& rhs) {
+  return lhs.t_.rfind(rhs) == (lhs.t_.size() - rhs.size());
+}
+
+const struct Regex_ {
+} Regex;
+
+struct RegexProxy {
+  RegexProxy(const std::string& t) : t_(t) {}
+  RegexProxy(const std::string_view& t) : t_{t} {}
+  std::string t_;
+};
+
+inline RegexProxy operator<(const std::string& lhs, const Regex_& rhs) {
+  return RegexProxy(lhs);
+}
+
+inline RegexProxy operator<(const std::string_view& lhs, const Regex_& rhs) {
+  return RegexProxy(lhs);
+}
+
+inline bool operator>(const RegexProxy& lhs, const std::string& rhs) {
+  LOG(INFO) << "matching: " << lhs.t_ << " with " << rhs << ",result: "
+            << std::to_string(std::regex_match(lhs.t_, std::regex(rhs)));
+  return std::regex_match(lhs.t_, std::regex(rhs));
+}
+
 // ShortestPath
 /*
 message ShortestPathExpand {
@@ -1066,6 +1249,11 @@ inline ostream& operator<<(ostream& os, const gs::Dist& g) {
   os << g.dist;
   return os;
 }
+
+inline ostream& operator<<(ostream& os, const gs::GlobalId& g) {
+  os << g.global_id;
+  return os;
+}
 }  // namespace std
 
 #endif  // ENGINES_HQPS_ENGINE_PARAMS_H_
diff --git a/flex/engines/hqps_db/core/sync_engine.h b/flex/engines/hqps_db/core/sync_engine.h
index 796a1e419b61..c5580ecf207f 100644
--- a/flex/engines/hqps_db/core/sync_engine.h
+++ b/flex/engines/hqps_db/core/sync_engine.h
@@ -44,6 +44,7 @@ template <typename GRAPH_INTERFACE>
 class SyncEngine : public BaseEngine {
   using label_id_t = typename GRAPH_INTERFACE::label_id_t;
   using vertex_id_t = typename GRAPH_INTERFACE::vertex_id_t;
+  using gid_t = typename GRAPH_INTERFACE::gid_t;
   using default_vertex_set_t = DefaultRowVertexSet<label_id_t, vertex_id_t>;
   using two_label_set_t =
       TwoLabelVertexSet<vertex_id_t, label_id_t, grape::EmptyType>;
@@ -114,7 +115,7 @@ class SyncEngine : public BaseEngine {
                                      num_labels != 2)>::type* = nullptr,
             typename COL_T =
                 GeneralVertexSet<vertex_id_t, label_id_t, grape::EmptyType>>
-  static Context<COL_T, 0, 0, grape::EmptyType> ScanVertex(
+  static Context<COL_T, -1, 0, grape::EmptyType> ScanVertex(
       const GRAPH_INTERFACE& graph,
       std::array<label_id_t, num_labels>&& v_labels,
       Filter<EXPR, SELECTOR...>&& filter) {
@@ -168,10 +169,9 @@ class SyncEngine : public BaseEngine {
                 nullptr,
             typename COL_T = default_vertex_set_t>
   static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithOid(
-      const GRAPH_INTERFACE& graph, LabelT v_label, OID_T oid) {
-    auto v_set_tuple =
-        Scan<GRAPH_INTERFACE>::ScanVertexWithOid(graph, v_label, oid);
-
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<OID_T>&& oids) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithOid(
+        graph, v_label, std::move(oids));
     return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
   }
 
@@ -180,10 +180,9 @@ class SyncEngine : public BaseEngine {
       typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
       typename COL_T = default_vertex_set_t>
   static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithOid(
-      const GRAPH_INTERFACE& graph, LabelT v_label, OID_T oid) {
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<OID_T>&& oids) {
     auto v_set_tuple = Scan<GRAPH_INTERFACE>::template ScanVertexWithOid<OID_T>(
-        graph, v_label, oid);
-
+        graph, v_label, std::move(oids));
     return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
   }
 
@@ -194,24 +193,181 @@ class SyncEngine : public BaseEngine {
       typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
   static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithOid(
       const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
-      OID_T oid) {
+      std::vector<OID_T>&& oids) {
     auto v_set_tuple =
         Scan<GRAPH_INTERFACE>::template ScanVertexWithOid<OID_T, num_labels>(
-            graph, v_labels, oid);
-
+            graph, v_labels, std::move(oids));
     return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
   }
 
   template <
-      AppendOpt append_opt, typename LabelT, size_t num_labels,
+      AppendOpt append_opt, typename OID_T, typename LabelT, size_t num_labels,
       typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
       typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
   static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithOid(
       const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
-      int64_t oid) {
+      std::vector<OID_T>&& oids) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithOid(
+        graph, v_labels, std::move(oids));
+    return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  ///////////////// ScanVertexWith Gid//////////////////
+  template <AppendOpt append_opt, typename GID_T, typename LabelT,
+            typename std::enable_if<(append_opt == AppendOpt::Persist)>::type* =
+                nullptr,
+            typename COL_T = default_vertex_set_t>
+  static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithGid(
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<GID_T>&& gids) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithGid(
+        graph, v_label, std::move(gids));
+    return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename GID_T, typename LabelT,
+      typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
+      typename COL_T = default_vertex_set_t>
+  static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithGid(
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<GID_T>&& gids) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::template ScanVertexWithGid(
+        graph, v_label, std::move(gids));
+    return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename GID_T, typename LabelT, size_t num_labels,
+      typename std::enable_if<(append_opt == AppendOpt::Persist)>::type* =
+          nullptr,
+      typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
+  static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithGid(
+      const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
+      std::vector<GID_T>&& gids) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::template ScanVertexWithGid(
+        graph, v_labels, std::move(gids));
+    return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename GID_T, typename LabelT, size_t num_labels,
+      typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
+      typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
+  static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithGid(
+      const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
+      std::vector<GID_T>&& gids) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithGid(
+        graph, v_labels, std::move(gids));
+    return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  ///////////////// ScanVertexWith Expr and oid//////////////////
+  template <AppendOpt append_opt, typename OID_T, typename LabelT,
+            typename EXPR, typename... SELECTOR,
+            typename std::enable_if<(append_opt == AppendOpt::Persist)>::type* =
+                nullptr,
+            typename COL_T = default_vertex_set_t>
+  static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithOidExpr(
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<OID_T>&& oids,
+      Filter<EXPR, SELECTOR...>&& filter) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithOidExpr(
+        graph, v_label, std::move(oids), std::move(filter));
+    return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename OID_T, typename LabelT, typename EXPR,
+      typename... SELECTOR,
+      typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
+      typename COL_T = default_vertex_set_t>
+  static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithOidExpr(
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<OID_T>&& oids,
+      Filter<EXPR, SELECTOR...>&& filter) {
+    auto v_set_tuple =
+        Scan<GRAPH_INTERFACE>::template ScanVertexWithOidExpr<OID_T>(
+            graph, v_label, std::move(oids), std::move(filter));
+    return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename OID_T, typename LabelT, size_t num_labels,
+      typename EXPR, typename... SELECTOR,
+      typename std::enable_if<(append_opt == AppendOpt::Persist)>::type* =
+          nullptr,
+      typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
+  static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithOidExpr(
+      const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
+      std::vector<OID_T>&& oids, Filter<EXPR, SELECTOR...>&& filter) {
     auto v_set_tuple =
-        Scan<GRAPH_INTERFACE>::ScanVertexWithOid(graph, v_labels, oid);
+        Scan<GRAPH_INTERFACE>::template ScanVertexWithOidExpr<OID_T,
+                                                              num_labels>(
+            graph, v_labels, std::move(oids), std::move(filter));
+    return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename OID_T, typename LabelT, size_t num_labels,
+      typename EXPR, typename... SELECTOR,
+      typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
+      typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
+  static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithOidExpr(
+      const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
+      std::vector<OID_T>&& oids, Filter<EXPR, SELECTOR...>&& filter) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithOidExpr(
+        graph, v_labels, std::move(oids), std::move(filter));
+    return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  ///////////////// ScanVertexWith Expr and gid//////////////////
+  template <AppendOpt append_opt, typename GID_T, typename LabelT,
+            typename EXPR, typename... SELECTOR,
+            typename std::enable_if<(append_opt == AppendOpt::Persist)>::type* =
+                nullptr,
+            typename COL_T = default_vertex_set_t>
+  static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithGidExpr(
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<GID_T>&& gids,
+      Filter<EXPR, SELECTOR...>&& filter) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithGidExpr(
+        graph, v_label, std::move(gids), std::move(filter));
+    return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
 
+  template <
+      AppendOpt append_opt, typename GID_T, typename LabelT, typename EXPR,
+      typename... SELECTOR,
+      typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
+      typename COL_T = default_vertex_set_t>
+  static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithGidExpr(
+      const GRAPH_INTERFACE& graph, LabelT v_label, std::vector<GID_T>&& gids,
+      Filter<EXPR, SELECTOR...>&& filter) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::template ScanVertexWithGidExpr(
+        graph, v_label, std::move(gids), std::move(filter));
+    return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename GID_T, typename LabelT, size_t num_labels,
+      typename EXPR, typename... SELECTOR,
+      typename std::enable_if<(append_opt == AppendOpt::Persist)>::type* =
+          nullptr,
+      typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
+  static Context<COL_T, 0, 0, grape::EmptyType> ScanVertexWithGidExpr(
+      const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
+      std::vector<GID_T>&& gids, Filter<EXPR, SELECTOR...>&& filter) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::template ScanVertexWithGidExp(
+        graph, v_labels, std::move(gids), std::move(filter));
+    return Context<COL_T, 0, 0, grape::EmptyType>(std::move(v_set_tuple));
+  }
+
+  template <
+      AppendOpt append_opt, typename GID_T, typename LabelT, size_t num_labels,
+      typename EXPR, typename... SELECTOR,
+      typename std::enable_if<(append_opt == AppendOpt::Temp)>::type* = nullptr,
+      typename COL_T = GeneralVertexSet<vertex_id_t, LabelT, grape::EmptyType>>
+  static Context<COL_T, -1, 0, grape::EmptyType> ScanVertexWithGidExpr(
+      const GRAPH_INTERFACE& graph, std::array<LabelT, num_labels> v_labels,
+      std::vector<GID_T>&& gids, Filter<EXPR, SELECTOR...>&& filter) {
+    auto v_set_tuple = Scan<GRAPH_INTERFACE>::ScanVertexWithGidExpr(
+        graph, v_labels, std::move(gids), std::move(filter));
     return Context<COL_T, -1, 0, grape::EmptyType>(std::move(v_set_tuple));
   }
 
@@ -256,11 +412,8 @@ class SyncEngine : public BaseEngine {
   /// @return
   template <AppendOpt append_opt, int input_col_id, typename CTX_HEAD_T,
             int cur_alias, int base_tag, typename... CTX_PREV,
-            typename EDGE_FILTER_T, typename... SELECTOR,
-            typename RES_T = typename ResultContextT<
-                append_opt, default_vertex_set_t, cur_alias, CTX_HEAD_T,
-                base_tag, CTX_PREV...>::result_t>
-  static RES_T EdgeExpandV(
+            typename EDGE_FILTER_T, typename... SELECTOR>
+  static auto EdgeExpandV(
       const GRAPH_INTERFACE& graph,
       Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
       EdgeExpandOpt<label_id_t, EDGE_FILTER_T, SELECTOR...>&& edge_expand_opt,
@@ -419,11 +572,8 @@ class SyncEngine : public BaseEngine {
   template <AppendOpt opt, int alias_to_use, typename VERTEX_FILTER_T,
             typename CTX_HEAD_T, int cur_alias, int base_tag,
             typename... CTX_PREV, typename LabelT, typename EDGE_FILTER_T,
-            typename... T, typename RES_SET_T = vertex_set_t<dist_t, T...>,
-            typename RES_T =
-                typename ResultContextT<opt, RES_SET_T, cur_alias, CTX_HEAD_T,
-                                        base_tag, CTX_PREV...>::result_t>
-  static RES_T PathExpandV(
+            typename... T, typename RES_SET_T = vertex_set_t<dist_t, T...>>
+  static auto PathExpandV(
       const GRAPH_INTERFACE& graph,
       Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
       PathExpandVOpt<LabelT, EDGE_FILTER_T, VERTEX_FILTER_T, T...>&&
@@ -622,8 +772,8 @@ class SyncEngine : public BaseEngine {
   //      typename RES_T = typename ProjectResT<
   // is_append, Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>,
   // PROJECT_OPT>::result_t
-  template <bool is_append, typename CTX_HEAD_T, int cur_alias, int base_tag,
-            typename... CTX_PREV, typename... ProjMapper>
+  template <ProjectDesc is_append, typename CTX_HEAD_T, int cur_alias,
+            int base_tag, typename... CTX_PREV, typename... ProjMapper>
   static auto Project(
       const GRAPH_INTERFACE& graph,
       Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
@@ -836,7 +986,7 @@ class SyncEngine : public BaseEngine {
     auto prop_getters_tuple =
         create_prop_getters_from_prop_desc(graph, ctx, prop_descs);
     for (auto iter : ctx) {
-      auto eles = iter.GetAllElement();
+      auto eles = iter.GetAllIndexElement();
       // if (expr(eles)) {
       // if (std::apply(expr, props)) {
       if (run_expr_filter(expr, prop_getters_tuple, eles)) {
@@ -855,6 +1005,97 @@ class SyncEngine : public BaseEngine {
     return std::move(ctx);
   }
 
+  //////////////////////////////////////Dedup/////////////////////////
+  // Special implementation to only dedup on one column.
+  // template <
+  //     int alias_to_use, typename CTX_HEAD_T, int cur_alias, int base_tag,
+  //     typename... CTX_PREV, typename PropT,
+  //     typename RES_T = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>>
+  // static RES_T Dedup(
+  //     const GRAPH_INTERFACE& graph,
+  //     Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
+  //     std::tuple<PropertySelector<PropT>>&& selector) {
+  //   if constexpr (alias_to_use != cur_alias) {
+  //     // When we dedup a intermediate node, we need to
+  //     // 1) first dedup current node, no duplicate in us.
+  //     // 2) then iterate whole context, for later nodes, we only preserve the
+  //     // first element met.
+  //     //
+  //     // the result context type should be same with previous.
+  //     // 1 -> (2, 3)
+  //     // 2 -> (4, 5), 3 -> (6, 7);
+  //     //
+  //     // dedup on col 2, then we 1 -> (2, 3), 2 -> 4, 3 -> 6;
+
+  //     // first remove all possible duplication introduced by later csr.
+  //     ctx.template Dedup<alias_to_use>();
+  //   }
+  //   auto& select_node = gs::Get<alias_to_use>(ctx);
+  //   // dedup inplace, and return the offset_array to old node.
+  //   auto offset_to_old_node = select_node.Dedup(std::get<0>(selector));
+  //   // The offset need to be changed.
+  //   ctx.template
+  //   UpdateChildNode<alias_to_use>(std::move(offset_to_old_node)); return ctx;
+  // }
+
+  // Dedup with multiple columns.
+  template <
+      int... alias_to_use, typename CTX_HEAD_T, int cur_alias, int base_tag,
+      typename... CTX_PREV, typename... PropT,
+      typename RES_T = Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>>
+  static RES_T Dedup(
+      const GRAPH_INTERFACE& graph,
+      Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>&& ctx,
+      std::tuple<PropertySelector<PropT>...>&& dedup_keys) {
+    auto prop_descs =
+        create_prop_descs_from_selectors<alias_to_use...>(dedup_keys);
+    LOG(INFO) << "Dedup with :" << demangle(prop_descs);
+    auto prop_getters_tuple =
+        create_prop_getters_from_prop_desc(graph, ctx, prop_descs);
+
+    std::vector<size_t> active_indices;
+    std::vector<size_t> new_offset;
+
+    auto& cur_ = ctx.GetMutableHead();
+    new_offset.reserve(cur_.Size());
+    new_offset.emplace_back(0);
+
+    dedup_impl(ctx, active_indices, new_offset, prop_getters_tuple);
+
+    cur_.SubSetWithIndices(active_indices);
+    ctx.merge_offset_with_back(new_offset);
+    return ctx;
+  }
+
+  template <typename CTX_T, typename... PropGetterT>
+  static void dedup_impl(CTX_T& ctx, std::vector<size_t>& active_indices,
+                         std::vector<size_t>& new_offset,
+                         std::tuple<PropGetterT...>& prop_getters_tuple) {
+    // prop_tuple_t is the tuple of got type from prop_getters_tuple, by
+    // applying to ctx_all_ele_t
+    // using ctx_all_ele_t = typename CTX_T::index_ele_tuples_t;
+    using prop_tuple_t = std::tuple<typename PropGetterT::prop_element_t...>;
+    std::unordered_set<prop_tuple_t, boost::hash<prop_tuple_t>> dedup_set;
+    auto index_seq = std::make_index_sequence<sizeof...(PropGetterT)>();
+    size_t cnt = 0;
+    for (auto iter : ctx) {
+      auto index_eles = iter.GetAllIndexElement();
+      // for each element of prop_getter_tuple, apply get_from_all_element
+      // to index_eles, and make result as a tuple
+      auto key_tuple = get_prop_from_index_ele_with_prop_getter(
+          prop_getters_tuple, index_eles, index_seq);
+
+      if (dedup_set.find(key_tuple) == dedup_set.end()) {
+        dedup_set.insert(key_tuple);
+        active_indices.emplace_back(cnt);
+      }
+      cnt += 1;
+      new_offset.emplace_back(active_indices.size());
+    }
+    LOG(INFO) << "Dedup active record: " << active_indices.size() << " out of "
+              << cnt;
+  }
+
   template <typename EXPR, typename... PROP_GETTER, typename... ELE>
   static inline bool run_expr_filter(
       const EXPR& expr, std::tuple<PROP_GETTER...>& prop_getter_tuple,
diff --git a/flex/engines/hqps_db/core/utils/hqps_utils.h b/flex/engines/hqps_db/core/utils/hqps_utils.h
index 7fff40f0b1d0..6aed619e8978 100644
--- a/flex/engines/hqps_db/core/utils/hqps_utils.h
+++ b/flex/engines/hqps_db/core/utils/hqps_utils.h
@@ -169,54 +169,6 @@ inline bool check_edge_dir_consist_vopt(const Direction& dir, VOpt vopt) {
   return false;
 }
 
-// customized operator
-// 0. WithIn
-const struct WithIn_ {
-} WithIn;
-
-template <typename T>
-struct WithProxy {
-  WithProxy(const T& t) : t_(t) {}
-  const T& t_;
-};
-
-template <typename T>
-WithProxy<T> operator<(const T& lhs, const WithIn_& rhs) {
-  return WithProxy<T>(lhs);
-}
-
-template <
-    typename T, size_t N,
-    typename std::enable_if<std::is_pod_v<T> && (N == 1)>::type* = nullptr>
-bool operator>(const WithProxy<T>& lhs, const std::array<T, N>& rhs) {
-  return lhs.t_ == rhs[0];
-}
-
-template <typename T, size_t N,
-          typename std::enable_if<std::is_pod_v<T> && (N > 1)>::type* = nullptr>
-bool operator>(const WithProxy<T>& lhs, const std::array<T, N>& rhs) {
-  return rhs.end() != std::find(rhs.begin(), rhs.end(), lhs.t_);
-}
-
-template <size_t N, typename std::enable_if<(N > 0)>::type* = nullptr>
-bool operator>(const WithProxy<LabelKey>& lhs,
-               const std::array<int64_t, N>& rhs) {
-  return rhs.end() != std::find(rhs.begin(), rhs.end(), lhs.t_.label_id);
-}
-
-template <size_t N, typename std::enable_if<(N == 0)>::type* = nullptr>
-bool operator>(const WithProxy<LabelKey>& lhs,
-               const std::array<int64_t, N>& rhs) {
-  return false;
-}
-
-template <
-    typename T, size_t N,
-    typename std::enable_if<std::is_pod_v<T> && (N == 0)>::type* = nullptr>
-bool operator>(const WithProxy<T>& lhs, const std::array<T, N>& rhs) {
-  return false;
-}
-
 template <
     std::size_t nth, std::size_t... Head, std::size_t... Tail,
     typename... Types,
@@ -767,6 +719,21 @@ struct to_string_impl<std::vector<T>> {
   }
 };
 
+template <typename K, typename V>
+struct to_string_impl<std::unordered_map<K, V>> {
+  static inline std::string to_string(const std::unordered_map<K, V>& vec) {
+    std::ostringstream ss;
+    // map{key:value, ...}
+    ss << "map{";
+    for (auto& [k, v] : vec) {
+      ss << to_string_impl<K>::to_string(k) << ":"
+         << to_string_impl<V>::to_string(v) << ",";
+    }
+    ss << "}";
+    return ss.str();
+  }
+};
+
 template <typename T, size_t N>
 struct to_string_impl<std::array<T, N>> {
   static inline std::string to_string(const std::array<T, N>& empty) {
@@ -1023,6 +990,12 @@ struct Edge<VID_T, grape::EmptyType> {
 template <typename VID_T>
 using DefaultEdge = Edge<VID_T, grape::EmptyType>;
 
+template <typename VID_T>
+inline bool operator==(const DefaultEdge<VID_T>& lhs,
+                       const DefaultEdge<VID_T>& rhs) {
+  return lhs.src == rhs.src && lhs.dst == rhs.dst;
+}
+
 struct QPSError {
   std::string message;
   explicit QPSError(std::string msg) : message(std::move(msg)) {}
@@ -1052,6 +1025,28 @@ struct function_traits<ReturnType (ClassType::*)(Args...) const>
   };
 };
 
+template <typename label_id_t>
+static std::tuple<label_id_t, label_id_t> get_graph_label_pair(
+    Direction& direction, label_id_t query_src_label,
+    label_id_t query_dst_label) {
+  label_id_t src_label, dst_label;
+  if (direction == Direction::In) {
+    src_label = query_dst_label;
+    dst_label = query_src_label;
+  } else {
+    src_label = query_src_label;
+    dst_label = query_dst_label;
+  }
+  return std::tuple{src_label, dst_label};
+}
+
 }  // namespace gs
 
+namespace boost {
+template <typename VID_T>
+inline std::size_t hash_value(const gs::DefaultEdge<VID_T>& key) {
+  return std::hash<VID_T>()(key.src) ^ std::hash<VID_T>()(key.dst);
+}
+}  // namespace boost
+
 #endif  // ENGINES_HQPS_ENGINE_HQPS_UTILS_H_
diff --git a/flex/engines/hqps_db/core/utils/keyed.h b/flex/engines/hqps_db/core/utils/keyed.h
index ea4e116c4762..af073841e258 100644
--- a/flex/engines/hqps_db/core/utils/keyed.h
+++ b/flex/engines/hqps_db/core/utils/keyed.h
@@ -47,6 +47,11 @@ struct AggFirst<TwoLabelVertexSetImpl<VID_T, LabelT, T...>> {
   using result_t = TwoLabelVertexSetImpl<VID_T, LabelT, T...>;
 };
 
+template <typename VID_T, typename LabelT, typename... T>
+struct AggFirst<GeneralVertexSet<VID_T, LabelT, T...>> {
+  using result_t = GeneralVertexSet<VID_T, LabelT, T...>;
+};
+
 /// @brief Helper to get keyed set type
 /// @tparam T
 /// @tparam ValueT Keyed prop type
@@ -81,8 +86,9 @@ struct KeyedT<TwoLabelVertexSet<VID_T, LabelT, T...>,
               PropertySelector<grape::EmptyType>> {
   using keyed_set_t = TwoLabelVertexSet<VID_T, LabelT, T...>;
   // // The builder type.
-  using keyed_builder_t = TwoLabelVertexSetImplBuilder<VID_T, LabelT, T...>;
-  using unkeyed_builder_t = TwoLabelVertexSetImplBuilder<LabelT, VID_T, T...>;
+  using keyed_builder_t =
+      TwoLabelVertexSetImplKeyedBuilder<VID_T, LabelT, T...>;
+  using unkeyed_builder_t = TwoLabelVertexSetImplBuilder<VID_T, LabelT, T...>;
 
   static keyed_builder_t create_keyed_builder(
       const TwoLabelVertexSet<VID_T, LabelT, T...>& set,
@@ -172,6 +178,19 @@ struct KeyedT<KeyedRowVertexSetImpl<LabelT, KEY_T, VID_T, SET_T...>,
 //       KeyedRowVertexSetBuilder<LabelT, VID_T, VID_T, grape::EmptyType>;
 // };
 
+template <typename VID_T, typename LabelT, typename... T>
+struct KeyedT<GeneralVertexSet<VID_T, LabelT, T...>,
+              PropertySelector<LabelKey>> {
+  using keyed_set_t = Collection<LabelKey>;
+  using keyed_builder_t = KeyedCollectionBuilder<LabelKey>;
+
+  static keyed_builder_t create_keyed_builder(
+      const GeneralVertexSet<VID_T, LabelT, T...>& set,
+      const PropertySelector<LabelKey>& selector) {
+    return keyed_builder_t();
+  }
+};
+
 template <typename T>
 struct KeyedT<Collection<T>, PropertySelector<grape::EmptyType>> {
   using keyed_set_t = Collection<T>;
@@ -182,7 +201,7 @@ struct KeyedT<Collection<T>, PropertySelector<grape::EmptyType>> {
   static keyed_builder_t create_keyed_builder(
       const Collection<T>& set,
       const PropertySelector<grape::EmptyType>& selector) {
-    return builder_t(set);
+    return keyed_builder_t(set);
   }
   static unkeyed_builder_t create_unkeyed_builder(
       const Collection<T>& set,
@@ -252,7 +271,7 @@ template <typename GI, typename LabelT, typename VID_T, typename... T,
 struct KeyedAggT<GI, RowVertexSet<LabelT, VID_T, T...>, AggFunc::COUNT_DISTINCT,
                  std::tuple<grape::EmptyType>,
                  std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<size_t>;
+  using agg_res_t = Collection<int64_t>;
   // build a counter array.
   using vertex_set_t = RowVertexSet<LabelT, VID_T, T...>;
   using aggregate_res_builder_t = DistinctCountBuilder<tag_id, vertex_set_t>;
@@ -264,11 +283,28 @@ struct KeyedAggT<GI, RowVertexSet<LabelT, VID_T, T...>, AggFunc::COUNT_DISTINCT,
   }
 };
 
+template <typename GI, typename LabelT, typename VID_T, typename... T,
+          int tag_id>
+struct KeyedAggT<GI, GeneralVertexSet<VID_T, LabelT, T...>,
+                 AggFunc::COUNT_DISTINCT, std::tuple<grape::EmptyType>,
+                 std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<int64_t>;
+  // build a counter array.
+  using vertex_set_t = GeneralVertexSet<VID_T, LabelT, T...>;
+  using aggregate_res_builder_t = DistinctCountBuilder<tag_id, vertex_set_t>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const vertex_set_t& set, const GI& graph,
+      std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
+    return aggregate_res_builder_t(set.GetBitsets(), set.GetLabels());
+  }
+};
+
 template <typename GI, typename VID_T, typename LabelT, typename... T,
           typename PropT, int tag_id>
 struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, T...>, AggFunc::COUNT,
                  std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<size_t>;
+  using agg_res_t = Collection<int64_t>;
   using index_ele_t =
       typename TwoLabelVertexSet<VID_T, LabelT, T...>::index_ele_tuple_t;
   // build a counter array.
@@ -290,7 +326,7 @@ template <typename GI, typename VID_T, typename LabelT, typename... T,
 struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, T...>, AggFunc::COUNT,
                  std::tuple<grape::EmptyType>,
                  std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<size_t>;
+  using agg_res_t = Collection<int64_t>;
   // build a counter array.
   using aggregate_res_builder_t = CountBuilder<tag_id>;
 
@@ -307,7 +343,7 @@ template <typename GI, typename VID_T, typename LabelT, typename... T,
 struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, T...>,
                  AggFunc::COUNT_DISTINCT, std::tuple<grape::EmptyType>,
                  std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<size_t>;
+  using agg_res_t = Collection<int64_t>;
   // build a counter array.
   using vertex_set_t = TwoLabelVertexSet<VID_T, LabelT, T...>;
   using aggregate_res_builder_t = DistinctCountBuilder<tag_id, vertex_set_t>;
@@ -324,7 +360,7 @@ template <typename GI, typename VID_T, typename LabelT, typename... SET_T,
           typename PropT, int tag_id>
 struct KeyedAggT<GI, GeneralVertexSet<VID_T, LabelT, SET_T...>, AggFunc::COUNT,
                  std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<size_t>;
+  using agg_res_t = Collection<int64_t>;
   using index_ele_t =
       typename GeneralVertexSet<VID_T, LabelT, SET_T...>::index_ele_tuple_t;
   using prop_getter_t = GeneralVertexSetPropGetter<
@@ -348,7 +384,7 @@ template <typename GI, typename VID_T, typename LabelT, typename... SET_T,
 struct KeyedAggT<GI, GeneralVertexSet<VID_T, LabelT, SET_T...>, AggFunc::COUNT,
                  std::tuple<grape::EmptyType>,
                  std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<size_t>;
+  using agg_res_t = Collection<int64_t>;
   // build a counter array.
   using aggregate_res_builder_t = CountBuilder<tag_id>;
 
@@ -363,13 +399,33 @@ template <typename GI, typename T, int tag_id>
 struct KeyedAggT<GI, Collection<T>, AggFunc::SUM, std::tuple<grape::EmptyType>,
                  std::integer_sequence<int32_t, tag_id>> {
   using agg_res_t = Collection<T>;
+  using prop_getter_t = CollectionPropGetter<tag_id, T>;
   // build a counter array.
-  using aggregate_res_builder_t = SumBuilder<T, tag_id>;
+  using aggregate_res_builder_t = SumBuilder<prop_getter_t, tag_id>;
 
   static aggregate_res_builder_t create_agg_builder(
       const Collection<T>& set, const GI& graph,
       std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
-    return aggregate_res_builder_t();
+    return aggregate_res_builder_t(prop_getter_t());
+  }
+};
+
+template <typename GI, typename VID_T, typename LabelT, typename... SET_T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, SET_T...>, AggFunc::SUM,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using index_ele_t =
+      typename TwoLabelVertexSet<VID_T, LabelT, SET_T...>::index_ele_tuple_t;
+  using prop_getter_t = TwoLabelVertexSetImplPropGetter<
+      tag_id, gs::mutable_csr_graph_impl::SinglePropGetter<PropT>, index_ele_t>;
+  using aggregate_res_builder_t = SumBuilder<prop_getter_t, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const TwoLabelVertexSet<VID_T, LabelT, SET_T...>& set, const GI& graph,
+      std::tuple<PropertySelector<PropT>>& selectors) {
+    return aggregate_res_builder_t(create_prop_getter_impl<tag_id, PropT>(
+        set, graph, std::get<0>(selectors).prop_name_));
   }
 };
 
@@ -426,33 +482,162 @@ struct KeyedAggT<GI, RowVertexSet<LabelT, VID_T, T...>, AggFunc::TO_LIST,
   }
 };
 
-// get min
-template <typename GI, typename T, typename PropT, int tag_id>
-struct KeyedAggT<GI, Collection<T>, AggFunc::MIN, std::tuple<PropT>,
+// Aggregate to_list for twolabel vertex set's internal id.
+template <typename GI, typename LabelT, typename VID_T, typename... T,
+          int tag_id>
+struct KeyedAggT<GI, RowVertexSet<LabelT, VID_T, T...>, AggFunc::TO_LIST,
+                 std::tuple<grape::EmptyType>,
                  std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<T>;
-  using aggregate_res_builder_t = MinBuilder<GI, T, tag_id>;
+  using agg_res_t = CollectionOfVec<GlobalId>;
+  using aggregate_res_builder_t =
+      CollectionOfVecBuilder<grape::EmptyType, GI,
+                             RowVertexSet<LabelT, VID_T, T...>, tag_id>;
 
   static aggregate_res_builder_t create_agg_builder(
-      const Collection<T>& set, const GI& graph,
+      const RowVertexSet<LabelT, VID_T, T...>& set, const GI& graph,
+      std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
+    return aggregate_res_builder_t(set.GetLabel());
+  }
+};
+
+template <typename GI, typename LabelT, typename VID_T, typename... T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, T...>, AggFunc::TO_LIST,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = CollectionOfVec<PropT>;
+  using aggregate_res_builder_t =
+      CollectionOfVecBuilder<PropT, GI, TwoLabelVertexSet<VID_T, LabelT, T...>,
+                             tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const TwoLabelVertexSet<VID_T, LabelT, T...>& set, const GI& graph,
       std::tuple<PropertySelector<PropT>>& selectors) {
     return aggregate_res_builder_t(set, graph,
                                    {std::get<0>(selectors).prop_name_});
   }
 };
 
+template <typename GI, typename LabelT, typename VID_T, typename... T,
+          int tag_id>
+struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, T...>, AggFunc::TO_LIST,
+                 std::tuple<grape::EmptyType>,
+                 std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = CollectionOfVec<GlobalId>;
+  using aggregate_res_builder_t =
+      CollectionOfVecBuilder<grape::EmptyType, GI,
+                             TwoLabelVertexSet<VID_T, LabelT, T...>, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const TwoLabelVertexSet<VID_T, LabelT, T...>& set, const GI& graph,
+      std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
+    return aggregate_res_builder_t(set.GetLabels());
+  }
+};
+
+// get min
+template <typename GI, typename PropT, int tag_id>
+struct KeyedAggT<GI, Collection<PropT>, AggFunc::MIN,
+                 std::tuple<grape::EmptyType>,
+                 std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using PropGetterT = CollectionPropGetter<tag_id, PropT>;
+  using aggregate_res_builder_t = MinBuilder<GI, PropGetterT, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const Collection<PropT>& set, const GI& graph,
+      std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
+    return aggregate_res_builder_t(PropGetterT());
+  }
+};
+
+template <typename GI, typename VID_T, typename LabelT, typename... SET_T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, RowVertexSet<LabelT, VID_T, SET_T...>, AggFunc::MIN,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using index_ele_t =
+      typename RowVertexSet<LabelT, VID_T, SET_T...>::index_ele_tuple_t;
+  using prop_getter_t = RowVertexSetPropGetter<
+      tag_id, gs::mutable_csr_graph_impl::SinglePropGetter<PropT>, index_ele_t>;
+  using aggregate_res_builder_t = MinBuilder<GI, prop_getter_t, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const RowVertexSet<LabelT, VID_T, SET_T...>& set, const GI& graph,
+      std::tuple<PropertySelector<PropT>>& selectors) {
+    return aggregate_res_builder_t(create_prop_getter_impl<tag_id, PropT>(
+        set, graph, std::get<0>(selectors).prop_name_));
+  }
+};
+
+template <typename GI, typename VID_T, typename LabelT, typename... SET_T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, SET_T...>, AggFunc::MIN,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using index_ele_t =
+      typename TwoLabelVertexSet<VID_T, LabelT, SET_T...>::index_ele_tuple_t;
+  using prop_getter_t = TwoLabelVertexSetImplPropGetter<
+      tag_id, gs::mutable_csr_graph_impl::SinglePropGetter<PropT>, index_ele_t>;
+  using aggregate_res_builder_t = MinBuilder<GI, prop_getter_t, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const TwoLabelVertexSet<VID_T, LabelT, SET_T...>& set, const GI& graph,
+      std::tuple<PropertySelector<PropT>>& selectors) {
+    return aggregate_res_builder_t(create_prop_getter_impl<tag_id, PropT>(
+        set, graph, std::get<0>(selectors).prop_name_));
+  }
+};
+
 // get max
-template <typename GI, typename T, typename PropT, int tag_id>
-struct KeyedAggT<GI, Collection<T>, AggFunc::MAX, std::tuple<PropT>,
+template <typename GI, typename PropT, int tag_id>
+struct KeyedAggT<GI, Collection<PropT>, AggFunc::MAX, std::tuple<PropT>,
                  std::integer_sequence<int32_t, tag_id>> {
-  using agg_res_t = Collection<T>;
-  using aggregate_res_builder_t = MaxBuilder<GI, T, tag_id>;
+  using agg_res_t = Collection<PropT>;
+  using prop_getter_t = CollectionPropGetter<tag_id, PropT>;
+  using aggregate_res_builder_t = MaxBuilder<GI, prop_getter_t, tag_id>;
 
   static aggregate_res_builder_t create_agg_builder(
-      const Collection<T>& set, const GI& graph,
+      const Collection<PropT>& set, const GI& graph,
       std::tuple<PropertySelector<PropT>>& selectors) {
-    return aggregate_res_builder_t(
-        set, graph, std::array{std::get<0>(selectors).prop_name_});
+    return aggregate_res_builder_t(prop_getter_t());
+  }
+};
+
+template <typename GI, typename LabelT, typename VID_T, typename... SET_T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, RowVertexSet<LabelT, VID_T, SET_T...>, AggFunc::MAX,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using index_ele_t =
+      typename RowVertexSet<LabelT, VID_T, SET_T...>::index_ele_tuple_t;
+  using prop_getter_t = RowVertexSetPropGetter<
+      tag_id, gs::mutable_csr_graph_impl::SinglePropGetter<PropT>, index_ele_t>;
+  using aggregate_res_builder_t = MaxBuilder<GI, prop_getter_t, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const RowVertexSet<LabelT, VID_T, SET_T...>& set, const GI& graph,
+      std::tuple<PropertySelector<PropT>>& selectors) {
+    return aggregate_res_builder_t(create_prop_getter_impl<tag_id, PropT>(
+        set, graph, std::get<0>(selectors).prop_name_));
+  }
+};
+
+template <typename GI, typename LabelT, typename VID_T, typename... SET_T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, SET_T...>, AggFunc::MAX,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using index_ele_t =
+      typename TwoLabelVertexSet<VID_T, LabelT, SET_T...>::index_ele_tuple_t;
+  using prop_getter_t = TwoLabelVertexSetImplPropGetter<
+      tag_id, gs::mutable_csr_graph_impl::SinglePropGetter<PropT>, index_ele_t>;
+  using aggregate_res_builder_t = MaxBuilder<GI, prop_getter_t, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const TwoLabelVertexSet<VID_T, LabelT, SET_T...>& set, const GI& graph,
+      std::tuple<PropertySelector<PropT>>& selectors) {
+    return aggregate_res_builder_t(create_prop_getter_impl<tag_id, PropT>(
+        set, graph, std::get<0>(selectors).prop_name_));
   }
 };
 
@@ -466,8 +651,8 @@ struct KeyedAggT<GI, Collection<T>, AggFunc::FIRST, std::tuple<PropT>,
   static aggregate_res_builder_t create_agg_builder(
       const Collection<T>& set, const GI& graph,
       std::tuple<PropertySelector<PropT>>& selectors) {
-    return aggregate_res_builder_t(
-        set, graph, std::array{std::get<0>(selectors).prop_name_});
+    return aggregate_res_builder_t(set, graph,
+                                   {std::get<0>(selectors).prop_name_});
   }
 };
 
@@ -486,7 +671,26 @@ struct KeyedAggT<GI, TwoLabelVertexSetImpl<VID_T, LabelT, T...>, AggFunc::FIRST,
   static aggregate_res_builder_t create_agg_builder(
       const old_set_t& set, const GI& graph,
       std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
-    auto labels = set.GetLabels();
+    return aggregate_res_builder_t(
+        set, graph, std::array{std::get<0>(selectors).prop_name_});
+  }
+};
+
+// Aggregate first for general vertex set
+template <typename GI, typename VID_T, typename LabelT, typename... T,
+          int tag_id>
+struct KeyedAggT<GI, GeneralVertexSet<VID_T, LabelT, T...>, AggFunc::FIRST,
+                 std::tuple<grape::EmptyType>,
+                 std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = GeneralVertexSet<VID_T, LabelT, T...>;
+  using old_set_t = GeneralVertexSet<VID_T, LabelT, T...>;
+  using aggregate_res_builder_t =
+      FirstBuilder<GI, GeneralVertexSet<VID_T, LabelT, T...>, grape::EmptyType,
+                   tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const old_set_t& set, const GI& graph,
+      std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
     return aggregate_res_builder_t(
         set, graph, std::array{std::get<0>(selectors).prop_name_});
   }
@@ -565,6 +769,20 @@ struct KeyedAggT<GI, PathSet<VID_T, LabelT>, AggFunc::COUNT,
   }
 };
 
+template <typename GI, typename T, int tag_id>
+struct KeyedAggT<GI, Collection<T>, AggFunc::COUNT,
+                 std::tuple<grape::EmptyType>,
+                 std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<size_t>;
+  using aggregate_res_builder_t = CountBuilder<tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const Collection<T>& set, const GI& graph,
+      std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
+    return CountBuilder<tag_id>();
+  }
+};
+
 // COUNT DISTINCT for EdgeSets.
 template <typename GI, typename VID_T, typename LabelT, int tag_id>
 struct KeyedAggT<GI, UnTypedEdgeSet<VID_T, LabelT, typename GI::sub_graph_t>,
@@ -644,6 +862,55 @@ struct KeyedAggT<GI, PathSet<VID_T, LabelT>, AggFunc::COUNT_DISTINCT,
   }
 };
 
+template <typename GI, typename T, int tag_id>
+struct KeyedAggT<GI, Collection<T>, AggFunc::COUNT_DISTINCT,
+                 std::tuple<grape::EmptyType>,
+                 std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<size_t>;
+  using set_t = Collection<T>;
+  using aggregate_res_builder_t = DistinctCountBuilder<tag_id, set_t>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const set_t& set, const GI& graph,
+      std::tuple<PropertySelector<grape::EmptyType>>& selectors) {
+    return aggregate_res_builder_t(set);
+  }
+};
+
+// aggregate average.
+
+template <typename GI, typename VID_T, typename LabelT, typename... SET_T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, RowVertexSetImpl<LabelT, VID_T, SET_T...>, AggFunc::AVG,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using set_t = RowVertexSetImpl<LabelT, VID_T, SET_T...>;
+  using aggregate_res_builder_t = AvgBuilder<PropT, GI, set_t, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const set_t& set, const GI& graph,
+      std::tuple<PropertySelector<PropT>>& selectors) {
+    return aggregate_res_builder_t(set, graph,
+                                   {std::get<0>(selectors).prop_name_});
+  }
+};
+
+template <typename GI, typename VID_T, typename LabelT, typename... SET_T,
+          typename PropT, int tag_id>
+struct KeyedAggT<GI, TwoLabelVertexSet<VID_T, LabelT, SET_T...>, AggFunc::AVG,
+                 std::tuple<PropT>, std::integer_sequence<int32_t, tag_id>> {
+  using agg_res_t = Collection<PropT>;
+  using set_t = TwoLabelVertexSet<VID_T, LabelT, SET_T...>;
+  using aggregate_res_builder_t = AvgBuilder<PropT, GI, set_t, tag_id>;
+
+  static aggregate_res_builder_t create_agg_builder(
+      const set_t& set, const GI& graph,
+      std::tuple<PropertySelector<PropT>>& selectors) {
+    return aggregate_res_builder_t(set, graph,
+                                   {std::get<0>(selectors).prop_name_});
+  }
+};
+
 template <typename GI, typename... SET_T, typename... PropSelectorT,
           int... tag_ids>
 struct KeyedAggMultiColT<GI, std::tuple<SET_T...>, AggFunc::COUNT_DISTINCT,
@@ -718,6 +985,16 @@ static inline auto insert_into_builder_v2_impl(BuilderT& builder,
   return builder.Insert(ele, data);
 }
 
+// insert into general vertex set builder
+template <typename BuilderT, typename ELE, typename DATA,
+          typename std::enable_if<
+              (BuilderT::is_general_vertex_set_builder)>::type* = nullptr>
+static inline auto insert_into_builder_v2_impl(BuilderT& builder,
+                                               const ELE& ele,
+                                               const DATA& data) {
+  return builder.Insert(ele, data);
+}
+
 template <typename VID_T, typename LabelT, typename EDATA_T, typename ELE,
           typename DATA>
 static inline auto insert_into_builder_v2_impl(
diff --git a/flex/engines/hqps_db/core/utils/props.h b/flex/engines/hqps_db/core/utils/props.h
index 6e46d0abce3c..f29b86b3ceaa 100644
--- a/flex/engines/hqps_db/core/utils/props.h
+++ b/flex/engines/hqps_db/core/utils/props.h
@@ -39,6 +39,10 @@ class SingleLabelEdgeSet;
 template <size_t N, typename GI, typename VID_T, typename LabelT, typename... T>
 class GeneralEdgeSet;
 
+// UntypedEdgeSet
+template <typename VID_T, typename LabelT, typename SUB_GRAPH_T>
+class UnTypedEdgeSet;
+
 // forward declare keyed_row_vertex_set
 template <typename LabelT, typename KEY_T, typename VID_T, typename... T>
 class KeyedRowVertexSetImpl;
@@ -100,28 +104,6 @@ static auto get_single_prop_getter_from_selector(
   return graph.template GetSinglePropGetter<PropT>(label, prop_name);
 }
 
-// get prop getter from multiple named property
-template <typename GRAPH_INTERFACE, typename LabelT, typename... NamedPropT>
-static auto get_prop_getters_from_named_property(
-    const GRAPH_INTERFACE& graph, const LabelT& label,
-    std::tuple<NamedPropT...> named_property) {
-  std::array<LabelT, 1> labels = {label};
-  return get_prop_getters_from_named_property(graph, labels, named_property);
-}
-
-template <typename GRAPH_INTERFACE, typename LabelT, size_t num_labels,
-          typename... NamedPropT, size_t... Is>
-static auto get_prop_getters_from_named_property(
-    const GRAPH_INTERFACE& graph, const std::array<LabelT, num_labels>& labels,
-    std::tuple<NamedPropT...> named_property, std::index_sequence<Is...>) {
-  using prop_getter_t = typename GRAPH_INTERFACE::template multi_prop_getter_t<
-      typename NamedPropT::prop_t...>;
-  std::array<prop_getter_t, num_labels> prop_getter_array{
-      get_prop_getter_from_named_property(graph, labels[Is],
-                                          named_property)...};
-  return prop_getter_array;
-}
-
 // Get prop getters from Selector.
 template <typename GRAPH_INTERFACE, typename LabelT, size_t num_labels,
           typename... SELECTOR, size_t... Is>
@@ -174,24 +156,62 @@ static auto get_prop_getters_from_selectors(
 template <int tag_id, typename VID_T>
 class InnerIdGetter {
  public:
+  using prop_element_t = VID_T;
   InnerIdGetter(const std::vector<VID_T>& vids) : vids_(vids) {}
 
   VID_T get_view(const std::tuple<size_t, VID_T>& ele) const {
     return std::get<1>(ele);
   }
 
+  VID_T get_view(const std::tuple<size_t, int32_t, VID_T>& ele) const {
+    return std::get<2>(ele);
+  }
+
+  VID_T get_view(const VID_T& ele) const { return ele; }
+
   template <typename ALL_ELE_T>
   inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
-    return gs::get_from_tuple<tag_id>(all_ele);
+    return get_view(gs::get_from_tuple<tag_id>(all_ele));
+  }
+
+ private:
+  const std::vector<VID_T>& vids_;
+};
+
+template <int tag_id, typename VID_T>
+class VertexGlobalIdGetter {
+ public:
+  using prop_element_t = GlobalId;
+  VertexGlobalIdGetter(const std::vector<VID_T>& vids,
+                       std::vector<LabelKey>&& label_vec)
+      : vids_(vids), label_vec_(std::move(label_vec)) {}
+
+  GlobalId get_view(const std::tuple<size_t, int32_t, VID_T>& ele) const {
+    return GlobalId(label_vec_[std::get<0>(ele)].label_id,
+                    vids_[std::get<0>(ele)]);
+  }
+
+  // two label set
+  GlobalId get_view(const std::tuple<size_t, VID_T>& ele) const {
+    return GlobalId(label_vec_[std::get<0>(ele)].label_id,
+                    vids_[std::get<0>(ele)]);
+  }
+
+  template <typename ALL_ELE_T>
+  inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
+    auto index = std::get<0>(gs::get_from_tuple<tag_id>(all_ele));
+    return GlobalId(label_vec_[index].label_id, vids_[index]);
   }
 
  private:
   const std::vector<VID_T>& vids_;
+  std::vector<LabelKey> label_vec_;
 };
 
 template <int tag_id>
 class VertexLabelGetter {
  public:
+  using prop_element_t = LabelKey;
   VertexLabelGetter(const std::vector<LabelKey>&& label_keys)
       : label_keys(std::move(label_keys)) {}
 
@@ -216,6 +236,7 @@ class VertexLabelGetter {
 template <int tag_id>
 class EdgeLabelGetter {
  public:
+  using prop_element_t = LabelKey;
   EdgeLabelGetter(const std::vector<LabelKey>&& label_keys)
       : label_keys(std::move(label_keys)) {}
 
@@ -240,6 +261,7 @@ class EdgeLabelGetter {
 template <int tag_id, typename VID_T, typename... DATA_T>
 class InnerIdDataGetter {
  public:
+  using prop_element_t = std::tuple<VID_T, std::tuple<DATA_T...>>;
   InnerIdDataGetter(const std::vector<VID_T>& vids,
                     const std::vector<std::tuple<DATA_T...>>& data)
       : vids_(vids), data_(data) {}
@@ -253,7 +275,7 @@ class InnerIdDataGetter {
   }
 
   template <typename TUPLE_T>
-  VID_T get_from_all_element(const TUPLE_T& tuple) const {
+  auto get_from_all_element(const TUPLE_T& tuple) const {
     auto ind_ele = gs::get_from_tuple<tag_id>(tuple);
     return ind_ele;
   }
@@ -266,23 +288,35 @@ class InnerIdDataGetter {
 template <int tag_id, typename PROP_GETTER_T, typename IND_ELE_T>
 class GeneralVertexSetPropGetter {
  public:
+  using prop_element_t = typename PROP_GETTER_T::value_type;
+  using ELE_TUPLE_T = std::tuple<label_t, vid_t>;
   GeneralVertexSetPropGetter(std::vector<PROP_GETTER_T>&& getters,
                              const std::vector<grape::Bitset>& bitset)
       : getters_(std::move(getters)), bitset_(bitset) {}
 
   inline auto get_view(const IND_ELE_T& ind_ele) const {
-    auto ind = std::get<0>(ind_ele);
-    for (size_t i = 0; i < bitset_.size(); ++i) {
-      CHECK(i < bitset_[i].cardinality());
-      if (bitset_[i].get_bit(ind)) {
-        return getters_[i].get_view(std::get<1>(ind_ele));
-      }
-    }
-    LOG(FATAL) << "should not reach here";
+    auto vid = std::get<2>(ind_ele);
+    auto label_ind = std::get<1>(ind_ele);
+    CHECK(std::get<0>(ind_ele) < bitset_[label_ind].cardinality());
+    CHECK(bitset_[label_ind].get_bit(std::get<0>(ind_ele)));
+    return getters_[label_ind].get_view(vid);
+  }
+
+  inline auto get_view(const ELE_TUPLE_T& ind_ele) const {
+    auto label_ind = std::get<0>(ind_ele);
+    auto vid = std::get<1>(ind_ele);
+    CHECK(label_ind < bitset_.size());
+    return getters_[label_ind].get_view(vid);
   }
 
   inline auto get_view() const { return get_view(ind_ele_); }
 
+  template <typename TUPLE_T>
+  auto get_from_all_element(const TUPLE_T& tuple) const {
+    auto ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    return get_view(ind_ele);
+  }
+
  private:
   IND_ELE_T ind_ele_;
   std::vector<PROP_GETTER_T> getters_;
@@ -293,31 +327,46 @@ class GeneralVertexSetPropGetter {
 template <int tag_id, typename VID_T, typename EDATA_T>
 class EdgeSetInnerIdGetter {
  public:
+  using prop_element_t = Edge<VID_T, grape::EmptyType>;
   EdgeSetInnerIdGetter() {}
 
   template <typename ALL_ELE_T>
   inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
-    auto tuple = gs::get_from_tuple<tag_id>(all_ele);
-    auto src_vid = std::get<0>(tuple);
-    auto dst_vid = std::get<1>(tuple);
-    return Edge<VID_T, grape::EmptyType>(src_vid, dst_vid);
+    auto& tuple = gs::get_from_tuple<tag_id>(all_ele);
+    using cur_tuple_t =
+        std::remove_const_t<std::remove_reference_t<decltype(tuple)>>;
+    if constexpr (std::tuple_size_v<cur_tuple_t> > 2) {
+      auto src_vid = std::get<0>(tuple);
+      auto dst_vid = std::get<1>(tuple);
+      return Edge<VID_T, grape::EmptyType>(src_vid, dst_vid);
+    } else if constexpr (std::tuple_size_v<cur_tuple_t> == 2) {
+      auto& actual_tuple = std::get<1>(tuple);
+      auto src_vid = std::get<0>(actual_tuple);
+      auto dst_vid = std::get<1>(actual_tuple);
+      return Edge<VID_T, grape::EmptyType>(src_vid, dst_vid);
+    } else {
+      LOG(FATAL) << "tuple size error";
+    }
   }
 };
 
 template <int tag_id, typename T>
 class CollectionPropGetter {
  public:
+  using prop_element_t = T;
   CollectionPropGetter() {}
 
   inline auto get_view(const std::tuple<size_t, T>& ele) const {
     return std::get<1>(ele);
   }
 
+  inline auto get_view(const T& ele) const { return ele; }
+
   inline auto get_view() const { return std::get<1>(ind_ele_); }
 
   template <typename ALL_ELE_T>
   inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
-    return gs::get_from_tuple<tag_id>(all_ele);
+    return get_view(gs::get_from_tuple<tag_id>(all_ele));
   }
 
   template <typename ALL_IND_ELE_T>
@@ -333,6 +382,7 @@ class CollectionPropGetter {
 template <int tag_id>
 class CollectionPropGetter<tag_id, LabelKey> {
  public:
+  using prop_element_t = LabelKey;
   CollectionPropGetter() {}
 
   inline auto get_view(const std::tuple<size_t, LabelKey>& ele) const {
@@ -359,6 +409,7 @@ class CollectionPropGetter<tag_id, LabelKey> {
 template <int tag_id, typename T>
 class CollectionPropGetter<tag_id, std::tuple<T>> {
  public:
+  using prop_element_t = T;
   CollectionPropGetter() {}
 
   inline T get_view(const std::tuple<size_t, T>& ele) const {
@@ -384,6 +435,9 @@ class CollectionPropGetter<tag_id, std::tuple<T>> {
 template <int tag_id, typename index_ele_tuple_t>
 class FlatEdgeSetPropGetter {
  public:
+  using prop_element_t = typename std::tuple_element_t<
+      0, typename std::tuple_element_t<
+             2, typename std::tuple_element_t<1, index_ele_tuple_t>>>;
   FlatEdgeSetPropGetter() {}
 
   inline auto get_view(const index_ele_tuple_t& ind_ele) const {
@@ -398,7 +452,7 @@ class FlatEdgeSetPropGetter {
   template <typename ALL_ELE_T>
   inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
     auto& my_ele = gs::get_from_tuple<tag_id>(all_ele);
-    return std::get<0>(std::get<2>(my_ele));
+    return get_view(my_ele);
   }
 
   template <typename ALL_IND_ELE_T>
@@ -410,9 +464,31 @@ class FlatEdgeSetPropGetter {
   index_ele_tuple_t ind_ele_;
 };
 
+template <int tag_id, typename PropT>
+class UnTypedEdgeSetPropGetter {
+ public:
+  using index_ele_tuple_t = std::tuple<size_t, vid_t, vid_t, Any>;
+  using prop_element_t = PropT;
+  UnTypedEdgeSetPropGetter() {}
+
+  inline auto get_view(const index_ele_tuple_t& ind_ele) const {
+    PropT prop;
+    ConvertAny<PropT>::to(std::get<3>(ind_ele), prop);
+    return prop;
+  }
+
+  template <typename ALL_ELE_T>
+  inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
+    auto& my_ele = gs::get_from_tuple<tag_id>(all_ele);
+    return get_view(my_ele);
+  }
+};
+
 template <int tag_id, typename index_ele_tuple_t>
 class GeneralEdgeSetPropGetter {
  public:
+  using adj_iterator_t = typename std::tuple_element_t<2, index_ele_tuple_t>;
+  using prop_element_t = typename adj_iterator_t::edge_property_t;
   GeneralEdgeSetPropGetter() {}
 
   inline auto get_view(const index_ele_tuple_t& ind_ele) const {
@@ -427,7 +503,7 @@ class GeneralEdgeSetPropGetter {
   template <typename ALL_ELE_T>
   inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
     auto& my_ele = gs::get_from_tuple<tag_id>(all_ele);
-    return std::get<0>(std::get<1>(my_ele).properties());
+    return get_view(my_ele);
   }
 
   template <typename ALL_IND_ELE_T>
@@ -442,6 +518,7 @@ class GeneralEdgeSetPropGetter {
 template <int tag_id, typename PROP_GETTER_T, typename IND_ELE_T>
 class TwoLabelVertexSetImplPropGetter {
  public:
+  using prop_element_t = typename PROP_GETTER_T::value_type;
   TwoLabelVertexSetImplPropGetter(std::array<PROP_GETTER_T, 2>&& getters)
       : getters_(std::move(getters)) {}
 
@@ -449,16 +526,19 @@ class TwoLabelVertexSetImplPropGetter {
     return getters_[std::get<1>(ind_ele)].get_view(std::get<2>(ind_ele));
   }
 
+  inline auto get_view(const std::pair<int32_t, vid_t>& ind_ele) const {
+    return getters_[ind_ele.first].get_view(ind_ele.second);
+  }
+
   inline auto get_view()
       const {  // const std::tuple<size_t, int32_t, VID_T>& ind_ele
     return getters_[std::get<1>(ind_ele_)].get_view(std::get<2>(ind_ele_));
   }
 
-  template <typename ALL_ELE_T>
-  inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
+  template <typename ALL_INDEX_ELE_T>
+  inline auto get_from_all_element(const ALL_INDEX_ELE_T& all_ele) const {
     auto& my_ele = gs::get_from_tuple<tag_id>(all_ele);
-    auto& getter = getters_[std::get<0>(my_ele)];
-    return getter.get_view(std::get<1>(my_ele));
+    return get_view(my_ele);
   }
 
   template <typename ELE_T>
@@ -479,6 +559,7 @@ class TwoLabelVertexSetImplPropGetter {
 template <int tag_id, typename PROP_GETTER_T, typename IND_ELE_T>
 class RowVertexSetPropGetter {
  public:
+  using prop_element_t = typename PROP_GETTER_T::value_type;
   RowVertexSetPropGetter(PROP_GETTER_T&& getter) : getter_(std::move(getter)) {}
 
   template <typename VID_T>
@@ -486,6 +567,11 @@ class RowVertexSetPropGetter {
     return getter_.get_view(std::get<1>(ind_ele));
   }
 
+  template <typename VID_T>
+  inline auto get_view(const VID_T& vid) const {
+    return getter_.get_view(vid);
+  }
+
   inline auto get_view() const {
     return getter_.get_view(std::get<1>(ind_ele_));
   }
@@ -493,7 +579,7 @@ class RowVertexSetPropGetter {
   template <typename ALL_ELE_T>
   inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
     auto& my_ele = gs::get_from_tuple<tag_id>(all_ele);
-    return getter_.get_view(my_ele);
+    return get_view(my_ele);
   }
 
   // get from ele
@@ -515,6 +601,7 @@ class RowVertexSetPropGetter {
 template <int tag_id, typename PROP_GETTER_T, typename IND_ELE_T>
 class KeyedRowVertexSetPropGetter {
  public:
+  using prop_element_t = typename PROP_GETTER_T::value_type;
   KeyedRowVertexSetPropGetter(PROP_GETTER_T&& getter)
       : getter_(std::move(getter)) {}
 
@@ -530,7 +617,7 @@ class KeyedRowVertexSetPropGetter {
   template <typename ALL_ELE_T>
   inline auto get_from_all_element(const ALL_ELE_T& all_ele) const {
     auto& my_ele = gs::get_from_tuple<tag_id>(all_ele);
-    return getter_.get_view(my_ele);
+    return get_view(my_ele);
   }
 
   template <typename ALL_IND_ELE_T>
@@ -546,6 +633,7 @@ class KeyedRowVertexSetPropGetter {
 template <int tag_id, typename IND_ELE_T>
 class DistGetter {
  public:
+  using prop_element_t = Dist;
   DistGetter(std::vector<Dist>&& dist) : dist_(std::move(dist)) {}
 
   template <typename VID_T>
@@ -789,6 +877,17 @@ static auto create_prop_getter_impl(
       typename FlatEdgeSet<VID_T, LabelT, EDATA_T>::index_ele_tuple_t>();
 }
 
+// get for common properties for UntypedEdgeSet
+template <int tag_id, typename prop_t, typename GRAPH_INTERFACE, typename VID_T,
+          typename LabelT, typename SUB_GRAPH_T,
+          typename std::enable_if<!(std::is_same_v<prop_t, LabelKey>)>::type* =
+              nullptr>
+static auto create_prop_getter_impl(
+    const UnTypedEdgeSet<VID_T, LabelT, SUB_GRAPH_T>& set,
+    const GRAPH_INTERFACE& graph, const std::string& prop_name) {
+  return UnTypedEdgeSetPropGetter<tag_id, prop_t>();
+}
+
 // get for common properties for Single label edge set.
 template <int tag_id, typename prop_t, typename GRAPH_INTERFACE, typename VID_T,
           typename LabelT, typename EDATA_T,
@@ -857,6 +956,18 @@ static auto create_prop_getter_from_prop_desc(
       set.GetVertices());
 }
 
+// create inner_id getter for two label vertex set
+template <typename GRAPH_INTERFACE, typename LabelT, typename... SET_Ts,
+          int tag_id>
+static auto create_prop_getter_from_prop_desc(
+    const GRAPH_INTERFACE& graph,
+    const GeneralVertexSet<typename GRAPH_INTERFACE::vertex_id_t, LabelT,
+                           SET_Ts...>& set,
+    const InnerIdProperty<tag_id>& inner_id_prop) {
+  return InnerIdGetter<tag_id, typename GRAPH_INTERFACE::vertex_id_t>(
+      set.GetVertices());
+}
+
 // create inner id getter for collection.
 template <typename GRAPH_INTERFACE, typename COL_T, int tag_id>
 static auto create_prop_getter_from_prop_desc(
@@ -885,6 +996,34 @@ static auto create_prop_getter_from_prop_desc(
   return EdgeSetInnerIdGetter<tag_id, VID_T, EDATA_T>();
 }
 
+// create GlobalId getter for vertex_set
+template <typename GRAPH_INTERFACE, typename VERTEX_SET_T, int tag_id,
+          typename std::enable_if<VERTEX_SET_T::is_vertex_set>::type* = nullptr>
+static auto create_global_id_prop_getter_from_prop_desc(
+    const GRAPH_INTERFACE& graph, const VERTEX_SET_T& set,
+    const GlobalIdProperty<tag_id>& global_id_prop) {
+  return VertexGlobalIdGetter<tag_id, typename GRAPH_INTERFACE::vertex_id_t>(
+      set.GetVertices(), set.GetLabelVec());
+}
+
+// create global id getter for collection
+template <typename GRAPH_INTERFACE, typename T, int tag_id>
+static auto create_global_id_prop_getter_from_prop_desc(
+    const GRAPH_INTERFACE& graph, const Collection<T>& set,
+    const GlobalIdProperty<tag_id>& global_id_prop) {
+  return InnerIdGetter<tag_id, T>(set.GetVector());
+}
+
+// create global id getter for EdgeSet
+template <typename GRAPH_INTERFACE, typename EDGE_SET_T, int tag_id,
+          typename std::enable_if<EDGE_SET_T::is_edge_set>::type* = nullptr>
+static auto create_global_id_prop_getter_from_prop_desc(
+    const GRAPH_INTERFACE& graph, const EDGE_SET_T& set,
+    const GlobalIdProperty<tag_id>& global_id_prop) {
+  return EdgeSetInnerIdGetter<tag_id, typename GRAPH_INTERFACE::vertex_id_t,
+                              typename EDGE_SET_T::edata_t>();
+}
+
 // get prop for inner id for idKey
 template <typename GRAPH_INTERFACE, typename CTX_HEAD_T, int cur_alias,
           int base_tag, typename... CTX_PREV, int tag_id>
@@ -896,6 +1035,18 @@ static auto create_prop_getter_from_prop_desc(
   return create_prop_getter_from_prop_desc(graph, set, inner_id_prop);
 }
 
+// get prop for global id for idKey
+template <typename GRAPH_INTERFACE, typename CTX_HEAD_T, int cur_alias,
+          int base_tag, typename... CTX_PREV, int tag_id>
+static auto create_prop_getter_from_prop_desc(
+    const GRAPH_INTERFACE& graph,
+    Context<CTX_HEAD_T, cur_alias, base_tag, CTX_PREV...>& ctx,
+    const GlobalIdProperty<tag_id>& global_id_property) {
+  auto& set = ctx.template GetNode<tag_id>();
+  return create_global_id_prop_getter_from_prop_desc(graph, set,
+                                                     global_id_property);
+}
+
 // get prop for common property.
 // return a single prop getter
 template <typename GRAPH_INTERFACE, typename CTX_T, typename T, int tag_id>
@@ -927,6 +1078,19 @@ auto create_prop_desc_from_selector(
   return InnerIdProperty<col_id>();
 }
 
+// For getting edge_set's edge
+template <int col_id, typename VID_T>
+auto create_prop_desc_from_selector(
+    const PropertySelector<const DefaultEdge<VID_T>&>& selector) {
+  return InnerIdProperty<col_id>();
+}
+
+template <int col_id>
+auto create_prop_desc_from_selector(
+    const PropertySelector<GlobalId>& selector) {
+  return GlobalIdProperty<col_id>();
+}
+
 template <int col_id, typename T>
 auto create_prop_desc_from_selector(const PropertySelector<T>& selector) {
   return NamedProperty<T, col_id>(selector.prop_name_);
@@ -940,6 +1104,14 @@ auto create_prop_descs_from_selectors(std::integer_sequence<int, in_col_id...>,
       create_prop_desc_from_selector<in_col_id>(std::get<Ind>(selectors))...);
 }
 
+template <AggFunc agg_func, typename PropT, int tag_id>
+auto create_prop_desc_from_aggregate_prop(
+    const AggregateProp<agg_func, std::tuple<PropT>,
+                        std::integer_sequence<int32_t, tag_id>>& agg_prop) {
+  return create_prop_desc_from_selector<tag_id>(
+      std::get<0>(agg_prop.selectors_));
+}
+
 template <int... in_col_id, typename... SELECTOR>
 auto create_prop_descs_from_selectors(
     const std::tuple<SELECTOR...>& selectors) {
@@ -962,6 +1134,14 @@ static auto create_prop_descs_from_group_keys(
       group_keys, std::make_index_sequence<sizeof...(GROUP_KEY)>());
 }
 
+template <typename... PropTGetter, typename... IndexEle, size_t... Is>
+auto get_prop_from_index_ele_with_prop_getter(
+    const std::tuple<PropTGetter...>& prop_getters,
+    const std::tuple<IndexEle...>& index_ele, std::index_sequence<Is...>) {
+  return std::make_tuple(
+      std::get<Is>(prop_getters).get_from_all_element(index_ele)...);
+}
+
 }  // namespace gs
 
 #endif  // ENGINES_HQPS_ENGINE_OPERATOR_PROP_UTILS_H_
\ No newline at end of file
diff --git a/flex/engines/hqps_db/database/adj_list.h b/flex/engines/hqps_db/database/adj_list.h
index 752f45eac12d..986bd30a0416 100644
--- a/flex/engines/hqps_db/database/adj_list.h
+++ b/flex/engines/hqps_db/database/adj_list.h
@@ -33,16 +33,24 @@ template <typename LabelT>
 class EdgeIter {
  public:
   using label_id_t = LabelT;
-  EdgeIter() : label_triplet_(), ptr1_(nullptr), prop_names_(nullptr) {}
+  EdgeIter()
+      : dir_(Direction::Out),
+        label_triplet_(),
+        ptr1_(nullptr),
+        prop_names_(nullptr) {}
 
   EdgeIter(const EdgeIter& other)
-      : label_triplet_(other.label_triplet_),
+      : dir_(other.dir_),
+        label_triplet_(other.label_triplet_),
         ptr1_(other.ptr1_),
         prop_names_(other.prop_names_) {}
-  EdgeIter(const std::array<LabelT, 3>& label_triplet,
+  EdgeIter(const Direction& dir, const std::array<LabelT, 3>& label_triplet,
            std::shared_ptr<CsrConstEdgeIterBase> ptr,
            const std::vector<std::string>* prop_names)
-      : label_triplet_(label_triplet), ptr1_(ptr), prop_names_(prop_names) {}
+      : dir_(dir),
+        label_triplet_(label_triplet),
+        ptr1_(ptr),
+        prop_names_(prop_names) {}
 
   inline void Next() const { ptr1_->next(); }
   inline vid_t GetDstId() const { return ptr1_->get_neighbor(); }
@@ -51,12 +59,15 @@ class EdgeIter {
 
   inline label_id_t GetSrcLabel() const { return label_triplet_[0]; }
 
+  inline Direction GetDirection() const { return dir_; }
+
   inline Any GetData() const { return ptr1_->get_data(); }
   inline bool IsValid() const { return ptr1_ && ptr1_->is_valid(); }
 
   const std::vector<std::string>& GetPropNames() const { return *prop_names_; }
 
   EdgeIter<LabelT>& operator=(const EdgeIter<LabelT>& rhs) {
+    this->dir_ = rhs.dir_;
     this->ptr1_ = rhs.ptr1_;
     this->label_triplet_ = rhs.label_triplet_;
     this->prop_names_ = rhs.prop_names_;
@@ -71,6 +82,7 @@ class EdgeIter {
   }
 
  private:
+  Direction dir_;
   std::array<LabelT, 3> label_triplet_;
   std::shared_ptr<CsrConstEdgeIterBase> ptr1_;
   const std::vector<std::string>* prop_names_;
@@ -84,20 +96,25 @@ class SubGraph {
   using iterator = EdgeIter<LabelT>;
   using label_id_t = LabelT;
   SubGraph(const CsrBase* first, const std::array<label_id_t, 3>& label_triplet,
-           const std::vector<std::string>& prop_names)
-      : first_(first), label_triplet_(label_triplet), prop_names_(prop_names) {}
+           const std::vector<std::string>& prop_names, Direction dir)
+      : first_(first),
+        label_triplet_(label_triplet),
+        prop_names_(prop_names),
+        dir_(dir) {}
 
   inline iterator get_edges(VID_T vid) const {
     if (first_) {
-      return iterator(label_triplet_, first_->edge_iter(vid), &prop_names_);
+      return iterator(dir_, label_triplet_, first_->edge_iter(vid),
+                      &prop_names_);
     }
-    return iterator(label_triplet_, nullptr, &prop_names_);
+    return iterator(dir_, label_triplet_, nullptr, &prop_names_);
   }
 
   // here the src, dst, refer the src, dst of the csr, not the direction.
   label_id_t GetSrcLabel() const { return label_triplet_[0]; }
   label_id_t GetEdgeLabel() const { return label_triplet_[2]; }
   label_id_t GetDstLabel() const { return label_triplet_[1]; }
+  Direction GetDirection() const { return dir_; }
 
   const std::vector<std::string>& GetPropNames() const { return prop_names_; }
 
@@ -106,6 +123,7 @@ class SubGraph {
   // We assume first is out edge, second is in edge.
   std::array<label_id_t, 3> label_triplet_;
   std::vector<std::string> prop_names_;
+  Direction dir_;
 };
 
 template <typename T>
@@ -236,6 +254,7 @@ class AdjList<T> {
   using nbr_t = MutableNbr<T>;
   class Iterator {
    public:
+    using edge_property_t = std::tuple<T>;
     Iterator()
         : cur_(),
           begin0_(nullptr),
diff --git a/flex/engines/hqps_db/database/mutable_csr_interface.h b/flex/engines/hqps_db/database/mutable_csr_interface.h
index b3b566ee9655..770831dde9b3 100644
--- a/flex/engines/hqps_db/database/mutable_csr_interface.h
+++ b/flex/engines/hqps_db/database/mutable_csr_interface.h
@@ -29,13 +29,22 @@
 
 namespace gs {
 
+template <typename T>
+bool exists_nullptr_in_tuple(const T& t) {
+  return std::apply([](auto&&... args) { return ((args == nullptr) || ...); },
+                    t);
+}
+
 template <size_t I = 0, typename... T>
 void get_tuple_from_column_tuple(
     size_t index, std::tuple<T...>& t,
     const std::tuple<std::shared_ptr<TypedRefColumn<T>>...>& columns) {
+  using cur_ele_t = std::tuple_element_t<I, std::tuple<T...>>;
   auto ptr = std::get<I>(columns);
   if (ptr) {
     std::get<I>(t) = ptr->get_view(index);
+  } else {
+    std::get<I>(t) = NullRecordCreator<cur_ele_t>::GetNull();
   }
 
   if constexpr (I + 1 < sizeof...(T)) {
@@ -47,8 +56,11 @@ template <size_t I = 0, typename... T, typename... COL_T>
 void get_tuple_from_column_tuple(size_t index, std::tuple<T...>& t,
                                  const std::tuple<COL_T...>& columns) {
   auto ptr = std::get<I>(columns);
+  using cur_ele_t = std::tuple_element_t<I, std::tuple<T...>>;
   if (ptr) {
     std::get<I>(t) = ptr->get_view(index);
+  } else {
+    std::get<I>(t) = NullRecordCreator<cur_ele_t>::GetNull();
   }
 
   if constexpr (I + 1 < sizeof...(T)) {
@@ -66,6 +78,7 @@ class MutableCSRInterface {
   const GraphDBSession& GetDBSession() const { return db_session_; }
 
   using vertex_id_t = vid_t;
+  using gid_t = uint64_t;
   using label_id_t = uint8_t;
 
   using nbr_list_array_t = mutable_csr_graph_impl::NbrListArray;
@@ -96,15 +109,10 @@ class MutableCSRInterface {
   MutableCSRInterface(const MutableCSRInterface&) = delete;
 
   MutableCSRInterface(MutableCSRInterface&& other)
-      : db_session_(other.db_session_) {
-    LOG(INFO) << "Move MutableCSRInterface";
-  }
+      : db_session_(other.db_session_) {}
 
   explicit MutableCSRInterface(const GraphDBSession& session)
-      : db_session_(session) {
-    LOG(INFO) << "Creating MutableCSRInterface";
-    LOG(INFO) << "person label num: " << db_session_.graph().vertex_num(1);
-  }
+      : db_session_(session) {}
 
   const Schema& schema() const { return db_session_.schema(); }
 
@@ -142,8 +150,8 @@ class MutableCSRInterface {
    */
   template <typename FUNC_T, typename... SELECTOR>
   void ScanVertices(const std::string& label,
-                    const std::tuple<SELECTOR...>& props,
-                    const FUNC_T& func) const {
+                    const std::tuple<SELECTOR...>& props, const FUNC_T& func,
+                    bool filter_null = false) const {
     auto label_id = db_session_.schema().get_vertex_label_id(label);
     return ScanVertices(label_id, props, func);
   }
@@ -160,7 +168,7 @@ class MutableCSRInterface {
   template <typename FUNC_T, typename... SELECTOR>
   void ScanVertices(const label_id_t& label_id,
                     const std::tuple<SELECTOR...>& selectors,
-                    const FUNC_T& func) const {
+                    const FUNC_T& func, bool filter_null = false) const {
     auto vnum = db_session_.graph().vertex_num(label_id);
     std::tuple<typename SELECTOR::prop_t...> t;
     if constexpr (sizeof...(SELECTOR) == 0) {
@@ -168,8 +176,15 @@ class MutableCSRInterface {
         func(v, t);
       }
     } else {
-      auto columns =
-          get_tuple_column_from_graph_with_property(label_id, selectors);
+      auto columns = GetPropertyColumnWithSelectors(label_id, selectors);
+      if (exists_nullptr_in_tuple(columns)) {
+        VLOG(10) << "When scanning for label " << std::to_string(label_id)
+                 << ", there is null column, using default NULL value";
+        if (filter_null) {
+          return;
+        }
+      }
+
       for (size_t v = 0; v != vnum; ++v) {
         get_tuple_from_column_tuple(v, t, columns);
         func(v, t);
@@ -304,10 +319,15 @@ class MutableCSRInterface {
       const std::vector<std::vector<int32_t>>& vid_inds,
       const std::array<std::string, std::tuple_size_v<std::tuple<T...>>>&
           prop_names) const {
-    std::vector<std::tuple<T...>> props(vids.size());
+    std::vector<std::tuple<T...>> props;
+    props.resize(vids.size());
     using column_tuple_t = std::tuple<std::shared_ptr<TypedRefColumn<T>>...>;
     std::vector<column_tuple_t> columns;
     columns.resize(label_ids.size());
+    VLOG(10) << "GetVertexProps from vid of label: " << gs::to_string(label_ids)
+             << ", " << vids.size() << ",: " << gs::to_string(vids)
+             << "vid ind size: " << vid_inds.size()
+             << ", vid ind: " << gs::to_string(vid_inds);
     for (size_t i = 0; i < label_ids.size(); ++i) {
       get_tuple_column_from_graph(label_ids[i], prop_names, columns[i]);
     }
@@ -340,22 +360,11 @@ class MutableCSRInterface {
       const grape::Bitset& bitset,
       const std::array<std::string, std::tuple_size_v<std::tuple<T...>>>&
           prop_names) const {
-    size_t total_size = vids.size();
-    std::vector<std::tuple<T...>> props(total_size);
-    std::vector<label_t> label_ids;
-    for (auto label : labels) {
-      label_ids.emplace_back(db_session_.schema().get_vertex_label_id(label));
-    }
-    using column_tuple_t = std::tuple<std::shared_ptr<TypedRefColumn<T>>...>;
-    std::vector<column_tuple_t> columns;
-    columns.resize(label_ids.size());
-    for (size_t i = 0; i < label_ids.size(); ++i) {
-      get_tuple_column_from_graph(label_ids[i], prop_names, columns[i]);
+    std::array<label_id_t, num_labels> label_ids;
+    for (size_t i = 0; i < num_labels; ++i) {
+      label_ids[i] = db_session_.schema().get_vertex_label_id(labels[i]);
     }
-
-    fetch_propertiesV2<0>(props, columns, vids, bitset);
-
-    return std::move(props);
+    return GetVertexPropsFromVidV2<T...>(vids, label_ids, bitset, prop_names);
   }
 
   /**
@@ -407,28 +416,23 @@ class MutableCSRInterface {
       auto& column_tuple1 = columns[1];
       auto ptr0 = std::get<Is>(column_tuple0);
       auto ptr1 = std::get<Is>(column_tuple1);
-      if (ptr0 && ptr1) {
-        for (size_t i = 0; i < vids.size(); ++i) {
-          if (bitset.get_bit(i)) {
+
+      for (size_t i = 0; i < vids.size(); ++i) {
+        if (bitset.get_bit(i)) {
+          if (ptr0) {
             std::get<Is>(props[i]) = ptr0->get_view(vids[i]);
           } else {
-            std::get<Is>(props[i]) = ptr1->get_view(vids[i]);
-          }
-        }
-      } else if (ptr0) {
-        for (size_t i = 0; i < vids.size(); ++i) {
-          if (bitset.get_bit(i)) {
-            std::get<Is>(props[i]) = ptr0->get_view(vids[i]);
+            std::get<Is>(props[i]) = NullRecordCreator<
+                std::tuple_element_t<Is, std::tuple<T...>>>::GetNull();
           }
-        }
-      } else if (ptr1) {
-        for (size_t i = 0; i < vids.size(); ++i) {
-          if (!bitset.get_bit(i)) {
+        } else {
+          if (ptr1) {
             std::get<Is>(props[i]) = ptr1->get_view(vids[i]);
+          } else {
+            std::get<Is>(props[i]) = NullRecordCreator<
+                std::tuple_element_t<Is, std::tuple<T...>>>::GetNull();
           }
         }
-      } else {
-        VLOG(10) << "skip for column " << Is;
       }
     }
     fetch_propertiesV2<Is + 1>(props, columns, vids, bitset);
@@ -445,6 +449,11 @@ class MutableCSRInterface {
       for (size_t i = 0; i < vids.size(); ++i) {
         std::get<Is>(props[i]) = cur_column->get_view(vids[i]);
       }
+    } else {
+      for (size_t i = 0; i < vids.size(); ++i) {
+        std::get<Is>(props[i]) = NullRecordCreator<
+            std::tuple_element_t<Is, std::tuple<T...>>>::GetNull();
+      }
     }
 
     if constexpr (Is + 1 < sizeof...(T)) {
@@ -471,14 +480,16 @@ class MutableCSRInterface {
     for (size_t i = 0; i < num_labels; ++i) {
       auto column_tuple = columns[i];
       auto ptr = std::get<Is>(column_tuple);
-      if (ptr) {
-        for (size_t j = 0; j < vid_inds[i].size(); ++j) {
-          auto vid_ind = vid_inds[i][j];
-          auto vid = vids[vid_ind];
+
+      for (size_t j = 0; j < vid_inds[i].size(); ++j) {
+        auto vid_ind = vid_inds[i][j];
+        auto vid = vids[vid_ind];
+        if (ptr) {
           std::get<Is>(props[vid_ind]) = ptr->get_view(vid);
+        } else {
+          std::get<Is>(props[vid_ind]) = NullRecordCreator<
+              std::tuple_element_t<Is, std::tuple<T...>>>::GetNull();
         }
-      } else {
-        VLOG(10) << "skip for column " << Is;
       }
     }
 
@@ -503,14 +514,20 @@ class MutableCSRInterface {
         direction_str == "OUT") {
       csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id,
                                            edge_label_id);
-      return std::vector<sub_graph_t>{sub_graph_t{
-          csr, {src_label_id, dst_label_id, edge_label_id}, prop_names}};
+      return std::vector<sub_graph_t>{
+          sub_graph_t{csr,
+                      {src_label_id, dst_label_id, edge_label_id},
+                      prop_names,
+                      Direction::Out}};
     } else if (direction_str == "in" || direction_str == "In" ||
                direction_str == "IN") {
       csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id,
                                            edge_label_id);
-      return std::vector<sub_graph_t>{sub_graph_t{
-          csr, {dst_label_id, src_label_id, edge_label_id}, prop_names}};
+      return std::vector<sub_graph_t>{
+          sub_graph_t{csr,
+                      {dst_label_id, src_label_id, edge_label_id},
+                      prop_names,
+                      Direction::In}};
     } else if (direction_str == "both" || direction_str == "Both" ||
                direction_str == "BOTH") {
       csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id,
@@ -518,15 +535,17 @@ class MutableCSRInterface {
       other_csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id,
                                                  edge_label_id);
       return std::vector<sub_graph_t>{
-          sub_graph_t{
-              csr, {src_label_id, dst_label_id, edge_label_id}, prop_names},
+          sub_graph_t{csr,
+                      {src_label_id, dst_label_id, edge_label_id},
+                      prop_names,
+                      Direction::Out},
           sub_graph_t{other_csr,
                       {dst_label_id, src_label_id, edge_label_id},
-                      prop_names}};
+                      prop_names,
+                      Direction::In}};
     } else {
       throw std::runtime_error("Not implemented - " + direction_str);
     }
-    if (csr && !other_csr) {}
   }
 
   template <typename... T>
@@ -620,7 +639,7 @@ class MutableCSRInterface {
       }
     } else if (direction_str == "in" || direction_str == "In" ||
                direction_str == "IN") {
-      auto csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id,
+      auto csr = db_session_.graph().get_ie_csr(src_label_id, dst_label_id,
                                                 edge_label_id);
       auto size = 0;
       for (size_t i = 0; i < vids.size(); ++i) {
@@ -642,7 +661,7 @@ class MutableCSRInterface {
       }
     } else if (direction_str == "both" || direction_str == "Both" ||
                direction_str == "BOTH") {
-      auto ie_csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id,
+      auto ie_csr = db_session_.graph().get_ie_csr(src_label_id, dst_label_id,
                                                    edge_label_id);
       auto oe_csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id,
                                                    edge_label_id);
@@ -713,7 +732,7 @@ class MutableCSRInterface {
       }
     } else if (direction_str == "in" || direction_str == "In" ||
                direction_str == "IN") {
-      auto csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id,
+      auto csr = db_session_.graph().get_ie_csr(src_label_id, dst_label_id,
                                                 edge_label_id);
       if (csr) {
         for (size_t i = 0; i < vids.size(); ++i) {
@@ -730,8 +749,15 @@ class MutableCSRInterface {
                direction_str == "BOTH") {
       auto ocsr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id,
                                                  edge_label_id);
-      auto icsr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id,
+      auto icsr = db_session_.graph().get_ie_csr(src_label_id, dst_label_id,
                                                  edge_label_id);
+      // the reason why we use (src_label_id, dst_label_id) is that, when the
+      // src_label and dst_label argument denote the labels on two sides.
+      // And src labels are vertices we have.
+      // And we will make sure the edge direction is at least from src to
+      // dst.(or src is same)
+      // So, to get the incoming edges of src_label, we need to edges originally
+      // from dst_label to src_label.
       for (size_t i = 0; i < vids.size(); ++i) {
         auto v = vids[i];
         auto& vec = ret.get_vector(i);
@@ -798,6 +824,9 @@ class MutableCSRInterface {
     if constexpr (std::is_same_v<T, LabelKey>) {
       return std::make_shared<TypedRefColumn<LabelKey>>(label_id);
     }
+    if constexpr (std::is_same_v<T, GlobalId>) {
+      return std::make_shared<TypedRefColumn<GlobalId>>(label_id);
+    }
     if (prop_name == "id" || prop_name == "ID" || prop_name == "Id") {
       column = std::dynamic_pointer_cast<TypedRefColumn<T>>(
           db_session_.get_vertex_id_column(label_id));
@@ -825,6 +854,13 @@ class MutableCSRInterface {
     }
   }
 
+  template <typename... SELECTOR>
+  inline auto GetPropertyColumnWithSelectors(
+      label_t label, const std::tuple<SELECTOR...>& selectors) const {
+    return get_tuple_column_from_graph_with_property_impl(
+        label, selectors, std::make_index_sequence<sizeof...(SELECTOR)>());
+  }
+
  private:
   std::shared_ptr<RefColumnBase> create_ref_column(
       std::shared_ptr<ColumnBase> column) const {
@@ -864,7 +900,6 @@ class MutableCSRInterface {
   auto get_single_column_from_graph_with_property(
       label_t label, const PropertySelector<PropT>& selector) const {
     auto res = GetTypedRefColumn<PropT>(label, selector.prop_name_);
-    CHECK(res) << "Property " << selector.prop_name_ << " not found";
     return res;
   }
 
@@ -876,13 +911,6 @@ class MutableCSRInterface {
         label, std::get<Is>(selectors))...);
   }
 
-  template <typename... SELECTOR>
-  inline auto get_tuple_column_from_graph_with_property(
-      label_t label, const std::tuple<SELECTOR...>& selectors) const {
-    return get_tuple_column_from_graph_with_property_impl(
-        label, selectors, std::make_index_sequence<sizeof...(SELECTOR)>());
-  }
-
   template <size_t I = 0, typename... T,
             typename std::enable_if<(sizeof...(T) > 0)>::type* = nullptr>
   void get_tuple_column_from_graph(
diff --git a/flex/engines/hqps_db/structures/collection.h b/flex/engines/hqps_db/structures/collection.h
index 33d80e6d25f2..73caf4ed0402 100644
--- a/flex/engines/hqps_db/structures/collection.h
+++ b/flex/engines/hqps_db/structures/collection.h
@@ -77,6 +77,9 @@ class CollectionBuilder {
 
   void Insert(const T& t) { vec_.push_back(t); }
 
+  // ele& data
+  void Insert(const T& t, const T& t2) { vec_.push_back(t); }
+
   // insert index ele tuple
   void Insert(const std::tuple<size_t, T>& t) {
     vec_.emplace_back(std::get<1>(t));
@@ -112,7 +115,10 @@ class KeyedCollectionBuilder {
   }
 
   // insert returning a unique index for the inserted element
-  size_t insert(const T& t) {
+  int32_t insert(const T& t) {
+    if (IsNull(t)) {
+      return -1;
+    }
     if (map_.find(t) == map_.end()) {
       map_[t] = vec_.size();
       vec_.push_back(t);
@@ -122,17 +128,13 @@ class KeyedCollectionBuilder {
     }
   }
 
-  size_t insert(T&& t) {
-    if (map_.find(t) == map_.end()) {
-      map_[t] = vec_.size();
-      vec_.emplace_back(std::move(t));
-      return vec_.size() - 1;
-    } else {
-      return map_[t];
-    }
+  int32_t insert(const std::tuple<size_t, T>& t, const T& data) {
+    return insert(std::get<1>(t));
   }
 
-  size_t Insert(const std::tuple<size_t, T>& t) {
+  int32_t insert(T&& t) { return insert(t); }
+
+  int32_t Insert(const std::tuple<size_t, T>& t) {
     return insert(std::get<1>(t));
   }
 
@@ -357,7 +359,9 @@ class Collection {
     LOG(WARNING) << " Not implemented";
   }
 
-  std::vector<offset_t> Dedup() {
+  template <typename PropT, typename std::enable_if_t<std::is_same_v<
+                                PropT, grape::EmptyType>>::type* = nullptr>
+  std::vector<offset_t> Dedup(const PropertySelector<PropT>& prop_selector) {
     std::vector<offset_t> offsets;
     std::vector<T> new_vec;
     std::unordered_map<T, size_t, boost::hash<T>> map;
@@ -397,7 +401,7 @@ class CountBuilder {
   // insert tuple at index ind.
   // if the ele_value equal to invalid_value, then do not insert.
   template <typename ELE_TUPLE, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE& tuple, const DATA_TUPLE& data) {
     // just count times.
     while (vec_.size() <= ind) {
       vec_.emplace_back(0);
@@ -417,15 +421,16 @@ class CountBuilder {
       VLOG(10) << "inc:" << ind << ", " << gs::to_string(tuple);
       ++vec_[ind];
     }
+    return true;
   }
 
-  Collection<size_t> Build() {
+  Collection<int64_t> Build() {
     // VLOG(10) << "Finish building counter" << gs::to_string(vec_);
-    return Collection<size_t>(std::move(vec_));
+    return Collection<int64_t>(std::move(vec_));
   }
 
  private:
-  std::vector<size_t> vec_;
+  std::vector<int64_t> vec_;
 };
 
 // Prop Count Builder.
@@ -438,15 +443,21 @@ class PropCountBuilder {
   // insert tuple at index ind.
   // if the ele_value equal to invalid_value, then do not insert.
   template <typename ELE_TUPLE, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE& tuple, const DATA_TUPLE& data) {
     // just count times.
     while (vec_.size() <= ind) {
       vec_.emplace_back(0);
     }
 
     auto& cur_ele = gs::get_from_tuple<tag>(tuple);
+    if (IsNull(cur_ele)) {
+      return false;
+    }
     // get prop from prop getter
     auto props = prop_getter_.get_view(cur_ele);
+    if (IsNull(props)) {
+      return false;
+    }
     // get the type of props
     using props_t = decltype(props);
 
@@ -456,12 +467,13 @@ class PropCountBuilder {
       VLOG(10) << "ele is null, ind: " << ind
                << "ele:" << gs::to_string(cur_ele);
     }
+    return true;
   }
 
-  Collection<size_t> Build() { return Collection<size_t>(std::move(vec_)); }
+  Collection<int64_t> Build() { return Collection<int64_t>(std::move(vec_)); }
 
  private:
-  std::vector<size_t> vec_;
+  std::vector<int64_t> vec_;
   PropGetterT prop_getter_;
 };
 
@@ -473,9 +485,12 @@ class MultiColCountBuilder {
   // insert tuple at index ind.
   // if the ele_value equal to invalid_value, then do not insert.
   template <typename ELE_TUPLE, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE& tuple, const DATA_TUPLE& data) {
     auto cur_ele_tuple =
         std::tuple_cat(tuple_slice<1>(gs::get_from_tuple<tag>(tuple))...);
+    if (IsNull(cur_ele_tuple)) {
+      return false;
+    }
     while (vec_.size() <= ind) {
       vec_.emplace_back(0);
     }
@@ -490,12 +505,13 @@ class MultiColCountBuilder {
     } else {
       VLOG(10) << "ele is null";
     }
+    return true;
   }
 
-  Collection<size_t> Build() { return Collection<size_t>(std::move(vec_)); }
+  Collection<int64_t> Build() { return Collection<int64_t>(std::move(vec_)); }
 
  private:
-  std::vector<size_t> vec_;
+  std::vector<int64_t> vec_;
 };
 
 template <int tag_id, typename T, typename Enable = void>
@@ -516,8 +532,11 @@ class DistinctCountBuilder<
   }
 
   template <typename ELE_TUPLE_T, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
     auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
     while (vec_.size() <= ind) {
       vec_.emplace_back(grape::Bitset(edges_num_));
     }
@@ -529,15 +548,16 @@ class DistinctCountBuilder<
       LOG(FATAL) << "Invalid edge index: " << cur_ind
                  << ", edges num: " << edges_num_;
     }
+    return true;
   }
 
-  Collection<size_t> Build() {
-    std::vector<size_t> res;
+  Collection<int64_t> Build() {
+    std::vector<int64_t> res;
     res.reserve(vec_.size());
     for (auto& bitset : vec_) {
       res.emplace_back(bitset.count());
     }
-    return Collection<size_t>(std::move(res));
+    return Collection<int64_t>(std::move(res));
   }
 
  private:
@@ -559,8 +579,11 @@ class DistinctCountBuilder<tag_id, RowVertexSetImpl<LabelT, VID_T, T...>> {
   }
 
   template <typename ELE_TUPLE_T, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
     auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
     static_assert(
         std::is_same_v<
             std::tuple_element_t<1, std::remove_const_t<std::remove_reference_t<
@@ -575,15 +598,16 @@ class DistinctCountBuilder<tag_id, RowVertexSetImpl<LabelT, VID_T, T...>> {
     cur_bitset.set_bit(cur_v - min_v);
     // VLOG(10) << "tag id: " << tag_id << "insert at ind: " << ind
     //          << ",value : " << cur_v << ", res: " << cur_bitset.count();
+    return true;
   }
 
-  Collection<size_t> Build() {
-    std::vector<size_t> res;
+  Collection<int64_t> Build() {
+    std::vector<int64_t> res;
     res.reserve(vec_.size());
     for (auto& bitset : vec_) {
       res.emplace_back(bitset.count());
     }
-    return Collection<size_t>(std::move(res));
+    return Collection<int64_t>(std::move(res));
   }
 
  private:
@@ -615,8 +639,11 @@ class DistinctCountBuilder<tag_id, TwoLabelVertexSetImpl<VID_T, LabelT, T...>> {
   }
 
   template <typename ELE_TUPLE_T, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
     auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
     static_assert(
         std::is_same_v<
             std::tuple_element_t<2, std::remove_const_t<std::remove_reference_t<
@@ -633,10 +660,11 @@ class DistinctCountBuilder<tag_id, TwoLabelVertexSetImpl<VID_T, LabelT, T...>> {
     cur_bitset.set_bit(cur_v - min_v[label_ind]);
     VLOG(10) << "tag id: " << tag_id << "insert at ind: " << ind
              << ",value : " << cur_v << ", res: " << cur_bitset.count();
+    return true;
   }
 
-  Collection<size_t> Build() {
-    std::vector<size_t> res;
+  Collection<int64_t> Build() {
+    std::vector<int64_t> res;
     auto max_ind = std::max(vec_[0].size(), vec_[1].size());
     res.resize(max_ind, 0);
     for (auto label_ind = 0; label_ind < 2; ++label_ind) {
@@ -644,7 +672,7 @@ class DistinctCountBuilder<tag_id, TwoLabelVertexSetImpl<VID_T, LabelT, T...>> {
         res[i] += vec_[label_ind][i].count();
       }
     }
-    return Collection<size_t>(std::move(res));
+    return Collection<int64_t>(std::move(res));
   }
 
  private:
@@ -652,6 +680,54 @@ class DistinctCountBuilder<tag_id, TwoLabelVertexSetImpl<VID_T, LabelT, T...>> {
   std::array<VID_T, 2> min_v, max_v, range_size;
 };
 
+// specialization for DistinctCountBuilder for num_labels=2
+template <int tag_id, typename VID_T, typename LabelT, typename... T>
+class DistinctCountBuilder<tag_id, GeneralVertexSet<VID_T, LabelT, T...>> {
+ public:
+  DistinctCountBuilder(const std::vector<grape::Bitset>& bitsets,
+                       const std::vector<LabelT>& labels)
+      : bitset_vec_(bitsets), labels_(labels) {
+    CHECK(bitset_vec_.size() == labels_.size());
+  }
+
+  template <typename ELE_TUPLE_T, typename DATA_TUPLE>
+  bool insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
+    auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
+    while (vec_.size() <= ind) {
+      vec_.emplace_back(std::unordered_set<GlobalId>());
+    }
+    auto ele_index = std::get<0>(cur_ind_ele);
+    int32_t label_ind = -1;
+    for (size_t i = 0; i < bitset_vec_.size(); ++i) {
+      if (bitset_vec_[i].get_bit(ele_index)) {
+        label_ind = i;
+        break;
+      }
+    }
+    CHECK(label_ind != -1);
+    auto global_id = GlobalId(labels_[label_ind], std::get<1>(cur_ind_ele));
+    vec_[ind].insert(global_id);
+    return true;
+  }
+
+  Collection<int64_t> Build() {
+    std::vector<int64_t> res;
+    res.reserve(vec_.size());
+    for (auto& set : vec_) {
+      res.emplace_back(set.size());
+    }
+    return Collection<int64_t>(std::move(res));
+  }
+
+ private:
+  std::vector<std::unordered_set<GlobalId>> vec_;
+  const std::vector<grape::Bitset>& bitset_vec_;
+  const std::vector<LabelT>& labels_;
+};
+
 // DistinctCountBuilder for PathSet
 template <int tag_id, typename PATH_SET_T>
 class DistinctCountBuilder<
@@ -662,11 +738,15 @@ class DistinctCountBuilder<
   using index_ele_t = typename path_set_t::index_ele_tuple_t;
   DistinctCountBuilder(const path_set_t& path_set) {
     paths_num_ = path_set.Size();
+    LOG(INFO) << "path num: " << paths_num_;
   }
 
   template <typename ELE_TUPLE_T, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
     auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(std::get<1>(cur_ind_ele))) {
+      return false;
+    }
     while (vec_.size() <= ind) {
       vec_.emplace_back(grape::Bitset(paths_num_));
     }
@@ -678,15 +758,16 @@ class DistinctCountBuilder<
       LOG(FATAL) << "Invalid path set index: " << cur_ind
                  << ", path set num: " << paths_num_;
     }
+    return true;
   }
 
-  Collection<size_t> Build() {
-    std::vector<size_t> res;
+  Collection<int64_t> Build() {
+    std::vector<int64_t> res;
     res.reserve(vec_.size());
     for (auto& bitset : vec_) {
       res.emplace_back(bitset.count());
     }
-    return Collection<size_t>(std::move(res));
+    return Collection<int64_t>(std::move(res));
   }
 
  private:
@@ -694,6 +775,48 @@ class DistinctCountBuilder<
   size_t paths_num_;
 };
 
+// DistinctCountBuilder for Collection
+template <int tag_id, typename T>
+class DistinctCountBuilder<tag_id, Collection<T>> {
+ public:
+  using set_t = Collection<T>;
+  using index_ele_t = typename set_t::index_ele_tuple_t;
+  DistinctCountBuilder(const set_t& set) { record_num_ = set.Size(); }
+
+  template <typename ELE_TUPLE_T, typename DATA_TUPLE>
+  bool insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
+    auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
+    while (vec_.size() <= ind) {
+      vec_.emplace_back(grape::Bitset(record_num_));
+    }
+    auto& cur_bitset = vec_[ind];
+    auto cur_ind = std::get<0>(cur_ind_ele);
+    if (cur_ind < record_num_) {
+      cur_bitset.set_bit(cur_ind);
+    } else {
+      LOG(FATAL) << "Invalid path set index: " << cur_ind
+                 << ", set num: " << record_num_;
+    }
+    return true;
+  }
+
+  Collection<int64_t> Build() {
+    std::vector<int64_t> res;
+    res.reserve(vec_.size());
+    for (auto& bitset : vec_) {
+      res.emplace_back(bitset.count());
+    }
+    return Collection<int64_t>(std::move(res));
+  }
+
+ private:
+  std::vector<grape::Bitset> vec_;
+  size_t record_num_;
+};
+
 // DistinctCountBuilder for multiple sets together
 template <typename SET_TUPLE_T, int... TAG_IDs>
 class MultiColDistinctCountBuilder;
@@ -705,11 +828,14 @@ class MultiColDistinctCountBuilder<std::tuple<SET_Ts...>, TAG_IDs...> {
   MultiColDistinctCountBuilder() {}
 
   template <typename ELE_TUPLE_T, typename DATA_TUPLE>
-  void insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const ELE_TUPLE_T& tuple, const DATA_TUPLE& data) {
     // construct the ref tuple from tuple,with TAG_IDS,
     // get element tuple from index_ele_tuple_t
     auto cur_ele_tuple =
         std::tuple_cat(tuple_slice<1>(gs::get_from_tuple<TAG_IDs>(tuple))...);
+    if (IsNull(cur_ele_tuple)) {
+      return false;
+    }
 
     while (vec_of_set_.size() <= ind) {
       vec_of_set_.emplace_back(
@@ -720,15 +846,16 @@ class MultiColDistinctCountBuilder<std::tuple<SET_Ts...>, TAG_IDs...> {
     VLOG(10) << "tuple: " << gs::to_string(cur_ele_tuple)
              << ",all ele: " << gs::to_string(tuple) << "insert at ind: " << ind
              << ", res: " << cur_set.size();
+    return true;
   }
 
-  Collection<size_t> Build() {
-    std::vector<size_t> res;
+  Collection<int64_t> Build() {
+    std::vector<int64_t> res;
     res.reserve(vec_of_set_.size());
     for (auto& set : vec_of_set_) {
       res.emplace_back(set.size());
     }
-    return Collection<size_t>(std::move(res));
+    return Collection<int64_t>(std::move(res));
   }
 
  private:
@@ -736,46 +863,64 @@ class MultiColDistinctCountBuilder<std::tuple<SET_Ts...>, TAG_IDs...> {
       vec_of_set_;
 };
 
-template <typename T, int tag_id>
+template <typename PropGetterT, int tag_id>
 class SumBuilder {
  public:
-  SumBuilder() {}
-  SumBuilder(size_t cap) { vec_.resize(cap, (T) 0); }
+  using T = typename PropGetterT::prop_element_t;
+  SumBuilder(PropGetterT&& prop_getter)
+      : prop_getter_(std::move(prop_getter)) {}
 
   // insert tuple at index ind.
   template <typename IND_ELE_TUPLE, typename DATA_TUPLE>
-  void insert(size_t ind, const IND_ELE_TUPLE& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const IND_ELE_TUPLE& tuple, const DATA_TUPLE& data) {
     const auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
     // just count times.
     while (vec_.size() <= ind) {
       vec_.emplace_back((T) 0);
+      is_null_.emplace_back(true);
+    }
+    // vec_[ind] += std::get<1>(cur_ind_ele);
+    auto view = prop_getter_.get_view(cur_ind_ele);
+    if (IsNull(view)) {
+      return false;
     }
-    vec_[ind] += std::get<1>(cur_ind_ele);
+    vec_[ind] += view;
+    is_null_[ind] = false;
+    return true;
   }
 
   Collection<T> Build() {
-    // VLOG(10) << "Finish building counter" << gs::to_string(vec_);
+    for (size_t i = 0; i < vec_.size(); ++i) {
+      if (is_null_[i]) {
+        vec_[i] = NullRecordCreator<T>::GetNull();
+      }
+    }
     return Collection<T>(std::move(vec_));
   }
 
  private:
+  PropGetterT prop_getter_;
   std::vector<T> vec_;
+  std::vector<bool> is_null_;
 };
 
-template <typename GI, typename T, int tag_id>
+template <typename GI, typename PropGetterT, int tag_id>
 class MinBuilder {
  public:
-  MinBuilder(const Collection<T>& set, const GI& graph,
-             PropNameArray<T> prop_names) {
-    // vec_.resize(set.Size(), std::numeric_limits<T>::max());
-  }
-  MinBuilder() {}
-  MinBuilder(size_t cap) { vec_.resize(cap, (T) 0); }
+  using T = typename PropGetterT::prop_element_t;
+  MinBuilder(PropGetterT&& prop_getter)
+      : prop_getter_(std::move(prop_getter)) {}
 
   // insert tuple at index ind.
   template <typename IND_ELE_TUPLE, typename DATA_TUPLE>
-  void insert(size_t ind, const IND_ELE_TUPLE& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const IND_ELE_TUPLE& tuple, const DATA_TUPLE& data) {
     const auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
     VLOG(10) << "Insert min with ind: " << ind
              << ", value: " << std::get<1>(cur_ind_ele)
              << ", vec size: " << vec_.size();
@@ -783,46 +928,60 @@ class MinBuilder {
     while (vec_.size() <= ind) {
       vec_.emplace_back(std::numeric_limits<T>::max());
     }
-    vec_[ind] = std::min(vec_[ind], std::get<1>(cur_ind_ele));
+    vec_[ind] = std::min(vec_[ind], prop_getter_.get_view(cur_ind_ele));
+    return true;
   }
 
   Collection<T> Build() {
-    // VLOG(10) << "Finish building counter" << gs::to_string(vec_);
+    // iterate over vec_, turn inf to null
+    for (auto& ele : vec_) {
+      if (ele == std::numeric_limits<T>::max()) {
+        ele = NullRecordCreator<T>::GetNull();
+      }
+    }
     return Collection<T>(std::move(vec_));
   }
 
  private:
+  PropGetterT prop_getter_;
   std::vector<T> vec_;
 };
 
-template <typename GI, typename T, int tag_id>
+template <typename GI, typename PropGetterT, int tag_id>
 class MaxBuilder {
  public:
-  MaxBuilder(const Collection<T>& set, const GI& graph,
-             PropNameArray<T> prop_names) {
-    vec_.resize(set.Size(), std::numeric_limits<T>::min());
-  }
-  MaxBuilder() {}
-  MaxBuilder(size_t cap) { vec_.resize(cap, (T) 0); }
+  using T = typename PropGetterT::prop_element_t;
+  MaxBuilder(PropGetterT&& prop_getter)
+      : prop_getter_(std::move(prop_getter)) {}
 
   // insert tuple at index ind.
   template <typename IND_ELE_TUPLE, typename DATA_TUPLE_T>
-  void insert(size_t ind, const IND_ELE_TUPLE& tuple,
+  bool insert(size_t ind, const IND_ELE_TUPLE& tuple,
               const DATA_TUPLE_T& data) {
     const auto& cur_ind_ele = gs::get_from_tuple<tag_id>(tuple);
+    if (IsNull(cur_ind_ele)) {
+      return false;
+    }
     // just count times.
     while (vec_.size() <= ind) {
-      vec_.emplace_back(std::numeric_limits<T>::max());
+      vec_.emplace_back(std::numeric_limits<T>::min());
     }
-    vec_[ind] = std::max(vec_[ind], std::get<1>(cur_ind_ele));
+    vec_[ind] = std::max(vec_[ind], prop_getter_.get_view(cur_ind_ele));
+    return true;
   }
 
   Collection<T> Build() {
-    // VLOG(10) << "Finish building counter" << gs::to_string(vec_);
+    // iterate over vec_, turn -inf to inf
+    for (auto& ele : vec_) {
+      if (ele == std::numeric_limits<T>::min()) {
+        ele = NullRecordCreator<T>::GetNull();
+      }
+    }
     return Collection<T>(std::move(vec_));
   }
 
  private:
+  PropGetterT prop_getter_;
   std::vector<T> vec_;
 };
 
@@ -841,16 +1000,17 @@ class FirstBuilder<GI, Collection<C_T>, grape::EmptyType, tag_id> {
   }
 
   template <typename IND_ELE_T, typename DATA_ELE_T>
-  void insert(size_t ind, const IND_ELE_T& tuple,
+  bool insert(size_t ind, const IND_ELE_T& tuple,
               const DATA_ELE_T& data_tuple) {
     if (ind < vec_.size()) {
-      return;
+      return false;
     } else if (ind == vec_.size()) {
       vec_.emplace_back(std::get<1>(gs::get_from_tuple<tag_id>(tuple)));
     } else {
       LOG(FATAL) << "Can not insert with ind: " << ind
                  << ", which cur size is : " << vec_.size();
     }
+    return true;
   }
 
   Collection<C_T> Build() { return Collection<C_T>(std::move(vec_)); }
@@ -872,16 +1032,17 @@ class FirstBuilder<GI, RowVertexSetImpl<LabelT, VID_T, OLD_T...>,
       : builder_(set.GetLabel(), set.GetPropNames()) {}
 
   template <typename IND_ELE_T, typename DATA_TUPLE_T>
-  void insert(size_t ind, const IND_ELE_T& tuple,
+  bool insert(size_t ind, const IND_ELE_T& tuple,
               const DATA_TUPLE_T& data_tuple) {
     if (ind < builder_.Size()) {
-      return;
+      return false;
     } else if (ind == builder_.Size()) {
       builder_.Insert(tuple, data_tuple);
     } else {
       LOG(FATAL) << "Can not insert with ind: " << ind
                  << ", which cur size is : " << builder_.size();
     }
+    return true;
   }
 
   set_t Build() { return builder_.Build(); }
@@ -901,16 +1062,17 @@ class FirstBuilder<GI, RowVertexSetImpl<LabelT, VID_T, grape::EmptyType>,
       : builder_(set.GetLabel(), set.GetPropNames()) {}
 
   template <typename IND_ELE_T, typename DATA_TUPLE_T>
-  void insert(size_t ind, const IND_ELE_T& tuple,
+  bool insert(size_t ind, const IND_ELE_T& tuple,
               const DATA_TUPLE_T& data_tuple) {
     if (ind < builder_.Size()) {
-      return;
+      return false;
     } else if (ind == builder_.Size()) {
       builder_.Insert(tuple);
     } else {
       LOG(FATAL) << "Can not insert with ind: " << ind
                  << ", which cur size is : " << builder_.Size();
     }
+    return true;
   }
 
   set_t Build() { return builder_.Build(); }
@@ -932,16 +1094,17 @@ class FirstBuilder<GI, TwoLabelVertexSetImpl<VID_T, LabelT, grape::EmptyType>,
       : builder_(set.Size(), set.GetLabels()) {}
 
   template <typename IND_ELE_T, typename DATA_TUPLE_T>
-  void insert(size_t ind, const IND_ELE_T& tuple,
+  bool insert(size_t ind, const IND_ELE_T& tuple,
               const DATA_TUPLE_T& data_tuple) {
     if (ind < builder_.Size()) {
-      return;
+      return false;
     } else if (ind == builder_.Size()) {
       builder_.Insert(gs::get_from_tuple<tag_id>(tuple));
     } else {
       LOG(FATAL) << "Can not insert with ind: " << ind
                  << ", which cur size is : " << builder_.Size();
     }
+    return true;
   }
 
   set_t Build() { return builder_.Build(); }
@@ -963,10 +1126,43 @@ class FirstBuilder<GI, TwoLabelVertexSetImpl<VID_T, LabelT, T...>,
       : builder_(set.Size(), set.GetLabels()) {}
 
   template <typename IND_ELE_T, typename DATA_TUPLE_T>
-  void insert(size_t ind, const IND_ELE_T& tuple,
+  bool insert(size_t ind, const IND_ELE_T& tuple,
               const DATA_TUPLE_T& data_tuple) {
     if (ind < builder_.Size()) {
-      return;
+      return false;
+    } else if (ind == builder_.Size()) {
+      builder_.Insert(gs::get_from_tuple<tag_id>(tuple),
+                      gs::get_from_tuple<tag_id>(data_tuple));
+    } else {
+      LOG(FATAL) << "Can not insert with ind: " << ind
+                 << ", which cur size is : " << builder_.Size();
+    }
+    return true;
+  }
+
+  set_t Build() { return builder_.Build(); }
+
+ private:
+  builder_t builder_;
+};
+
+template <typename GI, typename VID_T, typename LabelT, typename... T,
+          int tag_id>
+class FirstBuilder<GI, GeneralVertexSet<VID_T, LabelT, T...>, grape::EmptyType,
+                   tag_id> {
+ public:
+  using set_t = GeneralVertexSet<VID_T, LabelT, T...>;
+  using builder_t = GeneralVertexSetBuilder<VID_T, LabelT, T...>;
+  FirstBuilder(const set_t& set, const GI& graph,
+               PropNameArray<grape::EmptyType> prop_names)
+      // we should use a size which indicate the context size
+      : builder_(set.Size(), set.GetLabels()) {}
+
+  template <typename IND_ELE_T, typename DATA_TUPLE_T>
+  bool insert(size_t ind, const IND_ELE_T& tuple,
+              const DATA_TUPLE_T& data_tuple) {
+    if (ind < builder_.Size()) {
+      return false;
     } else if (ind == builder_.Size()) {
       builder_.Insert(gs::get_from_tuple<tag_id>(tuple),
                       gs::get_from_tuple<tag_id>(data_tuple));
@@ -974,6 +1170,7 @@ class FirstBuilder<GI, TwoLabelVertexSetImpl<VID_T, LabelT, T...>,
       LOG(FATAL) << "Can not insert with ind: " << ind
                  << ", which cur size is : " << builder_.Size();
     }
+    return true;
   }
 
   set_t Build() { return builder_.Build(); }
@@ -1006,31 +1203,33 @@ class CollectionOfSetBuilder<
 
   // insert tuple at index ind.
   template <typename IND_TUPLE>
-  void insert(size_t ind, IND_TUPLE& tuple) {
+  bool insert(size_t ind, IND_TUPLE& tuple) {
     while (vec_.size() <= ind) {
       vec_.emplace_back(std::vector<T>());
     }
     auto cur = gs::get_from_tuple<tag_id>(tuple);
     using ele_t = typename set_t::index_ele_tuple_t;
     if (NullRecordCreator<ele_t>::GetNull() == cur) {
-      return;
+      return false;
     }
 
     vec_[ind].emplace_back(prop_getter_.get_view(cur));
+    return true;
   }
 
   template <typename IND_TUPLE, typename DATA_TUPLE>
-  void insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
     while (vec_.size() <= ind) {
       vec_.emplace_back(std::vector<T>());
     }
     auto cur = gs::get_from_tuple<tag_id>(tuple);
     using ele_t = typename set_t::index_ele_tuple_t;
     if (NullRecordCreator<ele_t>::GetNull() == cur) {
-      return;
+      return false;
     }
 
     vec_[ind].emplace_back(prop_getter_.get_view(cur));
+    return true;
   }
 
   CollectionOfVec<T> Build() {
@@ -1060,7 +1259,7 @@ class CollectionOfVecBuilder<T, GI, Collection<T>, tag_id> {
 
   // insert tuple at index ind.
   template <typename IND_TUPLE>
-  void insert(size_t ind, IND_TUPLE& tuple) {
+  bool insert(size_t ind, IND_TUPLE& tuple) {
     auto cur = std::get<1>(gs::get_from_tuple<tag_id>(tuple));
     // just count times.
     while (vec_.size() <= ind) {
@@ -1068,14 +1267,15 @@ class CollectionOfVecBuilder<T, GI, Collection<T>, tag_id> {
     }
     using input_ele_t = typename std::remove_reference<decltype(cur)>::type;
     if (NullRecordCreator<input_ele_t>::GetNull() == cur) {
-      return;
+      return false;
     }
     // emplace the element to vector
     vec_[ind].emplace_back(cur);
+    return true;
   }
 
   template <typename IND_TUPLE, typename DATA_TUPLE>
-  void insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+  bool insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
     return insert(ind, tuple);
   }
 
@@ -1105,7 +1305,7 @@ class CollectionOfVecBuilder<PropT, GRAPH_INTERFACE,
 
   // insert tuple at index ind.
   template <typename IND_TUPLE>
-  void insert(size_t ind, IND_TUPLE& tuple) {
+  bool insert(size_t ind, IND_TUPLE& tuple) {
     auto cur = gs::get_from_tuple<tag_id>(tuple);
     // just count times.
     while (vec_.size() <= ind) {
@@ -1113,10 +1313,145 @@ class CollectionOfVecBuilder<PropT, GRAPH_INTERFACE,
     }
     using input_ele_t = typename std::remove_reference<decltype(cur)>::type;
     if (NullRecordCreator<input_ele_t>::GetNull() == cur) {
-      return;
+      return false;
+    }
+    // emplace the element to vector
+    vec_[ind].emplace_back(prop_getter_.get_view(cur));
+    return true;
+  }
+
+  template <typename IND_TUPLE, typename DATA_TUPLE>
+  bool insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+    return insert(ind, tuple);
+  }
+
+  CollectionOfVec<PropT> Build() {
+    return CollectionOfVec<PropT>(std::move(vec_));
+  }
+
+ private:
+  std::vector<std::vector<PropT>> vec_;
+  PROP_GETTER_T prop_getter_;
+};
+
+template <typename GRAPH_INTERFACE, typename LabelT, typename VID_T,
+          typename... VERTEX_SET_TT, int tag_id>
+class CollectionOfVecBuilder<grape::EmptyType, GRAPH_INTERFACE,
+                             RowVertexSetImpl<LabelT, VID_T, VERTEX_SET_TT...>,
+                             tag_id> {
+ public:
+  using set_t = RowVertexSetImpl<LabelT, VID_T, VERTEX_SET_TT...>;
+  CollectionOfVecBuilder(LabelT label) : label_(label) {}
+
+  // insert tuple at index ind.
+  template <typename IND_TUPLE>
+  bool insert(size_t ind, IND_TUPLE& tuple) {
+    auto cur = gs::get_from_tuple<tag_id>(tuple);
+    // just count times.
+    while (vec_.size() <= ind) {
+      vec_.emplace_back(std::vector<GlobalId>());
+    }
+    using input_ele_t = typename std::remove_reference<decltype(cur)>::type;
+    if (NullRecordCreator<input_ele_t>::GetNull() == cur) {
+      return false;
+    }
+    // emplace the element to vector
+    vec_[ind].emplace_back(GlobalId(label_, cur));
+    return true;
+  }
+
+  template <typename IND_TUPLE, typename DATA_TUPLE>
+  bool insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+    return insert(ind, tuple);
+  }
+
+  CollectionOfVec<GlobalId> Build() {
+    return CollectionOfVec<GlobalId>(std::move(vec_));
+  }
+
+ private:
+  LabelT label_;
+  std::vector<std::vector<GlobalId>> vec_;
+};
+
+template <typename GRAPH_INTERFACE, typename LabelT, typename VID_T,
+          typename... VERTEX_SET_TT, int tag_id>
+class CollectionOfVecBuilder<
+    grape::EmptyType, GRAPH_INTERFACE,
+    TwoLabelVertexSetImpl<VID_T, LabelT, VERTEX_SET_TT...>, tag_id> {
+ public:
+  using set_t = TwoLabelVertexSetImpl<VID_T, LabelT, VERTEX_SET_TT...>;
+  CollectionOfVecBuilder(std::array<LabelT, 2> labels) : labels_(labels) {}
+
+  // insert tuple at index ind.
+  template <typename IND_TUPLE>
+  bool insert(size_t ind, IND_TUPLE& tuple) {
+    auto cur = gs::get_from_tuple<tag_id>(tuple);
+    // just count times.
+    while (vec_.size() <= ind) {
+      vec_.emplace_back(std::vector<GlobalId>());
+    }
+    using input_ele_t = typename std::remove_reference<decltype(cur)>::type;
+    if (NullRecordCreator<input_ele_t>::GetNull() == cur) {
+      return false;
+    }
+    // emplace the element to vector
+    auto label_ind = std::get<1>(cur);
+    CHECK(label_ind < 2);
+    vec_[ind].emplace_back(GlobalId(labels_[label_ind], std::get<2>(cur)));
+    return true;
+  }
+
+  template <typename IND_TUPLE, typename DATA_TUPLE>
+  bool insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+    return insert(ind, tuple);
+  }
+
+  CollectionOfVec<GlobalId> Build() {
+    return CollectionOfVec<GlobalId>(std::move(vec_));
+  }
+
+ private:
+  std::array<LabelT, 2> labels_;
+  std::vector<std::vector<GlobalId>> vec_;
+};
+
+template <typename PropT, typename GRAPH_INTERFACE, typename LabelT,
+          typename VID_T, typename... VERTEX_SET_TT, int tag_id>
+class CollectionOfVecBuilder<
+    PropT, GRAPH_INTERFACE,
+    TwoLabelVertexSetImpl<VID_T, LabelT, VERTEX_SET_TT...>, tag_id> {
+ public:
+  using set_t = TwoLabelVertexSetImpl<VID_T, LabelT, VERTEX_SET_TT...>;
+  using graph_prop_getter_t =
+      typename GRAPH_INTERFACE::template single_prop_getter_t<PropT>;
+  using PROP_GETTER_T =
+      TwoLabelVertexSetImplPropGetter<tag_id, graph_prop_getter_t,
+                                      typename set_t::index_ele_tuple_t>;
+  CollectionOfVecBuilder(const set_t& set, const GRAPH_INTERFACE& graph,
+                         PropNameArray<PropT>& prop_names)
+      : prop_getter_(create_prop_getter_impl<tag_id, PropT>(set, graph,
+                                                            prop_names[0])) {}
+  // insert tuple at index ind.
+  template <typename IND_TUPLE>
+  bool insert(size_t ind, IND_TUPLE& tuple) {
+    auto cur = gs::get_from_tuple<tag_id>(tuple);
+    // just count times.
+    while (vec_.size() <= ind) {
+      vec_.emplace_back(std::vector<PropT>());
+    }
+    using input_ele_t = typename std::remove_reference<decltype(cur)>::type;
+    if (NullRecordCreator<input_ele_t>::GetNull() == cur) {
+      return false;
     }
     // emplace the element to vector
     vec_[ind].emplace_back(prop_getter_.get_view(cur));
+    return true;
+  }
+
+  template <typename IND_TUPLE, typename DATA_TUPLE>
+  void insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+    return insert(ind, tuple);
   }
 
   CollectionOfVec<PropT> Build() {
@@ -1124,7 +1459,121 @@ class CollectionOfVecBuilder<PropT, GRAPH_INTERFACE,
   }
 
  private:
+  PROP_GETTER_T prop_getter_;
   std::vector<std::vector<PropT>> vec_;
+};
+
+// Aggregating for average.
+template <typename PropT, typename GRAPH_INTERFACE, typename SET_T, int tag_id>
+class AvgBuilder;
+template <typename PropT, typename GRAPH_INTERFACE, typename LabelT,
+          typename VID_T, typename... VERTEX_SET_TT, int tag_id>
+class AvgBuilder<PropT, GRAPH_INTERFACE,
+                 TwoLabelVertexSetImpl<VID_T, LabelT, VERTEX_SET_TT...>,
+                 tag_id> {
+ public:
+  using set_t = TwoLabelVertexSetImpl<VID_T, LabelT, VERTEX_SET_TT...>;
+  using index_ele_t = typename set_t::index_ele_tuple_t;
+  using graph_prop_getter_t =
+      typename GRAPH_INTERFACE::template single_prop_getter_t<PropT>;
+  using PROP_GETTER_T =
+      TwoLabelVertexSetImplPropGetter<tag_id, graph_prop_getter_t,
+                                      typename set_t::index_ele_tuple_t>;
+
+  AvgBuilder(const set_t& set, const GRAPH_INTERFACE& graph,
+             PropNameArray<PropT> prop_names)
+      : prop_getter_(create_prop_getter_impl<tag_id, PropT>(set, graph,
+                                                            prop_names[0])) {}
+
+  template <typename IND_TUPLE, typename DATA_TUPLE>
+  bool insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+    while (vec_.size() <= ind) {
+      vec_.emplace_back(std::pair<size_t, PropT>(0, (PropT) 0));
+    }
+    auto cur = gs::get_from_tuple<tag_id>(tuple);
+    using ele_t = typename set_t::index_ele_tuple_t;
+    if (NullRecordCreator<ele_t>::GetNull() == cur) {
+      return false;
+    }
+    auto prop = prop_getter_.get_view(cur);
+    if (IsNull(prop)) {
+      return false;
+    }
+    vec_[ind].first += 1;
+    vec_[ind].second += prop;
+    return true;
+  }
+
+  Collection<PropT> Build() {
+    std::vector<PropT> res;
+    res.reserve(vec_.size());
+    for (auto& pair : vec_) {
+      if (pair.first == 0) {
+        res.emplace_back(NullRecordCreator<PropT>::GetNull());
+      } else {
+        res.emplace_back(pair.second / pair.first);
+      }
+    }
+    return Collection<PropT>(std::move(res));
+  }
+
+ private:
+  std::vector<std::pair<size_t, PropT>> vec_;
+  PROP_GETTER_T prop_getter_;
+};
+
+template <typename PropT, typename GRAPH_INTERFACE, typename LabelT,
+          typename VID_T, typename... VERTEX_SET_TT, int tag_id>
+class AvgBuilder<PropT, GRAPH_INTERFACE,
+                 RowVertexSetImpl<LabelT, VID_T, VERTEX_SET_TT...>, tag_id> {
+ public:
+  using set_t = RowVertexSetImpl<LabelT, VID_T, VERTEX_SET_TT...>;
+  using index_ele_t = typename set_t::index_ele_tuple_t;
+  using graph_prop_getter_t =
+      typename GRAPH_INTERFACE::template single_prop_getter_t<PropT>;
+  using PROP_GETTER_T =
+      RowVertexSetPropGetter<tag_id, graph_prop_getter_t,
+                             typename set_t::index_ele_tuple_t>;
+
+  AvgBuilder(const set_t& set, const GRAPH_INTERFACE& graph,
+             PropNameArray<PropT> prop_names)
+      : prop_getter_(create_prop_getter_impl<tag_id, PropT>(set, graph,
+                                                            prop_names[0])) {}
+
+  template <typename IND_TUPLE, typename DATA_TUPLE>
+  bool insert(size_t ind, const IND_TUPLE& tuple, const DATA_TUPLE& data) {
+    while (vec_.size() <= ind) {
+      vec_.emplace_back(std::pair<size_t, PropT>(0, (PropT) 0));
+    }
+    auto cur = gs::get_from_tuple<tag_id>(tuple);
+    using ele_t = typename set_t::index_ele_tuple_t;
+    if (NullRecordCreator<ele_t>::GetNull() == cur) {
+      return false;
+    }
+    auto prop = prop_getter_.get_view(cur);
+    if (IsNull(prop)) {
+      return false;
+    }
+    vec_[ind].first += 1;
+    vec_[ind].second += prop;
+    return true;
+  }
+
+  Collection<PropT> Build() {
+    std::vector<PropT> res;
+    res.reserve(vec_.size());
+    for (auto& pair : vec_) {
+      if (pair.first == 0) {
+        res.emplace_back(NullRecordCreator<PropT>::GetNull());
+      } else {
+        res.emplace_back(pair.second / pair.first);
+      }
+    }
+    return Collection<PropT>(std::move(res));
+  }
+
+ private:
+  std::vector<std::pair<size_t, PropT>> vec_;
   PROP_GETTER_T prop_getter_;
 };
 
diff --git a/flex/engines/hqps_db/structures/multi_edge_set/flat_edge_set.h b/flex/engines/hqps_db/structures/multi_edge_set/flat_edge_set.h
index 870133b25d2d..10ef6dfee768 100644
--- a/flex/engines/hqps_db/structures/multi_edge_set/flat_edge_set.h
+++ b/flex/engines/hqps_db/structures/multi_edge_set/flat_edge_set.h
@@ -44,18 +44,19 @@ class FlatEdgeSetBuilder {
   static constexpr bool is_row_vertex_set_builder = false;
   static constexpr bool is_general_edge_set_builder = false;
   static constexpr bool is_two_label_set_builder = false;
+  static constexpr bool is_general_vertex_set_builder = false;
 
   FlatEdgeSetBuilder(
-      std::vector<std::array<LabelT, 3>>&& label_triplet,
+      const std::vector<std::array<LabelT, 3>>& label_triplet,
       std::vector<std::vector<std::string>> prop_names,
       const std::vector<uint8_t>&
           label_triplet_ind,  // label_triplet_ind is needed to create
                               // new label_triplet_ind with index_ele
-      Direction direction)
-      : label_triplet_(std::move(label_triplet)),
+      std::vector<Direction> directions)
+      : label_triplet_(label_triplet),
         prop_names_(prop_names),
         label_triplet_ind_(label_triplet_ind),
-        direction_(direction) {}
+        directions_(directions) {}
 
   // There could be null record.
   void Insert(const index_ele_tuple_t& tuple) {
@@ -68,8 +69,8 @@ class FlatEdgeSetBuilder {
   }
 
   result_t Build() {
-    return result_t(std::move(vec_), label_triplet_, prop_names_,
-                    std::move(label_triplet_ind_new_), direction_);
+    return result_t(std::move(vec_), std::move(label_triplet_), prop_names_,
+                    std::move(label_triplet_ind_new_), std::move(directions_));
   }
 
  private:
@@ -78,7 +79,7 @@ class FlatEdgeSetBuilder {
   std::vector<std::vector<std::string>> prop_names_;
   std::vector<uint8_t> label_triplet_ind_;
   std::vector<uint8_t> label_triplet_ind_new_;
-  Direction direction_;
+  std::vector<Direction> directions_;
 };
 
 template <typename VID_T, typename LabelT, typename EDATA_T>
@@ -88,17 +89,29 @@ class FlatEdgeSetIter {
   using self_type_t = FlatEdgeSetIter<VID_T, LabelT, EDATA_T>;
   using index_ele_tuple_t = std::tuple<size_t, ele_tuple_t>;
   using data_tuple_t = ele_tuple_t;
-  FlatEdgeSetIter(const std::vector<ele_tuple_t>& vec, size_t ind,
+  FlatEdgeSetIter(const std::vector<Direction>& dirs,
+                  const std::vector<ele_tuple_t>& vec, size_t ind,
                   const std::vector<uint8_t>& label_triplet_ind,
                   const std::vector<std::array<LabelT, 3>>& label_triplet,
                   const std::vector<std::vector<std::string>>& prop_names)
-      : vec_(vec),
+      : dirs_(dirs),
+        vec_(vec),
         ind_(ind),
         label_triplet_(label_triplet),
         label_triplet_ind_(label_triplet_ind),
         prop_names_(prop_names) {}
 
-  ele_tuple_t GetElement() const { return vec_[ind_]; }
+  ele_tuple_t GetElement() const {
+    auto dir = dirs_[label_triplet_ind_[ind_]];
+    if (dir == Direction::Out) {
+      return vec_[ind_];
+    } else if (dir == Direction::In) {
+      return std::make_tuple(std::get<1>(vec_[ind_]), std::get<0>(vec_[ind_]),
+                             std::get<2>(vec_[ind_]));
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
+  }
 
   index_ele_tuple_t GetIndexElement() const {
     return std::make_tuple(ind_, GetElement());
@@ -107,16 +120,48 @@ class FlatEdgeSetIter {
     return label_triplet_[label_triplet_ind_[ind_]][2];
   }
 
-  VID_T GetSrc() const { return std::get<0>(vec_[ind_]); }
+  VID_T GetSrc() const {
+    auto dir = dirs_[label_triplet_ind_[ind_]];
+    if (dir == Direction::Out) {
+      return std::get<0>(vec_[ind_]);
+    } else if (dir == Direction::In) {
+      return std::get<1>(vec_[ind_]);
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
+  }
 
   LabelT GetSrcLabel() const {
-    return label_triplet_[label_triplet_ind_[ind_]][0];
+    auto dir = dirs_[label_triplet_ind_[ind_]];
+    if (dir == Direction::Out) {
+      return label_triplet_[label_triplet_ind_[ind_]][0];
+    } else if (dir == Direction::In) {
+      return label_triplet_[label_triplet_ind_[ind_]][1];
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
   }
 
-  VID_T GetDst() const { return std::get<1>(vec_[ind_]); }
+  VID_T GetDst() const {
+    auto dir = dirs_[label_triplet_ind_[ind_]];
+    if (dir == Direction::Out) {
+      return std::get<1>(vec_[ind_]);
+    } else if (dir == Direction::In) {
+      return std::get<0>(vec_[ind_]);
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
+  }
 
   LabelT GetDstLabel() const {
-    return label_triplet_[label_triplet_ind_[ind_]][1];
+    auto dir = dirs_[label_triplet_ind_[ind_]];
+    if (dir == Direction::Out) {
+      return label_triplet_[label_triplet_ind_[ind_]][1];
+    } else if (dir == Direction::In) {
+      return label_triplet_[label_triplet_ind_[ind_]][0];
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
   }
 
   const EDATA_T& GetData() const { return std::get<2>(vec_[ind_]); }
@@ -149,11 +194,12 @@ class FlatEdgeSetIter {
   inline const self_type_t* operator->() const { return this; }
 
  private:
+  const std::vector<Direction>& dirs_;
   const std::vector<ele_tuple_t>& vec_;
-  const std::vector<uint8_t>& label_triplet_ind_;
+  size_t ind_;
   const std::vector<std::array<LabelT, 3>>& label_triplet_;
+  const std::vector<uint8_t>& label_triplet_ind_;
   const std::vector<std::vector<std::string>>& prop_names_;
-  size_t ind_;
 };
 
 // for FlatEdgeSet with only one label triplet, we should use SingleLabelEdgeSet
@@ -178,29 +224,58 @@ class FlatEdgeSet {
               std::vector<std::array<LabelT, 3>>&& label_triplet,
               std::vector<std::vector<std::string>> prop_names,
               std::vector<uint8_t>&& label_triplet_ind, Direction direction)
+      : vec_(std::move(vec)),
+        label_triplet_(std::move(label_triplet)),
+        prop_names_(prop_names),
+        label_triplet_ind_(std::move(label_triplet_ind)) {
+    CHECK(label_triplet_ind_.size() == vec_.size());
+    CHECK(prop_names_.size() == label_triplet_.size());
+    if (label_triplet_.size() > 0) {
+      // make sure label_triplet_ind_ is valid
+      uint8_t max_ind = 0;
+      for (size_t i = 0; i < label_triplet_ind_.size(); ++i) {
+        max_ind = std::max(max_ind, label_triplet_ind_[i]);
+      }
+      CHECK(max_ind < label_triplet_.size())
+          << "max_ind: " << max_ind
+          << ", label_triplet_.size(): " << label_triplet_.size();
+    }
+    directions_.resize(label_triplet_.size(), direction);
+  }
+
+  FlatEdgeSet(std::vector<ele_tuple_t>&& vec,
+              std::vector<std::array<LabelT, 3>>&& label_triplet,
+              std::vector<std::vector<std::string>> prop_names,
+              std::vector<uint8_t>&& label_triplet_ind,
+              std::vector<Direction>&& direction)
       : vec_(std::move(vec)),
         label_triplet_(std::move(label_triplet)),
         prop_names_(prop_names),
         label_triplet_ind_(std::move(label_triplet_ind)),
-        direction_(direction) {
+        directions_(std::move(direction)) {
     CHECK(label_triplet_ind_.size() == vec_.size());
     CHECK(prop_names_.size() == label_triplet_.size());
-    uint8_t max_ind = 0;
-    for (size_t i = 0; i < label_triplet_ind_.size(); ++i) {
-      max_ind = std::max(max_ind, label_triplet_ind_[i]);
+    CHECK(directions_.size() == label_triplet_.size());
+    if (label_triplet_.size() > 0) {
+      // make sure label_triplet_ind_ is valid
+      uint8_t max_ind = 0;
+      for (size_t i = 0; i < label_triplet_ind_.size(); ++i) {
+        max_ind = std::max(max_ind, label_triplet_ind_[i]);
+      }
+      CHECK(max_ind < label_triplet_.size())
+          << "max_ind: " << max_ind
+          << ", label_triplet_.size(): " << label_triplet_.size();
     }
-    CHECK(max_ind < label_triplet_.size())
-        << "max_ind: " << max_ind
-        << ", label_triplet_.size(): " << label_triplet_.size();
   }
 
   iterator begin() const {
-    return iterator(vec_, 0, label_triplet_ind_, label_triplet_, prop_names_);
+    return iterator(directions_, vec_, 0, label_triplet_ind_, label_triplet_,
+                    prop_names_);
   }
 
   iterator end() const {
-    return iterator(vec_, vec_.size(), label_triplet_ind_, label_triplet_,
-                    prop_names_);
+    return iterator(directions_, vec_, vec_.size(), label_triplet_ind_,
+                    label_triplet_, prop_names_);
   }
 
   std::vector<LabelKey> GetLabelVec() const {
@@ -227,8 +302,10 @@ class FlatEdgeSet {
           label_triplet_ind_[std::get<0>(cur_ind_ele)]);
     }
     auto copied_label_triplet = label_triplet_;
+    auto copied_directions = directions_;
     return FlatEdgeSet(std::move(res), std::move(copied_label_triplet),
-                       prop_names_, std::move(label_triplet_ind), direction_);
+                       prop_names_, std::move(label_triplet_ind),
+                       std::move(copied_directions));
   }
 
   void fillBuiltinPropsImpl(std::vector<EDATA_T>& tuples,
@@ -280,7 +357,8 @@ class FlatEdgeSet {
 
   size_t Size() const { return vec_.size(); }
 
-  template <size_t num_labels, typename FILTER_T>
+  template <size_t num_labels, typename FILTER_T,
+            typename std::enable_if<(num_labels != 1)>::type* = nullptr>
   std::pair<GeneralVertexSet<VID_T, LabelT, grape::EmptyType>,
             std::vector<size_t>>
   GetVertices(VOpt v_opt, const std::array<LabelT, num_labels>& labels,
@@ -296,10 +374,23 @@ class FlatEdgeSet {
     {
       std::unordered_set<LabelT> valid_label_set;
       for (size_t i = 0; i < label_triplet_.size(); ++i) {
+        auto dir = directions_[i];
         if (v_opt == VOpt::Start) {
-          valid_label_set.emplace(label_triplet_[i][0]);
+          if (dir == Direction::Out) {
+            valid_label_set.emplace(label_triplet_[i][0]);
+          } else if (dir == Direction::In) {
+            valid_label_set.emplace(label_triplet_[i][1]);
+          } else {
+            LOG(FATAL) << "Unknown direction: " << dir;
+          }
         } else if (v_opt == VOpt::End || v_opt == VOpt::Other) {
-          valid_label_set.emplace(label_triplet_[i][1]);
+          if (dir == Direction::Out) {
+            valid_label_set.emplace(label_triplet_[i][1]);
+          } else if (dir == Direction::In) {
+            valid_label_set.emplace(label_triplet_[i][0]);
+          } else {
+            LOG(FATAL) << "Unknown direction: " << dir;
+          }
         } else {
           LOG(FATAL) << "Unknown v_opt: " << v_opt;
         }
@@ -332,22 +423,21 @@ class FlatEdgeSet {
         label_to_ind[i] = i;
       }
     }
-    size_t cur_ind = 0;
     for (auto iter : *this) {
       label_t cur_label;
       VID_T cur_vid;
       if (v_opt == VOpt::Start) {
-        cur_label = label_triplet_[iter.GetIndex()][0];
+        cur_label = iter.GetSrcLabel();
         cur_vid = iter.GetSrc();
       } else if (v_opt == VOpt::End || v_opt == VOpt::Other) {
-        cur_label = label_triplet_[iter.GetIndex()][1];
+        cur_label = iter.GetDstLabel();
         cur_vid = iter.GetDst();
       } else {
         LOG(FATAL) << "Unknown v_opt: " << v_opt;
       }
       auto ind = label_to_ind[cur_label];
       if (ind != -1) {
-        res_bitset[ind].set_bit(cur_ind);
+        res_bitset[ind].set_bit(vids.size());
         vids.emplace_back(cur_vid);
       }
       offsets.emplace_back(vids.size());
@@ -362,6 +452,38 @@ class FlatEdgeSet {
     return std::make_pair(std::move(set), std::move(offsets));
   }
 
+  template <size_t num_labels, typename FILTER_T,
+            typename std::enable_if<(num_labels == 1)>::type* = nullptr>
+  auto GetVertices(VOpt v_opt, const std::array<LabelT, num_labels>& labels,
+                   FILTER_T& expr) const {
+    std::vector<offset_t> offsets;
+    std::vector<VID_T> vids;
+    offsets.reserve(Size());
+    offsets.emplace_back(0);
+    label_t req_label = labels[0];
+
+    for (auto iter : *this) {
+      label_t cur_label;
+      VID_T cur_vid;
+      if (v_opt == VOpt::Start) {
+        cur_label = iter.GetSrcLabel();
+        cur_vid = iter.GetSrc();
+      } else if (v_opt == VOpt::End || v_opt == VOpt::Other) {
+        cur_label = iter.GetDstLabel();
+        cur_vid = iter.GetDst();
+      } else {
+        LOG(FATAL) << "Unknown v_opt: " << v_opt;
+      }
+      if (cur_label == req_label) {
+        vids.emplace_back(cur_vid);
+      }
+      offsets.emplace_back(vids.size());
+    }
+
+    auto set = make_default_row_vertex_set(std::move(vids), req_label);
+    return std::make_pair(std::move(set), std::move(offsets));
+  }
+
   // implement ProjectWithRepeatArray
   template <int tag_id, int Fs,
             typename std::enable_if<Fs == -1>::type* = nullptr>
@@ -386,10 +508,11 @@ class FlatEdgeSet {
       }
     }
     auto copied_label_triplet = label_triplet_;
+    auto copied_directions = directions_;
 
     return self_type_t(std::move(new_vec), std::move(copied_label_triplet),
                        prop_names_, std::move(new_label_triplet_ind),
-                       direction_);
+                       std::move(copied_directions));
   }
 
   void SubSetWithIndices(std::vector<size_t>& indices) {
@@ -427,7 +550,7 @@ class FlatEdgeSet {
 
   builder_t CreateBuilder() const {
     return builder_t(label_triplet_, prop_names_, label_triplet_ind_,
-                     direction_);
+                     directions_);
   }
 
  private:
@@ -435,7 +558,8 @@ class FlatEdgeSet {
   std::vector<std::array<label_t, 3>> label_triplet_;
   std::vector<std::vector<std::string>> prop_names_;
   std::vector<uint8_t> label_triplet_ind_;
-  Direction direction_;
+  // Direction direction_;
+  std::vector<Direction> directions_;
 };
 
 template <typename VID_T, typename LabelT, typename EDATA_T>
@@ -448,31 +572,32 @@ class SingleLabelEdgeSetBuilder {
   static constexpr bool is_row_vertex_set_builder = false;
   static constexpr bool is_general_edge_set_builder = false;
   static constexpr bool is_two_label_set_builder = false;
+  static constexpr bool is_general_vertex_set_builder = false;
 
   SingleLabelEdgeSetBuilder(const std::array<LabelT, 3>& label_triplet,
                             std::vector<std::string> prop_names,
-                            Direction direction)
+                            const std::vector<Direction>& direction)
       : label_triplet_(label_triplet),
         prop_names_(prop_names),
-        direction_(direction) {}
+        old_direction_(direction) {}
 
   // There could be null record.
   void Insert(const index_ele_tuple_t& tuple) {
     vec_.push_back(std::get<1>(tuple));
+    new_directions.push_back(old_direction_[std::get<0>(tuple)]);
   }
 
-  void Insert(const ele_tuple_t& tuple) { vec_.push_back(tuple); }
-
   result_t Build() {
     return result_t(std::move(vec_), std::move(label_triplet_), prop_names_,
-                    direction_);
+                    std::move(new_directions));
   }
 
  private:
   std::vector<ele_tuple_t> vec_;
   std::array<LabelT, 3> label_triplet_;
   std::vector<std::string> prop_names_;
-  Direction direction_;
+  const std::vector<Direction>& old_direction_;
+  std::vector<Direction> new_directions;
 };
 
 template <typename VID_T, typename LabelT, typename EDATA_T>
@@ -482,10 +607,12 @@ class SingleLabelEdgeSetIter {
   using self_type_t = SingleLabelEdgeSetIter<VID_T, LabelT, EDATA_T>;
   using index_ele_tuple_t = std::tuple<size_t, ele_tuple_t>;
   using data_tuple_t = ele_tuple_t;
-  SingleLabelEdgeSetIter(const std::vector<ele_tuple_t>& vec, size_t ind,
+  SingleLabelEdgeSetIter(const std::vector<Direction>& dir,
+                         const std::vector<ele_tuple_t>& vec, size_t ind,
                          const std::array<LabelT, 3>& label_triplet,
                          const std::vector<std::string>& prop_names)
-      : vec_(vec),
+      : dir_(dir),
+        vec_(vec),
         ind_(ind),
         label_triplet_(label_triplet),
         prop_names_(prop_names) {}
@@ -497,13 +624,51 @@ class SingleLabelEdgeSetIter {
   }
   LabelT GetEdgeLabel() const { return label_triplet_[2]; }
 
-  VID_T GetSrc() const { return std::get<0>(vec_[ind_]); }
+  VID_T GetSrc() const {
+    auto dir = get_direction();
+    if (dir == Direction::Out) {
+      return std::get<0>(vec_[ind_]);
+    } else if (dir == Direction::In) {
+      return std::get<1>(vec_[ind_]);
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
+  }
+
+  LabelT GetSrcLabel() const {
+    auto dir = get_direction();
+    if (dir == Direction::Out) {
+      return label_triplet_[0];
+    } else if (dir == Direction::In) {
+      return label_triplet_[1];
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
+  }
 
-  LabelT GetSrcLabel() const { return label_triplet_[0]; }
+  VID_T GetDst() const {
+    auto dir = get_direction();
+    if (dir == Direction::Out) {
+      return std::get<1>(vec_[ind_]);
+    } else if (dir == Direction::In) {
+      return std::get<0>(vec_[ind_]);
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
+  }
 
-  VID_T GetDst() const { return std::get<1>(vec_[ind_]); }
+  VID_T GetOther() const { return std::get<1>(vec_[ind_]); }
 
-  LabelT GetDstLabel() const { return label_triplet_[1]; }
+  LabelT GetDstLabel() const {
+    auto dir = get_direction();
+    if (dir == Direction::Out) {
+      return label_triplet_[1];
+    } else if (dir == Direction::In) {
+      return label_triplet_[0];
+    } else {
+      LOG(FATAL) << "Unknown direction: " << dir;
+    }
+  }
 
   const EDATA_T& GetData() const { return std::get<2>(vec_[ind_]); }
 
@@ -533,6 +698,8 @@ class SingleLabelEdgeSetIter {
   inline const self_type_t* operator->() const { return this; }
 
  private:
+  inline Direction get_direction() const { return dir_[ind_]; }
+  const std::vector<Direction>& dir_;
   const std::vector<ele_tuple_t>& vec_;
   size_t ind_;
   const std::array<LabelT, 3>& label_triplet_;
@@ -550,6 +717,7 @@ class SingleLabelEdgeSet {
   using flat_t = self_type_t;
   using data_tuple_t = ele_tuple_t;
   using builder_t = SingleLabelEdgeSetBuilder<VID_T, LabelT, EDATA_T>;
+  using edata_t = EDATA_T;
 
   static constexpr bool is_multi_label = false;
   static constexpr bool is_collection = false;
@@ -560,17 +728,28 @@ class SingleLabelEdgeSet {
   SingleLabelEdgeSet(std::vector<ele_tuple_t>&& vec,
                      std::array<LabelT, 3>&& label_triplet,
                      std::vector<std::string> prop_names, Direction direction)
+      : vec_(std::move(vec)),
+        label_triplet_(std::move(label_triplet)),
+        prop_names_(prop_names) {
+    directions_.resize(vec_.size(), direction);
+  }
+
+  SingleLabelEdgeSet(std::vector<ele_tuple_t>&& vec,
+                     std::array<LabelT, 3>&& label_triplet,
+                     std::vector<std::string> prop_names,
+                     std::vector<Direction>&& direction)
       : vec_(std::move(vec)),
         label_triplet_(std::move(label_triplet)),
         prop_names_(prop_names),
-        direction_(direction) {}
+        directions_(std::move(direction)) {}
 
   iterator begin() const {
-    return iterator(vec_, 0, label_triplet_, prop_names_);
+    return iterator(directions_, vec_, 0, label_triplet_, prop_names_);
   }
 
   iterator end() const {
-    return iterator(vec_, vec_.size(), label_triplet_, prop_names_);
+    return iterator(directions_, vec_, vec_.size(), label_triplet_,
+                    prop_names_);
   }
 
   std::vector<LabelKey> GetLabelVec() const {
@@ -592,7 +771,7 @@ class SingleLabelEdgeSet {
       res.emplace_back(std::get<1>(cur_ind_ele));
     }
     return SingleLabelEdgeSet(std::move(res), label_triplet_, prop_names_,
-                              direction_);
+                              directions_);
   }
 
   // we assume edata_t is tuple
@@ -668,8 +847,10 @@ class SingleLabelEdgeSet {
         VID_T cur_vid;
         if (v_opt == VOpt::Start) {
           cur_vid = iter.GetSrc();
-        } else if (v_opt == VOpt::End || v_opt == VOpt::Other) {
+        } else if (v_opt == VOpt::End) {
           cur_vid = iter.GetDst();
+        } else if (v_opt == VOpt::Other) {
+          cur_vid = iter.GetOther();
         } else {
           LOG(FATAL) << "Unknown v_opt: " << v_opt;
         }
@@ -704,8 +885,9 @@ class SingleLabelEdgeSet {
     }
 
     auto copy_label_triplet = label_triplet_;
+    auto copied_directions = directions_;
     return self_type_t(std::move(new_vec), std::move(copy_label_triplet),
-                       prop_names_, direction_);
+                       prop_names_, std::move(copied_directions));
   }
 
   void Repeat(std::vector<offset_t>& cur_offset,
@@ -734,14 +916,14 @@ class SingleLabelEdgeSet {
   }
 
   builder_t CreateBuilder() const {
-    return builder_t(label_triplet_, prop_names_, direction_);
+    return builder_t(label_triplet_, prop_names_, directions_);
   }
 
  private:
   std::vector<ele_tuple_t> vec_;
   std::array<label_t, 3> label_triplet_;
   std::vector<std::string> prop_names_;
-  Direction direction_;
+  std::vector<Direction> directions_;
 };
 }  // namespace gs
 
diff --git a/flex/engines/hqps_db/structures/multi_edge_set/general_edge_set.h b/flex/engines/hqps_db/structures/multi_edge_set/general_edge_set.h
index 204c6928a73f..6d959195871e 100644
--- a/flex/engines/hqps_db/structures/multi_edge_set/general_edge_set.h
+++ b/flex/engines/hqps_db/structures/multi_edge_set/general_edge_set.h
@@ -89,14 +89,14 @@ class GeneralEdgeSetBuilder<2, GI, VID_T, LabelT, std::tuple<T...>,
   }
 
  private:
-  std::vector<ele_tuple_t> vec_;
-  std::vector<LabelT> label_vec_;
+  const grape::Bitset& bitset_;
   std::vector<std::string> prop_names_;
   LabelT edge_label_;
   std::array<LabelT, 2> src_labels_;
   LabelT dst_label_;
-  const grape::Bitset& bitset_;
   Direction direction_;
+  std::vector<ele_tuple_t> vec_;
+  std::vector<LabelT> label_vec_;
 };
 
 template <typename GI, typename VID_T, typename LabelT>
@@ -266,13 +266,13 @@ class GeneralEdgeSetIter {
  private:
   const std::vector<VID_T>& vids_;
   const adj_list_array_t& adj_lists_;
-  adj_list_t cur_adj_list_;
   const grape::Bitset& bitsets_;
-  adj_list_iterator begin_, end_;
   const std::array<LabelT, 2>& src_labels_;
   LabelT dst_label_;
   LabelT edge_label_;
   size_t ind_;
+  adj_list_t cur_adj_list_;
+  adj_list_iterator begin_, end_;
 };
 
 template <typename GI, typename VID_T, typename LabelT>
@@ -465,12 +465,11 @@ class GeneralEdgeSet<2, GI, VID_T, LabelT, std::tuple<T...>, std::tuple<T...>> {
                  LabelT edge_label, std::array<LabelT, 2> src_labels,
                  LabelT dst_label, Direction dir)
       : vids_(std::move(vids)),
-
         adj_lists_(std::move(adj_lists)),
         prop_names_(prop_names),
         edge_label_(edge_label),
-        src_labels_(src_labels),
         dst_label_(dst_label),
+        src_labels_(src_labels),
         size_(0),
         dir_(dir) {
     bitsets_.swap(bitsets);
@@ -481,8 +480,8 @@ class GeneralEdgeSet<2, GI, VID_T, LabelT, std::tuple<T...>, std::tuple<T...>> {
         adj_lists_(std::move(other.adj_lists_)),
         prop_names_(other.prop_names_),
         edge_label_(other.edge_label_),
-        src_labels_(std::move(other.src_labels_)),
         dst_label_(other.dst_label_),
+        src_labels_(std::move(other.src_labels_)),
         size_(0),
         dir_(other.dir_) {
     bitsets_.swap(other.bitsets_);
@@ -673,16 +672,14 @@ class GeneralEdgeSet<2, GI, VID_T, LabelT, std::tuple<T...>, std::tuple<T...>> {
   }
 
  private:
-  mutable size_t size_;
-  LabelT edge_label_, dst_label_;
-  std::array<LabelT, 2> src_labels_;
-
-  std::vector<std::string> prop_names_;
-
   std::vector<VID_T> vids_;
   adj_list_array_t adj_lists_;
-  grape::Bitset bitsets_;  // bitset of src vertices.
+  std::vector<std::string> prop_names_;
+  LabelT edge_label_, dst_label_;
+  std::array<LabelT, 2> src_labels_;
+  mutable size_t size_;
   Direction dir_;
+  grape::Bitset bitsets_;  // bitset of src vertices.
 };
 
 // general edge set stores multi src labels but only one dst label
diff --git a/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h b/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h
index fc253e8c77b8..bb799e8b30e4 100644
--- a/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h
+++ b/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h
@@ -74,11 +74,53 @@ class UnTypedEdgeSetIter {
 
   inline LabelT GetEdgeLabel() const { return cur_iter_.GetEdgeLabel(); }
 
-  inline VID_T GetSrc() const { return src_vertices_[vid_ind_]; }
-  inline VID_T GetDst() const { return cur_iter_.GetDstId(); }
+  inline VID_T GetSrc() const {
+    auto dir = get_cur_direction();
+    if (dir == Direction::Out) {
+      return src_vertices_[vid_ind_];
+    } else if (dir == Direction::In) {
+      return cur_iter_.GetDstId();
+    } else {
+      LOG(FATAL) << "Not supported direction " << gs::to_string(dir);
+    }
+  }
+
+  inline VID_T GetDst() const {
+    auto dir = get_cur_direction();
+    if (dir == Direction::Out) {
+      return cur_iter_.GetDstId();
+    } else if (dir == Direction::In) {
+      return src_vertices_[vid_ind_];
+    } else {
+      LOG(FATAL) << "Not supported direction " << gs::to_string(dir);
+    }
+  }
 
-  inline label_t GetDstLabel() const { return cur_iter_.GetDstLabel(); }
-  inline label_t GetSrcLabel() const { return cur_iter_.GetSrcLabel(); }
+  inline label_t GetDstLabel() const {
+    auto dir = get_cur_direction();
+    if (dir == Direction::Out) {
+      return cur_iter_.GetDstLabel();
+    } else if (dir == Direction::In) {
+      return cur_iter_.GetSrcLabel();
+    } else {
+      LOG(FATAL) << "Not supported direction " << gs::to_string(dir);
+    }
+  }
+
+  inline label_t GetSrcLabel() const {
+    auto dir = get_cur_direction();
+    if (dir == Direction::Out) {
+      return cur_iter_.GetSrcLabel();
+    } else if (dir == Direction::In) {
+      return cur_iter_.GetDstLabel();
+    } else {
+      LOG(FATAL) << "Not supported direction " << gs::to_string(dir);
+    }
+  }
+
+  inline label_t GetOtherLabel() const { return cur_iter_.GetDstLabel(); }
+
+  inline VID_T GetOther() const { return cur_iter_.GetDstId(); }
 
   inline Any GetData() const { return cur_iter_.GetData(); }
   inline const std::vector<std::string>& GetPropNames() const {
@@ -123,6 +165,9 @@ class UnTypedEdgeSetIter {
   inline const self_type_t* operator->() const { return this; }
 
  private:
+  inline Direction get_cur_direction() const {
+    return cur_iter_.GetDirection();
+  }
   // try to find next valid edge
   inline void probe_for_next() {
     // assume cur_iter_ is invalid
@@ -146,7 +191,6 @@ class UnTypedEdgeSetIter {
       iter_ind_ = 0;
     }
   }
-
   const std::vector<vid_t>& src_vertices_;
   std::vector<std::vector<edge_iter_t>> adj_lists_;
   size_t vid_ind_, iter_ind_;
@@ -168,7 +212,7 @@ class UnTypedEdgeSet {
   using index_ele_tuple_t = typename iterator::index_ele_tuple_t;
   using ele_tuple_t = typename iterator::ele_tuple_t;
   using self_type_t = UnTypedEdgeSet<vid_t, label_t, sub_graph_t>;
-  using flat_t = UnTypedEdgeSet;
+  using flat_t = FlatEdgeSet<vid_t, label_t, Any>;
   using data_tuple_t = std::tuple<vid_t, vid_t, Any>;
 
   static constexpr bool is_edge_set = true;
@@ -222,6 +266,91 @@ class UnTypedEdgeSet {
     return res;
   }
 
+  template <size_t col_ind, typename... index_ele_tuple_t_>
+  flat_t Flat(
+      std::vector<std::tuple<index_ele_tuple_t_...>>& index_ele_tuple) const {
+    using res_ele_tuple_t = std::tuple<vid_t, vid_t, Any>;
+    std::vector<res_ele_tuple_t> dst_eles;
+    dst_eles.reserve(index_ele_tuple.size());
+    auto edge_label_triplets = get_edge_triplets();
+    auto edge_iters = generate_iters();
+    std::vector<uint8_t> label_triplet_indices;
+    label_triplet_indices.reserve(index_ele_tuple.size());
+    std::vector<size_t> sizes;
+    sizes.emplace_back(0);
+    for (size_t i = 0; i < edge_label_triplets.size(); ++i) {
+      sizes.emplace_back(sizes.back() + edge_label_triplets[i].size());
+    }
+
+    std::vector<size_t> selected_indices;
+    selected_indices.reserve(index_ele_tuple.size());
+    for (auto& t : index_ele_tuple) {
+      auto& cur_tuple = std::get<col_ind>(t);
+      CHECK(std::get<0>(cur_tuple) < Size());
+      selected_indices.emplace_back(std::get<0>(cur_tuple));
+    }
+    sort(selected_indices.begin(), selected_indices.end());
+
+    // 0,2,4
+    size_t cur_ind = 0;
+    size_t selected_ind = 0;
+    for (size_t i = 0; i < src_vertices_.size(); ++i) {
+      auto src_vid = src_vertices_[i];
+      auto& cur_edge_iters = edge_iters[i];
+      auto src_label_ind = label_indices_[i];
+      auto src_label = src_labels_[src_label_ind];
+
+      for (size_t j = 0; j < cur_edge_iters.size(); ++j) {
+        auto& cur_iter = cur_edge_iters[j];
+        while (cur_iter.IsValid()) {
+          if (cur_ind == selected_indices[selected_ind]) {
+            auto dst_vid = cur_iter.GetDstId();
+            auto data = cur_iter.GetData();
+            dst_eles.emplace_back(std::make_tuple(src_vid, dst_vid, data));
+            label_triplet_indices.emplace_back(sizes[src_label_ind] + j);
+            selected_ind += 1;
+            if (selected_ind == selected_indices.size()) {
+              break;
+            }
+          }
+          cur_iter.Next();
+          cur_ind += 1;
+        }
+        if (selected_ind == selected_indices.size()) {
+          break;
+        }
+      }
+    }
+    std::vector<std::array<LabelT, 3>> res_label_triplets;
+    // put edge_label_triplets into res_label_triplets
+    for (size_t i = 0; i < edge_label_triplets.size(); ++i) {
+      auto& cur_triplets_vec = edge_label_triplets[i];
+      for (size_t j = 0; j < cur_triplets_vec.size(); ++j) {
+        res_label_triplets.emplace_back(std::array{
+            std::get<0>(cur_triplets_vec[j]), std::get<1>(cur_triplets_vec[j]),
+            std::get<2>(cur_triplets_vec[j])});
+      }
+    }
+    std::vector<std::vector<std::string>> prop_names = get_prop_names();
+    CHECK(prop_names.size() == res_label_triplets.size());
+    if (direction_ != Direction::Both) {
+      return FlatEdgeSet<vid_t, label_t, Any>(
+          std::move(dst_eles), std::move(res_label_triplets), prop_names,
+          std::move(label_triplet_indices), direction_);
+    } else {
+      std::vector<Direction> directions;
+      for (size_t i = 0; i < edge_label_triplets.size(); ++i) {
+        auto& cur_triplets_vec = edge_label_triplets[i];
+        for (size_t j = 0; j < cur_triplets_vec.size(); ++j) {
+          directions.emplace_back(std::get<3>(cur_triplets_vec[j]));
+        }
+      }
+      return FlatEdgeSet<vid_t, label_t, Any>(
+          std::move(dst_eles), std::move(res_label_triplets), prop_names,
+          std::move(label_triplet_indices), std::move(directions));
+    }
+  }
+
   size_t Size() const {
     if (size_ == 0) {
       auto iter_vec = generate_iters();
@@ -252,7 +381,7 @@ class UnTypedEdgeSet {
     size_t cur_ind = 0;
     while (iter != end_iter) {
       auto edata = iter.GetData();
-      PropT prop;
+      PropT prop{};
       if (edata.type == TypeConverter<PropT>::property_type()) {
         ConvertAny<PropT>::to(edata, prop);
       }
@@ -283,7 +412,6 @@ class UnTypedEdgeSet {
       if (v_labels_vec.empty()) {
         v_labels_vec = src_labels_;
       }
-      return getSrcVertices(v_labels_vec);
     } else if ((v_opt == VOpt::Start && direction_ == Direction::In) ||
                (v_opt == VOpt::End && direction_ == Direction::Out) ||
                (v_opt == VOpt::Other)) {
@@ -292,55 +420,57 @@ class UnTypedEdgeSet {
         v_labels_vec =
             std::vector<label_t>(dst_label_set.begin(), dst_label_set.end());
       }
-      return getDstVertices(v_labels_vec);
     } else {
       LOG(FATAL) << "Not supported for " << gs::to_string(v_opt);
     }
+    return getVerticesImpl(v_labels_vec, v_opt);
   }
 
-  std::pair<GeneralVertexSet<vid_t, label_t, grape::EmptyType>,
-            std::vector<offset_t>>
-  getSrcVertices(const std::vector<label_t>& req_labels) const {
-    std::vector<vid_t> ret;
-    std::vector<offset_t> offset;
-    ret.reserve(Size());
-    offset.reserve(Size());
-    std::vector<label_t> res_label_vec;
-    std::unordered_map<label_t, size_t> label_to_ind;
-    std::tie(res_label_vec, label_to_ind) =
-        preprocess_getting_labels(req_labels, src_labels_);
-
-    std::vector<grape::Bitset> bitsets(res_label_vec.size());
-    for (size_t i = 0; i < bitsets.size(); ++i) {
-      bitsets[i].init(Size());
-    }
-    size_t cur_cnt = 0;
-    for (auto iter : *this) {
-      offset.emplace_back(cur_cnt);
-      auto label = iter.GetSrcLabel();
-      if (label_to_ind.find(label) != label_to_ind.end()) {
-        auto ind = label_to_ind[label];
-        auto vid = iter.GetSrc();
-        ret.emplace_back(vid);
-        CHECK(ind < res_label_vec.size());
-        bitsets[ind].set_bit(cur_cnt);
-        cur_cnt += 1;
-      }
-    }
-    CHECK(cur_cnt <= Size());
-    offset.emplace_back(cur_cnt);
-    for (size_t i = 0; i < bitsets.size(); ++i) {
-      bitsets[i].resize(cur_cnt);
-    }
-    LOG(INFO) << "After resize from " << Size() << " to " << cur_cnt;
-    auto general_set =
-        make_general_set(std::move(ret), res_label_vec, std::move(bitsets));
-    return std::make_pair(std::move(general_set), std::move(offset));
-  }
+  // std::pair<GeneralVertexSet<vid_t, label_t, grape::EmptyType>,
+  //           std::vector<offset_t>>
+  // getSrcVertices(const std::vector<label_t>& req_labels) const {
+  //   VLOG(10) << "getSrcVertices for UntypedEdgeSet";
+  //   std::vector<vid_t> ret;
+  //   std::vector<offset_t> offset;
+  //   ret.reserve(Size());
+  //   offset.reserve(Size());
+  //   std::vector<label_t> res_label_vec;
+  //   std::unordered_map<label_t, size_t> label_to_ind;
+  //   std::tie(res_label_vec, label_to_ind) =
+  //       preprocess_getting_labels(req_labels);
+
+  //   std::vector<grape::Bitset> bitsets(res_label_vec.size());
+  //   for (size_t i = 0; i < bitsets.size(); ++i) {
+  //     bitsets[i].init(Size());
+  //   }
+  //   size_t cur_cnt = 0;
+  //   for (auto iter : *this) {
+  //     offset.emplace_back(cur_cnt);
+  //     auto label = iter.GetSrcLabel();
+  //     if (label_to_ind.find(label) != label_to_ind.end()) {
+  //       auto ind = label_to_ind[label];
+  //       auto vid = iter.GetSrc();
+  //       ret.emplace_back(vid);
+  //       CHECK(ind < res_label_vec.size());
+  //       bitsets[ind].set_bit(cur_cnt);
+  //       cur_cnt += 1;
+  //     }
+  //   }
+  //   CHECK(cur_cnt <= Size());
+  //   offset.emplace_back(cur_cnt);
+  //   for (size_t i = 0; i < bitsets.size(); ++i) {
+  //     bitsets[i].resize(cur_cnt);
+  //   }
+  //   LOG(INFO) << "After resize from " << Size() << " to " << cur_cnt;
+  //   auto general_set =
+  //       make_general_set(std::move(ret), res_label_vec, std::move(bitsets));
+  //   return std::make_pair(std::move(general_set), std::move(offset));
+  // }
 
   std::pair<GeneralVertexSet<vid_t, label_t, grape::EmptyType>,
             std::vector<offset_t>>
-  getDstVertices(const std::vector<label_t>& req_labels) const {
+  getVerticesImpl(const std::vector<label_t>& req_labels, VOpt vopt) const {
+    VLOG(10) << "getVerticesImpl for UntypedEdgeSet: " << gs::to_string(vopt);
     std::vector<vid_t> ret;
     std::vector<offset_t> offset;
     ret.reserve(Size());
@@ -349,15 +479,14 @@ class UnTypedEdgeSet {
     std::unordered_map<label_t, size_t> label_to_ind;
 
     label_t max_label_id = 0;
-    std::vector<label_t> all_cur_labels;
+    std::tie(res_label_vec, label_to_ind) =
+        preprocess_getting_labels(req_labels);
     {
-      auto dst_label_set = get_dst_label_set();
-      for (auto iter : dst_label_set) {
-        all_cur_labels.emplace_back(iter);
+      VLOG(10) << "res label size: " << res_label_vec.size();
+      for (auto l : res_label_vec) {
+        VLOG(10) << "res label: " << gs::to_string(l);
       }
     }
-    std::tie(res_label_vec, label_to_ind) =
-        preprocess_getting_labels(req_labels, all_cur_labels);
 
     std::vector<grape::Bitset> bitsets(res_label_vec.size());
     for (size_t i = 0; i < bitsets.size(); ++i) {
@@ -376,14 +505,32 @@ class UnTypedEdgeSet {
     size_t cur_cnt = 0;
     for (auto iter : *this) {
       offset.emplace_back(cur_cnt);
-      auto label = iter.GetDstLabel();
+      VLOG(10) << "edge: " << iter.GetSrc() << "("
+               << gs::to_string(iter.GetSrcLabel()) << ") -> " << iter.GetDst()
+               << "(" << gs::to_string(iter.GetDstLabel());
+
+      label_t label;
+      vid_t vid;
+      if (vopt == VOpt::Start) {
+        label = iter.GetSrcLabel();
+        vid = iter.GetSrc();
+      } else if (vopt == VOpt::End) {
+        label = iter.GetDstLabel();
+        vid = iter.GetDst();
+      } else if (vopt == VOpt::Other) {
+        label = iter.GetOtherLabel();
+        vid = iter.GetOther();
+      } else {
+        LOG(FATAL) << "Not supported for " << gs::to_string(vopt);
+      }
       auto ind = label_to_ind_vec[label];
       if (ind != -1) {
         auto ind = label_to_ind[label];
-        auto vid = iter.GetDst();
         ret.emplace_back(vid);
         bitsets[ind].set_bit(cur_cnt);
         cur_cnt += 1;
+      } else {
+        VLOG(10) << "Skip edge with label " << gs::to_string(label);
       }
     }
     CHECK(cur_cnt <= Size());
@@ -452,27 +599,41 @@ class UnTypedEdgeSet {
     for (size_t i = 0; i < edge_label_triplets.size(); ++i) {
       auto& cur_triplets_vec = edge_label_triplets[i];
       for (size_t j = 0; j < cur_triplets_vec.size(); ++j) {
-        res_label_triplets.emplace_back(cur_triplets_vec[j]);
+        res_label_triplets.emplace_back(std::array{
+            std::get<0>(cur_triplets_vec[j]), std::get<1>(cur_triplets_vec[j]),
+            std::get<2>(cur_triplets_vec[j])});
       }
     }
     std::vector<std::vector<std::string>> prop_names = get_prop_names();
     CHECK(prop_names.size() == res_label_triplets.size());
-    return FlatEdgeSet<vid_t, label_t, Any>(
-        std::move(dst_eles), std::move(res_label_triplets), prop_names,
-        std::move(label_triplet_indices), direction_);
+
+    if (direction_ != Direction::Both) {
+      return FlatEdgeSet<vid_t, label_t, Any>(
+          std::move(dst_eles), std::move(res_label_triplets), prop_names,
+          std::move(label_triplet_indices), direction_);
+    } else {
+      std::vector<Direction> directions;
+      for (size_t i = 0; i < edge_label_triplets.size(); ++i) {
+        auto& cur_triplets_vec = edge_label_triplets[i];
+        for (size_t j = 0; j < cur_triplets_vec.size(); ++j) {
+          directions.emplace_back(std::get<3>(cur_triplets_vec[j]));
+        }
+      }
+      return FlatEdgeSet<vid_t, label_t, Any>(
+          std::move(dst_eles), std::move(res_label_triplets), prop_names,
+          std::move(label_triplet_indices), std::move(directions));
+    }
   }
 
  private:
   std::pair<std::vector<label_t>, std::unordered_map<label_t, size_t>>
-  preprocess_getting_labels(const std::vector<label_t>& req_labels,
-                            const std::vector<label_t>& all_cur_labels) const {
+  preprocess_getting_labels(const std::vector<label_t>& req_labels) const {
     std::unordered_map<label_t, size_t> label_to_ind;
     std::vector<label_t> res_label_vec;
-    for (size_t i = 0; i < all_cur_labels.size(); ++i) {
-      if (std::find(req_labels.begin(), req_labels.end(), all_cur_labels[i]) !=
-          req_labels.end()) {
-        res_label_vec.emplace_back(all_cur_labels[i]);
-        label_to_ind[all_cur_labels[i]] = res_label_vec.size() - 1;
+    for (size_t i = 0; i < req_labels.size(); ++i) {
+      if (label_to_ind.find(req_labels[i]) == label_to_ind.end()) {
+        label_to_ind[req_labels[i]] = res_label_vec.size();
+        res_label_vec.emplace_back(req_labels[i]);
       }
     }
     return std::make_pair(std::move(res_label_vec), std::move(label_to_ind));
@@ -515,18 +676,19 @@ class UnTypedEdgeSet {
     return edge_iter_vecs;
   }
 
-  std::vector<std::vector<std::array<LabelT, 3>>> get_edge_triplets() const {
-    std::vector<std::vector<std::array<LabelT, 3>>> ret;
+  std::vector<std::vector<std::tuple<LabelT, LabelT, LabelT, Direction>>>
+  get_edge_triplets() const {
+    std::vector<std::vector<std::tuple<LabelT, LabelT, LabelT, Direction>>> ret;
     for (auto src_label_ind = 0; src_label_ind < src_labels_.size();
          ++src_label_ind) {
       auto src_label = src_labels_[src_label_ind];
-      std::vector<std::array<LabelT, 3>> tmp;
+      std::vector<std::tuple<LabelT, LabelT, LabelT, Direction>> tmp;
       if (adj_lists_.find(src_label) != adj_lists_.end()) {
         auto& sub_graphs = adj_lists_.at(src_label);
         for (auto& sub_graph : sub_graphs) {
-          tmp.emplace_back(std::array<LabelT, 3>({sub_graph.GetSrcLabel(),
-                                                  sub_graph.GetDstLabel(),
-                                                  sub_graph.GetEdgeLabel()}));
+          tmp.emplace_back(std::make_tuple(
+              sub_graph.GetSrcLabel(), sub_graph.GetDstLabel(),
+              sub_graph.GetEdgeLabel(), sub_graph.GetDirection()));
         }
       }
       ret.emplace_back(std::move(tmp));
diff --git a/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h b/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h
index 4e16c373a797..72448a6f198c 100644
--- a/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h
+++ b/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h
@@ -42,47 +42,51 @@ auto general_project_vertices_impl(
   CHECK(prop_getters.size() == old_labels.size());
   auto old_num_labels = old_bit_sets.size();
   std::vector<grape::Bitset> res_bitsets(old_num_labels);
+  LOG(INFO) << "old labels: " << gs::to_string(old_labels)
+            << ", filter labels: " << gs::to_string(filter_labels);
   // reserve enough size for bitset.
   for (size_t i = 0; i < old_num_labels; ++i) {
     res_bitsets[i].init(old_vec.size());
   }
-  std::vector<size_t> select_label_id;
+  // std::vector<size_t> select_label_id;
+  std::unordered_set<LabelT> label_ind_set;
   if constexpr (filter_num_labels == 0) {
     for (size_t i = 0; i < old_labels.size(); ++i) {
-      select_label_id.emplace_back(i);
+      label_ind_set.insert(i);
     }
   } else {
-    std::unordered_set<LabelT> set;
-    for (auto l : filter_labels) {
-      set.insert(l);
-    }
     for (size_t i = 0; i < old_labels.size(); ++i) {
-      if (set.find(old_labels[i]) != set.end()) {
-        select_label_id.emplace_back(i);
+      auto it =
+          std::find(filter_labels.begin(), filter_labels.end(), old_labels[i]);
+      if (it != filter_labels.end()) {
+        label_ind_set.insert(i);
       }
     }
   }
-  VLOG(10) << "selected label ids: " << gs::to_string(select_label_id)
+  VLOG(10) << "selected label ids: " << label_ind_set.size()
            << ", out of size: " << old_labels.size();
   std::vector<offset_t> offset;
 
   offset.emplace_back(0);
   for (size_t i = 0; i < old_vec.size(); ++i) {
-    for (auto label_id : select_label_id) {
-      if (old_bit_sets[label_id].get_bit(i)) {
-        auto eles = prop_getters[label_id].get_view(old_vec[i]);
-        if constexpr (PROP_GETTER::prop_num == 0) {
-          if (expr()) {
-            res_bitsets[label_id].set_bit(res_vec.size());
-            res_vec.push_back(old_vec[i]);
-            break;
-          }
-        } else {
-          if (std::apply(expr, eles)) {
-            res_bitsets[label_id].set_bit(res_vec.size());
-            res_vec.push_back(old_vec[i]);
-            break;
-          }
+    size_t j = 0;
+    while (j < old_num_labels) {
+      if (old_bit_sets[j].get_bit(i)) {
+        break;
+      }
+      ++j;
+    }
+    CHECK(j < old_num_labels) << "no label is active at ind: " << i;
+    if constexpr (PROP_GETTER::prop_num == 0) {
+      if (expr()) {
+        res_bitsets[j].set_bit(res_vec.size());
+        res_vec.push_back(old_vec[i]);
+      }
+    } else {
+      if (label_ind_set.find(j) != label_ind_set.end()) {
+        if (std::apply(expr, prop_getters[j].get_view(old_vec[i]))) {
+          res_bitsets[j].set_bit(res_vec.size());
+          res_vec.push_back(old_vec[i]);
         }
       }
     }
@@ -96,7 +100,11 @@ auto general_project_vertices_impl(
     CHECK(flag) << "check fail at ind: " << i;
   }
   // resize bitset.
+  LOG(INFO) << "res bitset size: " << res_bitsets.size();
   for (size_t i = 0; i < old_num_labels; ++i) {
+    LOG(INFO) << "Shrink bitset: " << i
+              << ", from size: " << res_bitsets[i].cardinality()
+              << " to size: " << res_vec.size();
     res_bitsets[i].resize(res_vec.size());
   }
   for (size_t i = 0; i < res_vec.size(); ++i) {
@@ -413,12 +421,148 @@ auto generalSetFlatImpl(
   return std::make_pair(std::move(res_vids), std::move(res_bitsets));
 }
 
-template <typename VID_T, typename... T>
+template <typename VID_T, typename LabelT, typename... T>
+class GeneralVertexSet;
+
+template <typename VID_T, typename LabelT, typename... T>
+class GeneralVertexSetBuilder {
+ public:
+  using res_t = GeneralVertexSet<VID_T, LabelT, T...>;
+  using ele_tuple_t = typename res_t::ele_tuple_t;
+  using data_tuple_t = typename res_t::data_tuple_t;
+  using index_ele_tuple_t = typename res_t::index_ele_tuple_t;
+
+  static constexpr bool is_row_vertex_set_builder = false;
+  static constexpr bool is_flat_edge_set_builder = false;
+  static constexpr bool is_general_edge_set_builder = false;
+  static constexpr bool is_two_label_set_builder = false;
+  static constexpr bool is_collection_builder = false;
+  static constexpr bool is_general_vertex_set_builder = true;
+
+  GeneralVertexSetBuilder(
+      const std::vector<VID_T>& vec,
+      const std::vector<std::tuple<T...>>& data_vec,
+      const std::array<std::string, sizeof...(T)>& prop_names,
+      const std::vector<LabelT>& labels,
+      const std::vector<grape::Bitset>& bitsets)
+      : labels_(labels), prop_names_(prop_names) {
+    vec_.reserve(vec.size());
+    data_vec_.reserve(data_vec.size());
+    bitsets_.resize(bitsets.size());
+    for (size_t i = 0; i < bitsets.size(); ++i) {
+      bitsets_[i].init(vec.size());
+    }
+  }
+
+  void Insert(const index_ele_tuple_t& tuple, const data_tuple_t& data) {
+    vec_.emplace_back(std::get<2>(tuple));
+    data_vec_.emplace_back(data);
+    CHECK(std::get<1>(tuple) < bitsets_.size());
+    if (vec_.size() - 1 > bitsets_[std::get<1>(tuple)].cardinality()) {
+      VLOG(10) << "vec size: " << vec_.size() << ", bitset size: "
+               << bitsets_[std::get<1>(tuple)].cardinality();
+      for (size_t i = 0; i < bitsets_.size(); ++i) {
+        bitsets_[i].resize(2 * bitsets_[i].cardinality());
+      }
+    }
+    bitsets_[std::get<1>(tuple)].set_bit(vec_.size() - 1);
+  }
+
+  res_t Build() {
+    for (size_t i = 0; i < bitsets_.size(); ++i) {
+      bitsets_[i].resize(vec_.size());
+    }
+    return res_t(std::move(vec_), std::move(data_vec_), std::move(prop_names_),
+                 std::move(labels_), std::move(bitsets_));
+  }
+
+  size_t Size() const { return vec_.size(); }
+
+ private:
+  std::vector<VID_T> vec_;
+  std::vector<std::tuple<T...>> data_vec_;
+  std::array<std::string, sizeof...(T)> prop_names_;
+  std::vector<LabelT> labels_;
+  std::vector<grape::Bitset> bitsets_;
+};
+
+// Specialize for grape::EmptyType
+template <typename VID_T, typename LabelT>
+class GeneralVertexSetBuilder<VID_T, LabelT, grape::EmptyType> {
+ public:
+  using res_t = GeneralVertexSet<VID_T, LabelT, grape::EmptyType>;
+  using ele_tuple_t = typename res_t::ele_tuple_t;
+  using data_tuple_t = typename res_t::data_tuple_t;
+  using index_ele_tuple_t = typename res_t::index_ele_tuple_t;
+
+  static constexpr bool is_row_vertex_set_builder = false;
+  static constexpr bool is_flat_edge_set_builder = false;
+  static constexpr bool is_general_edge_set_builder = false;
+  static constexpr bool is_two_label_set_builder = false;
+  static constexpr bool is_collection_builder = false;
+  static constexpr bool is_general_vertex_set_builder = true;
+
+  GeneralVertexSetBuilder(const std::vector<VID_T>& vec,
+                          const std::vector<LabelT>& labels,
+                          const std::vector<grape::Bitset>& bitsets)
+      : labels_(labels) {
+    vec_.reserve(vec.size());
+    bitsets_.resize(bitsets.size());
+    for (size_t i = 0; i < bitsets.size(); ++i) {
+      CHECK(bitsets[i].cardinality() == vec.size());
+      bitsets_[i].init(vec.size());
+    }
+  }
+
+  GeneralVertexSetBuilder(size_t size, const std::vector<LabelT>& labels)
+      : labels_(labels) {
+    vec_.reserve(size);
+    bitsets_.resize(labels.size());
+    VLOG(10) << "Create general vertex builder : " << size
+             << ", labels: " << labels.size();
+    for (size_t i = 0; i < bitsets_.size(); ++i) {
+      bitsets_[i].init(size);
+    }
+  }
+
+  void Insert(const index_ele_tuple_t& tuple, const data_tuple_t& data) {
+    vec_.emplace_back(std::get<2>(tuple));
+    CHECK(std::get<1>(tuple) < bitsets_.size());
+    if (vec_.size() - 1 > bitsets_[std::get<1>(tuple)].cardinality()) {
+      VLOG(10) << "vec size: " << vec_.size() << ", bitset size: "
+               << bitsets_[std::get<1>(tuple)].cardinality();
+      for (size_t i = 0; i < bitsets_.size(); ++i) {
+        bitsets_[i].resize(2 * bitsets_[i].cardinality());
+      }
+    }
+    bitsets_[std::get<1>(tuple)].set_bit(vec_.size() - 1);
+  }
+
+  res_t Build() {
+    for (size_t i = 0; i < bitsets_.size(); ++i) {
+      LOG(INFO) << "Shrink bitset: " << i
+                << ", from size: " << bitsets_[i].cardinality()
+                << " to size: " << vec_.size();
+      bitsets_[i].resize(vec_.size());
+    }
+    return res_t(std::move(vec_), std::move(labels_), std::move(bitsets_));
+  }
+
+  size_t Size() const { return vec_.size(); }
+
+ private:
+  std::vector<VID_T> vec_;
+  std::vector<LabelT> labels_;
+  std::vector<grape::Bitset> bitsets_;
+};
+
+template <typename VID_T, typename LabelT, typename... T>
 class GeneralVertexSetIter {
  public:
   using lid_t = VID_T;
-  using self_type_t = GeneralVertexSetIter<VID_T, T...>;
-  using index_ele_tuple_t = std::tuple<size_t, VID_T, std::tuple<T...>>;
+  using self_type_t = GeneralVertexSetIter<VID_T, LabelT, T...>;
+  using index_ele_tuple_t = std::tuple<size_t, size_t, VID_T, std::tuple<T...>>;
+  using ele_tuple_t = std::tuple<size_t, VID_T, std::tuple<T...>>;
   using data_tuple_t = std::tuple<VID_T, std::tuple<T...>>;
 
   GeneralVertexSetIter(const std::vector<VID_T>& vec,
@@ -431,12 +575,28 @@ class GeneralVertexSetIter {
         bitsets_(bitsets),
         ind_(ind) {}
 
-  lid_t GetElement() const { return vec_[ind_]; }
+  ele_tuple_t GetElement() const {
+    size_t label_ind = 0;
+    for (size_t i = 0; i < bitsets_.size(); ++i) {
+      if (bitsets_[i].get_bit(ind_)) {
+        label_ind = i;
+        break;
+      }
+    }
+    return std::make_tuple(label_ind, vec_[ind_], data_vec_[ind_]);
+  }
 
   data_tuple_t GetData() const { return vec_[ind_]; }
 
   index_ele_tuple_t GetIndexElement() const {
-    return std::make_tuple(ind_, vec_[ind_], data_vec_[ind_]);
+    size_t label_ind = 0;
+    for (size_t i = 0; i < bitsets_.size(); ++i) {
+      if (bitsets_[i].get_bit(ind_)) {
+        label_ind = i;
+        break;
+      }
+    }
+    return std::make_tuple(ind_, label_ind, vec_[ind_], data_vec_[ind_]);
   }
 
   lid_t GetVertex() const { return vec_[ind_]; }
@@ -477,24 +637,41 @@ class GeneralVertexSetIter {
   size_t ind_;
 };
 
-template <typename VID_T>
-class GeneralVertexSetIter<VID_T, grape::EmptyType> {
+template <typename VID_T, typename LabelT>
+class GeneralVertexSetIter<VID_T, LabelT, grape::EmptyType> {
  public:
   using lid_t = VID_T;
-  using self_type_t = GeneralVertexSetIter<VID_T, grape::EmptyType>;
-  using index_ele_tuple_t = std::tuple<size_t, VID_T>;
+  using self_type_t = GeneralVertexSetIter<VID_T, LabelT, grape::EmptyType>;
+  using index_ele_tuple_t = std::tuple<size_t, size_t, VID_T>;
+  using ele_tuple_t = std::tuple<size_t, VID_T>;
   using data_tuple_t = std::tuple<VID_T>;
 
   GeneralVertexSetIter(const std::vector<VID_T>& vec,
                        const std::vector<grape::Bitset>& bitsets, size_t ind)
       : vec_(vec), bitsets_(bitsets), ind_(ind) {}
 
-  lid_t GetElement() const { return vec_[ind_]; }
+  ele_tuple_t GetElement() const {
+    size_t label_ind = 0;
+    for (size_t i = 0; i < bitsets_.size(); ++i) {
+      if (bitsets_[i].get_bit(ind_)) {
+        label_ind = i;
+        break;
+      }
+    }
+    return std::make_tuple(label_ind, vec_[ind_]);
+  }
 
   data_tuple_t GetData() const { return vec_[ind_]; }
 
   index_ele_tuple_t GetIndexElement() const {
-    return std::make_tuple(ind_, vec_[ind_]);
+    size_t label_ind = 0;
+    for (size_t i = 0; i < bitsets_.size(); ++i) {
+      if (bitsets_[i].get_bit(ind_)) {
+        label_ind = i;
+        break;
+      }
+    }
+    return std::make_tuple(ind_, label_ind, vec_[ind_]);
   }
 
   lid_t GetVertex() const { return vec_[ind_]; }
@@ -538,11 +715,13 @@ class GeneralVertexSet {
  public:
   using lid_t = VID_T;
   using self_type_t = GeneralVertexSet<VID_T, LabelT, T...>;
-  using iterator = GeneralVertexSetIter<VID_T, T...>;
-  using index_ele_tuple_t = std::tuple<size_t, VID_T, std::tuple<T...>>;
+  using iterator = GeneralVertexSetIter<VID_T, LabelT, T...>;
+  using index_ele_tuple_t = std::tuple<size_t, size_t, VID_T, std::tuple<T...>>;
+  using ele_tuple_t = std::tuple<size_t, VID_T, std::tuple<T...>>;
   using data_tuple_t = std::tuple<VID_T, std::tuple<T...>>;
   using flat_t = self_type_t;
   using EntityValueType = VID_T;
+  using builder_t = GeneralVertexSetBuilder<VID_T, LabelT, T...>;
 
   static constexpr bool is_vertex_set = true;
   static constexpr bool is_two_label_set = false;
@@ -610,6 +789,10 @@ class GeneralVertexSet {
     return iterator(vec_, data_vec_, prop_names_, bitsets_, vec_.size());
   }
 
+  builder_t CreateBuilder() const {
+    return builder_t(vec_, data_vec_, prop_names_, label_names_, bitsets_);
+  }
+
   template <typename EXPRESSION, size_t num_labels, typename PROP_GETTER,
             typename RES_SET_T = self_type_t,
             typename RES_T = std::pair<RES_SET_T, std::vector<offset_t>>>
@@ -653,7 +836,7 @@ class GeneralVertexSet {
 
   LabelT GetLabel(size_t i) const { return label_names_[i]; }
 
-  const std::vector<LabelKey> GetLabelVec() const {
+  std::vector<LabelKey> GetLabelVec() const {
     std::vector<LabelKey> res;
     // fill res with vertex labels
     for (size_t i = 0; i < vec_.size(); ++i) {
@@ -694,11 +877,11 @@ class GeneralVertexSet {
       return std::make_pair(std::vector<VID_T>(), std::vector<int32_t>());
     } else {
       auto ind = std::distance(label_names_.begin(), it);
-      return GetVerticesWithIndex(ind);
+      return GetVertices(ind);
     }
   }
 
-  std::pair<std::vector<VID_T>, std::vector<int32_t>> GetVerticesWithIndex(
+  std::pair<std::vector<VID_T>, std::vector<int32_t>> GetVertices(
       size_t ind) const {
     CHECK(ind < bitsets_.size());
     std::vector<VID_T> res;
@@ -729,14 +912,16 @@ class GeneralVertexSet {
     res_vec.reserve(indices.size());
     res_data_vec.reserve(indices.size());
     for (auto i : indices) {
-      res_vec.emplace_back(vec_[i]);
-      res_data_vec.emplace_back(data_vec_[i]);
-      for (size_t j = 0; j < bitsets_.size(); ++j) {
-        if (bitsets_[j].get_bit(i)) {
-          res_bitsets[j].set_bit(i);
+      size_t label_ind = 0;
+      for (; label_ind < bitsets_.size(); ++label_ind) {
+        if (bitsets_[label_ind].get_bit(i)) {
           break;
         }
       }
+      CHECK(label_ind < bitsets_.size());
+      res_bitsets[label_ind].set_bit(res_vec.size());
+      res_vec.emplace_back(vec_[i]);
+      res_data_vec.emplace_back(data_vec_[i]);
     }
     vec_.swap(res_vec);
     data_vec_.swap(res_data_vec);
@@ -917,11 +1102,13 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
  public:
   using lid_t = VID_T;
   using self_type_t = GeneralVertexSet<VID_T, LabelT, grape::EmptyType>;
-  using iterator = GeneralVertexSetIter<VID_T, grape::EmptyType>;
-  using index_ele_tuple_t = std::tuple<size_t, VID_T>;
+  using iterator = GeneralVertexSetIter<VID_T, LabelT, grape::EmptyType>;
+  using index_ele_tuple_t = std::tuple<size_t, size_t, VID_T>;
+  using ele_tuple_t = std::tuple<size_t, VID_T>;
   using data_tuple_t = std::tuple<VID_T>;
   using flat_t = self_type_t;
   using EntityValueType = VID_T;
+  using builder_t = GeneralVertexSetBuilder<VID_T, LabelT, grape::EmptyType>;
 
   static constexpr bool is_vertex_set = true;
   static constexpr bool is_two_label_set = false;
@@ -940,7 +1127,8 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
     if (bitsets_.size() > 0) {
       VLOG(10) << "[GeneralVertexSet], size: " << vec_.size()
                << ", bitset size: " << bitsets_[0].cardinality()
-               << ", ind_ele_t: " << demangle(index_ele_tuple_t());
+               << ", ind_ele_t: " << demangle(index_ele_tuple_t())
+               << ", label names: " << gs::to_string(label_names_);
     }
   }
 
@@ -953,7 +1141,8 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
     }
     if (bitsets_.size() > 0) {
       VLOG(10) << "[GeneralVertexSet], size: " << vec_.size()
-               << ", bitset size: " << bitsets_[0].cardinality();
+               << ", bitset size: " << bitsets_[0].cardinality()
+               << ", labels: " << gs::to_string(label_names_);
     }
   }
 
@@ -973,6 +1162,10 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
 
   iterator end() const { return iterator(vec_, bitsets_, vec_.size()); }
 
+  builder_t CreateBuilder() const {
+    return builder_t(vec_, label_names_, bitsets_);
+  }
+
   template <typename EXPRESSION, size_t num_labels, typename PROP_GETTER,
             typename RES_SET_T = self_type_t,
             typename RES_T = std::pair<RES_SET_T, std::vector<offset_t>>>
@@ -1011,7 +1204,7 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
 
   LabelT GetLabel(size_t i) const { return label_names_[i]; }
 
-  const std::vector<LabelKey> GetLabelVec() const {
+  std::vector<LabelKey> GetLabelVec() const {
     std::vector<LabelKey> res;
     // fill res with vertex labels
     for (size_t i = 0; i < vec_.size(); ++i) {
@@ -1052,11 +1245,11 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
       return std::make_pair(std::vector<VID_T>(), std::vector<int32_t>());
     } else {
       auto ind = std::distance(label_names_.begin(), it);
-      return GetVerticesWithIndex(ind);
+      return GetVertices(ind);
     }
   }
 
-  std::pair<std::vector<VID_T>, std::vector<int32_t>> GetVerticesWithIndex(
+  std::pair<std::vector<VID_T>, std::vector<int32_t>> GetVertices(
       size_t ind) const {
     CHECK(ind < bitsets_.size());
     std::vector<VID_T> res;
@@ -1085,13 +1278,15 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
     }
     res_vec.reserve(indices.size());
     for (auto i : indices) {
-      res_vec.emplace_back(vec_[i]);
-      for (size_t j = 0; j < bitsets_.size(); ++j) {
-        if (bitsets_[j].get_bit(i)) {
-          res_bitsets[j].set_bit(i);
+      size_t label_ind = 0;
+      for (; label_ind < bitsets_.size(); ++label_ind) {
+        if (bitsets_[label_ind].get_bit(i)) {
           break;
         }
       }
+      CHECK(label_ind < bitsets_.size());
+      res_bitsets[label_ind].set_bit(res_vec.size());
+      res_vec.emplace_back(vec_[i]);
     }
     vec_.swap(res_vec);
     for (size_t i = 0; i < bitsets_.size(); ++i) {
@@ -1109,7 +1304,8 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
       next_size += repeat_array[i];
     }
     VLOG(10) << "[GeneralVertexSet] size: " << Size()
-             << " Project self, next size: " << next_size;
+             << " Project self, next size: " << next_size
+             << "repeat array size: " << repeat_array.size();
 
     next_vids.reserve(next_size);
     std::vector<grape::Bitset> next_sets(bitsets_.size());
@@ -1118,16 +1314,16 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
     }
     VLOG(10) << "after init";
     for (size_t i = 0; i < repeat_array.size(); ++i) {
-      size_t ind = 0;
-      while (ind < bitsets_.size()) {
-        if (bitsets_[ind].get_bit(i)) {
+      size_t label_ind = 0;
+      while (label_ind < bitsets_.size()) {
+        if (bitsets_[label_ind].get_bit(i)) {
           break;
         }
-        ind += 1;
+        label_ind += 1;
       }
-      CHECK(ind < bitsets_.size());
+      CHECK(label_ind < bitsets_.size());
       for (size_t j = 0; j < repeat_array[i]; ++j) {
-        next_sets[ind].set_bit(next_vids.size());
+        next_sets[label_ind].set_bit(next_vids.size());
         next_vids.push_back(vec_[i]);
       }
     }
@@ -1137,6 +1333,47 @@ class GeneralVertexSet<VID_T, LabelT, grape::EmptyType> {
                        std::move(next_sets));
   }
 
+  // std::vector<offset_t> dedup_on_internal_id() {
+  //   std::unordered_set<uint64_t> appeared_vids;
+  //   std::vector<offset_t> offsets;
+  //   std::vector<VID_T> res_vec;
+  //   std::vector<grape::Bitset> res_bitsets(bitsets_.size());
+  //   for (auto& i : res_bitsets) {
+  //     i.init(vec_.size());
+  //   }
+  //   offsets.push_back(0);
+  //   auto label_indices = GenerateLabelIndices();
+  //   CHECK(label_indices.size() == vec_.size());
+  //   for (size_t i = 0; i < vec_.size(); ++i) {
+  //     uint64_t vid = vec_[i];
+  //     vid = vid | (static_cast<uint64_t>(label_indices[i]) << 56);
+  //     if (appeared_vids.find(vid) == appeared_vids.end()) {
+  //       appeared_vids.insert(vid);
+  //       res_vec.push_back(vec_[i]);
+  //       CHECK(label_indices[i] < res_bitsets.size());
+  //       res_bitsets[label_indices[i]].set_bit(res_vec.size() - 1);
+  //     }
+  //     offsets.push_back(res_vec.size());
+  //   }
+  //   vec_.swap(res_vec);
+  //   bitsets_.swap(res_bitsets);
+  //   for (auto i = 0; i < bitsets_.size(); ++i) {
+  //     bitsets_[i].resize(res_vec.size());
+  //   }
+  //   return offsets;
+  // }
+
+  // template <typename PropT>
+  // std::vector<offset_t> Dedup(const PropertySelector<PropT>& prop_selector) {
+  //   if constexpr (std::is_same_v<PropT, grape::EmptyType>) {
+  //     return dedup_on_internal_id();
+  //   } else if constexpr (std::is_same_v<PropT, LabelKey>) {
+  //     return dedup_on_label();
+  //   } else {
+  //     LOG(FATAL) << "Not supported: " << demangle(typeid(PropT).name());
+  //   }
+  // }
+
   void Repeat(std::vector<offset_t>& cur_offset,
               std::vector<offset_t>& repeat_vec) {
     CHECK(cur_offset.size() == repeat_vec.size());
@@ -1295,6 +1532,8 @@ static auto get_property_tuple_general(
                            SET_T...>& general_set,
     const std::array<std::string, sizeof...(T)>& prop_names) {
   auto label_vec = general_set.GetLabels();
+  VLOG(10) << "general set label vec: " << gs::to_string(label_vec)
+           << ", vid size: " << general_set.Size();
   auto vids_inds = bitsets_to_vids_inds(general_set.GetBitsets());
 
   auto data_tuples = graph.template GetVertexPropsFromVid<T...>(
diff --git a/flex/engines/hqps_db/structures/multi_vertex_set/keyed_row_vertex_set.h b/flex/engines/hqps_db/structures/multi_vertex_set/keyed_row_vertex_set.h
index a4cfb6c202f9..45da398e3f69 100644
--- a/flex/engines/hqps_db/structures/multi_vertex_set/keyed_row_vertex_set.h
+++ b/flex/engines/hqps_db/structures/multi_vertex_set/keyed_row_vertex_set.h
@@ -753,6 +753,7 @@ class KeyedRowVertexSetImpl<LabelT, KEY_T, VID_T, grape::EmptyType> {
   static constexpr bool is_collection = false;
   static constexpr bool is_general_set = false;
   static constexpr bool is_two_label_set = false;
+  static constexpr bool is_row_vertex_set = true;
 
   explicit KeyedRowVertexSetImpl(std::vector<key_t>&& keys,
                                  std::vector<lid_t>&& vids, LabelT v_label)
@@ -947,8 +948,11 @@ class KeyedRowVertexSetBuilderImpl {
         prop_names_(old_set.GetPropNames()),
         ind_(0) {}
 
-  size_t insert(std::tuple<size_t, VID_T> ele_tuple, data_tuple_t data_tuple) {
+  int32_t insert(std::tuple<size_t, VID_T> ele_tuple, data_tuple_t data_tuple) {
     auto key = std::get<1>(ele_tuple);
+    if (IsNull(key)) {
+      return -1;
+    }
     if (prop2ind_.find(key) != prop2ind_.end()) {
       return prop2ind_[key];
     } else {
@@ -960,7 +964,10 @@ class KeyedRowVertexSetBuilderImpl {
     }
   }
 
-  size_t insert(const VID_T& key, data_tuple_t data_tuple) {
+  int32_t insert(const VID_T& key, data_tuple_t data_tuple) {
+    if (IsNull(key)) {
+      return -1;
+    }
     if (prop2ind_.find(key) != prop2ind_.end()) {
       return prop2ind_[key];
     } else {
@@ -972,7 +979,7 @@ class KeyedRowVertexSetBuilderImpl {
     }
   }
 
-  size_t insert(const std::tuple<VID_T, data_tuple_t>& ele_tuple) {
+  int32_t insert(const std::tuple<VID_T, data_tuple_t>& ele_tuple) {
     return insert(std::get<0>(ele_tuple), std::get<1>(ele_tuple));
   }
 
@@ -1007,8 +1014,11 @@ class KeyedRowVertexSetBuilderImpl<LabelT, KEY_T, VID_T, grape::EmptyType> {
       const RowVertexSet<LabelT, VID_T, grape::EmptyType>& old_set)
       : label_(old_set.GetLabel()), ind_(0) {}
 
-  size_t insert(std::tuple<size_t, VID_T> ele_tuple) {
+  int32_t insert(std::tuple<size_t, VID_T> ele_tuple) {
     auto key = std::get<1>(ele_tuple);
+    if (IsNull(key)) {
+      return -1;
+    }
     if (prop2ind_.find(key) != prop2ind_.end()) {
       return prop2ind_[key];
     } else {
@@ -1019,7 +1029,10 @@ class KeyedRowVertexSetBuilderImpl<LabelT, KEY_T, VID_T, grape::EmptyType> {
     }
   }
 
-  size_t insert(const VID_T& key) {
+  int32_t insert(const VID_T& key) {
+    if (IsNull(key)) {
+      return -1;
+    }
     if (prop2ind_.find(key) != prop2ind_.end()) {
       return prop2ind_[key];
     } else {
diff --git a/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h b/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h
index 80adf0efbed8..5036e912d57c 100644
--- a/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h
+++ b/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h
@@ -20,6 +20,8 @@
 #include <unordered_set>
 #include <vector>
 
+#include "flex/engines/hqps_db/core/null_record.h"
+#include "flex/utils/property/types.h"
 #include "grape/util.h"
 #include "grape/utils/bitset.h"
 
@@ -28,6 +30,131 @@ namespace gs {
 template <typename VID_T, typename LabelT, typename... T>
 class TwoLabelVertexSetImpl;
 
+// Keyed two label vertex set builder
+template <typename VID_T, typename LabelT, typename... T>
+class TwoLabelVertexSetImplKeyedBuilder {
+ public:
+  using key_t = GlobalId;
+  using res_t = TwoLabelVertexSetImpl<VID_T, LabelT, T...>;
+  using ele_tuple_t = std::tuple<int32_t, VID_T>;
+  using data_tuple_t = std::tuple<T...>;
+  using index_ele_tuple_t = std::tuple<size_t, int32_t, VID_T>;
+
+  static constexpr bool is_row_vertex_set_builder = false;
+  static constexpr bool is_flat_edge_set_builder = false;
+  static constexpr bool is_general_edge_set_builder = false;
+  static constexpr bool is_two_label_set_builder = true;
+  static constexpr bool is_collection_builder = false;
+
+  TwoLabelVertexSetImplKeyedBuilder(
+      size_t size, const std::array<LabelT, 2>& labels,
+      const std::array<std::string, sizeof...(T)>& props)
+      : ind_(0), labels_(labels), props_(props) {
+    vec_.reserve(size);
+    data_.reserve(size);
+    bitset_.init(size);
+  }
+
+  TwoLabelVertexSetImplKeyedBuilder(const res_t& res)
+      : TwoLabelVertexSetImplKeyedBuilder(res.Size(), res.GetLabels(),
+                                          res.GetPropNames()) {}
+
+  int32_t insert(const index_ele_tuple_t& tuple, const data_tuple_t& data) {
+    auto global_id = GlobalId(std::get<1>(tuple), std::get<2>(tuple));
+    if (IsNull(global_id)) {
+      return -1;
+    }
+    if (global_id_map_.find(global_id) != global_id_map_.end()) {
+      return global_id_map_[global_id];
+    } else {
+      global_id_map_[global_id] = ind_;
+      data_.emplace_back(data);
+      if (std::get<1>(tuple) == 0) {
+        bitset_.set_bit(vec_.size());
+      }
+      vec_.emplace_back(std::get<2>(tuple));
+      return ind_++;
+    }
+  }
+
+  res_t Build() {
+    VLOG(10) << "Try to resize from " << bitset_.cardinality() << ", to "
+             << vec_.size();
+    bitset_.resize(vec_.size());
+    return res_t(std::move(vec_), std::move(data_), std::move(labels_),
+                 std::move(props_), std::move(bitset_));
+  }
+
+  size_t Size() const { return vec_.size(); }
+
+ private:
+  size_t ind_;
+  std::vector<VID_T> vec_;
+  std::vector<data_tuple_t> data_;
+  std::array<LabelT, 2> labels_;
+  std::array<std::string, sizeof...(T)> props_;
+  std::unordered_map<GlobalId, size_t> global_id_map_;
+  grape::Bitset bitset_;
+};
+
+template <typename VID_T, typename LabelT>
+class TwoLabelVertexSetImplKeyedBuilder<VID_T, LabelT, grape::EmptyType> {
+ public:
+  using key_t = GlobalId;
+  using res_t = TwoLabelVertexSetImpl<VID_T, LabelT, grape::EmptyType>;
+  using ele_tuple_t = std::tuple<int32_t, VID_T>;
+  using index_ele_tuple_t = std::tuple<size_t, int32_t, VID_T>;
+
+  static constexpr bool is_row_vertex_set_builder = false;
+  static constexpr bool is_flat_edge_set_builder = false;
+  static constexpr bool is_general_edge_set_builder = false;
+  static constexpr bool is_two_label_set_builder = true;
+  static constexpr bool is_collection_builder = false;
+
+  TwoLabelVertexSetImplKeyedBuilder(size_t size,
+                                    const std::array<LabelT, 2>& labels)
+      : ind_(0), labels_(labels) {
+    vec_.reserve(size);
+    bitset_.init(size);
+  }
+
+  TwoLabelVertexSetImplKeyedBuilder(const res_t& res)
+      : TwoLabelVertexSetImplKeyedBuilder(res.Size(), res.GetLabels()) {}
+
+  int32_t insert(const index_ele_tuple_t& tuple) {
+    auto global_id = GlobalId(std::get<1>(tuple), std::get<2>(tuple));
+    if (IsNull(global_id)) {
+      return -1;
+    }
+    if (global_id_map_.find(global_id) != global_id_map_.end()) {
+      return global_id_map_[global_id];
+    } else {
+      global_id_map_[global_id] = ind_;
+      if (std::get<1>(tuple) == 0) {
+        bitset_.set_bit(vec_.size());
+      }
+      vec_.emplace_back(std::get<2>(tuple));
+      return ind_++;
+    }
+  }
+
+  res_t Build() {
+    VLOG(10) << "Try to resize from " << bitset_.cardinality() << ", to "
+             << vec_.size();
+    bitset_.resize(vec_.size());
+    return res_t(std::move(vec_), std::move(labels_), std::move(bitset_));
+  }
+
+  size_t Size() const { return vec_.size(); }
+
+ private:
+  size_t ind_;
+  std::vector<VID_T> vec_;
+  std::array<LabelT, 2> labels_;
+  std::unordered_map<GlobalId, size_t> global_id_map_;
+  grape::Bitset bitset_;
+};
+
 template <typename VID_T, typename LabelT, typename... T>
 class TwoLabelVertexSetImplBuilder {
  public:
@@ -188,21 +315,22 @@ auto two_label_project_vertices_impl(
   offset.emplace_back(0);
   double t0 = -grape::GetCurrentTime();
   for (size_t i = 0; i < old_vec.size(); ++i) {
+    auto vid = old_vec[i];
     if (old_bit_set.get_bit(i) && label_flag[0]) {
-      auto vid = old_vec[i];
       if (std::apply(expr, prop_getters[0].get_view(vid))) {
         res_bitset.set_bit(res_vec.size());
         res_vec.emplace_back(old_vec[i]);
       }
-    } else if (label_flag[1] &&
-               std::apply(expr, prop_getters[1].get_view(old_vec[i]))) {
-      res_vec.emplace_back(old_vec[i]);
+    } else if (label_flag[1] && !old_bit_set.get_bit(i)) {
+      if (std::apply(expr, prop_getters[1].get_view(vid))) {
+        res_vec.emplace_back(old_vec[i]);
+      }
     }
-
     offset.emplace_back(res_vec.size());
   }
   t0 += grape::GetCurrentTime();
-  VLOG(10) << "expr + copy cost: " << t0;
+  VLOG(10) << "expr + copy cost: " << t0
+           << ", res_vec: " << gs::to_string(res_vec);
 
   res_bitset.resize(res_vec.size());
 
@@ -758,6 +886,19 @@ class TwoLabelVertexSetImpl {
     return res;
   }
 
+  std::vector<uint8_t> GenerateLabelIndices() const {
+    std::vector<uint8_t> label_indices;
+    label_indices.resize(vec_.size(), 255);
+    for (size_t i = 0; i < vec_.size(); ++i) {
+      if (bitset_.get_bit(i)) {
+        label_indices[i] = 0;
+      } else {
+        label_indices[i] = 1;
+      }
+    }
+    return label_indices;
+  }
+
   LabelT GetLabel(size_t i) const { return label_names_[i]; }
 
   const grape::Bitset& GetBitset() const { return bitset_; }
@@ -829,15 +970,12 @@ class TwoLabelVertexSetImpl {
     for (size_t i = 0; i < repeat_array.size(); ++i) {
       if (bitset_.get_bit(i)) {
         for (size_t j = 0; j < repeat_array[i]; ++j) {
-          // VLOG(10) << "Project: " << vids_[i];
           next_set.set_bit(next_vids.size());
           next_vids.push_back(vec_[i]);
           next_datas.push_back(data_tuple_[i]);
         }
       } else {
         for (size_t j = 0; j < repeat_array[i]; ++j) {
-          // VLOG(10) << "Project: " << vids_[i];
-          //   next_set.set_bit(next_vids.size());
           next_vids.push_back(vec_[i]);
           next_datas.push_back(data_tuple_[i]);
         }
@@ -1034,6 +1172,19 @@ class TwoLabelVertexSetImpl<VID_T, LabelT, grape::EmptyType> {
     return res;
   }
 
+  std::vector<uint8_t> GenerateLabelIndices() const {
+    std::vector<uint8_t> label_indices;
+    label_indices.resize(vec_.size(), 255);
+    for (size_t i = 0; i < vec_.size(); ++i) {
+      if (bitset_.get_bit(i)) {
+        label_indices[i] = 0;
+      } else {
+        label_indices[i] = 1;
+      }
+    }
+    return label_indices;
+  }
+
   LabelT GetLabel(size_t i) const { return label_names_[i]; }
 
   const grape::Bitset& GetBitset() const { return bitset_; }
@@ -1179,6 +1330,7 @@ class TwoLabelVertexSetImpl<VID_T, LabelT, grape::EmptyType> {
         ++i;
       }
     }
+    new_bitset.resize(new_vec.size());
     vec_.swap(new_vec);
     // safe?
     bitset_.swap(new_bitset);
diff --git a/flex/engines/hqps_db/structures/path.h b/flex/engines/hqps_db/structures/path.h
index 8b8b4d899737..234b2ea1619d 100644
--- a/flex/engines/hqps_db/structures/path.h
+++ b/flex/engines/hqps_db/structures/path.h
@@ -256,7 +256,7 @@ class PathSet {
       std::fill(label_set.begin(), label_set.end(), true);  // empty means all
     }
     size_t valid_labels = 0;
-    for (int i = 0; i < label_set.size(); ++i) {
+    for (size_t i = 0; i < label_set.size(); ++i) {
       if (label_set[i]) {
         labels.emplace_back(i);
         label_to_index[i] = valid_labels++;
@@ -265,7 +265,7 @@ class PathSet {
 
     std::vector<grape::Bitset> label_bitsets;
     label_bitsets.resize(labels.size());
-    for (auto i = 0; i < labels.size(); ++i) {
+    for (size_t i = 0; i < labels.size(); ++i) {
       label_bitsets[i].init(paths_.size());
     }
     for (auto iter : *this) {
@@ -289,7 +289,7 @@ class PathSet {
       offsets.emplace_back(vids.size());
     }
 
-    for (auto i = 0; i < labels.size(); ++i) {
+    for (size_t i = 0; i < labels.size(); ++i) {
       label_bitsets[i].resize(vids.size());
     }
     auto general_set = make_general_set(std::move(vids), std::move(labels),
diff --git a/flex/engines/http_server/codegen_proxy.h b/flex/engines/http_server/codegen_proxy.h
index 3e99343b80b3..012c63726583 100644
--- a/flex/engines/http_server/codegen_proxy.h
+++ b/flex/engines/http_server/codegen_proxy.h
@@ -109,6 +109,7 @@ class CodegenProxy {
                                    const std::string& query_name,
                                    const physical::PhysicalPlan& plan);
 
+  bool initialized_;
   std::string working_directory_;
   std::string codegen_bin_;
   std::string ir_compiler_prop_;
@@ -116,7 +117,6 @@ class CodegenProxy {
   std::mutex mutex_;
   std::condition_variable cv_;
   std::unordered_map<int32_t, StoredProcedureLibMeta> job_id_2_procedures_;
-  bool initialized_;
 };
 
 }  // namespace server
diff --git a/flex/interactive/bin/gs_interactive b/flex/interactive/bin/gs_interactive
index 208fb80fec25..64bbc6c2e29e 100755
--- a/flex/interactive/bin/gs_interactive
+++ b/flex/interactive/bin/gs_interactive
@@ -256,7 +256,10 @@ DATABASE_COMPILER_PLANNER_IS_ON="true"
 DATABASE_COMPILER_PLANNER_OPT="RBO"
 DATABASE_COMPILER_PLANNER_RULES="FilterIntoJoinRule,FilterMatchRule,NotMatchToAntiJoinRule"
 DATABASE_COMPILER_ENDPOINT_ADDRESS="localhost"
+DATABASE_COMPILER_BOLT_DISABLED="false"
 DATABASE_COMPILER_BOLT_PORT="7687"
+DATABASE_COMPILER_GREMLIN_DISABLED="true"
+DATABASE_COMPILER_GREMLIN_PORT="8182"
 DATABASE_COMPILER_QUERY_TIMEOUT="20000"
 
 ## hiactor related default configuration
@@ -401,7 +404,12 @@ function update_init_config_from_yaml(){
     x=$((x + 1))
   done
   # append compiler port and engine port to DATABASE_PORTS
-  DATABASE_PORTS="${DATABASE_COMPILER_BOLT_PORT}:${DATABASE_COMPILER_BOLT_PORT}"
+  if [ "${DATABASE_COMPILER_BOLT_DISABLED}" == "false" ]; then
+    DATABASE_PORTS="${DATABASE_COMPILER_BOLT_PORT}:${DATABASE_COMPILER_BOLT_PORT}"
+  fi
+  if [ "${DATABASE_COMPILER_GREMLIN_DISABLED}" == "false" ]; then
+    DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_COMPILER_GREMLIN_PORT}:${DATABASE_COMPILER_GREMLIN_PORT}"
+  fi
   DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_COMPUTE_ENGINE_PORT}:${DATABASE_COMPUTE_ENGINE_PORT}"
   DATABASE_PORTS="${DATABASE_PORTS},${DATABASE_ADMIN_PORT}:${DATABASE_ADMIN_PORT}"
 }
@@ -459,6 +467,15 @@ function update_engine_config_from_yaml(){
   if [[ -n "${compiler_endpoint_bolt_connector_port}" ]]; then
     DATABASE_COMPILER_BOLT_PORT="${compiler_endpoint_bolt_connector_port}"
   fi
+  if [[ -n "${compiler_endpoint_bolt_connector_disabled}" ]]; then
+    DATABASE_COMPILER_BOLT_DISABLED="${compiler_endpoint_bolt_connector_disabled}"
+  fi
+  if [[ -n "${compiler_endpoint_gremlin_connector_port}" ]]; then
+    DATABASE_COMPILER_GREMLIN_PORT="${compiler_endpoint_gremlin_connector_port}"
+  fi
+  if [[ -n "${compiler_endpoint_gremlin_connector_disabled}" ]]; then
+    DATABASE_COMPILER_GREMLIN_DISABLED="${compiler_endpoint_gremlin_connector_disabled}"
+  fi
   if [[ -n "${compiler_query_timeout}" ]]; then
     DATABASE_COMPILER_QUERY_TIMEOUT="${compiler_query_timeout}"
   fi
@@ -745,11 +762,11 @@ function do_gen_conf(){
   echo "  endpoint:"  >> ${output_config_file}
   echo "    default_listen_address: ${DATABASE_COMPILER_ENDPOINT_ADDRESS}" >> ${output_config_file}
   echo "    bolt_connector:" >> ${output_config_file}
-  echo "      disabled: false" >> ${output_config_file}
+  echo "      disabled: ${DATABASE_COMPILER_BOLT_DISABLED}" >> ${output_config_file}
   echo "      port: ${DATABASE_COMPILER_BOLT_PORT}" >> ${output_config_file}
   echo "    gremlin_connector:" >> ${output_config_file}
-  echo "      disabled: true" >> ${output_config_file}
-  echo "      port: 8182" >> ${output_config_file}
+  echo "      disabled: ${DATABASE_COMPILER_GREMLIN_DISABLED}" >> ${output_config_file}
+  echo "      port: ${DATABASE_COMPILER_GREMLIN_PORT}" >> ${output_config_file}
   echo "  query_timeout: ${DATABASE_COMPILER_QUERY_TIMEOUT}" >> ${output_config_file}
 
   #echo http_service
@@ -964,10 +981,10 @@ function do_init(){
   local username=$(id -un)
 
   GIE_DB_IMAGE_NAME_TAG="${GIE_DB_IMAGE_NAME}:${DATABASE_VERSION}"
-  cmd="docker run -it -d --privileged --name ${GIE_DB_CONTAINER_NAME}"
+  cmd="docker run --entrypoint bash -it -d --privileged --name ${GIE_DB_CONTAINER_NAME}"
 
   # create user in container
-  cmd="${cmd} ${port_cmd} ${mount_cmd} ${GIE_DB_IMAGE_NAME_TAG} bash"
+  cmd="${cmd} ${port_cmd} ${mount_cmd} ${GIE_DB_IMAGE_NAME_TAG}"
   
   info "Running cmd: ${cmd}"
   # eval $cmd, exit if failed
@@ -1439,8 +1456,18 @@ function do_start(){
   check_process_running_in_container ${GIE_DB_CONTAINER_NAME} ${DOCKER_DB_COMPILER_BIN} ", use gs_interactive service get_log -o [dir] to see more details"
   info "Successfully start compiler"
   # get cypher port from engine config file
-  # bolt_connector_port=$(parse_yaml "${engine_config_file}" | grep "compiler_endpoint_bolt_connector_port" | awk -F "=" '{print $2}')
-  emph "DataBase service is running, port is open on :${DATABASE_COMPILER_BOLT_PORT}"
+  eval $(parse_yaml "${real_engine_config_file}")
+
+  if [ "${compiler_endpoint_bolt_connector_disabled}" = "false" ]; then
+    emph "Cypher port is open on :${compiler_endpoint_bolt_connector_port}"
+  else
+    emph "Bolt connector is disabled, no bolt endpoint is available"
+  fi
+  if [ "${compiler_endpoint_gremlin_connector_disabled}" = "false" ]; then
+    emph "Gremlin port is running, port is open on :${compiler_endpoint_gremlin_connector_port}"
+  else
+    emph "Gremlin connector is disabled, no gremlin endpoint is available"
+  fi
 
   # if do_start success, we should write current args to ${HOST_DB_RUNNING_FILE}
   echo "GRAPH_NAME=${graph_name}" > ${HOST_DB_RUNNING_FILE}
@@ -1497,7 +1524,16 @@ function do_status() {
   # copy the engine config file to host's tmp directory
   docker cp "${GIE_DB_CONTAINER_NAME}:${docker_engine_config_file}" "${HOST_DB_TMP_DIR}/engine_config.yaml" || exit 1
   eval $(parse_yaml "${HOST_DB_TMP_DIR}/engine_config.yaml")
-  emph "Database service is running, port is open on :${compiler_endpoint_bolt_connector_port}"
+  if [ "${compiler_endpoint_bolt_connector_disabled}" = "false" ]; then
+    emph "Cypher port is open on :${compiler_endpoint_bolt_connector_port}"
+  else
+    emph "Bolt connector is disabled, no bolt endpoint is available"
+  fi
+  if [ "${compiler_endpoint_gremlin_connector_disabled}" = "false" ]; then
+    emph "Gremlin port is open on :${compiler_endpoint_gremlin_connector_port}"
+  else
+    emph "Gremlin connector is disabled, no gremlin endpoint is available"
+  fi
   # print current running graph name
   if [ -f "${HOST_DB_RUNNING_FILE}" ]; then
     local graph_name=$(sed -n '1p' ${HOST_DB_RUNNING_FILE} | cut -d '=' -f 2)
diff --git a/flex/interactive/examples/modern_graph/person_created_software.csv b/flex/interactive/examples/modern_graph/person_created_software.csv
index 248331e15956..2b1d262322b5 100644
--- a/flex/interactive/examples/modern_graph/person_created_software.csv
+++ b/flex/interactive/examples/modern_graph/person_created_software.csv
@@ -2,4 +2,4 @@ person.id|software.id|weight
 1|3|0.4
 4|3|0.4
 6|3|0.2
-2|5|1.0
\ No newline at end of file
+4|5|1.0
\ No newline at end of file
diff --git a/flex/resources/hqps/CMakeLists.txt.template b/flex/resources/hqps/CMakeLists.txt.template
index f8c0d8445445..d40f3e0544bd 100644
--- a/flex/resources/hqps/CMakeLists.txt.template
+++ b/flex/resources/hqps/CMakeLists.txt.template
@@ -33,7 +33,7 @@ endif()
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -Wl,-rpath,$ORIGIN -O3 -flto -Werror=unused-result -fPIC -no-pie")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -Wl,-rpath,$ORIGIN -O0 -flto -Werror=unused-result -fPIC -no-pie")
 
 add_library(${QUERY_NAME} SHARED ${PROJECT_SOURCE_DIR}/${QUERY_NAME}.cc)
 target_include_directories(${QUERY_NAME} PUBLIC ${FLEX_INCLUDE_PREFIX} ${FLEX_INCLUDE_PREFIX}/flex/build/engines/hqps_db/)
diff --git a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc
index 490374359e27..1debf80ac3bb 100644
--- a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc
+++ b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc
@@ -401,8 +401,7 @@ void CSVFragmentLoader::fillVertexReaderMeta(
     }
   }
 
-  VLOG(10) << "Include columns: " << included_col_names.size()
-           << gs::to_string(included_col_names);
+  VLOG(10) << "Include columns: " << included_col_names.size();
   // if empty, then means need all columns
   convert_options.include_columns = included_col_names;
 
diff --git a/flex/utils/property/column.h b/flex/utils/property/column.h
index 478364445d13..85d27e4f765e 100644
--- a/flex/utils/property/column.h
+++ b/flex/utils/property/column.h
@@ -661,6 +661,8 @@ class TypedRefColumn : public RefColumnBase {
                               : extra_buffer.get(index - basic_size);
   }
 
+  size_t size() const { return basic_size + extra_size; }
+
   Any get(size_t index) const override {
     return AnyConverter<T>::to_any(get_view(index));
   }
@@ -692,6 +694,27 @@ class TypedRefColumn<LabelKey> : public RefColumnBase {
   LabelKey label_key_;
 };
 
+template <>
+class TypedRefColumn<GlobalId> : public RefColumnBase {
+ public:
+  using label_t = typename LabelKey::label_data_type;
+  TypedRefColumn(label_t label_key) : label_key_(label_key) {}
+
+  ~TypedRefColumn() {}
+
+  inline GlobalId get_view(size_t index) const {
+    return GlobalId(label_key_, index);
+  }
+
+  Any get(size_t index) const override {
+    LOG(ERROR) << "GlobalId Column does not support get() to Any";
+    return Any();
+  }
+
+ private:
+  label_t label_key_;
+};
+
 }  // namespace gs
 
 #endif  // GRAPHSCOPE_PROPERTY_COLUMN_H_
diff --git a/flex/utils/property/types.cc b/flex/utils/property/types.cc
index 0b04865ecd5d..4ffc33ad1169 100644
--- a/flex/utils/property/types.cc
+++ b/flex/utils/property/types.cc
@@ -48,6 +48,10 @@ const PropertyType PropertyType::kString =
     PropertyType(impl::PropertyTypeImpl::kString);
 const PropertyType PropertyType::kStringMap =
     PropertyType(impl::PropertyTypeImpl::kStringMap);
+const PropertyType PropertyType::kVertexGlobalId =
+    PropertyType(impl::PropertyTypeImpl::kVertexGlobalId);
+const PropertyType PropertyType::kLabel =
+    PropertyType(impl::PropertyTypeImpl::kLabel);
 
 bool PropertyType::operator==(const PropertyType& other) const {
   if (type_enum == impl::PropertyTypeImpl::kVarChar &&
@@ -120,6 +124,14 @@ PropertyType PropertyType::Varchar(uint16_t max_length) {
   return PropertyType(impl::PropertyTypeImpl::kVarChar, max_length);
 }
 
+PropertyType PropertyType::VertexGlobalId() {
+  return PropertyType(impl::PropertyTypeImpl::kVertexGlobalId);
+}
+
+PropertyType PropertyType::Label() {
+  return PropertyType(impl::PropertyTypeImpl::kLabel);
+}
+
 grape::InArchive& operator<<(grape::InArchive& in_archive,
                              const PropertyType& value) {
   in_archive << value.type_enum;
@@ -164,6 +176,10 @@ grape::InArchive& operator<<(grape::InArchive& in_archive, const Any& value) {
     in_archive << value.type << value.value.day.to_u32();
   } else if (value.type == PropertyType::String()) {
     in_archive << value.type << value.value.s;
+  } else if (value.type == PropertyType::VertexGlobalId()) {
+    in_archive << value.type << value.value.vertex_gid;
+  } else if (value.type == PropertyType::Label()) {
+    in_archive << value.type << value.value.label_key;
   } else {
     in_archive << PropertyType::kEmpty;
   }
@@ -202,6 +218,10 @@ grape::OutArchive& operator>>(grape::OutArchive& out_archive, Any& value) {
     value.value.day.from_u32(val);
   } else if (value.type == PropertyType::String()) {
     out_archive >> value.value.s;
+  } else if (value.type == PropertyType::VertexGlobalId()) {
+    out_archive >> value.value.vertex_gid;
+  } else if (value.type == PropertyType::Label()) {
+    out_archive >> value.value.label_key;
   } else {
     value.type = PropertyType::kEmpty;
   }
@@ -225,6 +245,52 @@ grape::OutArchive& operator>>(grape::OutArchive& out_archive,
   return out_archive;
 }
 
+grape::InArchive& operator<<(grape::InArchive& in_archive,
+                             const GlobalId& value) {
+  in_archive << value.global_id;
+  return in_archive;
+}
+grape::OutArchive& operator>>(grape::OutArchive& out_archive, GlobalId& value) {
+  out_archive >> value.global_id;
+  return out_archive;
+}
+
+grape::InArchive& operator<<(grape::InArchive& in_archive,
+                             const LabelKey& value) {
+  in_archive << value.label_id;
+  return in_archive;
+}
+grape::OutArchive& operator>>(grape::OutArchive& out_archive, LabelKey& value) {
+  out_archive >> value.label_id;
+  return out_archive;
+}
+
+GlobalId::label_id_t GlobalId::get_label_id(gid_t gid) {
+  return static_cast<label_id_t>(gid >> label_id_offset);
+}
+
+GlobalId::vid_t GlobalId::get_vid(gid_t gid) {
+  return static_cast<vid_t>(gid & vid_mask);
+}
+
+GlobalId::GlobalId() : global_id(0) {}
+
+GlobalId::GlobalId(label_id_t label_id, vid_t vid) {
+  global_id = (static_cast<uint64_t>(label_id) << label_id_offset) | vid;
+}
+
+GlobalId::GlobalId(gid_t gid) : global_id(gid) {}
+
+GlobalId::label_id_t GlobalId::label_id() const {
+  return static_cast<label_id_t>(global_id >> label_id_offset);
+}
+
+GlobalId::vid_t GlobalId::vid() const {
+  return static_cast<vid_t>(global_id & vid_mask);
+}
+
+std::string GlobalId::to_string() const { return std::to_string(global_id); }
+
 Date::Date(int64_t x) : milli_second(x) {}
 
 std::string Date::to_string() const { return std::to_string(milli_second); }
diff --git a/flex/utils/property/types.h b/flex/utils/property/types.h
index d4f32544768d..8ba27b71053e 100644
--- a/flex/utils/property/types.h
+++ b/flex/utils/property/types.h
@@ -62,6 +62,8 @@ enum class PropertyTypeImpl {
   kUInt16,
   kStringMap,
   kVarChar,
+  kVertexGlobalId,
+  kLabel,
 };
 
 // Stores additional type information for PropertyTypeImpl
@@ -101,6 +103,8 @@ struct PropertyType {
   static PropertyType String();
   static PropertyType StringMap();
   static PropertyType Varchar(uint16_t max_length);
+  static PropertyType VertexGlobalId();
+  static PropertyType Label();
 
   static const PropertyType kEmpty;
   static const PropertyType kBool;
@@ -116,11 +120,51 @@ struct PropertyType {
   static const PropertyType kDay;
   static const PropertyType kString;
   static const PropertyType kStringMap;
+  static const PropertyType kVertexGlobalId;
+  static const PropertyType kLabel;
 
   bool operator==(const PropertyType& other) const;
   bool operator!=(const PropertyType& other) const;
 };
 
+// encoded with label_id and vid_t.
+struct GlobalId {
+  using label_id_t = uint8_t;
+  using vid_t = uint32_t;
+  using gid_t = uint64_t;
+  static constexpr int32_t label_id_offset = 64 - sizeof(label_id_t) * 8;
+  static constexpr uint64_t vid_mask = (1ULL << label_id_offset) - 1;
+
+  static label_id_t get_label_id(gid_t gid);
+  static vid_t get_vid(gid_t gid);
+
+  uint64_t global_id;
+
+  GlobalId();
+  GlobalId(label_id_t label_id, vid_t vid);
+  GlobalId(gid_t gid);
+
+  label_id_t label_id() const;
+  vid_t vid() const;
+
+  std::string to_string() const;
+};
+
+inline bool operator==(const GlobalId& lhs, const GlobalId& rhs) {
+  return lhs.global_id == rhs.global_id;
+}
+
+inline bool operator!=(const GlobalId& lhs, const GlobalId& rhs) {
+  return lhs.global_id != rhs.global_id;
+}
+inline bool operator<(const GlobalId& lhs, const GlobalId& rhs) {
+  return lhs.global_id < rhs.global_id;
+}
+
+inline bool operator>(const GlobalId& lhs, const GlobalId& rhs) {
+  return lhs.global_id > rhs.global_id;
+}
+
 struct __attribute__((packed)) Date {
   Date() = default;
   ~Date() = default;
@@ -202,14 +246,6 @@ struct LabelKey {
   int32_t label_id;
   LabelKey() = default;
   LabelKey(label_data_type id) : label_id(id) {}
-  LabelKey(LabelKey&&) = default;
-  LabelKey(const LabelKey&) = default;
-
-  // operator=
-  LabelKey& operator=(const LabelKey& other) {
-    label_id = other.label_id;
-    return *this;
-  }
 };
 
 union AnyValue {
@@ -222,6 +258,8 @@ union AnyValue {
   float f;
   int64_t l;
   uint64_t ul;
+  GlobalId vertex_gid;
+  LabelKey label_key;
 
   Date d;
   Day day;
@@ -277,6 +315,16 @@ struct Any {
     value.ul = v;
   }
 
+  void set_vertex_gid(GlobalId v) {
+    type = PropertyType::kVertexGlobalId;
+    value.vertex_gid = v;
+  }
+
+  void set_label_key(LabelKey v) {
+    type = PropertyType::kLabel;
+    value.label_key = v;
+  }
+
   void set_date(int64_t v) {
     type = PropertyType::kDate;
     value.d.milli_second = v;
@@ -345,6 +393,10 @@ struct Any {
       return value.b ? "true" : "false";
     } else if (type == PropertyType::kFloat) {
       return std::to_string(value.f);
+    } else if (type == PropertyType::kVertexGlobalId) {
+      return value.vertex_gid.to_string();
+    } else if (type == PropertyType::kLabel) {
+      return std::to_string(value.label_key.label_id);
     } else {
       LOG(FATAL) << "Unexpected property type: "
                  << static_cast<int>(type.type_enum);
@@ -407,6 +459,16 @@ struct Any {
     return value.day;
   }
 
+  const GlobalId& AsGlobalId() const {
+    assert(type == PropertyType::kVertexGlobalId);
+    return value.vertex_gid;
+  }
+
+  const LabelKey& AsLabelKey() const {
+    assert(type == PropertyType::kLabel);
+    return value.label_key;
+  }
+
   template <typename T>
   static Any From(const T& value) {
     return AnyConverter<T>::to_any(value);
@@ -436,6 +498,10 @@ struct Any {
         return value.b == other.value.b;
       } else if (type == PropertyType::kFloat) {
         return value.f == other.value.f;
+      } else if (type == PropertyType::kVertexGlobalId) {
+        return value.vertex_gid == other.value.vertex_gid;
+      } else if (type == PropertyType::kLabel) {
+        return value.label_key.label_id == other.value.label_key.label_id;
       } else if (type.type_enum == impl::PropertyTypeImpl::kVarChar) {
         if (other.type.type_enum != impl::PropertyTypeImpl::kVarChar) {
           return false;
@@ -473,6 +539,10 @@ struct Any {
         return value.b < other.value.b;
       } else if (type == PropertyType::kFloat) {
         return value.f < other.value.f;
+      } else if (type == PropertyType::kVertexGlobalId) {
+        return value.vertex_gid < other.value.vertex_gid;
+      } else if (type == PropertyType::kLabel) {
+        return value.label_key.label_id < other.value.label_key.label_id;
       } else {
         return false;
       }
@@ -534,6 +604,22 @@ struct ConvertAny<uint64_t> {
   }
 };
 
+template <>
+struct ConvertAny<GlobalId> {
+  static void to(const Any& value, GlobalId& out) {
+    CHECK(value.type == PropertyType::kVertexGlobalId);
+    out = value.value.vertex_gid;
+  }
+};
+
+template <>
+struct ConvertAny<LabelKey> {
+  static void to(const Any& value, LabelKey& out) {
+    CHECK(value.type == PropertyType::kLabel);
+    out = value.value.label_key;
+  }
+};
+
 template <>
 struct ConvertAny<Date> {
   static void to(const Any& value, Date& out) {
@@ -748,6 +834,32 @@ struct AnyConverter<uint64_t> {
   }
 };
 
+template <>
+struct AnyConverter<GlobalId> {
+  static PropertyType type() { return PropertyType::kVertexGlobalId; }
+
+  static Any to_any(const GlobalId& value) {
+    Any ret;
+    ret.set_vertex_gid(value);
+    return ret;
+  }
+
+  static AnyValue to_any_value(const GlobalId& value) {
+    AnyValue ret;
+    ret.vertex_gid = value;
+    return ret;
+  }
+
+  static const GlobalId& from_any(const Any& value) {
+    CHECK(value.type == PropertyType::kVertexGlobalId);
+    return value.value.vertex_gid;
+  }
+
+  static const GlobalId& from_any_value(const AnyValue& value) {
+    return value.vertex_gid;
+  }
+};
+
 template <>
 struct AnyConverter<Date> {
   static PropertyType type() { return PropertyType::kDate; }
@@ -936,6 +1048,32 @@ struct AnyConverter<float> {
   static const float& from_any_value(const AnyValue& value) { return value.f; }
 };
 
+template <>
+struct AnyConverter<LabelKey> {
+  static PropertyType type() { return PropertyType::kLabel; }
+
+  static Any to_any(const LabelKey& value) {
+    Any ret;
+    ret.set_label_key(value);
+    return ret;
+  }
+
+  static AnyValue to_any_value(const LabelKey& value) {
+    AnyValue ret;
+    ret.label_key = value;
+    return ret;
+  }
+
+  static const LabelKey& from_any(const Any& value) {
+    CHECK(value.type == PropertyType::kLabel);
+    return value.value.label_key;
+  }
+
+  static const LabelKey& from_any_value(const AnyValue& value) {
+    return value.label_key;
+  }
+};
+
 grape::InArchive& operator<<(grape::InArchive& in_archive,
                              const PropertyType& value);
 grape::OutArchive& operator>>(grape::OutArchive& out_archive,
@@ -949,11 +1087,23 @@ grape::InArchive& operator<<(grape::InArchive& in_archive,
 grape::OutArchive& operator>>(grape::OutArchive& out_archive,
                               std::string_view& value);
 
+grape::InArchive& operator<<(grape::InArchive& in_archive,
+                             const GlobalId& value);
+grape::OutArchive& operator>>(grape::OutArchive& out_archive, GlobalId& value);
+
 }  // namespace gs
 
 namespace boost {
 // override boost hash function for EmptyType
 inline std::size_t hash_value(const grape::EmptyType& value) { return 0; }
+inline std::size_t hash_value(const gs::GlobalId& value) {
+  return std::hash<uint64_t>()(value.global_id);
+}
+// overload hash_value for LabelKey
+inline std::size_t hash_value(const gs::LabelKey& key) {
+  return std::hash<int32_t>()(key.label_id);
+}
+
 }  // namespace boost
 
 namespace std {
@@ -1006,11 +1156,19 @@ inline ostream& operator<<(ostream& os, gs::PropertyType pt) {
     os << "string_map";
   } else if (pt.type_enum == gs::impl::PropertyTypeImpl::kVarChar) {
     os << "varchar(" << pt.additional_type_info.max_length << ")";
+  } else if (pt == gs::PropertyType::VertexGlobalId()) {
+    os << "vertex_global_id";
   } else {
     os << "unknown";
   }
   return os;
 }
+template <>
+struct hash<gs::GlobalId> {
+  size_t operator()(const gs::GlobalId& value) const {
+    return std::hash<uint64_t>()(value.global_id);
+  }
+};
 
 }  // namespace std
 
diff --git a/interactive_engine/compiler/ir_csr_ci.sh b/interactive_engine/compiler/ir_csr_ci.sh
index a54eb1a20e32..e2a98c797b23 100755
--- a/interactive_engine/compiler/ir_csr_ci.sh
+++ b/interactive_engine/compiler/ir_csr_ci.sh
@@ -5,7 +5,7 @@ cd ${base_dir}/../executor/ir/target/release &&
 RUST_LOG=info CSR_PATH=/tmp/gstest/modern_graph_csr_bin PARTITION_ID=0 ./start_rpc_server_csr --config ${base_dir}/../executor/ir/integrated/config &
 sleep 5s
 # start compiler service
-cd ${base_dir} && make run graph.store=csr &
+cd ${base_dir} && make run graph.store=rust-mcsr &
 sleep 5s
 # run gremlin standard tests
 cd ${base_dir} && make gremlin_test
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java
index b27cc62169fc..f5ee93fd4a9c 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java
@@ -179,8 +179,12 @@ private static GraphProperties getTestGraph(Configs configs) {
             case "exp":
                 testGraph = TestGraphFactory.EXPERIMENTAL;
                 break;
-            case "csr":
-                testGraph = TestGraphFactory.MCSR;
+            case "rust-mcsr":
+                testGraph = TestGraphFactory.RUST_MCSR;
+                break;
+            case "cpp-mcsr":
+                logger.info("using cpp-mcsr as test graph");
+                testGraph = TestGraphFactory.CPP_MCSR;
                 break;
             default:
                 throw new IllegalArgumentException("unknown graph store type");
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java
index b58f7aad881c..69f3137ea04e 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java
@@ -66,6 +66,15 @@ public class YamlConfigs extends Configs {
                                 return null;
                             }
                         })
+                .put(
+                        "graph.store",
+                        (Configs configs) -> {
+                            if (configs.get("compute_engine.store.type") != null) {
+                                return configs.get("compute_engine.store.type");
+                            } else {
+                                return "cpp-mcsr";
+                            }
+                        })
                 .put(
                         "graph.stored.procedures",
                         (Configs configs) -> {
@@ -200,7 +209,10 @@ public class YamlConfigs extends Configs {
                 .put("engine.type", (Configs configs) -> configs.get("compute_engine.type"))
                 .put(
                         "calcite.default.charset",
-                        (Configs configs) -> configs.get("compiler.calcite_default_charset"));
+                        (Configs configs) -> configs.get("compiler.calcite_default_charset"))
+                .put(
+                        "gremlin.script.language.name",
+                        (Configs configs) -> configs.get("compiler.gremlin_script_language_name"));
         valueGetterMap = mapBuilder.build();
     }
 
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java
index b1a3d59c8112..f6683b4673d6 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java
@@ -153,21 +153,27 @@ public static QueryContext get_graph_algo_test8() {
         return new QueryContext(query, expected);
     }
 
-    public static QueryContext get_graph_algo_test9() {
-        String query = "MATCH (a)-[b:WorkOn]-() return count(b) AS count;";
+    public static QueryContext get_graph_algo_test9_0() {
+        String query = "MATCH (a)-[b:WorkOn]->() return count(b) AS count;";
         List<String> expected = Arrays.asList("Record<{count: 123}>");
         return new QueryContext(query, expected);
     }
 
+    public static QueryContext get_graph_algo_test9_1() {
+        String query = "MATCH (a)-[b:WorkOn]-() return count(b) AS count;";
+        List<String> expected = Arrays.asList("Record<{count: 246}>");
+        return new QueryContext(query, expected);
+    }
+
     public static QueryContext get_graph_algo_test10() {
-        String query = "MATCH (a: Paper)-[]-() return count(a) AS count;";
-        List<String> expected = Arrays.asList("Record<{count: 336}>");
+        String query = "MATCH (a: Paper)-[]->() return count(a) AS count;";
+        List<String> expected = Arrays.asList("Record<{count: 685}>");
         return new QueryContext(query, expected);
     }
 
     public static QueryContext get_graph_algo_test11() {
         String query = "MATCH (a)-[b:Target]-() return count(b) AS count;";
-        List<String> expected = Arrays.asList("Record<{count: 69}>");
+        List<String> expected = Arrays.asList("Record<{count: 114}>");
         return new QueryContext(query, expected);
     }
 
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java
index 530c334a55fd..fdd1983522d7 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4x/visitor/ExpressionVisitor.java
@@ -203,7 +203,8 @@ public RexNode visitTraversalPredicate_outside(
     @Override
     public RexNode visitTraversalPredicate_startingWith(
             GremlinGSParser.TraversalPredicate_startingWithContext ctx) {
-        String posixRegex = "^" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral());
+        String posixRegex =
+                "^" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + ".*";
         return builder.call(
                 GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE,
                 propertyKey,
@@ -213,7 +214,8 @@ public RexNode visitTraversalPredicate_startingWith(
     @Override
     public RexNode visitTraversalPredicate_notStartingWith(
             GremlinGSParser.TraversalPredicate_notStartingWithContext ctx) {
-        String posixRegex = "^" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral());
+        String posixRegex =
+                "^" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + ".*";
         return builder.not(
                 builder.call(
                         GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE,
@@ -224,7 +226,8 @@ public RexNode visitTraversalPredicate_notStartingWith(
     @Override
     public RexNode visitTraversalPredicate_endingWith(
             GremlinGSParser.TraversalPredicate_endingWithContext ctx) {
-        String posixRegex = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "$";
+        String posixRegex =
+                ".*" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "$";
         return builder.call(
                 GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE,
                 propertyKey,
@@ -234,7 +237,8 @@ public RexNode visitTraversalPredicate_endingWith(
     @Override
     public RexNode visitTraversalPredicate_notEndingWith(
             GremlinGSParser.TraversalPredicate_notEndingWithContext ctx) {
-        String posixRegex = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "$";
+        String posixRegex =
+                ".*" + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()) + "$";
         return builder.not(
                 builder.call(
                         GraphStdOperatorTable.POSIX_REGEX_CASE_SENSITIVE,
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/calcite/RemoteTestGraph.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/calcite/RemoteTestGraph.java
index defe21a781b6..8fe552895195 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/calcite/RemoteTestGraph.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/calcite/RemoteTestGraph.java
@@ -1017,10 +1017,10 @@
 //        test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest",
 //        method = "g_VX4X_bothE",
 //        reason = "returned label is id")
-// @Graph.OptOut(
-//        test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest",
-//        method = "g_E_hasXlabelXknowsX",
-//        reason = "returned label is id")
+@Graph.OptOut(
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest",
+        method = "g_E_hasXlabelXknowsX",
+        reason = "Tested in IrGremlinQueryTest")
 // @Graph.OptOut(
 //        test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest",
 //        method = "g_VX2X_inE",
@@ -1311,9 +1311,10 @@
         method = "g_V_group_byXlabelX_byXlabel_countX",
         test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
         reason = "will be supported")
-// @Graph.OptOut(method="g_V_group_byXageX" ,
-// test="org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", reason = "will
-// be supported")
+@Graph.OptOut(
+        method = "g_V_group_byXageX",
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
+        reason = "tinkerpop v3.5 takes null key into account, but later version filter null key")
 
 // @Graph.OptOut(method="g_V_hasXp_neqXvXX" ,
 // test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of
@@ -1519,14 +1520,14 @@
 //        method = "g_V_valueMapXname_ageX",
 //        reason = "unsupported")
 
-// @Graph.OptOut(
-//        method = "g_V_unionXout__inX_name",
-//        test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
-//        reason = "union is unsupported yet")
-// @Graph.OptOut(
-//        test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
-//        method = "g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX",
-//        reason = "union is unsupported yet")
+@Graph.OptOut(
+        method = "g_V_unionXout__inX_name",
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
+        reason = "Tested in IrGremlinQueryTest")
+@Graph.OptOut(
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
+        method = "g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX",
+        reason = "Tested in IrGremlinQueryTest")
 // @Graph.OptOut(
 //        method = "g_V_haslabel_union_identity_out_values",
 //        test = "com.alibaba.graphscope.gremlin.integration.suite.standard.IrGremlinQueryTest",
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/TestGraphFactory.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/TestGraphFactory.java
index 96fe944cef2c..377dda31d7d7 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/TestGraphFactory.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/TestGraphFactory.java
@@ -79,7 +79,7 @@ public Map<String, Object> getProperties(Configs configs) {
             return JsonUtils.fromJson(json, new TypeReference<Map<String, Object>>() {});
         }
     },
-    MCSR {
+    RUST_MCSR {
         @Override
         public Map<String, Object> getProperties(Configs configs) {
             String json =
@@ -297,5 +297,60 @@ public Map<String, Object> getProperties(Configs configs) {
             }
             return JsonUtils.fromJson(json, new TypeReference<Map<String, Object>>() {});
         }
+    },
+    CPP_MCSR {
+        @Override
+        public Map<String, Object> getProperties(Configs configs) {
+            String json =
+                    "{\n"
+                            + "  \"vertex_properties\": {\n"
+                            + "    \"0\": {\n"
+                            + "      \"name\": \"marko\",\n"
+                            + "      \"age\": 29\n"
+                            + "    },\n"
+                            + "    \"1\": {\n"
+                            + "      \"name\": \"vadas\",\n"
+                            + "      \"age\": 27\n"
+                            + "    },\n"
+                            + "    \"2\": {\n"
+                            + "      \"name\": \"josh\",\n"
+                            + "      \"age\": 32\n"
+                            + "    },\n"
+                            + "    \"3\": {\n"
+                            + "      \"name\": \"peter\",\n"
+                            + "      \"age\": 35\n"
+                            + "    },\n"
+                            + "    \"72057594037927936\": {\n"
+                            + "      \"name\": \"lop\",\n"
+                            + "      \"lang\": \"java\"\n"
+                            + "    },\n"
+                            + "    \"72057594037927937\": {\n"
+                            + "      \"name\": \"ripple\",\n"
+                            + "      \"lang\": \"java\"\n"
+                            + "    }\n"
+                            + "  },\n"
+                            + "  \"edge_properties\": {\n"
+                            + "    \"1103806595072\": {\n"
+                            + "      \"weight\": 0.4\n" // 1-[created]->3
+                            + "    },\n"
+                            + "    \"1103808692224\": {\n"
+                            + "      \"weight\": 0.4\n" // 4-[created]->3
+                            + "    },\n"
+                            + "    \"1103809740800\": {\n"
+                            + "      \"weight\": 1.0\n" // 6-[created]->3
+                            + "    },\n"
+                            + "    \"1103808692225\": {\n"
+                            + "      \"weight\": 0.2\n" // 4-[created]->5
+                            + "    },\n"
+                            + "    \"1\": {\n"
+                            + "      \"weight\": 0.5\n" // 1-[knows]->2
+                            + "    },\n"
+                            + "    \"2\": {\n"
+                            + "      \"weight\": 1.0\n" // 2-[knows]->4
+                            + "    }\n"
+                            + "  }\n"
+                            + "}";
+            return JsonUtils.fromJson(json, new TypeReference<Map<String, Object>>() {});
+        }
     }
 }
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/suite/standard/IrGremlinQueryTest.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/suite/standard/IrGremlinQueryTest.java
index 8eaaab41c255..4a778fab3ac8 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/suite/standard/IrGremlinQueryTest.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/suite/standard/IrGremlinQueryTest.java
@@ -17,6 +17,7 @@
 package com.alibaba.graphscope.gremlin.integration.suite.standard;
 
 import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.*;
+import static org.junit.Assume.assumeFalse;
 
 import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversal;
 
@@ -34,6 +35,7 @@
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -124,6 +126,8 @@ public abstract class IrGremlinQueryTest extends AbstractGremlinProcessTest {
             get_g_VX1X_outEXknowsX_asXhereX_hasXweight_1X_inV_hasXname_joshX_selectXhereX(
                     final Object v1Id);
 
+    public abstract Traversal<Edge, Edge> get_g_E_hasLabelXknowsX();
+
     // g.V().out().union(out(), in(), in()).count()
     public abstract Traversal<Vertex, Long> get_g_V_out_union_out_in_in_count();
 
@@ -145,9 +149,17 @@ public abstract class IrGremlinQueryTest extends AbstractGremlinProcessTest {
     public abstract Traversal<Vertex, Long>
             get_g_V_out_union_inXunion_out_outX_inXunion_out_outX_count();
 
+    public abstract Traversal<Vertex, String> get_g_V_unionXout__inX_name();
+
+    public abstract Traversal<Vertex, Number>
+            get_g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX(
+                    final Object v1Id, final Object v2Id);
+
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_out_union_out_in_in_count() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported in hqps
         Traversal<Vertex, Long> traversal = this.get_g_V_out_union_out_in_in_count();
         this.printTraversalForm(traversal);
         Assert.assertEquals(26, traversal.next().intValue());
@@ -156,6 +168,8 @@ public void g_V_out_union_out_in_in_count() {
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_union_out_inXunion_out_outX_count() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported  in hqps
         Traversal<Vertex, Long> traversal = this.get_g_V_union_out_inXunion_out_outX_count();
         this.printTraversalForm(traversal);
         Assert.assertEquals(34, traversal.next().intValue());
@@ -164,6 +178,8 @@ public void g_V_union_out_inXunion_out_outX_count() {
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_out_union_out_inXunion_out_outX_count() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported in hqps
         Traversal<Vertex, Long> traversal = this.get_g_V_out_union_out_inXunion_out_outX_count();
         this.printTraversalForm(traversal);
         Assert.assertEquals(54, traversal.next().intValue());
@@ -172,6 +188,8 @@ public void g_V_out_union_out_inXunion_out_outX_count() {
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_out_union_outXunion_out_outX_inXunion_out_outX_count() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported  in hqps
         Traversal<Vertex, Long> traversal =
                 this.get_g_V_out_union_outXunion_out_outX_inXunion_out_outX_count();
         this.printTraversalForm(traversal);
@@ -181,6 +199,8 @@ public void g_V_out_union_outXunion_out_outX_inXunion_out_outX_count() {
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_union_inXunion_out_outX_inXunion_out_outX_count() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported in hqps
         Traversal<Vertex, Long> traversal =
                 this.get_g_V_union_inXunion_out_outX_inXunion_out_outX_count();
         this.printTraversalForm(traversal);
@@ -190,12 +210,47 @@ public void g_V_union_inXunion_out_outX_inXunion_out_outX_count() {
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_out_union_inXunion_out_outX_inXunion_out_outX_count() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported in hqps
         Traversal<Vertex, Long> traversal =
                 this.get_g_V_out_union_inXunion_out_outX_inXunion_out_outX_count();
         this.printTraversalForm(traversal);
         Assert.assertEquals(104, traversal.next().intValue());
     }
 
+    @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+    @Test
+    public void g_V_unionXout__inX_name() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported in hqps
+        final Traversal<Vertex, String> traversal = get_g_V_unionXout__inX_name();
+        printTraversalForm(traversal);
+        checkResults(
+                new HashMap<String, Long>() {
+                    {
+                        put("marko", 3L);
+                        put("lop", 3L);
+                        put("peter", 1L);
+                        put("ripple", 1L);
+                        put("josh", 3L);
+                        put("vadas", 1L);
+                    }
+                },
+                traversal);
+    }
+
+    @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+    @Test
+    public void g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported in hqps
+        final Traversal<Vertex, Number> traversal =
+                get_g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX(
+                        convertToVertexId("marko"), convertToVertexId("vadas"));
+        printTraversalForm(traversal);
+        checkResults(Arrays.asList(3L, 1.9d, 1L), traversal);
+    }
+
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_group_by_by_dedup_count_test() {
@@ -363,6 +418,7 @@ public void g_V_has_name_marko_label() {
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_E_has_weight_0_5_f_label() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
         final Traversal<Edge, String> traversal = get_g_E_has_weight_0_5_f_label();
         printTraversalForm(traversal);
         Assert.assertEquals("knows", traversal.next());
@@ -609,6 +665,8 @@ public void g_V_haslabel_as_identity_values() {
     @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
     @Test
     public void g_V_haslabel_union_identity_out_values() {
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        // union step is currently not supported in hqps
         Traversal<Vertex, Object> traversal = this.get_g_V_has_union_identity_out_values();
         this.printTraversalForm(traversal);
         int counter = 0;
@@ -773,6 +831,21 @@ public void g_VX1X_outEXknowsX_asXhereX_hasXweight_1X_inV_hasXname_joshX_selectX
         Assert.assertFalse(traversal.hasNext());
     }
 
+    @Test
+    @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+    public void g_E_hasLabelXknowsX() {
+        // Hqps engine doesn't support scan from edge index scan.
+        assumeFalse("hiactor".equals(System.getenv("ENGINE_TYPE")));
+        final Traversal<Edge, Edge> traversal = get_g_E_hasLabelXknowsX();
+        printTraversalForm(traversal);
+        int counter = 0;
+        while (traversal.hasNext()) {
+            counter++;
+            Assert.assertEquals("knows", traversal.next().label());
+        }
+        Assert.assertEquals(2, counter);
+    }
+
     public static class Traversals extends IrGremlinQueryTest {
 
         // g.V().out().union(out(), in(), in()).count()
@@ -813,6 +886,21 @@ public Traversal<Vertex, Long> get_g_V_union_inXunion_out_outX_inXunion_out_outX
             return g.V().out().union(in().union(out(), out()), in().union(out(), out())).count();
         }
 
+        @Override
+        public Traversal<Vertex, String> get_g_V_unionXout__inX_name() {
+            return g.V().union(out(), in()).values("name");
+        }
+
+        @Override
+        public Traversal<Vertex, Number> get_g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX(
+                final Object v1Id, final Object v2Id) {
+            return g.V(v1Id, v2Id)
+                    .union(
+                            outE().count(),
+                            inE().count(),
+                            (Traversal) outE().values("weight").sum());
+        }
+
         @Override
         public Traversal<Vertex, Map<Object, Long>> get_g_V_group_by_by_dedup_count_order_by_key() {
             return (IrCustomizedTraversal)
@@ -1060,5 +1148,10 @@ public Traversal<Vertex, Edge> get_g_VX1X_outE_asXhereX_inV_hasXname_vadasX_sele
                 final Object v1Id) {
             return g.V(v1Id).outE().as("here").inV().has("name", "vadas").select("here");
         }
+
+        @Override
+        public Traversal<Edge, Edge> get_g_E_hasLabelXknowsX() {
+            return g.E().hasLabel("knows");
+        }
     }
 }
diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java
index f01d46b1ed2a..16f9a7d45cc0 100644
--- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java
+++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java
@@ -115,7 +115,17 @@ public IrStandardOpProcessor(
         this.graph = graph;
         this.g = g;
         this.configs = configs;
-        this.rpcClient = new RpcClient(fetcher.fetch());
+        // hack implementation here: rpc client is the old way to submit job (gremlin -> traversal
+        // -> ir_core -> pegasus), we should remove it after replacing it with gremlin-calcite
+        // stack.
+        if (FrontendConfig.GREMLIN_SCRIPT_LANGUAGE_NAME
+                        .get(this.configs)
+                        .equals(AntlrGremlinScriptEngineFactory.LANGUAGE_NAME)
+                && FrontendConfig.ENGINE_TYPE.get(this.configs).equals("pegasus")) {
+            this.rpcClient = new RpcClient(fetcher.fetch());
+        } else {
+            this.rpcClient = null;
+        }
         this.metaQueryCallback = metaQueryCallback;
         this.idGenerator = idGenerator;
         this.queryCache = queryCache;


Test patch:
diff --git a/flex/tests/hqps/engine_config_test.yaml b/flex/tests/hqps/engine_config_test.yaml
index e58c86c7b2c9..e092baec9cbf 100644
--- a/flex/tests/hqps/engine_config_test.yaml
+++ b/flex/tests/hqps/engine_config_test.yaml
@@ -11,6 +11,8 @@ compute_engine:
   workers:
     - localhost:10000
   thread_num_per_worker: 1
+  store:
+    type: cpp-mcsr
 compiler:
   planner:
     is_on: true
@@ -25,9 +27,10 @@ compiler:
       disabled: false
       port: 7687
     gremlin_connector:
-      disabled: true
+      disabled: false
       port: 8182
   query_timeout: 40000
+  gremlin_script_language_name: antlr_gremlin_calcite
 http_service:
   default_listen_address: localhost
   admin_port: 7777
diff --git a/flex/tests/hqps/hqps_cypher_test.sh b/flex/tests/hqps/hqps_adhoc_test.sh
similarity index 72%
rename from flex/tests/hqps/hqps_cypher_test.sh
rename to flex/tests/hqps/hqps_adhoc_test.sh
index d295ca4e0fa7..3e37629b30a6 100644
--- a/flex/tests/hqps/hqps_cypher_test.sh
+++ b/flex/tests/hqps/hqps_adhoc_test.sh
@@ -20,24 +20,34 @@ SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server
 GIE_HOME=${FLEX_HOME}/../interactive_engine/
 
 # 
-if [ ! $# -eq 3 ]; then
-  echo "only receives: $# args, need 4"
-  echo "Usage: $0 <INTERACTIVE_WORKSPACE> <GRAPH_NAME> <ENGINE_CONFIG>"
+if [ $# -lt 3 ] || [ $# -gt 4 ]; then
+  echo "Receives: $# args, need 3 or 4 args"
+  echo "Usage: $0 <INTERACTIVE_WORKSPACE> <GRAPH_NAME> <ENGINE_CONFIG> [TEST_TYPE(cypher/gremlin/all)]"
   exit 1
 fi
 
 INTERACTIVE_WORKSPACE=$1
 GRAPH_NAME=$2
 ENGINE_CONFIG_PATH=$3
-if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then
-  echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists"
+if [ $# -eq 4 ]; then
+  TEST_TYPE=$4
+else
+  TEST_TYPE="cypher" # default run cypher tests
+fi
+
+# check TEST_TYPE is valid
+if [ ${TEST_TYPE} != "cypher" ] && [ ${TEST_TYPE} != "gremlin" ] && [ ${TEST_TYPE} != "all" ]; then
+  echo "TEST_TYPE: ${TEST_TYPE} not supported, use cypher or gremlin"
   exit 1
+else
+  echo "TEST_TYPE: ${TEST_TYPE}"
 fi
-# check graph is ldbc or movies
-if [ ${GRAPH_NAME} != "ldbc" ] && [ ${GRAPH_NAME} != "movies" ] && [ ${GRAPH_NAME} != "graph_algo" ]; then
-  echo "GRAPH_NAME: ${GRAPH_NAME} not supported, use movies or ldbc"
+
+if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then
+  echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists"
   exit 1
 fi
+
 if [ ! -d ${INTERACTIVE_WORKSPACE}/data/${GRAPH_NAME} ]; then
   echo "GRAPH: ${GRAPH_NAME} not exists"
   exit 1
@@ -158,19 +168,55 @@ run_graph_algo_test(){
   popd
 }
 
+run_cypher_test(){
+  # check graph is ldbc or movies
+  if [ ${GRAPH_NAME} != "ldbc" ] && [ ${GRAPH_NAME} != "movies" ] && [ ${GRAPH_NAME} != "graph_algo" ]; then
+    echo "GRAPH_NAME: ${GRAPH_NAME} not supported, use movies or ldbc"
+    exit 1
+  fi
+  if [ "${GRAPH_NAME}" == "ldbc" ]; then
+    run_ldbc_test
+    run_simple_test
+  elif [ "${GRAPH_NAME}" == "movies" ]; then
+    run_movie_test
+  elif [ "${GRAPH_NAME}" == "graph_algo" ]; then
+    run_graph_algo_test
+  else
+    echo "GRAPH_NAME: ${GRAPH_NAME} not supported, use movies, ldbc or graph_algo"
+  fi
+}
+
+
+run_gremlin_test(){
+  if [ "${GRAPH_NAME}" != "modern_graph" ]; then
+    echo "GRAPH_NAME: ${GRAPH_NAME} not supported, only support modern_graph"
+    exit 1
+  fi
+  echo "run gremlin test"
+  pushd ${GIE_HOME}/compiler
+  cmd="mvn test -Dtest=com.alibaba.graphscope.gremlin.integration.standard.calcite.IrGremlinTest"
+  echo "Start gremlin test: ${cmd}"
+  ${cmd}
+  info "Finish gremlin test"
+  popd
+}
+
 kill_service
 start_engine_service
 start_compiler_service
-# if GRAPH_NAME equals ldbc
-if [ "${GRAPH_NAME}" == "ldbc" ]; then
-  run_ldbc_test
-  run_simple_test
-elif [ "${GRAPH_NAME}" == "movies" ]; then
-  run_movie_test
-elif [ "${GRAPH_NAME}" == "graph_algo" ]; then
-  run_graph_algo_test
+
+
+if [ "${TEST_TYPE}" == "cypher" ]; then
+  run_cypher_test
+elif [ "${TEST_TYPE}" == "gremlin" ]; then
+  info "run gremlin test"
+  run_gremlin_test
+elif [ "${TEST_TYPE}" == "all" ]; then
+  run_cypher_test
+  run_gremlin_test
 else
-  echo "GRAPH_NAME: ${GRAPH_NAME} not supported, use movies, ldbc or graph_algo"
+  echo "TEST_TYPE: ${TEST_TYPE} not supported, use cypher/gremlin/all"
+  exit 1
 fi
 
 kill_service
diff --git a/flex/tests/rt_mutable_graph/modern_graph_string_edge.yaml b/flex/tests/rt_mutable_graph/modern_graph_string_edge.yaml
index 17a4e62e6fc4..205465af850f 100644
--- a/flex/tests/rt_mutable_graph/modern_graph_string_edge.yaml
+++ b/flex/tests/rt_mutable_graph/modern_graph_string_edge.yaml
@@ -61,7 +61,7 @@ schema:
       vertex_type_pair_relations:
         - source_vertex: person
           destination_vertex: software
-          relation: MANY_TO_ONE
+          relation: MANY_TO_MANY
       properties:
         - property_id: 0
           property_name: weight
diff --git a/flex/tests/rt_mutable_graph/string_edge_property_test.cc b/flex/tests/rt_mutable_graph/string_edge_property_test.cc
index 01875daff0df..4b20e7e1039f 100644
--- a/flex/tests/rt_mutable_graph/string_edge_property_test.cc
+++ b/flex/tests/rt_mutable_graph/string_edge_property_test.cc
@@ -61,13 +61,15 @@ class TestStringEdgeProperty {
     CHECK(db_.graph().get_lid(dst_label_, dst, dst_lid));
 
     {
-      auto graph_view = txn.GetOutgoingSingleGraphView<std::string_view>(
+      auto graph_view = txn.GetOutgoingGraphView<std::string_view>(
           src_label_, dst_label_, edge_label_);
-      auto oe = graph_view.get_edge(src_lid);
-      LOG(INFO) << oe.get_data() << "\n";
-      CHECK(oe.get_data() == "0.4")
-          << "Inconsistent value, Excepted : 0.4, Got " << oe.get_data()
-          << "\n";
+      auto oes = graph_view.get_edges(src_lid);
+      for (auto& oe : oes) {
+        LOG(INFO) << oe.get_data() << "\n";
+        CHECK(oe.get_data() == "0.4")
+            << "Inconsistent value, Excepted : 0.4, Got " << oe.get_data()
+            << "\n";
+      }
     }
     {
       auto graph_view = txn.GetIncomingGraphView<std::string_view>(
@@ -103,12 +105,14 @@ class TestStringEdgeProperty {
     }
     {
       auto txn = db_.GetReadTransaction();
-      auto graph_view = txn.GetOutgoingSingleGraphView<std::string_view>(
+      auto graph_view = txn.GetOutgoingGraphView<std::string_view>(
           src_label_, dst_label_, edge_label_);
-      auto oe = graph_view.get_edge(src_lid);
-      CHECK(oe.get_data() == "test")
-          << "Inconsistent value, Excepted: test, Got " << oe.get_data()
-          << "\n";
+      auto oes = graph_view.get_edges(src_lid);
+      for (auto& oe : oes) {
+        CHECK(oe.get_data() == "test")
+            << "Inconsistent value, Excepted: test, Got " << oe.get_data()
+            << "\n";
+      }
     }
     LOG(INFO) << "Finish test add edge\n";
   }
diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java
index 4f10b2f02df7..98e1a8cfc6a4 100644
--- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java
+++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/graphAlgo/GraphAlgoTest.java
@@ -95,8 +95,15 @@ public void run_graph_query8_test() {
     }
 
     @Test
-    public void run_graph_query9_test() {
-        QueryContext testQuery = GraphAlgoQueries.get_graph_algo_test9();
+    public void run_graph_query9_0_test() {
+        QueryContext testQuery = GraphAlgoQueries.get_graph_algo_test9_0();
+        Result result = session.run(testQuery.getQuery());
+        Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString());
+    }
+
+    @Test
+    public void run_graph_query9_1_test() {
+        QueryContext testQuery = GraphAlgoQueries.get_graph_algo_test9_1();
         Result result = session.run(testQuery.getQuery());
         Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString());
     }
diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java
index 6f71d6bc40c7..c478ac2fbb54 100644
--- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java
+++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java
@@ -238,7 +238,7 @@ public void g_V_has_name_endingWith_test() {
                 "GraphLogicalProject(DEFAULT=[DEFAULT], isAppend=[false])\n"
                     + "  GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}],"
                     + " alias=[DEFAULT], fusedFilter=[[POSIX REGEX CASE SENSITIVE(DEFAULT.name,"
-                    + " _UTF-8'mar$')]], opt=[VERTEX])",
+                    + " _UTF-8'.*mar$')]], opt=[VERTEX])",
                 node.explain().trim());
     }
 
@@ -249,7 +249,7 @@ public void g_V_has_name_not_endingWith_test() {
                 "GraphLogicalProject(DEFAULT=[DEFAULT], isAppend=[false])\n"
                     + "  GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}],"
                     + " alias=[DEFAULT], fusedFilter=[[NOT(POSIX REGEX CASE SENSITIVE(DEFAULT.name,"
-                    + " _UTF-8'mar$'))]], opt=[VERTEX])",
+                    + " _UTF-8'.*mar$'))]], opt=[VERTEX])",
                 node.explain().trim());
     }
 
@@ -300,7 +300,7 @@ public void g_V_has_name_startingWith_test() {
                 "GraphLogicalProject(DEFAULT=[DEFAULT], isAppend=[false])\n"
                     + "  GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}],"
                     + " alias=[DEFAULT], fusedFilter=[[POSIX REGEX CASE SENSITIVE(DEFAULT.name,"
-                    + " _UTF-8'^mar')]], opt=[VERTEX])",
+                    + " _UTF-8'^mar.*')]], opt=[VERTEX])",
                 node.explain().trim());
     }
 
@@ -311,7 +311,7 @@ public void g_V_has_name_not_startingWith_test() {
                 "GraphLogicalProject(DEFAULT=[DEFAULT], isAppend=[false])\n"
                     + "  GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}],"
                     + " alias=[DEFAULT], fusedFilter=[[NOT(POSIX REGEX CASE SENSITIVE(DEFAULT.name,"
-                    + " _UTF-8'^mar'))]], opt=[VERTEX])",
+                    + " _UTF-8'^mar.*'))]], opt=[VERTEX])",
                 node.explain().trim());
     }