Repository: tapdata/tapdata (language: python)

Issue:
fix: TAP-3457 #7495 【V3.6.0】数据转换，mysql-->mongodb,追加合并，报错：Unable to wr…
…ite data to mongodb due to violationof unique constraint.


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 372d3582dd..fbf6b11b60 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -258,6 +258,7 @@ jobs:
           echo "${{ secrets.RSYNC_PASSWORD }}" > /tmp/rsync.passwd && chmod 600 /tmp/rsync.passwd
           mkdir -p output
           rsync -r --links --progress --password-file=/tmp/rsync.passwd rsync://root@192.168.1.184:873/data/temp/${{ needs.Set-Branch.outputs.TAG_NAME }}/ output/
+          echo '${{ secrets.HARBOR_PASS }}' | docker login harbor.internal.tapdata.io --username=${{ secrets.HARBOR_USER }} --password-stdin
           bash build/build.sh -o docker -t ${{ needs.Set-Branch.outputs.TAG_NAME }}
       - name: Print Image Info
         run: |
@@ -310,8 +311,11 @@ jobs:
           echo "${{ secrets.RSYNC_PASSWORD }}" > /tmp/rsync.passwd && chmod 600 /tmp/rsync.passwd
           mkdir -p output
           rsync -r --links --progress --password-file=/tmp/rsync.passwd rsync://root@192.168.1.184:873/data/temp/${{ needs.Set-Branch.outputs.TAG_NAME }}/ output/
-          bash build/build.sh -o tar -t ${{ needs.Set-Branch.outputs.TAG_NAME }}
-          rsync -r --progress --password-file=/tmp/rsync.passwd output/tapdata-${{ needs.Set-Branch.outputs.TAG_NAME }}.tar.gz rsync://root@58.251.34.123:873/data/enterprise-artifact/gz/
+          bash build/build.sh -o tar -t ${{ needs.Set-Branch.outputs.TAG_NAME }} -m x86_64
+          rsync -r --progress --password-file=/tmp/rsync.passwd output/tapdata-x86_64-${{ needs.Set-Branch.outputs.TAG_NAME }}.tar.gz rsync://root@58.251.34.123:873/data/enterprise-artifact/gz/
+          rm -rf output/tapdata-x86_64-${{ needs.Set-Branch.outputs.TAG_NAME }}.tar.gz
+          bash build/build.sh -o tar -t ${{ needs.Set-Branch.outputs.TAG_NAME }} -m arm64
+          rsync -r --progress --password-file=/tmp/rsync.passwd output/tapdata-arm64-${{ needs.Set-Branch.outputs.TAG_NAME }}.tar.gz rsync://root@58.251.34.123:873/data/enterprise-artifact/gz/
 
   Clean-Build-Temp:
     runs-on: office-scan
diff --git a/.github/workflows/mr-ci.yaml b/.github/workflows/mr-ci.yaml
index 2d82464259..4ef49d19dd 100644
--- a/.github/workflows/mr-ci.yaml
+++ b/.github/workflows/mr-ci.yaml
@@ -148,10 +148,14 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.TAPDATA_ENT_CICD_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
+          update-alternatives --set java /usr/java/jdk-11.0.23/bin/java
+          export JAVA_HOME=/usr/java/jdk-11.0.23
           cd tapdata && mvn clean test -T1C -Dmaven.compile.fork=true -P idaas && mvn sonar:sonar \
             -Dsonar.projectKey=tapdata -Dsonar.host.url=${{ secrets.SONAR_HOST }} \
             -Dsonar.login=${{ secrets.SONAR_USER }} -Dsonar.password=${{ secrets.SONAR_PASS }} \
             -Dsonar.branch.name=${{ needs.Get-Stable-Branch.outputs.TAPDATA_BRANCH }}
+          update-alternatives --set java /usr/java/jdk1.8.0_361/bin/java
+          export JAVA_HOME=/usr/java/jdk1.8.0_361
       - name: Prevent Skip Unittest
         run: |
           cd tapdata
diff --git a/.gitignore b/.gitignore
index a4bc03e5e3..942a0367bf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -134,4 +134,6 @@ iengine/test
 py-lib
 sybase-poc/
 sybase-poc-temp/
-.tmStartMsg.json
\ No newline at end of file
+.tmStartMsg.json
+
+run-resources
\ No newline at end of file
diff --git a/RELEASE b/RELEASE
new file mode 100644
index 0000000000..ff0e70c2bf
--- /dev/null
+++ b/RELEASE
@@ -0,0 +1,2 @@
+LST: v3.7.0
+DEV: v3.8.0-dev
\ No newline at end of file
diff --git a/build/.connectors_list b/build/.connectors_list
index 87cabf958d..3def1d6d82 100644
--- a/build/.connectors_list
+++ b/build/.connectors_list
@@ -11,4 +11,6 @@ mongodb-atlas-connector-v1.0-SNAPSHOT.jar
 mysql-connector-v1.0-SNAPSHOT.jar
 postgres-connector-v1.0-SNAPSHOT.jar
 redis-connector-v1.0-SNAPSHOT.jar
-tdengine-connector-v1.0-SNAPSHOT.jar
\ No newline at end of file
+tdengine-connector-v1.0-SNAPSHOT.jar
+mock-source-connector-v1.0-SNAPSHOT.jar
+mock-target-connector-v1.0-SNAPSHOT.jar
\ No newline at end of file
diff --git a/build/build.sh b/build/build.sh
index 18c558b10c..aad101befa 100755
--- a/build/build.sh
+++ b/build/build.sh
@@ -31,8 +31,10 @@ PACKAGE_COMPONENTS=""
 OUTPUT_TYPE=""
 # filter connectors list
 CONNECTORS_LIST=$(cat $SCRIPT_BASE_DIR/.connectors_list)
+# platform
+PLATFORM=$(uname -m)
 
-while getopts 'c:l:u:p:t:o:' OPT; do
+while getopts 'c:l:u:p:t:o:m:' OPT; do
 	case "$OPT" in
 	c)
     COMPONENT_NAME="$OPTARG"
@@ -52,6 +54,9 @@ while getopts 'c:l:u:p:t:o:' OPT; do
   o)
     OUTPUT_TYPE="$OPTARG"
     ;;
+  m)
+    PLATFORM="$OPTARG"
+    ;;
   esac
 done
 
@@ -63,6 +68,7 @@ cat <<EOF
   BUILD_ARGS:         $BUILD_ARGS
   TAG_NAME:           $TAG_NAME
   OUTPUT_TYPE:        $OUTPUT_TYPE
+  PLATFORM:           $PLATFORM
 EOF
 
 IFS=" " read -r -a COMPONENTS <<<"$(echo "$COMPONENT_NAME" | tr -d ' ' | tr ',' ' ')"
@@ -144,8 +150,16 @@ make_tar() {
   cp $TAPDATA_DIR/build/image/docker-entrypoint.sh ./start.sh
   rsync -a $TAPDATA_DIR/build/image/bin/ ./
   rsync -a $TAPDATA_DIR/build/image/supervisor ./
+  # download async-profile
+  if [[ $PLATFORM == "x86_64" ]]; then
+    wget https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz -O async-profiler.tar.gz
+  else
+    wget https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-arm64.tar.gz -O async-profiler.tar.gz
+  fi
+  tar -xzf async-profiler.tar.gz -C ./components/
+  mv ./components/async-profiler-* ./components/async-profiler
   chmod +x start.sh stop.sh status.sh
-  tar cfz tapdata-$TAG_NAME.tar.gz *
+  tar cfz tapdata-$PLATFORM-$TAG_NAME.tar.gz *
 }
 
 # make output
diff --git a/build/image/Dockerfile b/build/image/Dockerfile
index 8863cf6642..c2a60bdc8b 100644
--- a/build/image/Dockerfile
+++ b/build/image/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/tapdata/base:0.1
+FROM ghcr.io/tapdata/base:0.2
 
 ENV JAVA_TOOL_OPTIONS -Dfile.encoding=UTF8
 COPY etc/ /tapdata/apps/etc
@@ -10,6 +10,16 @@ COPY bin/ /tapdata/apps
 
 RUN apt install supervisor -y
 
+RUN apt install wget -y && \
+    if [[ `uname -p` == "aarch64" ]]; then \
+      wget https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-arm64.tar.gz -O /tapdata/apps/components/async-profiler.tar.gz; \
+    else \
+      wget https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz -O /tapdata/apps/components/async-profiler.tar.gz; \
+    fi && \
+    tar -xzf /tapdata/apps/components/async-profiler.tar.gz -C /tapdata/apps/components/ && \
+    rm -f /tapdata/apps/components/async-profiler.tar.gz && \
+    mv /tapdata/apps/components/async-profiler-* /tapdata/apps/components/async-profiler
+
 COPY supervisor/ /tapdata/apps/supervisor/
 
 ENV TAPDATA_WORK_DIR /tapdata/apps
diff --git a/iengine/api/src/main/java/io/tapdata/ConverterProvider.java b/iengine/api/src/main/java/io/tapdata/ConverterProvider.java
index 3bc08e525e..dfb4563933 100644
--- a/iengine/api/src/main/java/io/tapdata/ConverterProvider.java
+++ b/iengine/api/src/main/java/io/tapdata/ConverterProvider.java
@@ -7,7 +7,6 @@
 import io.tapdata.common.SettingService;
 import io.tapdata.entity.ConvertLog;
 import io.tapdata.exception.ConvertException;
-import jdk.nashorn.internal.runtime.Undefined;
 import org.apache.commons.collections.MapUtils;
 import org.apache.kafka.connect.data.SchemaBuilder;
 
@@ -68,16 +67,6 @@ default void javaTypeConverter(RelateDatabaseField relateDatabaseField) throws C
 	Object targetValueConverter(Object data) throws ConvertException;
 
 	default Object commTargetValueConverter(Object data) throws ConvertException {
-
-		try {
-			if (data instanceof Undefined) {
-				data = null;
-			}
-		} catch (Exception e) {
-			throw new ConvertException(ConvertLog.ERR_COMMON_TARGET_0001.getErrCode(),
-					String.format(ConvertLog.ERR_COMMON_TARGET_0001.getMessage(), data, "Undefined", "null", e.getMessage()));
-		}
-
 		return data;
 	}
 
diff --git a/iengine/api/src/main/java/io/tapdata/aspect/taskmilestones/EngineDeductionAspect.java b/iengine/api/src/main/java/io/tapdata/aspect/taskmilestones/EngineDeductionAspect.java
new file mode 100644
index 0000000000..c80be12dd0
--- /dev/null
+++ b/iengine/api/src/main/java/io/tapdata/aspect/taskmilestones/EngineDeductionAspect.java
@@ -0,0 +1,37 @@
+package io.tapdata.aspect.taskmilestones;
+
+import io.tapdata.entity.aspect.Aspect;
+import lombok.Getter;
+
+
+public class EngineDeductionAspect extends Aspect {
+    public static final int DEDUCTION_START = 10;
+    public static final int DEDUCTION_END = 20;
+    public static final int DEDUCTION_ERROR = 30;
+
+    @Getter
+    private int state;
+    public Long endTime;
+
+    public EngineDeductionAspect start() {
+        this.state = DEDUCTION_START;
+        return this;
+    }
+
+    public EngineDeductionAspect end() {
+        this.state = DEDUCTION_END;
+        this.endTime = System.currentTimeMillis();
+        return this;
+    }
+
+    @Getter
+    private Throwable error;
+
+    public EngineDeductionAspect error(Throwable error) {
+        this.error = error;
+        this.state = DEDUCTION_ERROR;
+        this.endTime = System.currentTimeMillis();
+        return this;
+    }
+
+}
diff --git a/iengine/api/src/main/java/io/tapdata/inspect/InspectService.java b/iengine/api/src/main/java/io/tapdata/inspect/InspectService.java
index 5920656e88..7a3770f45d 100644
--- a/iengine/api/src/main/java/io/tapdata/inspect/InspectService.java
+++ b/iengine/api/src/main/java/io/tapdata/inspect/InspectService.java
@@ -20,7 +20,7 @@ default void startInspect(Inspect inspect) {
     void updateStatus(String id, InspectStatus status, String msg);
     void upsertInspectResult(InspectResult inspectResult, boolean excludeInspect);
     void upsertInspectResult(InspectResult inspectResult);
-    InspectResult getLastDifferenceInspectResult(String firstCheckId);
+    InspectResult getLastDifferenceInspectResult(String inspectId, String firstCheckId);
     InspectResult getLastInspectResult(String inspectId);
     InspectResult getInspectResultById(String inspectResultId);
     List<Connections> getInspectConnectionsById(Inspect inspect);
@@ -30,4 +30,6 @@ default void startInspect(Inspect inspect) {
     void inspectHeartBeat(String id);
     Inspect getInspectById(String id);
 
+    void handleDiffInspect(Inspect inspect,long syncTaskDelay,InspectResult inspectResult);
+
 }
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/common/DAGDataEngineServiceImpl.java b/iengine/iengine-app/src/main/java/io/tapdata/common/DAGDataEngineServiceImpl.java
new file mode 100644
index 0000000000..118cdc3d25
--- /dev/null
+++ b/iengine/iengine-app/src/main/java/io/tapdata/common/DAGDataEngineServiceImpl.java
@@ -0,0 +1,256 @@
+package io.tapdata.common;
+
+import com.alibaba.fastjson.JSON;
+import com.tapdata.constant.BeanUtil;
+import com.tapdata.constant.ConfigurationCenter;
+import com.tapdata.constant.ConnectorConstant;
+import com.tapdata.entity.Connections;
+import com.tapdata.entity.schema.SchemaApplyResult;
+import com.tapdata.mongo.ClientMongoOperator;
+import com.tapdata.tm.autoinspect.utils.GZIPUtil;
+import com.tapdata.tm.commons.dag.DAGDataServiceImpl;
+import com.tapdata.tm.commons.dag.Node;
+import com.tapdata.tm.commons.dag.logCollector.LogCollecotrConnConfig;
+import com.tapdata.tm.commons.dag.logCollector.LogCollectorNode;
+import com.tapdata.tm.commons.dag.process.MergeTableNode;
+import com.tapdata.tm.commons.dag.vo.MigrateJsResultVo;
+import com.tapdata.tm.commons.schema.*;
+import com.tapdata.tm.commons.schema.bean.SourceTypeEnum;
+import com.tapdata.tm.commons.task.dto.Message;
+import com.tapdata.tm.commons.task.dto.ParentTaskDto;
+import com.tapdata.tm.commons.task.dto.TaskDto;
+import com.tapdata.tm.commons.util.FilterMetadataInstanceUtil;
+import com.tapdata.tm.commons.util.JsonUtil;
+import com.tapdata.tm.commons.util.PdkSchemaConvert;
+import io.tapdata.common.sharecdc.ShareCdcUtil;
+import io.tapdata.entity.conversion.PossibleDataTypes;
+import io.tapdata.entity.schema.TapTable;
+import io.tapdata.entity.schema.type.TapType;
+import io.tapdata.flow.engine.V2.node.hazelcast.data.HazelcastSchemaTargetNode;
+import io.tapdata.flow.engine.V2.task.TaskClient;
+import io.tapdata.flow.engine.V2.task.TaskService;
+import io.tapdata.observable.logging.ObsLogger;
+import io.tapdata.observable.logging.ObsLoggerFactory;
+import io.tapdata.schema.TapTableMap;
+import io.tapdata.websocket.handler.DeduceSchemaHandler;
+import org.apache.commons.collections.CollectionUtils;
+import org.springframework.data.mongodb.core.query.Query;
+
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static org.springframework.data.mongodb.core.query.Criteria.where;
+
+
+public class DAGDataEngineServiceImpl extends DAGDataServiceImpl {
+    private final ObsLogger obsLogger;
+    private final TaskService<TaskDto> taskService;
+    private final TaskDto taskDto;
+
+    private final boolean engineDeduction;
+
+    private final ClientMongoOperator clientMongoOperator;
+
+    private final String uuid ;
+
+    private final Map<String,TapTableMap<String, TapTable>> tapTableMapHashMap;
+
+    public DAGDataEngineServiceImpl(TransformerWsMessageDto transformerWsMessageDto, TaskService<TaskDto> taskService,
+                                    Map<String,TapTableMap<String, TapTable>> tapTableMapHashMap,ClientMongoOperator clientMongoOperator) {
+        super(transformerWsMessageDto);
+        this.obsLogger = ObsLoggerFactory.getInstance().getObsLogger(transformerWsMessageDto.getTaskDto());
+        this.taskService = taskService;
+        this.taskDto = transformerWsMessageDto.getTaskDto();
+        this.engineDeduction = true;
+        this.clientMongoOperator = clientMongoOperator;
+        this.uuid = transformerWsMessageDto.getOptions().getUuid();
+        this.tapTableMapHashMap = tapTableMapHashMap;
+
+    }
+
+    public DAGDataEngineServiceImpl(DeduceSchemaHandler.DeduceSchemaRequest request,TaskService<TaskDto> taskService, ClientMongoOperator clientMongoOperator) {
+        super(request.getMetadataInstancesDtoList(), request.getDataSourceMap(), request.getDefinitionDtoMap(),request.getUserId(), request.getUserName(), request.getTaskDto(), request.getTransformerDtoMap());
+        this.obsLogger = ObsLoggerFactory.getInstance().getObsLogger(request.getTaskDto());
+        this.taskService = taskService;
+        this.tapTableMapHashMap = new HashMap<>();
+        this.clientMongoOperator = clientMongoOperator;
+        this.taskDto = request.getTaskDto();
+        this.engineDeduction = false;
+        this.uuid = request.getOptions().getUuid();
+    }
+
+    @Override
+    public TapTable loadTapTable(String nodeId, String virtualId, TaskDto taskDto) {
+        
+        try {
+            // 跑任务加载js模型
+            String schemaKey = taskDto.getId() + "-" + virtualId;
+            long startTs = System.currentTimeMillis();
+            TaskClient<TaskDto> taskClient = execTask(taskDto);
+
+            obsLogger.info("load tapTable task {} {}, cost {}ms", schemaKey, taskClient.getStatus(), (System.currentTimeMillis() - startTs));
+            //成功
+            TapTable tapTable = HazelcastSchemaTargetNode.getTapTable(schemaKey);
+            if (obsLogger.isDebugEnabled()) {
+                obsLogger.debug("derivation results: {}", JSON.toJSONString(tapTable));
+            }
+            return tapTable;
+        } catch (Exception e) {
+            obsLogger.error("An error occurred while obtaining the results of model deduction", e);
+        }
+        return null;
+    }
+
+    @Override
+    public List<MigrateJsResultVo> getJsResult(String jsNodeId, String virtualTargetId, TaskDto taskDto) {
+        try {
+            String schemaKey = taskDto.getId() + "-" + virtualTargetId;
+            long startTs = System.currentTimeMillis();
+
+            TaskClient<TaskDto> taskClient = execTask(taskDto);
+
+            obsLogger.info("load MigrateJsResultVos task {} {}, cost {}ms", schemaKey, taskClient.getStatus(), (System.currentTimeMillis() - startTs));
+            //成功
+            List<SchemaApplyResult> schemaApplyResultList = HazelcastSchemaTargetNode.getSchemaApplyResultList(schemaKey);
+            if (obsLogger.isDebugEnabled()) {
+                obsLogger.debug("derivation results: {}", JSON.toJSONString(schemaApplyResultList));
+            }
+
+            if (CollectionUtils.isNotEmpty(schemaApplyResultList)) {
+                return schemaApplyResultList.stream().map(s -> new MigrateJsResultVo(s.getOp(), s.getFieldName(), s.getTapField(), s.getTapIndex()))
+                        .collect(Collectors.toList());
+            }
+        } catch (Exception e) {
+            obsLogger.error("An error occurred while obtaining the results of model deduction", e);
+        }
+        return new ArrayList<>();
+    }
+
+    protected TaskClient<TaskDto> execTask(TaskDto taskDto) {
+        taskDto.setType(ParentTaskDto.TYPE_INITIAL_SYNC);
+        TaskClient<TaskDto> taskClient = taskService.startTestTask(taskDto);
+        taskClient.join();
+        return taskClient;
+    }
+    @Override
+    public void initializeModel(Boolean isSync) {
+        Map<String, List<MetadataInstancesDto>> updateMetadataInstancesDtos = getBatchMetadataUpdateMap().values().stream()
+                .filter(metadataInstancesDto -> SourceTypeEnum.VIRTUAL.name().equals(metadataInstancesDto.getSourceType()))
+                .collect(Collectors.groupingBy(MetadataInstancesDto::getNodeId));
+
+        Map<String, List<MetadataInstancesDto>> insertMetadataInstancesDtos = getBatchInsertMetaDataList().stream()
+                .filter(metadataInstancesDto -> SourceTypeEnum.VIRTUAL.name().equals(metadataInstancesDto.getSourceType()))
+                .collect(Collectors.groupingBy(MetadataInstancesDto::getNodeId));
+
+        insertMetadataInstancesDtos.forEach((nodeId, metadataInstancesDtos) -> updateMetadataInstancesDtos.merge(nodeId,metadataInstancesDtos,(v1, v2)->{
+            List<MetadataInstancesDto> metadataInstancesDtoList = new ArrayList<>(v1);
+            metadataInstancesDtoList.addAll(v2);
+            return metadataInstancesDtoList;
+        }));
+
+        for (Map.Entry<String, List<MetadataInstancesDto>> entry : updateMetadataInstancesDtos.entrySet()) {
+            Map<String, String> tableNameQualifiedNameMap = new HashMap<>();
+            List<MetadataInstancesDto> filteredList = entry.getValue();
+            TapTableMap<String, TapTable> tapTableMap = tapTableMapHashMap.computeIfAbsent(entry.getKey(), key -> TapTableMap.create(null, entry.getKey(), tableNameQualifiedNameMap,taskDto.getTmCurrentTime()));
+            filteredList.forEach(metadataInstancesDto -> {
+                TapTable tapTable = convertTapTable(metadataInstancesDto);
+                if (tapTable != null) {
+                    if(isSync && metadataInstancesDto.getMetaType().equals("processor_node")){
+                        tapTableMap.putNew(metadataInstancesDto.getNodeId(), tapTable, metadataInstancesDto.getQualifiedName());
+                    }else{
+                        tapTableMap.putNew(metadataInstancesDto.getOriginalName(), tapTable, metadataInstancesDto.getQualifiedName());
+                    }
+                }
+            });
+        }
+        if (taskDto.getDag() != null) {
+            List<Node> nodes = taskDto.getDag().getNodes();
+            if (CollectionUtils.isNotEmpty(nodes)) {
+                nodes.forEach(f -> {
+                    if(f instanceof MergeTableNode){
+                        List<Node> predecessors = taskDto.getDag().predecessors(f.getId());
+                        TapTableMap<String, TapTable> tapTableMap = tapTableMapHashMap.get(f.getId());
+                        if(tapTableMap == null)return;
+                        predecessors.forEach(node -> {
+                            TapTableMap<String, TapTable> preTapTableMap = tapTableMapHashMap.get(node.getId());
+                            mergeTableMap(tapTableMap,preTapTableMap);
+                        });
+                    }else if(f instanceof LogCollectorNode){
+                        LogCollectorNode logCollectorNode = (LogCollectorNode) f;
+                        Map<String, LogCollecotrConnConfig> connConfigs = logCollectorNode.getLogCollectorConnConfigs();
+                        TapTableMap<String, TapTable> tapTableMap = tapTableMapHashMap.computeIfAbsent(f.getId(), key -> TapTableMap.create(null, f.getId(), new HashMap<>(),taskDto.getTmCurrentTime()));
+                        Map<String,MetadataInstancesDto> metadataInstancesDtoHashMap = getLogCollectorMetadataInstancesDto().stream()
+                                .collect(Collectors.toMap(MetadataInstancesDto::getOriginalName
+                                        , metadataInstancesDto -> metadataInstancesDto, (m1, m2) -> m1));;
+                        if (null != connConfigs && !connConfigs.isEmpty()) {
+                            Map<String,List<String>> connections = ShareCdcUtil.getConnectionIds(f, ids -> {
+                                Query connectionQuery = new Query(where("_id").in(ids));
+                                connectionQuery.fields().include("_id").include("namespace");
+                                return clientMongoOperator.find(connectionQuery, ConnectorConstant.CONNECTION_COLLECTION, Connections.class);
+                            }).stream().collect(Collectors.toMap(Connections::getId,Connections::getNamespace,(value1, value2) -> value1));
+                            metadataInstancesDtoHashMap.values().forEach(metadataInstancesDto -> {
+                                if(null != connections.get(metadataInstancesDto.getSource().get_id())){
+                                    String namespace =String.join(".",connections.get(metadataInstancesDto.getSource().get_id()));
+                                    String originalName = String.join(".", namespace, metadataInstancesDto.getOriginalName());
+                                    tapTableMap.putNew(originalName,convertTapTable(metadataInstancesDto),metadataInstancesDto.getQualifiedName());
+                                }
+                            });
+                        }else{
+                            for(MetadataInstancesDto metadataInstancesDto :metadataInstancesDtoHashMap.values()){
+                                tapTableMap.putNew(metadataInstancesDto.getOriginalName(),convertTapTable(metadataInstancesDto),metadataInstancesDto.getQualifiedName());
+                            }
+                        }
+                    }
+                    f.setSchema(null);
+                    f.setOutputSchema(null);
+                });
+            }
+        }
+        clearTransformer();
+    }
+
+    @Override
+    public Boolean whetherEngineDeduction() {
+        return this.engineDeduction;
+    }
+
+
+    protected TapTable convertTapTable(MetadataInstancesDto item){
+        FilterMetadataInstanceUtil.filterMetadataInstancesFields(item);
+        return PdkSchemaConvert.toPdk(item);
+    }
+
+    @Override
+    public void uploadModel(Map<String, List<Message>> transformSchema){
+        TransformerWsMessageResult wsMessageResult = new TransformerWsMessageResult();
+        wsMessageResult.setBatchMetadataUpdateMap(getBatchMetadataUpdateMap());
+        wsMessageResult.setBatchInsertMetaDataList(getBatchInsertMetaDataList());
+        wsMessageResult.setUpsertItems(getUpsertItems());
+        wsMessageResult.setUpsertTransformer(getUpsertTransformer());
+        wsMessageResult.setTransformSchema(transformSchema);
+        wsMessageResult.setTaskId(taskDto.getId().toHexString());
+        wsMessageResult.setTransformUuid(uuid);
+        if (taskDto.getDag() != null) {
+            List<Node> nodes = taskDto.getDag().getNodes();
+            if (CollectionUtils.isNotEmpty(nodes)) {
+                nodes.forEach(f -> {
+                    f.setSchema(null);
+                    f.setOutputSchema(null);
+                });
+            }
+        }
+        wsMessageResult.setDag(taskDto.getDag());
+        String jsonResult = JsonUtil.toJsonUseJackson(wsMessageResult);
+        byte[] gzip = GZIPUtil.gzip(jsonResult.getBytes());
+        byte[] encode = Base64.getEncoder().encode(gzip);
+        String dataString = new String(encode, StandardCharsets.UTF_8);
+        clientMongoOperator.insertOne(dataString, ConnectorConstant.TASK_COLLECTION + "/transformer/resultV2");
+    }
+
+    protected void mergeTableMap(TapTableMap<String, TapTable> tapTableMap , TapTableMap<String, TapTable> preTapTableMap) {
+        for(String key : preTapTableMap.keySet()){
+            tapTableMap.putNew(key,preTapTableMap.get(key),preTapTableMap.getQualifiedName(key));
+        }
+    }
+}
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/error/TaskTargetProcessorExCode_15.java b/iengine/iengine-app/src/main/java/io/tapdata/error/TaskTargetProcessorExCode_15.java
index 71cedcb0ae..831111fed7 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/error/TaskTargetProcessorExCode_15.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/error/TaskTargetProcessorExCode_15.java
@@ -109,4 +109,14 @@ public interface TaskTargetProcessorExCode_15 {
 			recoverable = true
 	)
 	String WRITE_RECORD_COMMON_FAILED = "15019";
+	@TapExCode(
+			describe = "Create index event's tableId cannot be empty",
+			describeCN = "创建索引事件的tableId不能为空"
+	)
+	String CREATE_INDEX_EVENT_TABLE_ID_EMPTY = "15020";
+	@TapExCode(
+			describe = "When creating an index, checking if the index exists fails",
+			describeCN = "创建索引时，检查索引是否存在失败"
+	)
+	String CREATE_INDEX_QUERY_EXISTS_INDEX_FAILED = "15021";
 }
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNode.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNode.java
index 10fbc1382c..4ae58e67aa 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNode.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNode.java
@@ -48,6 +48,7 @@
 import io.tapdata.error.TapProcessorUnknownException;
 import io.tapdata.error.TaskProcessorExCode_11;
 import io.tapdata.exception.TapCodeException;
+import io.tapdata.exception.TapPdkBaseException;
 import io.tapdata.flow.engine.V2.exception.ErrorHandleException;
 import io.tapdata.flow.engine.V2.monitor.MonitorManager;
 import io.tapdata.flow.engine.V2.monitor.impl.JetJobStatusMonitor;
@@ -62,6 +63,8 @@
 import io.tapdata.flow.engine.util.TaskDtoUtil;
 import io.tapdata.observable.logging.ObsLogger;
 import io.tapdata.observable.logging.ObsLoggerFactory;
+import io.tapdata.pdk.core.error.TapPdkRunnerExCode_18;
+import io.tapdata.pdk.core.error.TapPdkRunnerUnknownException;
 import io.tapdata.pdk.core.utils.CommonUtils;
 import io.tapdata.schema.TapTableMap;
 import io.tapdata.task.skipError.SkipError;
@@ -80,6 +83,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 /**
@@ -923,4 +927,19 @@ protected boolean isJetJobRunning() {
 	public ObsLogger getObsLogger() {
 		return obsLogger;
 	}
+	public Consumer<RuntimeException> buildErrorConsumer(String tableName){
+		return err -> {
+			Throwable pdkBaseThrowable = CommonUtils.matchThrowable(err, TapPdkBaseException.class);
+			if (null != pdkBaseThrowable) {
+				((TapPdkBaseException)err).setTableName(tableName);
+				throw err;
+			}
+			Throwable pdkUnknownThrowable = CommonUtils.matchThrowable(err, TapPdkRunnerUnknownException.class);
+			if (null != pdkUnknownThrowable) {
+				((TapPdkRunnerUnknownException)err).setTableName(tableName);
+				throw err;
+			}
+			throw err;
+		};
+	}
 }
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSampleSourcePdkDataNode.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSampleSourcePdkDataNode.java
index 8f257bf762..6b82437fe4 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSampleSourcePdkDataNode.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSampleSourcePdkDataNode.java
@@ -69,7 +69,6 @@ public void startSourceRunner() {
 			int rows = 1;
 			if (node instanceof DatabaseNode) {
 				rows = ((DatabaseNode) node).getRows() == null ? 1 : ((DatabaseNode) node).getRows();
-				tables = ((DatabaseNode) node).getTableNames();
 			} else if (node instanceof TableNode) {
 				rows = ((TableNode) node).getRows() == null ? 1 : ((TableNode) node).getRows();
 			}
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNode.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNode.java
index 2672a1afda..40e30b845e 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNode.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNode.java
@@ -477,7 +477,7 @@ private void handleCustomCommandResult(Object result, String tableName, BiConsum
 		}
 	}
 
-	private void createTargetIndex(List<String> updateConditionFields, boolean createUnique, String tableId, TapTable tapTable) {
+	protected void createTargetIndex(List<String> updateConditionFields, boolean createUnique, String tableId, TapTable tapTable) {
 		CreateIndexFunction createIndexFunction = getConnectorNode().getConnectorFunctions().getCreateIndexFunction();
 		if (null == createIndexFunction) {
 			return;
@@ -513,7 +513,7 @@ private void createTargetIndex(List<String> updateConditionFields, boolean creat
 						.createIndexEvent(indexEvent.get())
 						.start(), createIndexFuncAspect -> PDKInvocationMonitor.invoke(getConnectorNode(),
 						PDKMethod.TARGET_CREATE_INDEX,
-						() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, indexEvent.get()), TAG));
+						() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, indexEvent.get()), TAG, buildErrorConsumer(tableId)));
 				LoadSchemaRunner loadSchemaRunner = new LoadSchemaRunner(dataProcessorContext.getConnections(), clientMongoOperator, 1);
 				loadSchemaRunner.run();
 			}
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNode.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNode.java
index 5c14f010b5..668321f69d 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNode.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNode.java
@@ -149,6 +149,7 @@ public abstract class HazelcastTargetPdkBaseNode extends HazelcastPdkBaseNode {
 
 	protected Boolean unwindProcess = false;
 	protected boolean illegalDateAcceptable = false;
+	protected ConcurrentHashMap<String, Boolean> everHandleTapTablePrimaryKeysMap;
 
 	public HazelcastTargetPdkBaseNode(DataProcessorContext dataProcessorContext) {
 		super(dataProcessorContext);
@@ -178,6 +179,7 @@ protected void doInit(@NotNull Context context) throws TapCodeException {
 		});
 		Thread.currentThread().setName(String.format("Target-Process-%s[%s]", getNode().getName(), getNode().getId()));
 		checkUnwindConfiguration();
+		everHandleTapTablePrimaryKeysMap = new ConcurrentHashMap<>();
 	}
 
 	@Override
@@ -187,6 +189,7 @@ protected void doInitWithDisableNode(@NotNull Context context) throws TapCodeExc
 			createPdkAndInit(context);
 		});
 		Thread.currentThread().setName(String.format("Target-Process-%s[%s]", getNode().getName(), getNode().getId()));
+		everHandleTapTablePrimaryKeysMap = new ConcurrentHashMap<>();
 	}
 
 	protected void initExactlyOnceWriteIfNeed() {
@@ -211,7 +214,7 @@ protected void initExactlyOnceWriteIfNeed() {
 			TapCreateIndexEvent indexEvent = createIndexEvent(exactlyOnceTable.getId(), exactlyOnceTable.getIndexList());
 			PDKInvocationMonitor.invoke(
 					getConnectorNode(), PDKMethod.TARGET_CREATE_INDEX,
-					() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), exactlyOnceTable, indexEvent), TAG);
+					() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), exactlyOnceTable, indexEvent), TAG, buildErrorConsumer(exactlyOnceTable.getId()));
 		}
 		Node node = getNode();
 		if (node instanceof TableNode) {
@@ -268,12 +271,12 @@ protected boolean createTable(TapTable tapTable, AtomicBoolean succeed) {
 			CreateTableV2Function createTableV2Function = getConnectorNode().getConnectorFunctions().getCreateTableV2Function();
 			createdTable = createTableV2Function != null || createTableFunction != null;
 			TapTable finalTapTable = new TapTable();
-			BeanUtil.copyProperties(tapTable,finalTapTable);
-			if(unwindProcess){
-				ignorePksAndIndices(finalTapTable, null);
-			}
 			if (createdTable) {
-				handleTapTablePrimaryKeys(finalTapTable);
+				handleTapTablePrimaryKeys(tapTable);
+				BeanUtil.copyProperties(tapTable,finalTapTable);
+				if(unwindProcess){
+					ignorePksAndIndices(finalTapTable, null);
+				}
 				tapCreateTableEvent.set(createTableEvent(finalTapTable));
 				executeDataFuncAspect(CreateTableFuncAspect.class, () -> new CreateTableFuncAspect()
 						.createTableEvent(tapCreateTableEvent.get())
@@ -289,16 +292,19 @@ protected boolean createTable(TapTable tapTable, AtomicBoolean succeed) {
 							} else {
 								createTableFunction.createTable(getConnectorNode().getConnectorContext(), tapCreateTableEvent.get());
 							}
-						}, TAG)));
+						}, TAG,buildErrorConsumer(tapCreateTableEvent.get().getTableId()))));
+				clientMongoOperator.insertOne(Collections.singletonList(finalTapTable),
+						ConnectorConstant.CONNECTION_COLLECTION + "/load/part/tables/" + dataProcessorContext.getTargetConn().getId());
 			} else {
 				// only execute start function aspect so that it would be cheated as input
 				AspectUtils.executeAspect(new CreateTableFuncAspect()
 						.createTableEvent(tapCreateTableEvent.get())
 						.connectorContext(getConnectorNode().getConnectorContext())
 						.dataProcessorContext(dataProcessorContext).state(NewFieldFuncAspect.STATE_START));
+				clientMongoOperator.insertOne(Collections.singletonList(tapTable),
+						ConnectorConstant.CONNECTION_COLLECTION + "/load/part/tables/" + dataProcessorContext.getTargetConn().getId());
 			}
-			clientMongoOperator.insertOne(Collections.singletonList(finalTapTable),
-					ConnectorConstant.CONNECTION_COLLECTION + "/load/part/tables/" + dataProcessorContext.getTargetConn().getId());
+
 		} catch (Throwable throwable) {
 			Throwable matched = CommonUtils.matchThrowable(throwable, TapCodeException.class);
 			if (null != matched) {
@@ -1038,6 +1044,7 @@ private void flushSyncProgressMap(TapdataEvent tapdataEvent) {
 	}
 
 	protected void handleTapTablePrimaryKeys(TapTable tapTable) {
+		everHandleTapTablePrimaryKeysMap.computeIfAbsent(tapTable.getId(), (value) -> {
 		if (writeStrategy.equals(com.tapdata.tm.commons.task.dto.MergeTableProperties.MergeType.updateOrInsert.name())) {
 			List<String> updateConditionFields = updateConditionFieldsMap.get(tapTable.getId());
 			if (CollectionUtils.isNotEmpty(updateConditionFields)) {
@@ -1058,6 +1065,8 @@ protected void handleTapTablePrimaryKeys(TapTable tapTable) {
 			// 没有关联条件，清空主键信息
 			ignorePksAndIndices(tapTable, null);
 		}
+		return true;
+		});
 	}
 
 	protected static void ignorePksAndIndices(TapTable tapTable, List<String> logicPrimaries) {
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNode.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNode.java
index 6a2fce815f..d3f20f8704 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNode.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNode.java
@@ -221,7 +221,7 @@ private List<String> getUpdateConditionFields(Node<?> node, TapTable tapTable) {
 		}
 	}
 
-	private void createTargetIndex(List<String> updateConditionFields, boolean createUnique, String tableId, TapTable tapTable, boolean createdTable) {
+	protected void createTargetIndex(List<String> updateConditionFields, boolean createUnique, String tableId, TapTable tapTable, boolean createdTable) {
 
 		if (writeStrategy.equals(com.tapdata.tm.commons.task.dto.MergeTableProperties.MergeType.appendWrite.name())) {
 			return;
@@ -268,7 +268,7 @@ private void createTargetIndex(List<String> updateConditionFields, boolean creat
 						.createIndexEvent(indexEvent.get())
 						.start(), createIndexFuncAspect -> PDKInvocationMonitor.invoke(getConnectorNode(),
 						PDKMethod.TARGET_CREATE_INDEX,
-						() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, indexEvent.get()), TAG));
+						() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, indexEvent.get()), TAG, buildErrorConsumer(tableId)));
 			}
 		} catch (Throwable throwable) {
 			Throwable matched = CommonUtils.matchThrowable(throwable, TapCodeException.class);
@@ -328,34 +328,14 @@ protected void syncIndex(String tableId, TapTable tapTable, boolean autoCreateTa
 				tapIndex.setCluster(index.getCluster());
 				indexList.add(tapIndex);
 			});
-			//query exist index
-			queryIndexesFunction.query(getConnectorNode().getConnectorContext(), tapTable, (tapIndexList)->{
-				tapIndexList.forEach(existsIndex -> {
-					// If the index already exists, it will no longer be created; Having the same name is considered as existence; Fields with the same order are also considered to exist
-					for (TapIndex tapIndex : indexList) {
-						if (tapIndex.getName().equals(existsIndex.getName())) {
-							indexList.remove(tapIndex);
-							obsLogger.info("Table: {} already exists Index: {} and will no longer create index", tableId, tapIndex.getName());
-							break;
-						}
-						if (tapIndex.getIndexFields().size() == existsIndex.getIndexFields().size()) {
-							boolean same = true;
-							for (int i = 0; i < tapIndex.getIndexFields().size(); i++) {
-								if (!tapIndex.getIndexFields().get(i).getName().equals(existsIndex.getIndexFields().get(i).getName())
-										|| !Objects.equals(tapIndex.getIndexFields().get(i).getFieldAsc(), existsIndex.getIndexFields().get(i).getFieldAsc())) {
-									same = false;
-									break;
-								}
-							}
-							if (same) {
-								indexList.remove(tapIndex);
-								obsLogger.info("Table: {} already exists Index: {} and will no longer create index", tableId, tapIndex.getName());
-								break;
-							}
-						}
-					}
+			List<TapIndex> existsIndexes = queryExistsIndexes(tapTable, indexList);
+			if(CollectionUtils.isNotEmpty(existsIndexes)){
+				existsIndexes.forEach(i -> {
+					obsLogger.info("Table: {} already exists Index: {} and will no longer create index", tableId, i.getName());
+					indexList.remove(i);
 				});
-			});
+				return;
+			}
 			if (CollectionUtils.isEmpty(indexList)) {
 				obsLogger.info("Table: {} already exists Index list: {}", tableId, indices);
 				return;
@@ -371,7 +351,7 @@ protected void syncIndex(String tableId, TapTable tapTable, boolean autoCreateTa
 						.createIndexEvent(indexEvent.get())
 						.start(), createIndexFuncAspect -> PDKInvocationMonitor.invoke(getConnectorNode(),
 						PDKMethod.TARGET_CREATE_INDEX,
-						() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, indexEvent.get()), TAG));
+						() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, indexEvent.get()), TAG, buildErrorConsumer(tableId)));
 				long currentIndexEnd = System.currentTimeMillis();
 				obsLogger.info("Table: {} create Index: {} successfully, cost {}ms", indexEvent.get().getTableId(), index.getName(), currentIndexEnd-currentIndexStart);
 			});
@@ -387,6 +367,40 @@ protected void syncIndex(String tableId, TapTable tapTable, boolean autoCreateTa
 		long end = System.currentTimeMillis();
 		obsLogger.info("Table: {} synchronize indexes completed, cost {}ms totally", tableId, end-start);
 	}
+
+	protected List<TapIndex> queryExistsIndexes(TapTable tapTable, List<TapIndex> indexList) throws Throwable {
+		QueryIndexesFunction queryIndexesFunction = getConnectorNode().getConnectorFunctions().getQueryIndexesFunction();
+		if (null == queryIndexesFunction) {
+			return new ArrayList<>();
+		}
+		List<TapIndex> existsIndexes = new ArrayList<>();
+		queryIndexesFunction.query(getConnectorNode().getConnectorContext(), tapTable, (tapIndexList)->{
+			tapIndexList.forEach(existsIndex -> {
+				// If the index already exists, it will no longer be created; Having the same name is considered as existence; Fields with the same order are also considered to exist
+				for (TapIndex tapIndex : indexList) {
+					if (null != tapIndex.getName() && null != existsIndex.getName() && tapIndex.getName().equals(existsIndex.getName())) {
+						existsIndexes.add(tapIndex);
+						continue;
+					}
+					if (tapIndex.getIndexFields().size() == existsIndex.getIndexFields().size()) {
+						boolean same = true;
+						for (int i = 0; i < tapIndex.getIndexFields().size(); i++) {
+							if (!tapIndex.getIndexFields().get(i).getName().equals(existsIndex.getIndexFields().get(i).getName())
+									|| !Objects.equals(tapIndex.getIndexFields().get(i).getFieldAsc(), existsIndex.getIndexFields().get(i).getFieldAsc())) {
+								same = false;
+								break;
+							}
+						}
+						if (same) {
+							existsIndexes.add(tapIndex);
+						}
+					}
+				}
+			});
+		});
+		return existsIndexes;
+	}
+
 	protected boolean checkSyncIndexOpen(){
 		Node node = getNode();
 		if (node instanceof DatabaseNode || node instanceof TableNode) {
@@ -398,7 +412,7 @@ protected boolean checkSyncIndexOpen(){
 		return false;
 	}
 
-	private void clearData(ExistsDataProcessEnum existsDataProcessEnum, String tableId) {
+	protected void clearData(ExistsDataProcessEnum existsDataProcessEnum, String tableId) {
 		if (SyncTypeEnum.CDC == syncType || existsDataProcessEnum != ExistsDataProcessEnum.REMOVE_DATE) return;
 
 		AtomicReference<TapClearTableEvent> tapClearTableEvent = new AtomicReference<>();
@@ -411,7 +425,7 @@ private void clearData(ExistsDataProcessEnum existsDataProcessEnum, String table
 						.connectorContext(getConnectorNode().getConnectorContext())
 						.dataProcessorContext(dataProcessorContext)
 						.start(), clearTableFuncAspect ->
-						PDKInvocationMonitor.invoke(getConnectorNode(), PDKMethod.TARGET_CLEAR_TABLE, () -> func.clearTable(getConnectorNode().getConnectorContext(), tapClearTableEvent.get()), TAG));
+						PDKInvocationMonitor.invoke(getConnectorNode(), PDKMethod.TARGET_CLEAR_TABLE, () -> func.clearTable(getConnectorNode().getConnectorContext(), tapClearTableEvent.get()), TAG, buildErrorConsumer(tapClearTableEvent.get().getTableId())));
 			});
 		} catch (Throwable throwable) {
 			Throwable matched = CommonUtils.matchThrowable(throwable, TapCodeException.class);
@@ -424,7 +438,7 @@ private void clearData(ExistsDataProcessEnum existsDataProcessEnum, String table
 		}
 	}
 
-	private void dropTable(ExistsDataProcessEnum existsDataProcessEnum, String tableId) {
+	protected void dropTable(ExistsDataProcessEnum existsDataProcessEnum, String tableId) {
 		if (SyncTypeEnum.CDC == syncType || existsDataProcessEnum != ExistsDataProcessEnum.DROP_TABLE) return;
 
 		AtomicReference<TapDropTableEvent> tapDropTableEvent = new AtomicReference<>();
@@ -437,7 +451,7 @@ private void dropTable(ExistsDataProcessEnum existsDataProcessEnum, String table
 						.connectorContext(getConnectorNode().getConnectorContext())
 						.dataProcessorContext(dataProcessorContext)
 						.start(), (dropTableFuncAspect ->
-						PDKInvocationMonitor.invoke(getConnectorNode(), PDKMethod.TARGET_DROP_TABLE, () -> dropTableFunction.dropTable(getConnectorNode().getConnectorContext(), tapDropTableEvent.get()), TAG)));
+						PDKInvocationMonitor.invoke(getConnectorNode(), PDKMethod.TARGET_DROP_TABLE, () -> dropTableFunction.dropTable(getConnectorNode().getConnectorContext(), tapDropTableEvent.get()), TAG, buildErrorConsumer(tapDropTableEvent.get().getTableId()))));
 			} else {
 				// only execute start function aspect so that it would be cheated as input
 				AspectUtils.executeAspect(new DropTableFuncAspect()
@@ -522,7 +536,7 @@ private void writeDDL(List<TapEvent> events) {
 		uploadDagService.compareAndSet(false, true);
 	}
 
-	private boolean executeNewFieldFunction(TapNewFieldEvent tapNewFieldEvent) {
+	protected boolean executeNewFieldFunction(TapNewFieldEvent tapNewFieldEvent) {
 		TapTable tapTable = dataProcessorContext.getTapTableMap().get(tapNewFieldEvent.getTableId());
 		if (null == tapTable) {
 			throw new TapEventException(TaskTargetProcessorExCode_15.ADD_NEW_FIELD_GET_TAP_TABLE_FAILED, "Table id: " + tapNewFieldEvent.getTableId())
@@ -560,7 +574,7 @@ private boolean executeNewFieldFunction(TapNewFieldEvent tapNewFieldEvent) {
 					.dataProcessorContext(dataProcessorContext)
 					.start(), (newFieldFuncAspect ->
 					PDKInvocationMonitor.invoke(connectorNode, pdkMethod,
-							() -> function.newField(connectorNode.getConnectorContext(), tapNewFieldEvent), TAG)));
+							() -> function.newField(connectorNode.getConnectorContext(), tapNewFieldEvent), TAG, buildErrorConsumer(tapTable.getId()))));
 		} catch (Exception e) {
 			Throwable matched = CommonUtils.matchThrowable(e, TapCodeException.class);
 			if (null != matched) {
@@ -583,7 +597,7 @@ private List<String> executeAlterFieldNameFunction(List<String> fields, ValueCha
 		return fields;
 	}
 
-	private boolean executeAlterFieldNameFunction(TapAlterFieldNameEvent tapAlterFieldNameEvent) {
+	protected boolean executeAlterFieldNameFunction(TapAlterFieldNameEvent tapAlterFieldNameEvent) {
 		// 字段名变更
 		Optional.ofNullable(tapAlterFieldNameEvent.getNameChange()
 		).ifPresent(nameChange -> {
@@ -615,7 +629,7 @@ private boolean executeAlterFieldNameFunction(TapAlterFieldNameEvent tapAlterFie
 					.start(), (alterFieldNameFuncAspect) ->
 					PDKInvocationMonitor.invoke(connectorNode, PDKMethod.ALTER_FIELD_NAME,
 							() -> function.alterFieldName(connectorNode.getConnectorContext(), tapAlterFieldNameEvent),
-							TAG));
+							TAG, buildErrorConsumer(tapAlterFieldNameEvent.getTableId())));
 		} catch (Exception e) {
 			Throwable matched = CommonUtils.matchThrowable(e, TapCodeException.class);
 			if (null != matched) {
@@ -628,7 +642,7 @@ private boolean executeAlterFieldNameFunction(TapAlterFieldNameEvent tapAlterFie
 		return true;
 	}
 
-	private boolean executeAlterFieldAttrFunction(TapAlterFieldAttributesEvent tapAlterFieldAttributesEvent) {
+	protected boolean executeAlterFieldAttrFunction(TapAlterFieldAttributesEvent tapAlterFieldAttributesEvent) {
 		TapTable tapTable = dataProcessorContext.getTapTableMap().get(tapAlterFieldAttributesEvent.getTableId());
 		if (null == tapTable) {
 			throw new TapEventException(TaskTargetProcessorExCode_15.ALTER_FIELD_ATTR_CANNOT_GET_TAP_TABLE, String.format("Table id: %s", tapAlterFieldAttributesEvent.getTableId()))
@@ -662,7 +676,7 @@ private boolean executeAlterFieldAttrFunction(TapAlterFieldAttributesEvent tapAl
 					.start(), (alterFieldAttributesFuncAspect ->
 					PDKInvocationMonitor.invoke(connectorNode, PDKMethod.ALTER_FIELD_ATTRIBUTES,
 							() -> function.alterFieldAttributes(connectorNode.getConnectorContext(), tapAlterFieldAttributesEvent),
-							TAG)));
+							TAG, buildErrorConsumer(tapTable.getId()))));
 		} catch (Exception e) {
 			Throwable matched = CommonUtils.matchThrowable(e, TapCodeException.class);
 			if (null != matched) {
@@ -675,7 +689,7 @@ private boolean executeAlterFieldAttrFunction(TapAlterFieldAttributesEvent tapAl
 		return true;
 	}
 
-	private boolean executeDropFieldFunction(TapDropFieldEvent tapDropFieldEvent) {
+	protected boolean executeDropFieldFunction(TapDropFieldEvent tapDropFieldEvent) {
 		ConnectorNode connectorNode = getConnectorNode();
 		DropFieldFunction function = connectorNode.getConnectorFunctions().getDropFieldFunction();
 		PDKMethod pdkMethod = PDKMethod.DROP_FIELD;
@@ -696,7 +710,7 @@ private boolean executeDropFieldFunction(TapDropFieldEvent tapDropFieldEvent) {
 					.start(), (dropFieldFuncAspect ->
 					PDKInvocationMonitor.invoke(connectorNode, PDKMethod.DROP_FIELD,
 							() -> function.dropField(connectorNode.getConnectorContext(), tapDropFieldEvent),
-							TAG)));
+							TAG, buildErrorConsumer(tapDropFieldEvent.getTableId()))));
 		} catch (Exception e) {
 			Throwable matched = CommonUtils.matchThrowable(e, TapCodeException.class);
 			if (null != matched) {
@@ -715,39 +729,48 @@ private boolean executeCreateTableFunction(TapCreateTableEvent tapCreateTableEve
 		return createTable(tgtTapTable, new AtomicBoolean());
 	}
 
-	private boolean executeCreateIndexFunction(TapCreateIndexEvent tapCreateIndexEvent) {
+	protected boolean executeCreateIndexFunction(TapCreateIndexEvent tapCreateIndexEvent) {
 		TapTableMap<String, TapTable> tapTableMap = dataProcessorContext.getTapTableMap();
+		String tableId = tapCreateIndexEvent.getTableId();
+		if (StringUtils.isBlank(tableId)) {
+			throw new TapEventException(TaskTargetProcessorExCode_15.CREATE_INDEX_EVENT_TABLE_ID_EMPTY).addEvent(tapCreateIndexEvent);
+		}
+		TapTable tapTable = tapTableMap.get(tableId);
+		if (null == tapTable) {
+			throw new TapEventException(TaskTargetProcessorExCode_15.CREATE_INDEX_TABLE_NOT_FOUND, String.format("Table id: %s", tableId))
+					.addEvent(tapCreateIndexEvent);
+		}
 		CreateIndexFunction createIndexFunction = getConnectorNode().getConnectorFunctions().getCreateIndexFunction();
 		if (null == createIndexFunction) {
-			return true;
+			return false;
+		}
+		List<TapIndex> existsIndexes;
+		try {
+			existsIndexes = queryExistsIndexes(tapTable, tapCreateIndexEvent.getIndexList());
+		} catch (Throwable e) {
+			throw new TapEventException(TaskTargetProcessorExCode_15.CREATE_INDEX_QUERY_EXISTS_INDEX_FAILED, e)
+					.addEvent(tapCreateIndexEvent);
+		}
+		if (CollectionUtils.isNotEmpty(existsIndexes)) {
+			existsIndexes.forEach(tapCreateIndexEvent.getIndexList()::remove);
 		}
-		for (String tableId : tapTableMap.keySet()) {
-			if (!isRunning()) {
-				return true;
-			}
-			TapTable tapTable = tapTableMap.get(tableId);
-			if (null == tapTable) {
-				throw new TapEventException(TaskTargetProcessorExCode_15.CREATE_INDEX_TABLE_NOT_FOUND, String.format("Table id: %s", tableId))
-						.addEvent(tapCreateIndexEvent);
-			}
 
-			try {
-				executeDataFuncAspect(CreateIndexFuncAspect.class, () -> new CreateIndexFuncAspect()
-						.table(tapTable)
-						.connectorContext(getConnectorNode().getConnectorContext())
-						.dataProcessorContext(dataProcessorContext)
-						.createIndexEvent(tapCreateIndexEvent)
-						.start(), createIndexFuncAspect -> PDKInvocationMonitor.invoke(getConnectorNode(),
-						PDKMethod.TARGET_CREATE_INDEX,
-						() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, tapCreateIndexEvent), TAG));
-			} catch (Exception e) {
-				Throwable matched = CommonUtils.matchThrowable(e, TapCodeException.class);
-				if (null != matched) {
-					throw (TapCodeException) matched;
-				}else {
-					throw new TapEventException(TaskTargetProcessorExCode_15.CREATE_INDEX_EXECUTE_FAILED, String.format("Execute PDK method: %s", PDKMethod.TARGET_CREATE_INDEX), e)
-							.addEvent(tapCreateIndexEvent);
-				}
+		try {
+			executeDataFuncAspect(CreateIndexFuncAspect.class, () -> new CreateIndexFuncAspect()
+					.table(tapTable)
+					.connectorContext(getConnectorNode().getConnectorContext())
+					.dataProcessorContext(dataProcessorContext)
+					.createIndexEvent(tapCreateIndexEvent)
+					.start(), createIndexFuncAspect -> PDKInvocationMonitor.invoke(getConnectorNode(),
+					PDKMethod.TARGET_CREATE_INDEX,
+					() -> createIndexFunction.createIndex(getConnectorNode().getConnectorContext(), tapTable, tapCreateIndexEvent), TAG, buildErrorConsumer(tableId)));
+		} catch (Exception e) {
+			Throwable matched = CommonUtils.matchThrowable(e, TapCodeException.class);
+			if (null != matched) {
+				throw (TapCodeException) matched;
+			}else {
+				throw new TapEventException(TaskTargetProcessorExCode_15.CREATE_INDEX_EXECUTE_FAILED, String.format("Execute PDK method: %s", PDKMethod.TARGET_CREATE_INDEX), e)
+						.addEvent(tapCreateIndexEvent);
 			}
 		}
 		return true;
@@ -992,6 +1015,7 @@ public void doClose() throws TapCodeException {
 
 	@Override
 	protected void updateDAG(TapdataEvent tapdataEvent) {
+		super.updateDAG(tapdataEvent);
 		final TapEvent tapEvent = tapdataEvent.getTapEvent();
 		if (tapEvent instanceof TapAlterFieldNameEvent) {
 			TapAlterFieldNameEvent tapAlterFieldNameEvent = (TapAlterFieldNameEvent) tapEvent;
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNode.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNode.java
index 4423629f0d..1dee58ef5d 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNode.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNode.java
@@ -26,6 +26,8 @@
 import io.tapdata.entity.schema.TapTable;
 import io.tapdata.entity.schema.type.TapString;
 import io.tapdata.entity.schema.value.DateTime;
+import io.tapdata.entity.schema.value.TapArrayValue;
+import io.tapdata.entity.schema.value.TapMapValue;
 import io.tapdata.entity.schema.value.TapStringValue;
 import io.tapdata.entity.utils.DataMap;
 import io.tapdata.error.TapEventException;
@@ -2146,6 +2148,10 @@ protected void recursiveMergeInfoTransformToTapValue(List<MergeLookupResult> mer
 						return new TapStringValue(value.toString())
 								.tapType(new TapString(24L, true))
 								.originType(BsonType.OBJECT_ID.name());
+					} else if (value instanceof Document) {
+						return new TapMapValue((Document) value);
+					} else if (value instanceof List) {
+						return new TapArrayValue((List<Object>) value);
 					}
 					return value;
 				});
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/schedule/TapdataTaskScheduler.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/schedule/TapdataTaskScheduler.java
index 5dec2e5f14..f89dd753a3 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/schedule/TapdataTaskScheduler.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/schedule/TapdataTaskScheduler.java
@@ -198,12 +198,16 @@ protected Runnable getHandleTaskOperationRunnable(TaskOperation taskOperation) {
 					Thread.currentThread().setName(String.format("Start-Task-Operation-Handler-%s[%s]", startTaskOperation.getTaskDto().getName(), startTaskOperation.getTaskDto().getId()));
 					taskId = startTaskOperation.getTaskDto().getId().toHexString();
 					TaskDto taskDto = startTaskOperation.getTaskDto();
-					taskLock.run(taskId, ()-> startTask(taskDto));
+					if (!taskLock.tryRun(taskId, ()-> startTask(taskDto), 1L, TimeUnit.SECONDS)) {
+						logger.warn("Start task {} failed because of task lock, will ignored", taskDto.getName());
+					}
 				} else if (taskOperation instanceof StopTaskOperation) {
 					StopTaskOperation stopTaskOperation = (StopTaskOperation) taskOperation;
 					Thread.currentThread().setName(String.format("Stop-Task-Operation-Handler-%s", stopTaskOperation.getTaskId()));
 					taskId = stopTaskOperation.getTaskId();
-					taskLock.run(taskId, () -> stopTask(taskId));
+					if (!taskLock.tryRun(taskId, () -> stopTask(taskId), 1L, TimeUnit.SECONDS)) {
+						logger.warn("Stop task {} failed because of task lock, will retry later", taskId);
+					}
 				}
 				logger.info("Handled task operation: {}", taskOperation);
 			} catch (Exception e) {
@@ -402,17 +406,15 @@ public void errorOrStopTask() {
 					continue;
 				}
 				if (!taskClient.isRunning()) {
-					try {
-						taskLock.lock(taskId);
-
+					taskLock.tryRun(taskId, () -> {
 						StopTaskResource stopTaskResource = null;
 						TerminalMode terminalMode = taskClient.getTerminalMode();
 						if (TerminalMode.STOP_GRACEFUL == terminalMode) {
 							stopTaskResource = StopTaskResource.STOPPED;
 						} else if (TerminalMode.COMPLETE == terminalMode) {
 							stopTaskResource = StopTaskResource.COMPLETE;
-						} else if(TerminalMode.INTERNAL_STOP == terminalMode){
-							if(taskClient.stop()){
+						} else if (TerminalMode.INTERNAL_STOP == terminalMode) {
+							if (taskClient.stop()) {
 								clearTaskCacheAfterStopped(taskClient);
 								clearTaskRetryCache(taskId);
 							}
@@ -429,7 +431,7 @@ public void errorOrStopTask() {
 										ObsLoggerFactory.getInstance().getObsLogger(taskClient.getTask()).info("Resume task[{}]", taskClient.getTask().getName());
 										long retryStartTime = System.currentTimeMillis();
 										sendStartTask(taskDto);
-										taskRetryTimeMap.put(taskId,retryStartTime);
+										taskRetryTimeMap.put(taskId, retryStartTime);
 									}
 								} else {
 									stopTaskResource = StopTaskResource.RUN_ERROR;
@@ -446,9 +448,7 @@ public void errorOrStopTask() {
 							taskClient.getTask().setSnapShotInterrupt(true);
 							stopTaskAndClear(taskClient, stopTaskResource, taskId);
 						}
-					} finally {
-						taskLock.unlock(taskId);
-					}
+					}, 1L, TimeUnit.SECONDS);
 				}
 			}
 		} catch (Exception e) {
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskClient.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskClient.java
index 3cb56d833d..6752827355 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskClient.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskClient.java
@@ -60,6 +60,10 @@ public class HazelcastTaskClient implements TaskClient<TaskDto> {
 	private final AtomicInteger retryCounter;
 	private AtomicBoolean retrying;
 
+	public static HazelcastTaskClient create(TaskDto taskDto, ClientMongoOperator clientMongoOperator, ConfigurationCenter configurationCenter, HazelcastInstance hazelcastInstance) {
+		return new HazelcastTaskClient(null, taskDto, clientMongoOperator, configurationCenter, hazelcastInstance);
+	}
+
 	public HazelcastTaskClient(Job job, TaskDto taskDto, ClientMongoOperator clientMongoOperator, ConfigurationCenter configurationCenter, HazelcastInstance hazelcastInstance) {
 		this.job = job;
 		this.taskDto = taskDto;
@@ -130,8 +134,11 @@ public TerminalMode getTerminalMode() {
 
 	@Override
 	public synchronized boolean stop() {
+		if(null == job) {
+			return false;
+		}
 		for (int i = 0; i < WAIT_JET_JOB_RUNNING_WHEN_STARTING_STATUS_TIME; i++) {
-			if (job.getStatus() == JobStatus.STARTING) {
+			if (getJetStatus() == JobStatus.STARTING) {
 				try {
 					TimeUnit.SECONDS.sleep(1L);
 				} catch (InterruptedException e) {
@@ -149,39 +156,43 @@ public synchronized boolean stop() {
 		}
 
 		if (job.getStatus().isTerminal()) {
-			ObsLogger obsLogger = ObsLoggerFactory.getInstance().getObsLogger(taskDto);
-			CommonUtils.handleAnyError(
-					() -> {
-						monitorManager.close();
-						obsLogger.info(String.format("Closed task monitor(s)\n%s", monitorManager));
-					},
-					err -> {
-						obsLogger.warn(String.format("Close task monitor(s) failed, error: %s\n  %s", err.getMessage(), Log4jUtil.getStackString(err)));
-					}
-			);
-			CommonUtils.handleAnyError(
-					() -> {
-						AspectUtils.executeAspect(new TaskStopAspect().task(taskDto).error(error));
-						obsLogger.info("Stopped task aspect(s)");
-					},
-					err -> {
-						obsLogger.warn(String.format("Stop task aspect(s) failed, error: %s\n  %s", err.getMessage(), Log4jUtil.getStackString(err)));
-					}
-			);
-			CommonUtils.handleAnyError(
-					() -> {
-						if (SnapshotOrderService.getInstance().removeController(taskDto.getId().toHexString())) {
-							obsLogger.info("Snapshot order controller have been removed");
-						}
-					},
-					error -> obsLogger.warn("Remove snapshot order controller failed, error: %s\n %s", error.getMessage(), Log4jUtil.getStackString(error))
-			);
-			CommonUtils.ignoreAnyError(() -> TaskGlobalVariable.INSTANCE.removeTask(taskDto.getId().toHexString()), TAG);
+			close();
 			return true;
 		}
 		return false;
 	}
 
+	public void close() {
+		ObsLogger obsLogger = ObsLoggerFactory.getInstance().getObsLogger(taskDto);
+		CommonUtils.handleAnyError(
+				() -> {
+					monitorManager.close();
+					obsLogger.info(String.format("Closed task monitor(s)\n%s", monitorManager));
+				},
+				err -> {
+					obsLogger.warn(String.format("Close task monitor(s) failed, error: %s\n  %s", err.getMessage(), Log4jUtil.getStackString(err)));
+				}
+		);
+		CommonUtils.handleAnyError(
+				() -> {
+					AspectUtils.executeAspect(new TaskStopAspect().task(taskDto).error(error));
+					obsLogger.info("Stopped task aspect(s)");
+				},
+				err -> {
+					obsLogger.warn(String.format("Stop task aspect(s) failed, error: %s\n  %s", err.getMessage(), Log4jUtil.getStackString(err)));
+				}
+		);
+		CommonUtils.handleAnyError(
+				() -> {
+					if (SnapshotOrderService.getInstance().removeController(taskDto.getId().toHexString())) {
+						obsLogger.info("Snapshot order controller have been removed");
+					}
+				},
+				error -> obsLogger.warn("Remove snapshot order controller failed, error: %s\n %s", error.getMessage(), Log4jUtil.getStackString(error))
+		);
+		CommonUtils.ignoreAnyError(() -> TaskGlobalVariable.INSTANCE.removeTask(taskDto.getId().toHexString()), TAG);
+	}
+
 	@Override
 	public void join() {
 		this.job.join();
@@ -249,4 +260,8 @@ public boolean resume() {
 	public int getRetryTime() {
 		return retryCounter.get();
 	}
+
+	public void setJob(Job job) {
+		this.job = job;
+	}
 }
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskService.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskService.java
index 265058bf0a..bc8ebc1f0b 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskService.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskService.java
@@ -27,9 +27,11 @@
 import com.tapdata.entity.task.config.TaskRetryConfig;
 import com.tapdata.entity.task.context.DataProcessorContext;
 import com.tapdata.entity.task.context.ProcessorBaseContext;
+import com.tapdata.entity.task.error.TaskServiceExCode_23;
 import com.tapdata.mongo.ClientMongoOperator;
 import com.tapdata.tm.autoinspect.constants.AutoInspectConstants;
 import com.tapdata.tm.autoinspect.exception.AutoInspectException;
+import com.tapdata.tm.commons.dag.DAGDataServiceImpl;
 import com.tapdata.tm.commons.dag.Edge;
 import com.tapdata.tm.commons.dag.Element;
 import com.tapdata.tm.commons.dag.Node;
@@ -46,12 +48,15 @@
 import com.tapdata.tm.commons.dag.process.ProcessorNode;
 import com.tapdata.tm.commons.dag.process.TableRenameProcessNode;
 import com.tapdata.tm.commons.dag.vo.ReadPartitionOptions;
+import com.tapdata.tm.commons.schema.*;
 import com.tapdata.tm.commons.task.dto.ErrorEvent;
 import com.tapdata.tm.commons.task.dto.TaskDto;
 import io.tapdata.aspect.TaskStartAspect;
 import io.tapdata.aspect.TaskStopAspect;
+import io.tapdata.aspect.taskmilestones.EngineDeductionAspect;
 import io.tapdata.aspect.utils.AspectUtils;
 import io.tapdata.autoinspect.utils.AutoInspectNodeUtil;
+import io.tapdata.common.DAGDataEngineServiceImpl;
 import io.tapdata.common.SettingService;
 import io.tapdata.common.sharecdc.ShareCdcUtil;
 import io.tapdata.dao.MessageDao;
@@ -100,6 +105,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.DependsOn;
 import org.springframework.data.mongodb.core.query.Query;
@@ -168,8 +174,6 @@ public static HazelcastInstance getHazelcastInstance() {
 
 	@Override
 	public TaskClient<TaskDto> startTask(TaskDto taskDto) {
-//        TaskThreadGroup threadGroup = new TaskThreadGroup(taskDto);
-//        try (ThreadPoolExecutorEx threadPoolExecutorEx = AsyncUtils.createThreadPoolExecutor("RootTask-" + taskDto.getName(), 1, threadGroup, TAG)) {
 		try {
 			taskDto.setDag(taskDto.getDag());
 			ObsLogger obsLogger = ObsLoggerFactory.getInstance().getObsLogger(taskDto);
@@ -178,16 +182,14 @@ public TaskClient<TaskDto> startTask(TaskDto taskDto) {
 			cleanAllUnselectedError(taskDto, obsLogger);
 
 			AspectUtils.executeAspect(new TaskStartAspect().task(taskDto).log(InstanceFactory.instance(LogFactory.class).getLog(taskDto)));
-//            return threadPoolExecutorEx.submitSync(() -> {
 			JobConfig jobConfig = new JobConfig();
 			jobConfig.setName(taskDto.getName() + "-" + taskDto.getId().toHexString());
 			jobConfig.setProcessingGuarantee(ProcessingGuarantee.NONE);
 			JetService jet = hazelcastInstance.getJet();
-			final JetDag jetDag = task2HazelcastDAG(taskDto);
-			obsLogger.info("The engine receives " + taskDto.getName() + " task data from TM and will continue to run tasks by jet");
-			Job job = jet.newJob(jetDag.getDag(), jobConfig);
-			return new HazelcastTaskClient(job, taskDto, clientMongoOperator, configurationCenter, hazelcastInstance);
-//            });
+			HazelcastTaskClient hazelcastTaskClient = HazelcastTaskClient.create(taskDto, clientMongoOperator, configurationCenter, hazelcastInstance);
+			Job job = startJetJob(taskDto, obsLogger, jet, jobConfig, hazelcastTaskClient);
+			hazelcastTaskClient.setJob(job);
+			return hazelcastTaskClient;
 		} catch (Throwable throwable) {
 			ObsLoggerFactory.getInstance().getObsLogger(taskDto).error(throwable);
 			AspectUtils.executeAspect(new TaskStopAspect().task(taskDto).error(throwable));
@@ -195,6 +197,19 @@ public TaskClient<TaskDto> startTask(TaskDto taskDto) {
 		}
 	}
 
+	private @NotNull Job startJetJob(TaskDto taskDto, ObsLogger obsLogger, JetService jet, JobConfig jobConfig, HazelcastTaskClient hazelcastTaskClient) {
+		Job job;
+		try {
+			final JetDag jetDag = task2HazelcastDAG(taskDto,true);
+			obsLogger.info("The engine receives " + taskDto.getName() + " task data from TM and will continue to run tasks by jet");
+			job = jet.newJob(jetDag.getDag(), jobConfig);
+		} catch (Exception e) {
+			hazelcastTaskClient.close();
+			throw e;
+		}
+		return job;
+	}
+
 	protected void cleanAllUnselectedError(TaskDto taskDto, ObsLogger obsLogger) {
 		List<ErrorEvent> errorEvents = taskDto.getErrorEvents();
 		if (!CollUtil.isEmpty(errorEvents)) {
@@ -213,7 +228,7 @@ public TaskClient<TaskDto> startTestTask(TaskDto taskDto) {
 			taskDto.setDag(taskDto.getDag());
 			AspectUtils.executeAspect(new TaskStartAspect().task(taskDto).log(new TapLog()));
 			long startTs = System.currentTimeMillis();
-			final JetDag jetDag = task2HazelcastDAG(taskDto);
+			final JetDag jetDag = task2HazelcastDAG(taskDto,false);
 			JobConfig jobConfig = new JobConfig();
 			jobConfig.setProcessingGuarantee(ProcessingGuarantee.NONE);
 			logger.info("task2HazelcastDAG cost {}ms", (System.currentTimeMillis() - startTs));
@@ -232,7 +247,7 @@ public TaskClient<TaskDto> startTestTask(TaskDto taskDto, AtomicReference<Object
 			taskDto.setDag(taskDto.getDag());
 			AspectUtils.executeAspect(new TaskStartAspect().task(taskDto).info("KYE_OF_SCRIPT_RUN_RESULT", result).log(new TapLog()));
 			long startTs = System.currentTimeMillis();
-			final JetDag jetDag = task2HazelcastDAG(taskDto);
+			final JetDag jetDag = task2HazelcastDAG(taskDto,false);
 			JobConfig jobConfig = new JobConfig();
 			jobConfig.setProcessingGuarantee(ProcessingGuarantee.NONE);
 			logger.info("task2HazelcastDAG cost {}ms", (System.currentTimeMillis() - startTs));
@@ -247,8 +262,13 @@ public TaskClient<TaskDto> startTestTask(TaskDto taskDto, AtomicReference<Object
 	}
 
 	@SneakyThrows
-	private JetDag task2HazelcastDAG(TaskDto taskDto) {
-
+	protected JetDag task2HazelcastDAG(TaskDto taskDto,Boolean deduce) {
+		Map<String,TapTableMap<String, TapTable>> tapTableMapHashMap;
+		if(deduce){
+			tapTableMapHashMap = engineTransformSchema(taskDto);
+		}else {
+			tapTableMapHashMap = new HashMap<>();
+		}
 		DAG dag = new DAG();
 		AtomicReference<TaskDto> taskDtoAtomicReference = new AtomicReference<>(taskDto);
 
@@ -301,8 +321,9 @@ private JetDag task2HazelcastDAG(TaskDto taskDto) {
 			AtomicBoolean needFilterEvent = new AtomicBoolean(true);
 			for (Node node : nodes) {
 				Connections connection = null;
+				TableNode tableNode = null;
 				DatabaseTypeEnum.DatabaseType databaseType = null;
-				TapTableMap<String, TapTable> tapTableMap = getTapTableMap(taskDto, tmCurrentTime, node);
+				TapTableMap<String, TapTable> tapTableMap = getTapTableMap(taskDto, tmCurrentTime, node,tapTableMapHashMap);
 				if (CollectionUtils.isEmpty(tapTableMap.keySet())
 						&& !(node instanceof CacheNode)
 						&& !(node instanceof HazelCastImdgNode)
@@ -319,6 +340,7 @@ private JetDag task2HazelcastDAG(TaskDto taskDto) {
 				if (node instanceof DataParentNode) {
 					connection = getConnection(((DataParentNode<?>) node).getConnectionId());
 					databaseType = ConnectionUtil.getDatabaseType(clientMongoOperator, connection.getPdkHash());
+					tableNode = node instanceof TableNode ? (TableNode) node : null;
 				} else if (node.isLogCollectorNode()) {
 					LogCollectorNode logCollectorNode = (LogCollectorNode) node;
 					String connectionId = logCollectorNode.getConnectionIds().get(0);
@@ -370,7 +392,7 @@ private JetDag task2HazelcastDAG(TaskDto taskDto) {
 
 				vertex.localParallelism(1);
 				dag.vertex(vertex);
-				this.singleTaskFilterEventDataIfNeed(connection, needFilterEvent);
+				this.singleTaskFilterEventDataIfNeed(connection, needFilterEvent, tableNode);
 			}
 
 			handleEdge(dag, edges, nodeMap, vertexMap);
@@ -379,15 +401,16 @@ private JetDag task2HazelcastDAG(TaskDto taskDto) {
 		return new JetDag(dag, hazelcastBaseNodeMap, typeConvertMap);
 	}
 
-	private void singleTaskFilterEventDataIfNeed(Connections conn, AtomicBoolean needFilterEvent) {
+	protected void singleTaskFilterEventDataIfNeed(Connections conn, AtomicBoolean needFilterEvent, TableNode tableNode) {
 		if (null == conn || null == needFilterEvent) return;
 		List<String> tags = conn.getDefinitionTags();
 		if (Boolean.TRUE.equals(needFilterEvent.get())) {
-			needFilterEvent.set(null == tags || !tags.contains("schema-free"));
+			boolean isCustomCommand = null != tableNode && tableNode.isEnableCustomCommand();
+			needFilterEvent.set(null == tags || (!tags.contains("schema-free") && !isCustomCommand));
 		}
 	}
 
-	private static void initSnapshotOrder(AtomicReference<TaskDto> taskDtoAtomicReference) {
+	protected static void initSnapshotOrder(AtomicReference<TaskDto> taskDtoAtomicReference) {
 		try {
 			SnapshotOrderService snapshotOrderService = SnapshotOrderService.getInstance();
 			SnapshotOrderController snapshotOrderController = snapshotOrderService.addController(taskDtoAtomicReference.get());
@@ -398,7 +421,7 @@ private static void initSnapshotOrder(AtomicReference<TaskDto> taskDtoAtomicRefe
 		}
 	}
 
-	private static TapTableMap<String, TapTable> getTapTableMap(TaskDto taskDto, Long tmCurrentTime, Node node) {
+	protected static TapTableMap<String, TapTable> getTapTableMap(TaskDto taskDto, Long tmCurrentTime, Node node, Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap) {
 		TapTableMap<String, TapTable> tapTableMap;
 		if (node instanceof AutoInspectNode) {
 			tapTableMap = TapTableUtil.getTapTableMapByNodeId(AutoInspectConstants.MODULE_NAME, ((AutoInspectNode) node).getTargetNodeId(), System.currentTimeMillis());
@@ -408,8 +431,15 @@ private static TapTableMap<String, TapTable> getTapTableMap(TaskDto taskDto, Lon
 			tapTableMap = TapTableUtil.getTapTableMap(node, tmCurrentTime);
 		} else if (node instanceof VirtualTargetNode) {
 			tapTableMap = TapTableMap.create(node.getId());
-		} else {
+		} else if(StringUtils.equalsAnyIgnoreCase(taskDto.getSyncType(),TaskDto.SYNC_TYPE_TEST_RUN)){
 			tapTableMap = TapTableUtil.getTapTableMapByNodeId(node.getId(), tmCurrentTime);
+		} else {
+			if(tapTableMapHashMap.containsKey(node.getId())){
+				tapTableMap = tapTableMapHashMap.get(node.getId());
+			}else{
+				tapTableMap = TapTableMap.create(node.getId());
+			}
+
 		}
 		return tapTableMap;
 	}
@@ -900,7 +930,7 @@ private Connections getConnection(String connectionId, List<String> includeField
 		return connections;
 	}
 
-	private TaskConfig getTaskConfig(TaskDto taskDto) {
+	protected TaskConfig getTaskConfig(TaskDto taskDto) {
 		return TaskConfig.create()
 				.taskDto(taskDto)
 				.taskRetryConfig(getTaskRetryConfig())
@@ -916,7 +946,7 @@ protected TaskRetryConfig getTaskRetryConfig() {
 				.maxRetryTimeSecond(maxRetryTimeSecond);
 	}
 
-	private void initSourceInitialCounter(TaskDto taskDto) {
+	protected void initSourceInitialCounter(TaskDto taskDto) {
 		String type = taskDto.getType();
 		com.tapdata.tm.commons.dag.DAG dag = taskDto.getDag();
 		List<Node> sourceNodes = dag.getSourceNodes();
@@ -925,4 +955,27 @@ private void initSourceInitialCounter(TaskDto taskDto) {
 			taskGlobalVariable.put(TaskGlobalVariable.SOURCE_INITIAL_COUNTER_KEY, new AtomicInteger(sourceNodes.size()));
 		}
 	}
+
+	protected Map<String,TapTableMap<String, TapTable>> engineTransformSchema(TaskDto taskDto){
+		AspectUtils.executeAspect(new EngineDeductionAspect().start());
+		Map<String,TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+		try {
+			com.tapdata.tm.commons.dag.DAG dag = taskDto.getDag().clone();
+			TransformerWsMessageDto transformerWsMessageDto = clientMongoOperator.findOne(new Query(),
+					ConnectorConstant.TASK_COLLECTION + "/transformAllParam/" + taskDto.getId().toHexString(),
+					TransformerWsMessageDto.class);
+			transformerWsMessageDto.getTaskDto().setDag(dag);
+			DAGDataServiceImpl dagDataService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,tapTableMapHashMap,clientMongoOperator);
+			dag.transformSchema(null, dagDataService, transformerWsMessageDto.getOptions(),(e)->{
+				throw new RuntimeException(e);
+            });
+			dagDataService.initializeModel((StringUtils.equalsAnyIgnoreCase(taskDto.getSyncType(), TaskDto.SYNC_TYPE_SYNC)));
+			AspectUtils.executeAspect(new EngineDeductionAspect().end());
+		}catch (Exception e){
+			AspectUtils.executeAspect(new EngineDeductionAspect().error(e));
+			throw new TapCodeException(TaskServiceExCode_23.TASK_FAILED_TO_LOAD_TABLE_STRUCTURE,"reason:"+e.getMessage(),e);
+		}
+		return tapTableMapHashMap;
+	}
+
 }
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/util/SingleLockWithKey.java b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/util/SingleLockWithKey.java
index 673bb0c1b2..485b54844c 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/util/SingleLockWithKey.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/flow/engine/V2/util/SingleLockWithKey.java
@@ -1,8 +1,12 @@
 package io.tapdata.flow.engine.V2.util;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -12,7 +16,7 @@
 public class SingleLockWithKey {
     private final Map<String, AtomicBoolean> locks = new HashMap<>();
 
-    public void lock(String key) throws InterruptedException {
+    private void lock(String key) throws InterruptedException {
         while (!Thread.interrupted()) {
             AtomicBoolean lock;
             synchronized (locks) {
@@ -30,11 +34,34 @@ public void lock(String key) throws InterruptedException {
         }
     }
 
-    public void unlock(String key) {
+    private boolean tryLock(String key, long timeout, TimeUnit timeUnit) throws InterruptedException {
+        long timeoutMills = timeUnit.toMillis(timeout);
+        long s = System.currentTimeMillis();
+        while (!Thread.interrupted()) {
+            AtomicBoolean locked = new AtomicBoolean(false);
+            synchronized (locks) {
+                locks.computeIfAbsent(key, k -> {
+                    locked.compareAndSet(false, true);
+                    return new AtomicBoolean(true);
+                });
+                if (Boolean.TRUE.equals(locked.get())) {
+                    locks.put(key, new AtomicBoolean(true));
+                    return true;
+                }
+            }
+            if ((System.currentTimeMillis() - s) > timeoutMills) {
+                break;
+            }
+			TimeUnit.MILLISECONDS.sleep(1L);
+        }
+        return false;
+    }
+
+    private void unlock(String key) {
         AtomicBoolean lock;
         synchronized (locks) {
             lock = locks.remove(key);
-            if (null == lock) return;
+			if (null == lock) return;
         }
         synchronized (lock) {
             lock.set(false);
@@ -60,43 +87,28 @@ public void run(String key, Runnable runnable) throws InterruptedException {
         }
     }
 
+    public boolean tryRun(String key, Runnable runnable, long timeout, TimeUnit timeUnit) throws InterruptedException {
+        if (tryLock(key, timeout, timeUnit)) {
+            try {
+                runnable.run();
+                return true;
+            } finally {
+                unlock(key);
+            }
+        }
+        return false;
+    }
+
     public Runnable wrap(String key, Runnable runnable) {
         return () -> {
             try {
                 lock(key);
                 runnable.run();
             } catch (InterruptedException ignore) {
+                Thread.currentThread().interrupt();
             } finally {
                 unlock(key);
             }
         };
     }
-
-//    public static void main(String[] args) throws Exception {
-//        int dataSize = 100000, keySize = 10, threadCounts = 10, interval = 10;
-//
-//        SingleLockWithKey singleLockWithKey = new SingleLockWithKey();
-//        ExecutorService executorService = Executors.newFixedThreadPool(threadCounts);
-//
-//        CountDownLatch countDownLatch = new CountDownLatch(dataSize);
-//        for (int i = 0; i < dataSize; i++) {
-//            int finalIndex = i;
-//            executorService.execute(() -> {
-//                try {
-//                    singleLockWithKey.call("task-" + (finalIndex % keySize), () -> {
-////                        TimeUnit.MILLISECONDS.sleep(interval);
-//                        logger.info("run {} with key: {}", finalIndex, (finalIndex % keySize));
-//                        return null;
-//                    });
-//                } catch (Exception e) {
-//                    logger.warn("return with interrupted: {}", finalIndex, e);
-//                } finally {
-//                    countDownLatch.countDown();
-//                }
-//            });
-//        }
-//
-//        countDownLatch.await();
-//        executorService.shutdown();
-//    }
 }
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/services/FlameGraphService.java b/iengine/iengine-app/src/main/java/io/tapdata/services/FlameGraphService.java
new file mode 100644
index 0000000000..f1add00924
--- /dev/null
+++ b/iengine/iengine-app/src/main/java/io/tapdata/services/FlameGraphService.java
@@ -0,0 +1,94 @@
+package io.tapdata.services;
+
+import io.tapdata.service.skeleton.annotation.RemoteService;
+
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.commons.io.FileUtils;
+
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.concurrent.TimeUnit;
+
+@RemoteService
+public class FlameGraphService {
+	public static final String TAG = FlameGraphService.class.getSimpleName();
+	private String asyncProfilerPath = "./async-profiler";
+
+	public FlameGraphService() {
+		// 寻找 async-profiler 路径, 可能在当前路径, 也可能在父路径
+		if (!new File(asyncProfilerPath).exists()) {
+			asyncProfilerPath = "../async-profiler";
+		}
+	}
+
+	@Data
+	private static class Request {
+	}
+
+	@Data
+	@NoArgsConstructor
+	private static class Response {
+		private byte[] content;
+		public Response(byte[] content) {
+			this.content = content;
+		}
+	}
+
+	public Response memory() throws Throwable {
+		// 删除 memory.html 文件
+		FileUtils.deleteQuietly(new File("memory.html"));
+
+		String name = ManagementFactory.getRuntimeMXBean().getName();
+		// 提取 PID
+		String pid = name.split("@")[0];
+		// 1. 执行命令:
+		String command = asyncProfilerPath+"/bin/asprof -e alloc -d " + 15 + " -f ./memory.html " + pid;
+
+		byte[] content = null;
+		try {
+			Process process = Runtime.getRuntime().exec(command);
+			process.waitFor( 20, TimeUnit.SECONDS);
+			content = FileUtils.readFileToByteArray(new File("memory.html"));
+		} catch (Exception e) {
+			content = e.getMessage().getBytes();
+		}
+		return new Response(content);
+	}
+
+	public Response cpu() throws Throwable {
+		// 删除 cpu.html 文件
+		FileUtils.deleteQuietly(new File("cpu.html"));
+
+		String name = ManagementFactory.getRuntimeMXBean().getName();
+		// 提取 PID
+		String pid = name.split("@")[0];
+		// 1. 执行命令:
+		String command = asyncProfilerPath+"/bin/asprof -e cpu -d " + 15 + " -f ./cpu.html " + pid;
+
+		byte[] content = null;
+		try {
+			Process process = Runtime.getRuntime().exec(command);
+			process.waitFor(20, TimeUnit.SECONDS);
+			content = FileUtils.readFileToByteArray(new File("cpu.html"));
+		} catch (Exception e) {
+			content = e.getMessage().getBytes();
+		}
+		return new Response(content);
+	}
+
+	public Response jstack() throws Throwable {
+		ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+		boolean lockedMonitors = false;
+		boolean lockedSynchronizers = false;
+		ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(lockedMonitors, lockedSynchronizers);
+		String threadDump = "";
+		for (ThreadInfo threadInfo : threadInfos) {
+			threadDump += threadInfo.toString();
+			threadDump += "\n";
+		}
+		return new Response(threadDump.getBytes());
+	}
+}
\ No newline at end of file
diff --git a/iengine/iengine-app/src/main/java/io/tapdata/websocket/handler/DeduceSchemaHandler.java b/iengine/iengine-app/src/main/java/io/tapdata/websocket/handler/DeduceSchemaHandler.java
index e56e0eb756..3045e65a8f 100644
--- a/iengine/iengine-app/src/main/java/io/tapdata/websocket/handler/DeduceSchemaHandler.java
+++ b/iengine/iengine-app/src/main/java/io/tapdata/websocket/handler/DeduceSchemaHandler.java
@@ -1,10 +1,7 @@
 package io.tapdata.websocket.handler;
 
-import com.alibaba.fastjson.JSON;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.tapdata.constant.ConnectorConstant;
-import com.tapdata.entity.schema.SchemaApplyResult;
 import com.tapdata.mongo.ClientMongoOperator;
 import com.tapdata.tm.autoinspect.utils.GZIPUtil;
 import com.tapdata.tm.commons.base.convert.DagDeserialize;
@@ -13,29 +10,20 @@
 import com.tapdata.tm.commons.base.convert.ObjectIdSerialize;
 import com.tapdata.tm.commons.dag.DAG;
 import com.tapdata.tm.commons.dag.DAGDataServiceImpl;
-import com.tapdata.tm.commons.dag.Node;
-import com.tapdata.tm.commons.dag.vo.MigrateJsResultVo;
 import com.tapdata.tm.commons.schema.DataSourceConnectionDto;
 import com.tapdata.tm.commons.schema.DataSourceDefinitionDto;
 import com.tapdata.tm.commons.schema.MetadataInstancesDto;
 import com.tapdata.tm.commons.schema.MetadataTransformerDto;
-import com.tapdata.tm.commons.schema.TransformerWsMessageResult;
 import com.tapdata.tm.commons.task.dto.Message;
-import com.tapdata.tm.commons.task.dto.ParentTaskDto;
 import com.tapdata.tm.commons.task.dto.TaskDto;
 import com.tapdata.tm.commons.util.JsonUtil;
+import io.tapdata.common.DAGDataEngineServiceImpl;
 import io.tapdata.common.SettingService;
-import io.tapdata.entity.schema.TapTable;
-import io.tapdata.flow.engine.V2.node.hazelcast.data.HazelcastSchemaTargetNode;
-import io.tapdata.flow.engine.V2.task.TaskClient;
 import io.tapdata.flow.engine.V2.task.TaskService;
-import io.tapdata.observable.logging.ObsLogger;
-import io.tapdata.observable.logging.ObsLoggerFactory;
 import io.tapdata.websocket.EventHandlerAnnotation;
 import io.tapdata.websocket.WebSocketEventHandler;
 import io.tapdata.websocket.WebSocketEventResult;
 import lombok.Data;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bson.types.ObjectId;
@@ -44,7 +32,6 @@
 import java.util.Base64;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 @EventHandlerAnnotation(type = "deduceSchema")
 public class DeduceSchemaHandler implements WebSocketEventHandler<WebSocketEventResult> {
@@ -75,115 +62,20 @@ public WebSocketEventResult handle(Map event) {
 		DeduceSchemaRequest request = JsonUtil.parseJsonUseJackson(json, DeduceSchemaRequest.class);
 		DAG dagAgo = request.getTaskDto().getDag();
 		request.getTaskDto().setDag(dagAgo);
-		DAGDataServiceImpl dagDataService = new DAGDataServiceImpl(
-				request.getMetadataInstancesDtoList(),
-				request.getDataSourceMap(),
-				request.getDefinitionDtoMap(),
-				request.getUserId(),
-				request.getUserName(),
-				request.getTaskDto(),
-				request.getTransformerDtoMap()
-		) {
-
-			@Override
-			public TapTable loadTapTable(String nodeId, String virtualId, TaskDto taskDto) {
-				ObsLogger obsLogger = ObsLoggerFactory.getInstance().getObsLogger(taskDto);
-				try {
-					// 跑任务加载js模型
-					String schemaKey = taskDto.getId() + "-" + virtualId;
-					long startTs = System.currentTimeMillis();
-					TaskClient<TaskDto> taskClient = execTask(taskDto);
-
-					obsLogger.info("load tapTable task {} {}, cost {}ms", schemaKey, taskClient.getStatus(), (System.currentTimeMillis() - startTs));
-					//成功
-					TapTable tapTable = HazelcastSchemaTargetNode.getTapTable(schemaKey);
-					if (obsLogger.isDebugEnabled()) {
-						obsLogger.debug("derivation results: {}", JSON.toJSONString(tapTable));
-					}
-					return tapTable;
-				} catch (Exception e) {
-					obsLogger.error("An error occurred while obtaining the results of model deduction", e);
-				}
-				return null;
-			}
-
-			@Override
-			public List<MigrateJsResultVo> getJsResult(String jsNodeId, String virtualTargetId, TaskDto taskDto) {
-				ObsLogger obsLogger = ObsLoggerFactory.getInstance().getObsLogger(taskDto);
-				try {
-					String schemaKey = taskDto.getId() + "-" + virtualTargetId;
-					long startTs = System.currentTimeMillis();
-
-					TaskClient<TaskDto> taskClient = execTask(taskDto);
-
-					obsLogger.info("load MigrateJsResultVos task {} {}, cost {}ms", schemaKey, taskClient.getStatus(), (System.currentTimeMillis() - startTs));
-					//成功
-					List<SchemaApplyResult> schemaApplyResultList = HazelcastSchemaTargetNode.getSchemaApplyResultList(schemaKey);
-					if (obsLogger.isDebugEnabled()) {
-						obsLogger.debug("derivation results: {}", JSON.toJSONString(schemaApplyResultList));
-					}
-
-					if (CollectionUtils.isNotEmpty(schemaApplyResultList)) {
-						return schemaApplyResultList.stream().map(s -> new MigrateJsResultVo(s.getOp(), s.getFieldName(), s.getTapField(), s.getTapIndex()))
-								.collect(Collectors.toList());
-					}
-				} catch (Exception e) {
-					obsLogger.error("An error occurred while obtaining the results of model deduction", e);
-				}
-				return null;
-			}
-
-			private TaskClient<TaskDto> execTask(TaskDto taskDto) {
-				taskDto.setType(ParentTaskDto.TYPE_INITIAL_SYNC);
-				TaskClient<TaskDto> taskClient = taskService.startTestTask(taskDto);
-				taskClient.join();
-				return taskClient;
-			}
-		};
-
+		DAGDataServiceImpl dagDataService = new DAGDataEngineServiceImpl(
+				request,
+				taskService,
+				clientMongoOperator
+		);
 		long start = System.currentTimeMillis();
 		Map<String, List<Message>> transformSchema = request.getTaskDto().getDag().transformSchema(null, dagDataService, request.getOptions());
 		logger.info("transformed cons={}", System.currentTimeMillis() - start + "ms");
-
-		TransformerWsMessageResult wsMessageResult = new TransformerWsMessageResult();
-		wsMessageResult.setBatchMetadataUpdateMap(dagDataService.getBatchMetadataUpdateMap());
-		wsMessageResult.setBatchInsertMetaDataList(dagDataService.getBatchInsertMetaDataList());
-		wsMessageResult.setUpsertItems(dagDataService.getUpsertItems());
-		wsMessageResult.setUpsertTransformer(dagDataService.getUpsertTransformer());
-		wsMessageResult.setTransformSchema(transformSchema);
-		wsMessageResult.setTaskId(request.getTaskDto().getId().toHexString());
-		wsMessageResult.setTransformUuid(request.getOptions().getUuid());
-		UpdateTaskDagDto updateTaskDagDto = new UpdateTaskDagDto();
-		updateTaskDagDto.setId(request.getTaskDto().getId());
-		updateTaskDagDto.setDag(request.getTaskDto().getDag());
-		if (dagAgo != null) {
-			List<Node> nodes = dagAgo.getNodes();
-			if (CollectionUtils.isNotEmpty(nodes)) {
-				nodes.forEach(f -> {
-					f.setOutputSchema(null);
-					f.setSchema(null);
-				});
-			}
-		}
-
-		wsMessageResult.setDag(dagAgo);
-
-		String jsonResult = JsonUtil.toJsonUseJackson(wsMessageResult);
-		byte[] gzip = GZIPUtil.gzip(jsonResult.getBytes());
-		byte[] encode = Base64.getEncoder().encode(gzip);
-		String dataString = new String(encode, StandardCharsets.UTF_8);
-
-		//返回结果调用接口返回
-		clientMongoOperator.insertOne(dataString, ConnectorConstant.TASK_COLLECTION + "/transformer/resultV2");
-
-		// Update task's dag
-
-		//clientMongoOperator.insertOne(updateTaskDagDto, ConnectorConstant.TASK_COLLECTION + "/dagNotHistory");
+		dagDataService.uploadModel(transformSchema);
 
 		return WebSocketEventResult.handleSuccess(WebSocketEventResult.Type.DEDUCE_SCHEMA, true);
 	}
 
-	private static class DeduceSchemaRequest {
+	public static class DeduceSchemaRequest {
 
 		private TaskDto taskDto;
 
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/constant/Krb5Util.java b/iengine/iengine-common/src/main/java/com/tapdata/constant/Krb5Util.java
deleted file mode 100644
index 6c98bcc3a0..0000000000
--- a/iengine/iengine-common/src/main/java/com/tapdata/constant/Krb5Util.java
+++ /dev/null
@@ -1,274 +0,0 @@
-package com.tapdata.constant;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import sun.security.krb5.Config;
-import sun.security.krb5.KrbException;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-/**
- * Kerberos 工具类
- *
- * @author <a href="mailto:harsen_lin@163.com">Harsen</a>
- * @version v1.0 2021/11/17 下午5:04 Create
- */
-public class Krb5Util {
-
-	private static Logger logger = LogManager.getLogger(Krb5Util.class);
-
-	private static String decodeConf(String base64Conf) {
-		byte[] bytes = Base64.getUrlDecoder().decode(base64Conf);
-		return new String(bytes, StandardCharsets.UTF_8);
-	}
-
-	private static String storeDir() {
-		String dir = System.getenv("TAPDATA_WORK_DIR");
-		if (null == dir) {
-			dir = ".";
-		}
-		return paths(dir, "krb5");
-	}
-
-	private static String paths(String... paths) {
-		return String.join(File.separator, paths);
-	}
-
-	/**
-	 * 保存文件
-	 *
-	 * @param data         数据
-	 * @param savePath     保存路径
-	 * @param deleteExists 是否存在删除
-	 * @throws IOException 异常
-	 */
-	private static void save(byte[] data, String savePath, boolean deleteExists) throws IOException {
-		File file = new File(savePath);
-		if (file.exists()) {
-			if (!deleteExists) {
-				throw new RuntimeException("Save config is exists: " + savePath);
-			}
-			file.delete();
-		} else {
-			File dir = file.getParentFile();
-			if (!dir.exists()) {
-				dir.mkdirs();
-			}
-		}
-		try (FileOutputStream fos = new FileOutputStream(file)) {
-			fos.write(data);
-		}
-	}
-
-	private static Map<String, Map<String, String>> getRealms(String krb5Conf) {
-		try {
-			String[] kv;
-			boolean inRealms = false, begin = true;
-			Map<String, String> kvMap = null;
-			Map<String, Map<String, String>> map = new HashMap<>();
-			for (String line : krb5Conf.split("\n")) {
-				if (line.startsWith("#")) continue;
-				line = line.replaceAll("#.*", "").trim();
-				if (line.isEmpty()) continue;
-
-				if (!inRealms) {
-					if (Pattern.matches("^[\\[]*\\[realms].*", line)) {
-						inRealms = true;
-					}
-					continue;
-				} else if (line.startsWith("[")) {
-					break;
-				}
-
-				if (begin) {
-					kvMap = new HashMap<>();
-					map.put(line.replaceAll("([^\\s]+).*", "$1"), kvMap);
-					begin = false;
-					continue;
-				} else if (line.contains("}")) {
-					begin = true;
-					continue;
-				}
-				kv = line.split("=");
-				if (kv.length != 2) continue;
-				kvMap.put(kv[0].trim(), kv[1].trim());
-			}
-			return map;
-		} catch (Exception e) {
-			throw new RuntimeException("Parse [realms] failed: " + e.getMessage(), e);
-		}
-	}
-
-	/**
-	 * 获取配置
-	 *
-	 * @param krb5Path      授权路径
-	 * @param krb5Principal 主体
-	 * @return 配置
-	 */
-	private static String saslJaasConfig(String krb5Path, String krb5Principal) {
-		return "com.sun.security.auth.module.Krb5LoginModule required\n" +
-				"    useKeyTab=true\n" +
-				"    storeKey=true\n" +
-				"    useTicketCache=true\n" +
-				"    keyTab=\"" + keytabPath(krb5Path) + "\"\n" +
-				"    principal=\"" + krb5Principal + "\";";
-	}
-
-	/**
-	 * 获取密钥路径
-	 *
-	 * @param dir 配置目录
-	 * @return 密钥路径
-	 */
-	public static String keytabPath(String dir) {
-		return paths(dir, "krb5.keytab");
-	}
-
-	/**
-	 * 获取配置路径
-	 *
-	 * @param dir 配置目录
-	 * @return 配置路径
-	 */
-	public static String confPath(String dir) {
-		return paths(dir, "krb5.conf");
-	}
-
-	/**
-	 * 检查配置域
-	 *
-	 * @param conf 配置
-	 * @throws UnknownHostException 异常
-	 */
-	public static void checkKDCDomains(String conf) throws UnknownHostException {
-		Map<String, Map<String, String>> realms = getRealms(conf);
-		for (Map<String, String> realm : realms.values()) {
-			for (Map.Entry<String, String> en : realm.entrySet()) {
-				if (StringUtils.containsAny(en.getKey(), "kdc", "master_kdc", "admin_server", "default_domain")
-						&& null != en.getValue()) {
-					String s = en.getValue().split(":")[0].trim();
-					if (!s.isEmpty()) {
-						InetAddress.getAllByName(s);
-					}
-				}
-			}
-		}
-	}
-
-	/**
-	 * 检查配置域
-	 *
-	 * @param base64Conf base64配置
-	 * @throws UnknownHostException 异常
-	 */
-	public static void checkKDCDomainsBase64(String base64Conf) throws UnknownHostException {
-		String conf = decodeConf(base64Conf);
-		checkKDCDomains(conf);
-	}
-
-	/**
-	 * 根据类别保存
-	 *
-	 * @param catalog      类别
-	 * @param keytab       密钥
-	 * @param conf         配置
-	 * @param deleteExists 存在删除
-	 * @return 配置路径
-	 */
-	public static String saveByCatalog(String catalog, String keytab, String conf, boolean deleteExists) {
-		byte[] bytes;
-		String savePath = null;
-		String dir = paths(storeDir(), catalog);
-		try {
-			savePath = keytabPath(dir);
-			bytes = Base64.getDecoder().decode(keytab);
-			save(bytes, savePath, deleteExists);
-		} catch (Exception e) {
-			throw new RuntimeException("Save kerberos keytab failed: " + savePath, e);
-		}
-		try {
-			savePath = confPath(dir);
-			bytes = Base64.getDecoder().decode(conf);
-			save(bytes, savePath, deleteExists);
-		} catch (Exception e) {
-			throw new RuntimeException("Save kerberos conf failed: " + savePath, e);
-		}
-		return dir;
-	}
-
-	/**
-	 * 更新 Kafka 配置
-	 *
-	 * @param serviceName 服务名
-	 * @param principal   主体
-	 * @param krb5Path    授权配置目录
-	 * @param krb5Conf    授权配置
-	 * @param conf        kafka配置
-	 */
-	public static void updateKafkaConf(String serviceName, String principal, String krb5Path, String krb5Conf, Map<String, Object> conf) {
-		try {
-			String krb5ConfPath = confPath(krb5Path);
-			String saslJaasConfig = saslJaasConfig(krb5Path, principal);
-
-			conf.put("security.protocol", "SASL_PLAINTEXT");
-			conf.put("sasl.mechanism", "GSSAPI");
-			conf.put("sasl.kerberos.service.name", serviceName);
-			conf.put("sasl.jaas.config", saslJaasConfig);
-			System.setProperty("java.security.krb5.conf", krb5ConfPath);
-
-			String realm = null;
-			if (null != principal) {
-				String[] arr = principal.split("@");
-				if (arr.length == 2) realm = arr[1].trim();
-			}
-			if (null == realm || realm.isEmpty()) {
-				logger.warn("Parse krb5 realm failed: " + principal);
-				return;
-			}
-
-			krb5Conf = decodeConf(krb5Conf);
-			Map<String, Map<String, String>> realms = getRealms(krb5Conf);
-			Map<String, String> currentRealms = realms.get(realm);
-			if (null != currentRealms && currentRealms.containsKey("kdc")) {
-				System.setProperty("java.security.krb5.realm", realm);
-				System.setProperty("java.security.krb5.kdc", currentRealms.get("kdc"));
-
-				Config.refresh();
-			} else {
-				logger.warn("Not found kdc in realm '{}' >> {}", realm, krb5Conf);
-			}
-		} catch (KrbException e) {
-			throw new RuntimeException("Refresh krb5 config failed", e);
-		}
-	}
-
-
-	public static void main(String[] args) throws Exception {
-		String conf;
-		try (FileInputStream fis = new FileInputStream("/Users/lhs/Downloads/krb5-test.conf")) {
-			try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-				int len;
-				byte[] buf = new byte[1024];
-				while (-1 != (len = fis.read(buf, 0, 1024))) {
-					baos.write(buf, 0, len);
-				}
-				conf = baos.toString("UTF-8");
-			}
-		}
-		System.out.println(conf);
-		checkKDCDomains(conf);
-	}
-}
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/constant/MapUtil.java b/iengine/iengine-common/src/main/java/com/tapdata/constant/MapUtil.java
index 949eec8aa5..b50b9c690a 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/constant/MapUtil.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/constant/MapUtil.java
@@ -4,10 +4,6 @@
 import com.tapdata.exception.MapUtilException;
 import io.tapdata.annotation.Ignore;
 import jdk.nashorn.api.scripting.ScriptObjectMirror;
-import jdk.nashorn.internal.objects.NativeBoolean;
-import jdk.nashorn.internal.objects.NativeNumber;
-import jdk.nashorn.internal.objects.NativeString;
-import jdk.nashorn.internal.runtime.ScriptObject;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.SerializationUtils;
@@ -297,18 +293,6 @@ public static void copyToNewMap(Map map, Map newMap) {
 				ListUtil.copyList(list, newObject);
 				newMap.put(key, newObject);
 			} else if (value instanceof Map) {
-				if (value instanceof ScriptObjectMirror) {
-					try {
-						ScriptObject sobj = (ScriptObject) FieldUtils.readField(value, "sobj", true);
-						if (sobj instanceof NativeNumber || sobj instanceof NativeBoolean || sobj instanceof NativeString) {
-							Object newObject = FieldUtils.readField(sobj, "value", true);
-							newMap.put(key, newObject);
-							continue;
-						}
-					} catch (IllegalAccessException e) {
-						logger.warn("get new obj error, skip it...", e);
-					}
-				}
 				Map newObject = mapInstance((Map<String, Object>) value);
 				copyToNewMap((Map) value, newObject);
 				newMap.put(key, newObject);
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/constant/MongodbUtil.java b/iengine/iengine-common/src/main/java/com/tapdata/constant/MongodbUtil.java
index 3141449df6..8eb134e8f7 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/constant/MongodbUtil.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/constant/MongodbUtil.java
@@ -47,7 +47,6 @@
 import com.tapdata.mongo.FloatCodec;
 import com.tapdata.mongo.ObjectIdCodec;
 import com.tapdata.mongo.StringCodec;
-import com.tapdata.mongo.UndifinedCodec;
 import io.tapdata.exception.BaseDatabaseUtilException;
 import io.tapdata.exception.MongodbException;
 import org.apache.commons.collections.CollectionUtils;
@@ -335,7 +334,7 @@ public static CodecRegistry getForJavaCoedcRegistry() {
 
 		CodecRegistry codecRegistry = MongodbUtil.customCodecRegistry(
 				Arrays.asList(
-						new BigIntegerCodec(), new BigDecimalCodec(), new FloatCodec(), new UndifinedCodec(), new ByteArrayCodec(),
+						new BigIntegerCodec(), new BigDecimalCodec(), new FloatCodec(), new ByteArrayCodec(),
 						new DateCodec(), new StringCodec()
 				),
 				replacments
@@ -359,7 +358,7 @@ public static CodecRegistry getForNodejsCoedcRegistry() {
 
 		CodecRegistry codecRegistry = MongodbUtil.customCodecRegistry(
 				Arrays.asList(
-						new BigIntegerCodec(), new BigDecimalCodec(), new FloatCodec(), new UndifinedCodec(), new ByteArrayCodec(),
+						new BigIntegerCodec(), new BigDecimalCodec(), new FloatCodec(), new ByteArrayCodec(),
 						new DateCodec(), new StringCodec(), new ObjectIdCodec()
 				),
 				replacments
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/entity/AppType.java b/iengine/iengine-common/src/main/java/com/tapdata/entity/AppType.java
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/Inspect.java b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/Inspect.java
index a0628d5414..9f851f2c21 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/Inspect.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/Inspect.java
@@ -53,6 +53,11 @@ public class Inspect extends BaseEntity implements Serializable {
 
 	private String inspectResultId;     // 重新校验时，指定 inspect result id
 
+	private Integer diffInspectTimes;
+
+
+
+
 	/**
 	 * Inspect mode
 	 */
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectMethod.java b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectMethod.java
index 7ee6f455ee..5ed88337c1 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectMethod.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectMethod.java
@@ -19,22 +19,26 @@ public enum InspectMethod {
 	HASH("hash")
 	;
 
-	private final String name;
+	private final String code;
 
-	InspectMethod(String name) {
-		this.name = name;
+	InspectMethod(String code) {
+		this.code = code;
 	}
 
 	private static Map<String, InspectMethod> map = new HashMap<>();
 
 	static {
 		for (InspectMethod value : InspectMethod.values()) {
-			map.put(value.name, value);
+			map.put(value.code, value);
 		}
 	}
 
-	public boolean equalsString(String name) {
-		return this.name.equals(name);
+    public String getCode() {
+        return code;
+    }
+
+    public boolean equalsString(String name) {
+		return this.code.equals(name);
 	}
 
 	public static InspectMethod get(String name) {
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectResult.java b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectResult.java
index 43c8bb74dc..a1ba4eddff 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectResult.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectResult.java
@@ -45,4 +45,6 @@ public class InspectResult extends BaseEntity implements Serializable {
 	private String parentId; // 父校验结果编号，表示此校验基于 parentId 结果做的二次校验
 
 	private boolean partStats;  // 标识是否只上报了一部分 stats 信息
+
+	private boolean repeat;
 }
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectTask.java b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectTask.java
index d9ac176150..c69eca14c0 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectTask.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/entity/inspect/InspectTask.java
@@ -52,4 +52,13 @@ public class InspectTask implements Serializable {
 	 * 是否开启高级校验
 	 */
 	private boolean showAdvancedVerification;
+
+
+	private String flowId;
+
+	private String id;
+
+	private boolean repeat;
+
+	private String mode;
 }
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/entity/task/error/TaskServiceExCode_23.java b/iengine/iengine-common/src/main/java/com/tapdata/entity/task/error/TaskServiceExCode_23.java
index b75dd1a68d..a142d5d198 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/entity/task/error/TaskServiceExCode_23.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/entity/task/error/TaskServiceExCode_23.java
@@ -18,4 +18,10 @@ public interface TaskServiceExCode_23 {
 			describe = "Failed to initialize task global variables, task ID cannot be empty"
 	)
 	String TASK_GLOBAL_VARIABLE_INIT_TASK_ID_EMPTY = "23002";
+
+	@TapExCode(
+			describeCN = "引擎推演模型失败",
+			describe = "Engine deduction model failed"
+	)
+	String TASK_FAILED_TO_LOAD_TABLE_STRUCTURE = "23004";
 }
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/exception/NonsupportMethodDifferenceInspectException.java b/iengine/iengine-common/src/main/java/com/tapdata/exception/NonsupportMethodDifferenceInspectException.java
new file mode 100644
index 0000000000..51436308c1
--- /dev/null
+++ b/iengine/iengine-common/src/main/java/com/tapdata/exception/NonsupportMethodDifferenceInspectException.java
@@ -0,0 +1,21 @@
+package com.tapdata.exception;
+
+import lombok.Getter;
+
+/**
+ * 不支持差异校验
+ *
+ * @author <a href="mailto:harsen_lin@163.com">Harsen</a>
+ * @version v1.0 2024/6/15 15:28 Create
+ */
+@Getter
+public class NonsupportMethodDifferenceInspectException extends RuntimeException {
+    private final String inspectId;
+    private final String method;
+
+    public NonsupportMethodDifferenceInspectException(String inspectId, String method) {
+        super("Nonsupport difference inspect by method: '" + method + "'");
+        this.inspectId = inspectId;
+        this.method = method;
+    }
+}
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/exception/NotfoundLastResultInspectException.java b/iengine/iengine-common/src/main/java/com/tapdata/exception/NotfoundLastResultInspectException.java
new file mode 100644
index 0000000000..15112dec56
--- /dev/null
+++ b/iengine/iengine-common/src/main/java/com/tapdata/exception/NotfoundLastResultInspectException.java
@@ -0,0 +1,22 @@
+package com.tapdata.exception;
+
+import lombok.Getter;
+
+/**
+ * 不支持差异校验
+ *
+ * @author <a href="mailto:harsen_lin@163.com">Harsen</a>
+ * @version v1.0 2024/6/15 15:28 Create
+ */
+@Getter
+public class NotfoundLastResultInspectException extends RuntimeException {
+    private final String inspectId;
+    private final String firstCheckId;
+
+    public NotfoundLastResultInspectException(String inspectId, String firstCheckId) {
+        super("Not found last difference inspect result in '" + firstCheckId + "'");
+        this.inspectId = inspectId;
+        this.firstCheckId = firstCheckId;
+    }
+
+}
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/mongo/UndifinedCodec.java b/iengine/iengine-common/src/main/java/com/tapdata/mongo/UndifinedCodec.java
index 34d8b06a5c..895393c638 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/mongo/UndifinedCodec.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/mongo/UndifinedCodec.java
@@ -1,3 +1,4 @@
+/*
 package com.tapdata.mongo;
 
 import jdk.nashorn.internal.runtime.Undefined;
@@ -24,3 +25,4 @@ public Class<Undefined> getEncoderClass() {
 		return Undefined.class;
 	}
 }
+*/
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/processor/ScriptUtil.java b/iengine/iengine-common/src/main/java/com/tapdata/processor/ScriptUtil.java
index 2e9fc48abc..ad620d71e3 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/processor/ScriptUtil.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/processor/ScriptUtil.java
@@ -214,7 +214,8 @@ public static Object invokeScript(
 			Connections targetConn,
 			Job job,
 			Map<String, Object> context,
-			Logger logger
+			Logger logger,
+			String tag
 	) throws Exception {
 		ProcessContext processContext = message.getProcessContext();
 		if (message.getProcessContext() == null) {
@@ -256,6 +257,15 @@ public static Object invokeScript(
 
 		Object o;
 		Map<String, Object> record = MapUtils.isNotEmpty(message.getAfter()) ? message.getAfter() : message.getBefore();
+		if (null != tag){
+			switch (tag) {
+				case "before":
+					record = MapUtils.isNotEmpty(message.getBefore()) ? message.getBefore() : message.getAfter();
+					break;
+                default:
+					break;
+            }
+		}
 		try {
 			if (engine instanceof GraalJSScriptEngine) {
 				o = engine.invokeFunction(functionName, ProxyObject.fromMap(record));
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/FieldDataFlowProcessor.java b/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/FieldDataFlowProcessor.java
index a4aa600e15..0c55b8281f 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/FieldDataFlowProcessor.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/FieldDataFlowProcessor.java
@@ -160,8 +160,12 @@ record = new HashMap<>();
 					FieldProcessUtil.filedProcess(after, fieldProcesses, fieldsNameTransform, deleteAllFields);
 					message.setAfter(after);
 				}
-				fieldScript(message, before);
-				fieldScript(message, after);
+				if (null != before){
+					fieldScript(message, before, "before");
+				}
+				if (null != after){
+					fieldScript(message, after, "after");
+				}
 			} else if (CollectionUtils.isNotEmpty(cloneFieldProcesses)) {
 				// cluster clone field process
 				CloneFieldProcess cloneFieldProcess = cloneFieldProcesses.parallelStream()
@@ -239,17 +243,17 @@ private List<FieldProcess> getFieldProcesses(MessageEntity message) {
 	}
 
 
-	private void fieldScript(MessageEntity message, Map<String, Object> record) {
+	protected void fieldScript(MessageEntity message, Map<String, Object> originRecord, String tag) {
 		if (MapUtils.isNotEmpty(fieldScriptEngine)) {
 			for (Map.Entry<String, Invocable> entry : fieldScriptEngine.entrySet()) {
 				try {
 					String fieldName = entry.getKey();
 					// 字段在源记录里不存在不做处理（兼容脏数据）
 
-					Object valueByKey = MapUtilV2.getValueByKeyV2(record, fieldName);
+					Object valueByKey = MapUtilV2.getValueByKeyV2(originRecord, fieldName);
 					Invocable engine = entry.getValue();
 					if (valueByKey instanceof TapList) {
-						Map<String, Object> finalRecord = record;
+						Map<String, Object> finalRecord = originRecord;
 						MessageEntity finalMessage = new MessageEntity();
 						BeanUtils.copyProperties(message, finalMessage);
 						finalMessage.setAfter(finalRecord);
@@ -257,7 +261,7 @@ private void fieldScript(MessageEntity message, Map<String, Object> record) {
 								o -> {
 									try {
 										return ScriptUtil.invokeScript(engine, ScriptUtil.FUNCTION_NAME, finalMessage, context.getSourceConn(),
-												context.getTargetConn(), context.getJob(), processContext, logger);
+												context.getTargetConn(), context.getJob(), processContext, logger, tag);
 									} catch (Exception e) {
 										context.getJob().jobError(e, false, OffsetUtil.getSyncStage(finalMessage.getOffset()), logger, ConnectorConstant.WORKER_TYPE_CONNECTOR,
 												TapLog.PROCESSOR_ERROR_0005.getMsg(), null, finalRecord, e.getMessage());
@@ -265,15 +269,15 @@ private void fieldScript(MessageEntity message, Map<String, Object> record) {
 										return o;
 									}
 								});
-						MapUtilV2.putValueInMap(record, fieldName, valueByKey);
+						MapUtilV2.putValueInMap(originRecord, fieldName, valueByKey);
 					} else {
 						Object o = ScriptUtil.invokeScript(engine, ScriptUtil.FUNCTION_NAME, message, context.getSourceConn(),
-								context.getTargetConn(), context.getJob(), processContext, logger);
-						MapUtilV2.putValueInMap(record, fieldName, o);
+								context.getTargetConn(), context.getJob(), processContext, logger, tag);
+						MapUtilV2.putValueInMap(originRecord, fieldName, o);
 					}
 				} catch (Exception e) {
 					context.getJob().jobError(e, false, OffsetUtil.getSyncStage(message.getOffset()), logger, ConnectorConstant.WORKER_TYPE_CONNECTOR,
-							TapLog.PROCESSOR_ERROR_0005.getMsg(), null, record, e.getMessage());
+							TapLog.PROCESSOR_ERROR_0005.getMsg(), null, originRecord, e.getMessage());
 				}
 			}
 		}
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/RowFilterProcessor.java b/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/RowFilterProcessor.java
index e6aff2d589..d5a292cd2d 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/RowFilterProcessor.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/RowFilterProcessor.java
@@ -87,7 +87,7 @@ public MessageEntity process(MessageEntity message) {
 			Map<String, Object> record = MapUtils.isNotEmpty(message.getAfter()) ? message.getAfter() : message.getBefore();
 			try {
 				if (MapUtils.isNotEmpty(record)) {
-					Object o = ScriptUtil.invokeScript(engine, SCRIPT_FUNCTION_NAME, message, context.getSourceConn(), context.getTargetConn(), context.getJob(), processContext, logger);
+					Object o = ScriptUtil.invokeScript(engine, SCRIPT_FUNCTION_NAME, message, context.getSourceConn(), context.getTargetConn(), context.getJob(), processContext, logger, null);
 					if (o instanceof Boolean) {
 						Boolean result = (Boolean) o;
 						// 满足条件处理
diff --git a/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/ScriptDataFlowProcessor.java b/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/ScriptDataFlowProcessor.java
index 422afbce11..6a1c6025d1 100644
--- a/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/ScriptDataFlowProcessor.java
+++ b/iengine/iengine-common/src/main/java/com/tapdata/processor/dataflow/ScriptDataFlowProcessor.java
@@ -85,7 +85,7 @@ public List<MessageEntity> process(MessageEntity message) {
 			try {
 				if (MapUtils.isNotEmpty(record)) {
 
-					Object o = ScriptUtil.invokeScript(engine, ScriptUtil.FUNCTION_NAME, message, context.getSourceConn(), context.getTargetConn(), context.getJob(), processContext, logger);
+					Object o = ScriptUtil.invokeScript(engine, ScriptUtil.FUNCTION_NAME, message, context.getSourceConn(), context.getTargetConn(), context.getJob(), processContext, logger, null);
 
 					if (o == null) {
 						return null;
diff --git a/iengine/iengine-oss/pom.xml b/iengine/iengine-oss/pom.xml
index 2535c82b50..e5111db482 100644
--- a/iengine/iengine-oss/pom.xml
+++ b/iengine/iengine-oss/pom.xml
@@ -24,6 +24,38 @@
             <artifactId>api</artifactId>
             <version>0.5.2-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>5.8.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
+    <build>
+    <plugins>
+    <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <executions>
+            <!--first execution : for preparing JaCoCo runtime agent-->
+            <execution>
+                <id>prepare-agent</id>
+                <goals>
+                    <goal>prepare-agent</goal>
+                </goals>
+            </execution>
+            <!--second execution : for creating code coverage reports-->
+            <execution>
+                <id>report</id>
+                <phase>test</phase>
+                <goals>
+                    <goal>report</goal>
+                </goals>
+            </execution>
+        </executions>
+    </plugin>
+    </plugins>
+    </build>
+
 
 </project>
\ No newline at end of file
diff --git a/iengine/iengine-oss/src/main/java/io/tapdata/inspect/InspectServiceImpl.java b/iengine/iengine-oss/src/main/java/io/tapdata/inspect/InspectServiceImpl.java
index b5e861c76e..ed7b1eaf5f 100644
--- a/iengine/iengine-oss/src/main/java/io/tapdata/inspect/InspectServiceImpl.java
+++ b/iengine/iengine-oss/src/main/java/io/tapdata/inspect/InspectServiceImpl.java
@@ -31,7 +31,7 @@ public void upsertInspectResult(InspectResult inspectResult) {
     }
 
     @Override
-    public InspectResult getLastDifferenceInspectResult(String firstCheckId) {
+    public InspectResult getLastDifferenceInspectResult(String inspectId, String firstCheckId) {
         throw new TapOssNonsupportFunctionException();
     }
 
@@ -74,4 +74,9 @@ public void inspectHeartBeat(String id) {
     public Inspect getInspectById(String id) {
         throw new TapOssNonsupportFunctionException();
     }
+
+    @Override
+    public void handleDiffInspect(Inspect inspect, long syncTaskDelay,InspectResult inspectResult) {
+        throw new TapOssNonsupportFunctionException();
+    }
 }
diff --git a/iengine/modules/milestone-module/src/main/java/io/tapdata/milestone/MilestoneAspectTask.java b/iengine/modules/milestone-module/src/main/java/io/tapdata/milestone/MilestoneAspectTask.java
index dd10da9c25..d83bb19418 100644
--- a/iengine/modules/milestone-module/src/main/java/io/tapdata/milestone/MilestoneAspectTask.java
+++ b/iengine/modules/milestone-module/src/main/java/io/tapdata/milestone/MilestoneAspectTask.java
@@ -35,6 +35,7 @@
 import static io.tapdata.aspect.TableInitFuncAspect.STATE_END;
 import static io.tapdata.aspect.TableInitFuncAspect.STATE_START;
 import static io.tapdata.aspect.TableInitFuncAspect.STATE_PROCESS;
+import static io.tapdata.aspect.taskmilestones.EngineDeductionAspect.*;
 
 /**
  * 设计要点：
@@ -66,6 +67,7 @@ public class MilestoneAspectTask extends AbstractAspectTask {
     protected static final String KPI_SNAPSHOT_WRITE = "SNAPSHOT_WRITE";
     protected static final String KPI_CDC_WRITE = "CDC_WRITE";
     protected static final String KPI_TABLE_INIT = "TABLE_INIT";
+    protected static final String KPI_DEDUCTION = "DEDUCTION";
 
     private final Map<String, MilestoneEntity> milestones = new ConcurrentHashMap<>();
     private final Map<String, Map<String, MilestoneEntity>> nodeMilestones = new ConcurrentHashMap<>();
@@ -88,7 +90,7 @@ protected void init() {
         observerHandlers.register(ProcessorNodeInitAspect.class, this::handleProcessNodeInit);
         observerHandlers.register(ProcessorNodeCloseAspect.class, this::handleProcessNodeClose);
         observerHandlers.register(TableInitFuncAspect.class, this::handleTableInit);
-
+        observerHandlers.register(EngineDeductionAspect.class,this::handleEngineDeduction);
         nodeRegister(SnapshotReadBeginAspect.class, KPI_SNAPSHOT_READ, (aspect, m) -> {
             m.setProgress(0L);
             m.setTotals((long) aspect.getTables().size());
@@ -155,6 +157,7 @@ protected void init() {
     public void onStart(TaskStartAspect startAspect) {
         log.info("Start task milestones: {}({})", task.getId().toHexString(), task.getName());
         taskMilestone(KPI_TASK, this::setFinish);
+        taskMilestone(KPI_DEDUCTION,null);
         if (!TaskDto.SYNC_TYPE_LOG_COLLECTOR.equals(task.getSyncType())) {
             taskMilestone(KPI_TABLE_INIT, null);
         }
@@ -276,6 +279,23 @@ protected synchronized Void handleTableInit(TableInitFuncAspect aspect) {
         return null;
     }
 
+    protected Void handleEngineDeduction(EngineDeductionAspect aspect) {
+        switch (aspect.getState()){
+            case DEDUCTION_START:
+                taskMilestone(KPI_DEDUCTION, this::setRunning);
+                break;
+            case DEDUCTION_END:
+                taskMilestone(KPI_DEDUCTION, this::setFinish);
+                break;
+            case DEDUCTION_ERROR:
+                taskMilestone(KPI_DEDUCTION, (m)->{setError(aspect,m);});
+                break;
+            default:
+                break;
+        }
+        return null;
+    }
+
     protected void storeMilestoneWhenTargetNodeTableInit(MilestoneEntity milestone, String nid, AtomicLong totals, AtomicLong completed) {
         nodeMilestones(nid, KPI_TABLE_INIT, m -> {
             milestone.setBegin(Math.min(milestone.getBegin(), m.getBegin()));
@@ -449,6 +469,11 @@ protected <T extends AbsDataNodeErrorAspect<T>> void setError(T aspect, Mileston
         m.setStatus(MilestoneStatus.ERROR);
         m.setErrorMessage(Optional.ofNullable(aspect.getError()).map(Throwable::getMessage).orElse(null));
     }
+    protected <T extends EngineDeductionAspect> void setError(T aspect, MilestoneEntity m) {
+        m.setEnd(System.currentTimeMillis());
+        m.setStatus(MilestoneStatus.ERROR);
+        m.setErrorMessage(Optional.ofNullable(aspect.getError()).map(Throwable::getMessage).orElse(null));
+    }
 
     protected Consumer<MilestoneEntity> getErrorConsumer(String errorMessage) {
         return m -> {
diff --git a/manager/tm-api/pom.xml b/manager/tm-api/pom.xml
index 526f9a4555..74c648edbb 100644
--- a/manager/tm-api/pom.xml
+++ b/manager/tm-api/pom.xml
@@ -124,6 +124,11 @@
             <version>1.4.1</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.21</version>
+        </dependency>
         <!-- Test Dependencies -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
diff --git a/manager/tm-api/src/main/java/com/tapdata/tm/task/entity/TaskEntity.java b/manager/tm-api/src/main/java/com/tapdata/tm/task/entity/TaskEntity.java
index df3528a3cc..dcc61cbc32 100644
--- a/manager/tm-api/src/main/java/com/tapdata/tm/task/entity/TaskEntity.java
+++ b/manager/tm-api/src/main/java/com/tapdata/tm/task/entity/TaskEntity.java
@@ -258,6 +258,7 @@ public class TaskEntity extends BaseEntity implements IDataPermissionEntity {
     /** 使用的共享挖掘任务停止 */
     private Boolean shareCdcStop;
     private String shareCdcStopMessage;
+    private long delayTime;
 
     /**
      * 开启动态调整队列内存使用
diff --git a/manager/tm-api/src/main/java/com/tapdata/tm/task/service/TaskService.java b/manager/tm-api/src/main/java/com/tapdata/tm/task/service/TaskService.java
index 00492f38fa..1fe41640c7 100644
--- a/manager/tm-api/src/main/java/com/tapdata/tm/task/service/TaskService.java
+++ b/manager/tm-api/src/main/java/com/tapdata/tm/task/service/TaskService.java
@@ -1,5 +1,7 @@
 package com.tapdata.tm.task.service;
 
+import cn.hutool.http.server.HttpServerRequest;
+import cn.hutool.http.server.HttpServerResponse;
 import com.mongodb.client.result.UpdateResult;
 import com.tapdata.tm.base.dto.*;
 import com.tapdata.tm.base.service.BaseService;
@@ -29,8 +31,10 @@
 import lombok.NonNull;
 import org.bson.Document;
 import org.bson.types.ObjectId;
+import org.springframework.core.io.InputStreamResource;
 import org.springframework.data.mongodb.core.aggregation.AggregationResults;
 import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletRequest;
@@ -178,6 +182,8 @@ public abstract TaskDto createShareCacheTask(SaveShareCacheParam saveShareCacheP
 
     public abstract void batchLoadTask(HttpServletResponse response, List<String> taskIds, UserDetail user);
 
+    public abstract ResponseEntity<InputStreamResource> analyzeTask(HttpServletRequest request, HttpServletResponse response, String taskId, UserDetail user) throws IOException;
+
     public abstract void importRmProject(MultipartFile multipartFile, UserDetail user, boolean cover, List<String> tags, String source, String sink) throws IOException;
 
     public abstract void checkJsProcessorTestRun(UserDetail user, List<TaskDto> tpTasks);
@@ -287,6 +293,7 @@ public <T extends BaseDto> T convertToDto(TaskEntity entity, Class<T> dtoClass,
     public abstract int runningTaskNum(UserDetail userDetail);
 
     public abstract boolean checkCloudTaskLimit(ObjectId taskId, UserDetail user, boolean checkCurrentTask);
+    public abstract void updateDelayTime(ObjectId taskId, long delayTime);
 
     @Data
     @AllArgsConstructor
diff --git a/manager/tm-api/src/main/java/com/tapdata/tm/utils/QueryUtil.java b/manager/tm-api/src/main/java/com/tapdata/tm/utils/QueryUtil.java
index d99f883646..019a9c69b3 100644
--- a/manager/tm-api/src/main/java/com/tapdata/tm/utils/QueryUtil.java
+++ b/manager/tm-api/src/main/java/com/tapdata/tm/utils/QueryUtil.java
@@ -1,8 +1,14 @@
 package com.tapdata.tm.utils;
 
+import com.tapdata.tm.base.dto.Filter;
 import com.tapdata.tm.base.dto.Where;
 import org.bson.Document;
+import org.springframework.data.domain.Sort;
 import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+
+import java.util.ArrayList;
+import java.util.List;
 
 public class QueryUtil {
     private QueryUtil() {}
@@ -16,4 +22,27 @@ public static Criteria parseWhereToCriteria(Where where) {
         Document doc = new Document(where);
         return Criteria.matchingDocumentStructure(() -> doc);
     }
+
+    public static void parsePageParam(Filter filter, Query query) {
+        int limit = filter.getLimit();
+        int skip = filter.getSkip();
+        query.skip(skip).limit(limit);
+    }
+
+    public static List<Sort> parseOrder(Filter filter) {
+        List<String> sort = filter.getSort();
+        List<Sort> sortList = new ArrayList<>();
+        for (String s : sort) {
+            String[] split = s.split(" ");
+            if (split.length == 2) {
+                String type = String.valueOf(split[1]).toUpperCase();
+                switch (type) {
+                    case "DESC": sortList.add(Sort.by(Sort.Order.desc(split[0])));break;
+                    case "ASC": sortList.add(Sort.by(Sort.Order.asc(split[0])));break;
+                    default: sortList.add(Sort.by(Sort.Order.by(split[0])));
+                }
+            }
+        }
+        return sortList;
+    }
 }
diff --git a/manager/tm-api/src/main/java/com/tapdata/tm/worker/service/WorkerService.java b/manager/tm-api/src/main/java/com/tapdata/tm/worker/service/WorkerService.java
index 2a27ea3cbc..486896d0ce 100644
--- a/manager/tm-api/src/main/java/com/tapdata/tm/worker/service/WorkerService.java
+++ b/manager/tm-api/src/main/java/com/tapdata/tm/worker/service/WorkerService.java
@@ -102,4 +102,6 @@ public Page<WorkerDto> find(Filter filter){
     public abstract Long getLastCheckAvailableAgentCount();
 
     public abstract String getWorkerCurrentTime(UserDetail userDetail);
+
+    public abstract Boolean checkEngineVersion(UserDetail userDetail);
 }
diff --git a/manager/tm-common/pom.xml b/manager/tm-common/pom.xml
index 926646376a..5747cc965c 100644
--- a/manager/tm-common/pom.xml
+++ b/manager/tm-common/pom.xml
@@ -110,7 +110,7 @@
     <dependency>
       <groupId>io.github.openlg</groupId>
       <artifactId>graphlib</artifactId>
-      <version>0.0.3</version>
+      <version>1.0.1</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.datatype</groupId>
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAG.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAG.java
index 5749532d32..3d8e0702a3 100644
--- a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAG.java
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAG.java
@@ -47,6 +47,7 @@
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -485,12 +486,15 @@ private void setLevel(Map<String, Integer> levelMap, List<Node> successors, int
     public Map<String, List<Message>> transformSchema(String nodeId, DAGDataService dagDataService) {
         return transformSchema(nodeId, dagDataService, null);
     }
+    public Map<String, List<Message>> transformSchema(String nodeId, DAGDataService dagDataService, Options options) {
+        return transformSchema(nodeId, dagDataService, options,null);
+    }
     /**
      * 模型推演入口
      * @param nodeId 节点id，不指定默认全部推演一遍
      * @return 错误消息列表，推演成功返回 0 长度map，推演失败返回错误消息列表
      */
-    public Map<String, List<Message>> transformSchema(String nodeId, DAGDataService dagDataService, Options options) {
+    public Map<String, List<Message>> transformSchema(String nodeId, DAGDataService dagDataService, Options options, Consumer<Exception> consumer) {
         try {
 
             long start = System.currentTimeMillis();
@@ -598,6 +602,7 @@ public List<SchemaTransformerResult> getSchemaTransformResult(String nodeId) {
             logger.debug("Transform schema cost {}ms", System.currentTimeMillis() - start);
         } catch (Exception e) {
             logger.error("transformSchema error:" + ThrowableUtils.getStackTraceByPn(e));
+            if(consumer != null)consumer.accept(e);
             Map<String, List<Message>> msg = Maps.newHashMap();
             msg.put(taskId.toHexString(), Lists.newArrayList(new Message("error", e.getMessage(), JSON.toJSONString(e.getStackTrace()), null)));
             return msg;
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataService.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataService.java
index eeffa2f019..776b919efd 100644
--- a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataService.java
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataService.java
@@ -4,11 +4,13 @@
 import com.tapdata.tm.commons.schema.DataSourceConnectionDto;
 import com.tapdata.tm.commons.schema.MetadataInstancesDto;
 import com.tapdata.tm.commons.schema.Schema;
+import com.tapdata.tm.commons.task.dto.Message;
 import com.tapdata.tm.commons.task.dto.TaskDto;
 import io.tapdata.entity.schema.TapTable;
 import org.bson.types.ObjectId;
 
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author lg&lt;lirufei0808@gmail.com&gt;
@@ -99,4 +101,15 @@ default List<MigrateJsResultVo> getJsResult(String jsNodeId, String virtualTarge
     ObjectId getTaskId();
 
     List<MetadataInstancesDto> findByNodeId(String id);
+
+    default void initializeModel(Boolean isSync){
+        return;
+    }
+
+    default Boolean whetherEngineDeduction(){
+        return false;
+    }
+
+    default void uploadModel(Map<String, List<Message>> transformSchema) {
+    }
 }
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataServiceImpl.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataServiceImpl.java
index c02d258003..56a77670f9 100644
--- a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataServiceImpl.java
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/DAGDataServiceImpl.java
@@ -1270,4 +1270,24 @@ public List<MetadataInstancesDto> findByNodeId(String nodeId) {
 
         return metadataMap.values().stream().filter(m -> nodeId.equals(m.getNodeId())).distinct().collect(Collectors.toList());
     }
+
+    public void clearTransformer() {
+        upsertTransformer.clear();
+        batchMetadataUpdateMap.clear();
+        batchInsertMetaDataList.clear();
+        batchRemoveMetaDataList.clear();
+        upsertItems.clear();
+    }
+
+    public List<MetadataInstancesDto> getLogCollectorMetadataInstancesDto(){
+        HashMap<String,MetadataInstancesDto> metadataInstancesDtos = new HashMap<>();
+        Collection<MetadataInstancesDto> values = metadataMap.values();
+        if (CollectionUtils.isEmpty(values)) {
+            return Lists.newArrayList();
+        }
+        for(MetadataInstancesDto metadataInstancesDto : values){
+            metadataInstancesDtos.put(metadataInstancesDto.getQualifiedName(),metadataInstancesDto);
+        }
+        return new ArrayList<>(metadataInstancesDtos.values());
+    }
 }
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/nodes/DatabaseNode.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/nodes/DatabaseNode.java
index 56a1fd2428..6c16cd94f4 100644
--- a/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/nodes/DatabaseNode.java
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/dag/nodes/DatabaseNode.java
@@ -227,12 +227,15 @@ public void transformSchema(DAG.Options options) {
             List<String> includes = new ArrayList<>();
             options.setIncludes(includes);
             List<List<String>> partition = ListUtils.partition(tables, options.getBatchNum());
-            partition.forEach(list -> {
+            for (int i = 0; i < partition.size(); i++){
                 includes.clear();
-                includes.addAll(list);
+                includes.addAll(partition.get(i));
                 this.setSchema(null);
                 super.transformSchema(options);
-            });
+                if(service.whetherEngineDeduction()){
+                    service.initializeModel(false);
+                }
+            }
         }
     }
 
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/metrics/MeasurementEntity.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/metrics/MeasurementEntity.java
new file mode 100644
index 0000000000..5061c3bf27
--- /dev/null
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/metrics/MeasurementEntity.java
@@ -0,0 +1,43 @@
+package com.tapdata.tm.commons.metrics;
+
+import lombok.Data;
+import org.bson.codecs.pojo.annotations.BsonId;
+import org.springframework.data.mongodb.core.mapping.Field;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+
+@Data
+public class MeasurementEntity  implements Serializable {
+    public static final String COLLECTION_NAME = "AgentMeasurementV2";
+    public static final String FIELD_ID = "_id";
+    public static final String FIELD_GRANULARITY = "grnty";
+    public static final String FIELD_DATE = "date";
+    public static final String FIELD_SAMPLE_SIZE = "size";
+    public static final String FIELD_FIRST = "first";
+    public static final String FIELD_LAST = "last";
+    public static final String FIELD_TAGS = "tags";
+    public static final String FIELD_SAMPLES = "ss";
+
+    @BsonId
+    private String id;
+
+    @Field(FIELD_GRANULARITY)
+    private String granularity;
+    @Field(FIELD_DATE)
+    private Date date;
+    @Field(FIELD_SAMPLE_SIZE)
+    private Long sampleSize;
+    @Field(FIELD_FIRST)
+    private Date first;
+    @Field(FIELD_LAST)
+    private Date last;
+
+    @Field(FIELD_TAGS)
+    private Map<String, String> tags;
+    @Field(FIELD_SAMPLES)
+    private List<SampleVO> samples;
+}
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/metrics/SampleVO.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/metrics/SampleVO.java
new file mode 100644
index 0000000000..1d39b9dbed
--- /dev/null
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/metrics/SampleVO.java
@@ -0,0 +1,40 @@
+package com.tapdata.tm.commons.metrics;
+
+import lombok.Data;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+public class SampleVO {
+    public static final String FIELD_DATE = "date";
+    public static final String FIELD_VALUES = "vs";
+
+    private Map<String, Number> vs;
+    private Date date;
+
+
+    public Map<String, Object> toMap() {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put(FIELD_DATE, date);
+        map.put(FIELD_VALUES, vs);
+        return map;
+    }
+
+    @Override
+    public String toString() {
+        return "Sample{" +
+                "values=" + vs +
+                ", date=" + date +
+                '}';
+    }
+
+    public Map initVsValue() {
+        Map<String, Number> vsMap = this.getVs();
+        vsMap.forEach((key, value) -> {
+            vsMap.put(key, 0);
+        });
+        return vsMap;
+    }
+}
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/MetadataInstancesDto.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/MetadataInstancesDto.java
index 51db6ab52a..29edc1d854 100644
--- a/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/MetadataInstancesDto.java
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/MetadataInstancesDto.java
@@ -1,5 +1,6 @@
 package com.tapdata.tm.commons.schema;
 
+import com.alibaba.fastjson.JSON;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.tapdata.tm.commons.base.dto.BaseDto;
@@ -10,6 +11,7 @@
 import com.tapdata.tm.commons.schema.bean.*;
 import io.tapdata.entity.conversion.PossibleDataTypes;
 import io.tapdata.entity.result.ResultItem;
+import io.tapdata.entity.schema.type.TapType;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import org.apache.commons.collections4.CollectionUtils;
@@ -150,5 +152,6 @@ public static void sortField(List<Field> fields) {
             fields.sort(Comparator.comparing(Field::getColumnPosition));
         }
     }
+
 }
 
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/TransformerWsMessageResult.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/TransformerWsMessageResult.java
index 88f8690b69..0f67c403c1 100644
--- a/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/TransformerWsMessageResult.java
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/schema/TransformerWsMessageResult.java
@@ -27,4 +27,6 @@ public class TransformerWsMessageResult {
     @JsonDeserialize( using = DagDeserialize.class)
     private DAG dag;
     private String transformUuid;
+    private Boolean isLastBatch = true;
+    private String agentId;
 }
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/task/dto/TaskDto.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/task/dto/TaskDto.java
index 783104fb97..d8f678ff2e 100644
--- a/manager/tm-common/src/main/java/com/tapdata/tm/commons/task/dto/TaskDto.java
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/task/dto/TaskDto.java
@@ -170,6 +170,7 @@ public boolean isAutoInspect() {
     /** 使用的共享挖掘任务停止 */
     private Boolean shareCdcStop;
     private String shareCdcStopMessage;
+    private long delayTime;
 
     /**
      * 同构或异构标记
diff --git a/manager/tm-common/src/main/java/com/tapdata/tm/commons/util/FilterMetadataInstanceUtil.java b/manager/tm-common/src/main/java/com/tapdata/tm/commons/util/FilterMetadataInstanceUtil.java
new file mode 100644
index 0000000000..f42b6823b9
--- /dev/null
+++ b/manager/tm-common/src/main/java/com/tapdata/tm/commons/util/FilterMetadataInstanceUtil.java
@@ -0,0 +1,64 @@
+package com.tapdata.tm.commons.util;
+
+import com.alibaba.fastjson.JSON;
+import com.tapdata.tm.commons.schema.Field;
+import com.tapdata.tm.commons.schema.MetadataInstancesDto;
+import com.tapdata.tm.commons.schema.TableIndex;
+import com.tapdata.tm.commons.schema.TableIndexColumn;
+import io.tapdata.entity.conversion.PossibleDataTypes;
+import io.tapdata.entity.schema.type.TapType;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class FilterMetadataInstanceUtil {
+
+    private FilterMetadataInstanceUtil() {
+        throw new IllegalStateException("Utility class");
+    }
+    public static void filterMetadataInstancesFields(MetadataInstancesDto metadataInstancesDto) {
+        List<Field> fields = metadataInstancesDto.getFields();
+        if (CollectionUtils.isNotEmpty(fields)){
+
+            Map<String, PossibleDataTypes> dataTypes = metadataInstancesDto.getFindPossibleDataTypes();
+            if (Objects.nonNull(dataTypes)) {
+                fields.forEach(field -> {
+                    if (Objects.nonNull(dataTypes.get(field.getFieldName())) && org.apache.commons.collections4.CollectionUtils.isEmpty(dataTypes.get(field.getFieldName()).getDataTypes())) {
+                        field.setDeleted(true);
+                    }
+                    TapType tapType = JSON.parseObject(field.getTapType(), TapType.class);
+                    if (TapType.TYPE_RAW == tapType.getType()) {
+                        field.setDeleted(true);
+                    }
+                });
+            }
+
+            List<String> deleteFieldNames = fields.stream().filter(Field::isDeleted).map(Field::getFieldName).collect(Collectors.toList());
+            metadataInstancesDto.setFields(fields.stream().filter(f->!f.isDeleted()).collect(Collectors.toList()));
+            List<TableIndex> indices = metadataInstancesDto.getIndices();
+            List<TableIndex> newIndices = new ArrayList<>();
+
+            if(indices != null) {
+                for (TableIndex index : indices) {
+                    List<TableIndexColumn> columns = index.getColumns();
+                    List<TableIndexColumn> newIndexColums = new ArrayList<>();
+                    for (TableIndexColumn column : columns) {
+                        if (!deleteFieldNames.contains(column.getColumnName())) {
+                            newIndexColums.add(column);
+                        }
+                    }
+                    if (newIndexColums.size() > 0) {
+                        index.setColumns(newIndexColums);
+                        newIndices.add(index);
+                    }
+                }
+            }
+
+            metadataInstancesDto.setIndices(newIndices);
+        }
+    }
+}
diff --git a/manager/tm-oss/src/main/java/com/tapdata/tm/inspect/service/InspectServiceImpl.java b/manager/tm-oss/src/main/java/com/tapdata/tm/inspect/service/InspectServiceImpl.java
index 4ffbb5a7f2..02ce5fef03 100644
--- a/manager/tm-oss/src/main/java/com/tapdata/tm/inspect/service/InspectServiceImpl.java
+++ b/manager/tm-oss/src/main/java/com/tapdata/tm/inspect/service/InspectServiceImpl.java
@@ -16,8 +16,11 @@
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.bson.types.ObjectId;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -51,7 +54,7 @@ public Map<String, Long> delete(String id, UserDetail user) {
 
     @Override
     public Page<InspectDto> list(Filter filter, UserDetail userDetail) {
-        throw new BizException(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION);
+        return new Page<>(0, new ArrayList<>());
     }
 
     @Override
@@ -71,12 +74,13 @@ public InspectDto createCheckByTask(TaskDto taskDto, UserDetail userDetail) {
 
     @Override
     public List<InspectDto> findByTaskIdList(List<String> taskIdList) {
-        throw new BizException(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION);
+        Query query = Query.query(Criteria.where("flowId").in(taskIdList).and("is_deleted").ne(true));
+        return findAll(query);
     }
 
     @Override
     public UpdateResult deleteByTaskId(String taskId) {
-        throw new BizException(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION);
+        return null;
     }
 
     @Override
@@ -101,7 +105,7 @@ public void importData(String json, String upsert, UserDetail userDetail) {
 
     @Override
     public void setRepeatInspectTask() {
-        throw new BizException(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION);
+        // do nothing
     }
 
     @Override
@@ -130,7 +134,7 @@ public List<InspectDto> findByResult(boolean passed) {
 
     @Override
     public void cleanDeadInspect() {
-        throw new BizException(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION);
+        //do nothing
     }
 
     @Override
diff --git a/manager/tm/pom.xml b/manager/tm/pom.xml
index 58f11da080..883cafa45c 100644
--- a/manager/tm/pom.xml
+++ b/manager/tm/pom.xml
@@ -506,7 +506,6 @@
 			<groupId>org.mockito</groupId>
 			<artifactId>mockito-core</artifactId>
 			<version>${mockito.version}</version>
-			<scope>test</scope>
 		</dependency>
 		<!---->
 	</dependencies>
diff --git a/manager/tm/src/main/java/com/tapdata/tm/init/patches/daas/v3_8_1_AccessToken_TTL_Index.java b/manager/tm/src/main/java/com/tapdata/tm/init/patches/daas/v3_8_1_AccessToken_TTL_Index.java
new file mode 100644
index 0000000000..b4ed38ef36
--- /dev/null
+++ b/manager/tm/src/main/java/com/tapdata/tm/init/patches/daas/v3_8_1_AccessToken_TTL_Index.java
@@ -0,0 +1,41 @@
+package com.tapdata.tm.init.patches.daas;
+
+import com.tapdata.tm.init.PatchType;
+import com.tapdata.tm.init.PatchVersion;
+import com.tapdata.tm.init.patches.AbsPatch;
+import com.tapdata.tm.init.patches.PatchAnnotation;
+import com.tapdata.tm.utils.SpringContextHelper;
+import io.tapdata.utils.AppType;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.core.env.Environment;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.index.Index;
+
+import java.util.concurrent.TimeUnit;
+@PatchAnnotation(appType = AppType.DAAS, version = "3.8-1")
+public class v3_8_1_AccessToken_TTL_Index extends AbsPatch {
+    private Long accessTokenTtl;
+    private static final Logger logger = LogManager.getLogger(v3_8_1_AccessToken_TTL_Index.class);
+    public v3_8_1_AccessToken_TTL_Index(PatchType type, PatchVersion version) {
+        super(type, version);
+    }
+
+    @Override
+    public void run() {
+        logger.info("Execute java patch: {}...", getClass().getName());
+        String collectionName = "AccessToken";
+        MongoTemplate mongoTemplate = SpringContextHelper.getBean(MongoTemplate.class);
+
+        Environment environment = SpringContextHelper.getBean(Environment.class);
+        accessTokenTtl = Long.parseLong(environment.getProperty("access.token.ttl"));
+
+        long buffer = TimeUnit.DAYS.toSeconds(1L);
+        long ttlValue = accessTokenTtl + buffer;
+
+        Index index = new Index().on("last_updated", Sort.Direction.DESC).expire(ttlValue, TimeUnit.SECONDS).background();
+        mongoTemplate.indexOps(collectionName).ensureIndex(index);
+    }
+
+}
diff --git a/manager/tm/src/main/java/com/tapdata/tm/init/patches/dfs/v3_8_2_AccessToken_TTL_Index.java b/manager/tm/src/main/java/com/tapdata/tm/init/patches/dfs/v3_8_2_AccessToken_TTL_Index.java
new file mode 100644
index 0000000000..c7e345cb81
--- /dev/null
+++ b/manager/tm/src/main/java/com/tapdata/tm/init/patches/dfs/v3_8_2_AccessToken_TTL_Index.java
@@ -0,0 +1,41 @@
+package com.tapdata.tm.init.patches.dfs;
+
+import com.tapdata.tm.init.PatchType;
+import com.tapdata.tm.init.PatchVersion;
+import com.tapdata.tm.init.patches.AbsPatch;
+import com.tapdata.tm.init.patches.PatchAnnotation;
+import com.tapdata.tm.utils.SpringContextHelper;
+import io.tapdata.utils.AppType;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.core.env.Environment;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.index.Index;
+
+import java.util.concurrent.TimeUnit;
+
+@PatchAnnotation(appType = AppType.DFS, version = "3.8-2")
+public class v3_8_2_AccessToken_TTL_Index extends AbsPatch {
+    private Long accessTokenTtl;
+    private static final Logger logger = LogManager.getLogger(v3_8_2_AccessToken_TTL_Index.class);
+    public v3_8_2_AccessToken_TTL_Index(PatchType type, PatchVersion version) {
+        super(type, version);
+    }
+
+    @Override
+    public void run() {
+        logger.info("Execute java patch: {}...", getClass().getName());
+        String collectionName = "AccessToken";
+        MongoTemplate mongoTemplate = SpringContextHelper.getBean(MongoTemplate.class);
+
+        Environment environment = SpringContextHelper.getBean(Environment.class);
+        accessTokenTtl = Long.parseLong(environment.getProperty("access.token.ttl"));
+
+        long buffer = TimeUnit.DAYS.toSeconds(1L);
+        long ttlValue = accessTokenTtl + buffer;
+
+        Index index = new Index().on("last_updated", Sort.Direction.DESC).expire(ttlValue, TimeUnit.SECONDS).background();
+        mongoTemplate.indexOps(collectionName).ensureIndex(index);
+    }
+}
diff --git a/manager/tm/src/main/java/com/tapdata/tm/inspect/controller/InspectController.java b/manager/tm/src/main/java/com/tapdata/tm/inspect/controller/InspectController.java
index c9ee9a3922..aab0388c2e 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/inspect/controller/InspectController.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/inspect/controller/InspectController.java
@@ -12,6 +12,7 @@
 import com.tapdata.tm.commons.task.dto.TaskDto;
 import com.tapdata.tm.commons.util.JsonUtil;
 import com.tapdata.tm.config.security.UserDetail;
+import com.tapdata.tm.inspect.bean.Task;
 import com.tapdata.tm.inspect.constant.InspectStatusEnum;
 import com.tapdata.tm.inspect.dto.InspectDto;
 import com.tapdata.tm.inspect.service.InspectService;
@@ -129,12 +130,17 @@ public ResponseMessage<String> dataPermissionTaskSign(@PathVariable String curre
     @Operation(summary = "Create a new instance of the model and persist it into the data source")
     @PostMapping
     public ResponseMessage<InspectDto> save(@RequestBody InspectDto inspect) {
-        List task = inspect.getTasks();
+        List<Task> task = inspect.getTasks();
         PlatformInfo platformInfo = inspect.getPlatformInfo();
         if (CollectionUtils.isEmpty(task)) {
             throw new BizException("Inspect.task.null");
         }
-        inspectService.fieldHandler(task,getLoginUser());
+
+        if(StringUtils.isNotEmpty(inspect.getInspectMethod()) &&
+                (inspect.getInspectMethod().equals("field") || inspect.getInspectMethod().equals("jointField"))){
+            inspectService.fieldHandler(task,getLoginUser());
+        }
+
         if (null == platformInfo || StringUtils.isEmpty(platformInfo.getAgentType())) {
             throw new BizException("Inspect.agentTag.null");
         }
diff --git a/manager/tm/src/main/java/com/tapdata/tm/livedataplatform/controller/LiveDataPlatformController.java b/manager/tm/src/main/java/com/tapdata/tm/livedataplatform/controller/LiveDataPlatformController.java
index abc53a2684..7ff93e4a32 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/livedataplatform/controller/LiveDataPlatformController.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/livedataplatform/controller/LiveDataPlatformController.java
@@ -88,6 +88,10 @@ public ResponseMessage<LiveDataPlatformDto> findOne(
         if (filter.getOrder() == null) {
             filter.setOrder("_id desc");
         }
+
+        // Only query yourself.
+        filter.where("userId", getLoginUser().getUserId());
+
         return success(liveDataPlatformService.findOneData(filter, getLoginUser()));
     }
 
diff --git a/manager/tm/src/main/java/com/tapdata/tm/metadatainstance/dto/MetadataInstancesDto.java b/manager/tm/src/main/java/com/tapdata/tm/metadatainstance/dto/MetadataInstancesDto.java
deleted file mode 100644
index 26250f4d8c..0000000000
--- a/manager/tm/src/main/java/com/tapdata/tm/metadatainstance/dto/MetadataInstancesDto.java
+++ /dev/null
@@ -1,119 +0,0 @@
-//package com.tapdata.tm.metadatainstance.dto;
-//
-//import com.fasterxml.jackson.annotation.JsonInclude;
-//import com.fasterxml.jackson.annotation.JsonProperty;
-//import com.tapdata.tm.commons.base.dto.BaseDto;
-//import com.tapdata.tm.ds.bean.FileProperty;
-//import com.tapdata.tm.commons.schema.bean.Schema;
-//import com.tapdata.tm.ds.bean.TableIndex;
-//import com.tapdata.tm.ds.bean.Tag;
-//import com.tapdata.tm.metadatainstance.bean.DataRules;
-//import com.tapdata.tm.metadatainstance.bean.Field;
-//import com.tapdata.tm.metadatainstance.bean.Relation;
-//import com.tapdata.tm.metadatainstance.bean.SourceDto;
-//import com.tapdata.tm.metadatainstance.vo.SourceTypeEnum;
-//import lombok.Data;
-//import lombok.EqualsAndHashCode;
-//
-//import java.util.*;
-//import java.util.stream.Collectors;
-//
-//
-///**
-// * 数据源模型
-// */
-//@EqualsAndHashCode(callSuper = true)
-//@Data
-//public class MetadataInstancesDto extends BaseDto {
-//    @JsonProperty("qualified_name")
-//    private String qualifiedName;
-//    @JsonProperty("meta_type")
-//    private String metaType;
-//    @JsonProperty("is_deleted")
-//    private boolean isDeleted = false;
-//    @JsonProperty("original_name")
-//    private String originalName;
-//    @JsonProperty("dev_version")
-//    private Integer devVersion;
-//    private String databaseId;
-//    private String schemaVersion;
-//    private Integer version;
-//    private String comment;
-//    private String name;
-//    private String lienage;
-//    private List<Relation> relation;
-//    @JsonProperty("fields_lienage")
-//    private String fieldsLienage;
-//    private List<Field> fields;
-//    private Object indexes;
-//    private SourceDto source;
-//    private String createSource;
-//    private Boolean virtual;
-//    private List<Tag> classifications;
-//    @JsonProperty("last_user_name")
-//    private String lastUserName;
-//    private List<TableIndex> indices;
-//    private Set<Integer> partitionSet;
-//    private FileProperty fileProperty;
-//    private List<String> fromFile;
-//    @JsonProperty("alias_name")
-//    private String aliasName;
-//    @JsonProperty("custom_properties")
-//    private Map<String, Object> customProperties;
-//
-//    //下面几个只有历史版本中存在的字段
-//    @JsonProperty("version_user_id")
-//    private String versionUserId;
-//    @JsonProperty("version_user_name")
-//    private String versionUserName;
-//    @JsonProperty("version_description")
-//    private String versionDescription;
-//
-//    @JsonProperty("version_time")
-//    private Date versionTime;
-//    private Object pipline;
-//    private Schema schema;
-//    //添加修改的时候，可能会作为数据源的id传参，不会入库
-//    private String connectionId;
-//
-//    //查询'database', 'directory', 'ftp' 等metatype的时候需要设置的属性
-//    private List<MetadataInstancesDto> collections;
-//
-//    //查询'collection', 'table', 'view', 'mongo_view' 等metatype的时候需要设置的属性
-//    private String database;
-//    private String username;
-//
-//
-//    private List<MetadataInstancesDto> histories;
-//    @JsonProperty("data_rules")
-//    private DataRules dataRules;
-//    private Long lastUpdate;
-//    private String storageEngine;
-//    private String charset;
-//    protected String pdkId;
-//    protected String pdkGroup;
-//    protected String pdkVersion;
-//
-//    /**
-//     * 是否是虚拟表 'virtual' 'source'
-//     */
-//    @JsonInclude(JsonInclude.Include.ALWAYS)
-//    private String sourceType= SourceTypeEnum.SOURCE.name();
-//
-//
-//    public static void sortField(List<Field> fields) {
-//        if (org.apache.commons.collections.CollectionUtils.isNotEmpty(fields)) {
-//            List<com.tapdata.tm.metadatainstance.bean.Field> noPrimarys = fields.stream().filter(f -> {
-//                Boolean primaryKey = f.getPrimaryKey();
-//                return primaryKey == null || !primaryKey;
-//            }).collect(Collectors.toList());
-//            fields.removeAll(noPrimarys);
-//            fields.sort(Comparator.comparing(com.tapdata.tm.metadatainstance.bean.Field::getFieldName));
-//            if (org.apache.commons.collections.CollectionUtils.isNotEmpty(noPrimarys)) {
-//                noPrimarys.sort(Comparator.comparing(com.tapdata.tm.metadatainstance.bean.Field::getFieldName));
-//                fields.addAll(noPrimarys);
-//            }
-//        }
-//    }
-//}
-//
diff --git a/manager/tm/src/main/java/com/tapdata/tm/metadatainstance/service/MetadataInstancesServiceImpl.java b/manager/tm/src/main/java/com/tapdata/tm/metadatainstance/service/MetadataInstancesServiceImpl.java
index 7ed8391567..c68bd1e875 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/metadatainstance/service/MetadataInstancesServiceImpl.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/metadatainstance/service/MetadataInstancesServiceImpl.java
@@ -2013,48 +2013,9 @@ public Page<TapTable> getTapTable(Filter filter, UserDetail loginUser) {
         tapTablePage.setTotal(list.getTotal());
         List<TapTable> tapTables = new ArrayList<>();
         List<MetadataInstancesDto> items = list.getItems();
-        for (MetadataInstancesDto item : items) {
-            List<Field> fields = item.getFields();
-            if (null == fields) continue;
-
-            //fields not support : dataTypes null
-            Map<String, PossibleDataTypes> dataTypes = item.getFindPossibleDataTypes();
-            if (Objects.nonNull(dataTypes)) {
-                fields.forEach(field -> {
-                    if (Objects.nonNull(dataTypes.get(field.getFieldName())) && CollectionUtils.isEmpty(dataTypes.get(field.getFieldName()).getDataTypes())) {
-                        field.setDeleted(true);
-                    }
-                    TapType tapType = JSON.parseObject(field.getTapType(), TapType.class);
-                    if (TapType.TYPE_RAW == tapType.getType()) {
-                        field.setDeleted(true);
-                    }
-                });
-            }
 
-            List<String> deleteFieldNames = fields.stream().filter(Field::isDeleted).map(Field::getFieldName).collect(Collectors.toList());
-            item.setFields(fields.stream().filter(f->!f.isDeleted()).collect(Collectors.toList()));
-            List<TableIndex> indices = item.getIndices();
-            List<TableIndex> newIndices = new ArrayList<>();
-
-            if(indices != null) {
-                for (TableIndex index : indices) {
-                    List<TableIndexColumn> columns = index.getColumns();
-                    List<TableIndexColumn> newIndexColums = new ArrayList<>();
-                    for (TableIndexColumn column : columns) {
-                        if (!deleteFieldNames.contains(column.getColumnName())) {
-                            newIndexColums.add(column);
-                        }
-                    }
-                    if (newIndexColums.size() > 0) {
-                        index.setColumns(newIndexColums);
-                        newIndices.add(index);
-                    }
-                }
-            }
+        items.forEach(FilterMetadataInstanceUtil::filterMetadataInstancesFields);
 
-            item.setIndices(newIndices);
-
-        }
         if (CollectionUtils.isNotEmpty(items)) {
             tapTables = items.stream().map(PdkSchemaConvert::toPdk).collect(Collectors.toList());
         }
diff --git a/manager/tm/src/main/java/com/tapdata/tm/monitor/controller/MeasureController.java b/manager/tm/src/main/java/com/tapdata/tm/monitor/controller/MeasureController.java
index fcb0808946..8879985d18 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/monitor/controller/MeasureController.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/monitor/controller/MeasureController.java
@@ -1,11 +1,14 @@
 package com.tapdata.tm.monitor.controller;
 
 import com.tapdata.tm.base.controller.BaseController;
+import com.tapdata.tm.base.dto.Filter;
 import com.tapdata.tm.base.dto.Page;
 import com.tapdata.tm.base.dto.ResponseMessage;
+import com.tapdata.tm.base.dto.Where;
 import com.tapdata.tm.config.security.UserDetail;
 import com.tapdata.tm.monitor.dto.BatchRequestDto;
 import com.tapdata.tm.monitor.dto.TableSyncStaticDto;
+import com.tapdata.tm.monitor.entity.MeasurementEntity;
 import com.tapdata.tm.monitor.param.AggregateMeasurementParam;
 import com.tapdata.tm.monitor.param.MeasurementQueryParam;
 import com.tapdata.tm.monitor.param.SyncStatusStatisticsParam;
@@ -34,7 +37,7 @@
 import java.util.concurrent.ExecutionException;
 
 @RestController
-@RequestMapping(value = "/api/measurement")
+@RequestMapping({"/api/measurement","/api/AgentMeasurementV2"})
 @Slf4j
 @Tag(name = "可观测性")
 @Setter(onMethod_ = {@Autowired})
@@ -95,4 +98,15 @@ public ResponseMessage<List<SyncStatusStatisticsVo>> queryTableSyncStatusStatist
 		) {
 			return success(measurementServiceV2.queryTableSyncStatusStatistics(new SyncStatusStatisticsParam(taskId, taskRecordId)));
 		}
+
+
+    @Operation(summary = "任务分钟级延迟查询")
+    @GetMapping()
+    public ResponseMessage<MeasurementEntity> findLastMinuteByTaskId(@RequestParam(value = "filter", required = false) String filterJson) {
+        Filter filter = parseFilter(filterJson);
+
+        Where where = filter.getWhere();
+        String taskId = where.get("taskId").toString();
+        return success(measurementServiceV2.findLastMinuteByTaskId(taskId));
+    }
 }
diff --git a/manager/tm/src/main/java/com/tapdata/tm/monitor/service/MeasurementServiceV2Impl.java b/manager/tm/src/main/java/com/tapdata/tm/monitor/service/MeasurementServiceV2Impl.java
index 16de223506..44847e2cbd 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/monitor/service/MeasurementServiceV2Impl.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/monitor/service/MeasurementServiceV2Impl.java
@@ -77,14 +77,18 @@
 @Service
 @Slf4j
 public class MeasurementServiceV2Impl implements MeasurementServiceV2 {
+    public static final String REPLICATE_LAG = "replicateLag";
+    public static final String TASK_ID = "taskId";
     private final MongoTemplate mongoOperations;
     private final MetadataInstancesService metadataInstancesService;
     private final TaskService taskService;
+    private Map<String, Long> taskDelayTimeMap;
 
     public MeasurementServiceV2Impl(@Qualifier(value = "obsMongoTemplate") CompletableFuture<MongoTemplate> mongoTemplateCompletableFuture, MetadataInstancesService metadataInstancesService, TaskService taskService) throws ExecutionException, InterruptedException {
         this.mongoOperations = mongoTemplateCompletableFuture.get();
         this.metadataInstancesService = metadataInstancesService;
         this.taskService = taskService;
+        this.taskDelayTimeMap = new HashMap<>();
     }
 
     @Override
@@ -140,29 +144,21 @@ private void addBulkAgentMeasurement(List<SampleRequest> sampleRequestList) {
             }
 
             bulkOperations.upsert(query, update);
+            if ("task".equals(tags.get("type"))) {
+                Map<String, Object> vs = (Map) sampleMap.get("vs");
+                Object replicateLag = Optional.ofNullable(vs.get(REPLICATE_LAG)).orElse(0);
+                Long taskDelayTime = taskDelayTimeMap.get(tags.get(TASK_ID));
+                long delayTime = Long.parseLong(replicateLag.toString());
+                if (null == taskDelayTime || taskDelayTime != delayTime){
+                    taskDelayTimeMap.put(tags.get(TASK_ID), delayTime);
+                    taskService.updateDelayTime(new ObjectId(tags.get(TASK_ID)), delayTime);
+                }
+            }
         }
 
         bulkOperations.execute();
     }
 
-    private Sample supplyKeyData(Sample requestSample, Map<String, Number> data, Map<String, Number> requestMap) {
-        List<String> list = Lists.newArrayList( "timeCostAvg", "targetWriteTimeCostAvg", "replicateLag");
-
-        for (String key : list) {
-            Number value = data.get(key);
-            if (requestMap.containsKey(key)
-                    && Objects.nonNull(requestMap.get(key))
-                    && requestMap.get(key).doubleValue() == 0
-                    && Objects.nonNull(value)
-                    && value.doubleValue() > 0) {
-                requestSample.getVs().put(key, value);
-            } else if (!requestMap.containsKey(key) && data.containsKey(key)) {
-                requestSample.getVs().put(key, value);
-            }
-        }
-        return requestSample;
-    }
-
     private static final String TAG_FORMAT = String.format("%s.%%s", MeasurementEntity.FIELD_TAGS);
     private static final String FIELD_FORMAT = String.format("%s.%s.%%s",
             MeasurementEntity.FIELD_SAMPLES, Sample.FIELD_VALUES);
@@ -440,7 +436,7 @@ public Document toDocument(AggregationOperationContext context) {
                 // 按照延迟逻辑,源端无事件时,应该为全量同步开始到现在的时间差
                 if (Objects.isNull(currentEventTimestamp) && Objects.nonNull(snapshotStartAt)) {
                     Number maxRep = Math.abs(System.currentTimeMillis() - snapshotStartAt.longValue());
-                    values.put("replicateLag", maxRep);
+                    values.put(REPLICATE_LAG, maxRep);
                 }
 
             }
@@ -1124,8 +1120,8 @@ public void queryTableMeasurement(String taskId, TableStatusInfoDto tableStatusI
             Sample sample = samples.get(0);
             Long cdcDelayTime = null;
             Date lastData = null;
-            if (sample.getVs().get("replicateLag") != null) {
-                cdcDelayTime = Long.valueOf(sample.getVs().get("replicateLag").toString());
+            if (sample.getVs().get(REPLICATE_LAG) != null) {
+                cdcDelayTime = Long.valueOf(sample.getVs().get(REPLICATE_LAG).toString());
             }
             tableStatusInfoDto.setCdcDelayTime(cdcDelayTime);
             if (sample.getVs().get("currentEventTimestamp") != null) {
diff --git a/manager/tm/src/main/java/com/tapdata/tm/task/controller/TaskController.java b/manager/tm/src/main/java/com/tapdata/tm/task/controller/TaskController.java
index 1929ade606..4757fe3726 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/task/controller/TaskController.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/task/controller/TaskController.java
@@ -4,6 +4,9 @@
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.http.ResponseEntity;
+
 import com.tapdata.tm.base.controller.BaseController;
 import com.tapdata.tm.base.dto.*;
 import com.tapdata.tm.commons.dag.DAG;
@@ -62,7 +65,7 @@
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
+import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.time.LocalDateTime;
 import java.util.*;
@@ -136,6 +139,7 @@ public ResponseMessage<TaskDto> save(@RequestBody TaskDto task) {
         UserDetail user = getLoginUser();
 
         task.setId(null);
+        taskSaveService.supplementAlarm(task, user);
         return success(taskService.create(task, user));
     }
 
@@ -322,6 +326,21 @@ public ResponseMessage<TaskDto> confirmStart(@PathVariable("id") String id, @Req
         return success(taskDto);
     }
 
+    /**
+     * collect task info to do performance analyze
+     *
+     * @param taskId taskId
+     * @return tar file
+     */
+    @Operation(summary = "collect task info to do performance analyze")
+    @PostMapping("analyze/{id}")
+    public ResponseEntity<InputStreamResource> analyzeTask(
+            HttpServletRequest request,
+            @PathVariable("id") String taskId,
+            HttpServletResponse response) throws IOException {
+        return taskService.analyzeTask(request, response, taskId, getLoginUser());
+    }
+
     /**
      * Find a model instance by {{id}} from the data source
      *
diff --git a/manager/tm/src/main/java/com/tapdata/tm/task/service/CustomSqlService.java b/manager/tm/src/main/java/com/tapdata/tm/task/service/CustomSqlService.java
deleted file mode 100644
index 5e1b0fe16e..0000000000
--- a/manager/tm/src/main/java/com/tapdata/tm/task/service/CustomSqlService.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.tapdata.tm.task.service;
-
-import com.tapdata.tm.commons.task.dto.TaskDto;
-import com.tapdata.tm.config.security.UserDetail;
-
-public interface CustomSqlService {
-
-    void checkCustomSqlTask(TaskDto taskDto, UserDetail user);
-
-}
diff --git a/manager/tm/src/main/java/com/tapdata/tm/task/service/TaskServiceImpl.java b/manager/tm/src/main/java/com/tapdata/tm/task/service/TaskServiceImpl.java
index 782c0e99e2..b2d14c958c 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/task/service/TaskServiceImpl.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/task/service/TaskServiceImpl.java
@@ -13,6 +13,7 @@
 import com.mongodb.client.result.UpdateResult;
 import com.tapdata.tm.Settings.service.SettingsServiceImpl;
 import com.tapdata.tm.agent.service.AgentGroupService;
+import com.tapdata.tm.async.AsyncContextManager;
 import com.tapdata.tm.autoinspect.constants.AutoInspectConstants;
 import com.tapdata.tm.autoinspect.entity.AutoInspectProgress;
 import com.tapdata.tm.autoinspect.service.TaskAutoInspectResultsService;
@@ -24,6 +25,13 @@
 import com.tapdata.tm.base.dto.ResponseMessage;
 import com.tapdata.tm.base.dto.TmPageable;
 import com.tapdata.tm.base.dto.Where;
+import com.tapdata.tm.monitor.param.MeasurementQueryParam;
+import com.tapdata.tm.monitor.service.BatchService;
+import com.tapdata.tm.task.bean.*;
+import com.tapdata.tm.task.vo.*;
+import io.tapdata.pdk.core.api.PDKIntegration;
+import org.apache.commons.io.FileUtils;
+import org.mockito.Mockito;
 import com.tapdata.tm.base.exception.BizException;
 import com.tapdata.tm.base.handler.ExceptionHandler;
 import com.tapdata.tm.commons.base.dto.BaseDto;
@@ -50,13 +58,7 @@
 import com.tapdata.tm.commons.dag.vo.SyncObjects;
 import com.tapdata.tm.commons.dag.vo.TestRunDto;
 import com.tapdata.tm.commons.externalStorage.ExternalStorageDto;
-import com.tapdata.tm.commons.schema.DataSourceConnectionDto;
-import com.tapdata.tm.commons.schema.DataSourceDefinitionDto;
-import com.tapdata.tm.commons.schema.MetadataInstancesDto;
-import com.tapdata.tm.commons.schema.MetadataTransformerDto;
-import com.tapdata.tm.commons.schema.MetadataTransformerItemDto;
-import com.tapdata.tm.commons.schema.Tag;
-import com.tapdata.tm.commons.schema.TransformerWsMessageDto;
+import com.tapdata.tm.commons.schema.*;
 import com.tapdata.tm.commons.task.constant.NotifyEnum;
 import com.tapdata.tm.commons.task.dto.Dag;
 import com.tapdata.tm.commons.task.dto.DataSyncMq;
@@ -101,9 +103,12 @@
 import com.tapdata.tm.metadatadefinition.service.MetadataDefinitionService;
 import com.tapdata.tm.metadatainstance.service.MetaDataHistoryService;
 import com.tapdata.tm.metadatainstance.service.MetadataInstancesServiceImpl;
+import com.tapdata.tm.monitor.dto.BatchRequestDto;
 import com.tapdata.tm.monitor.entity.MeasurementEntity;
 import com.tapdata.tm.monitor.param.IdParam;
 import com.tapdata.tm.monitor.service.MeasurementServiceV2;
+import com.tapdata.tm.monitor.service.impl.BatchServiceImpl;
+import com.tapdata.tm.monitoringlogs.param.MonitoringLogQueryParam;
 import com.tapdata.tm.monitoringlogs.service.MonitoringLogsService;
 import com.tapdata.tm.permissions.DataPermissionHelper;
 import com.tapdata.tm.permissions.constants.DataPermissionActionEnums;
@@ -116,14 +121,6 @@
 import com.tapdata.tm.statemachine.enums.DataFlowEvent;
 import com.tapdata.tm.statemachine.model.StateMachineResult;
 import com.tapdata.tm.statemachine.service.StateMachineService;
-import com.tapdata.tm.task.bean.Chart6Vo;
-import com.tapdata.tm.task.bean.FullSyncVO;
-import com.tapdata.tm.task.bean.LogCollectorResult;
-import com.tapdata.tm.task.bean.RunTimeInfo;
-import com.tapdata.tm.task.bean.SampleTaskVo;
-import com.tapdata.tm.task.bean.SyncTaskStatusDto;
-import com.tapdata.tm.task.bean.TableStatusInfoDto;
-import com.tapdata.tm.task.bean.TaskUpAndLoadDto;
 import com.tapdata.tm.task.constant.InputNumCache;
 import com.tapdata.tm.task.constant.SyncStatus;
 import com.tapdata.tm.task.constant.SyncType;
@@ -142,31 +139,34 @@
 import com.tapdata.tm.task.service.batchup.BatchUpChecker;
 import com.tapdata.tm.task.service.chart.ChartViewService;
 import com.tapdata.tm.task.service.utils.TaskServiceUtil;
-import com.tapdata.tm.task.vo.ShareCacheDetailVo;
-import com.tapdata.tm.task.vo.ShareCacheVo;
-import com.tapdata.tm.task.vo.TaskDetailVo;
-import com.tapdata.tm.task.vo.TaskStatsDto;
 import com.tapdata.tm.transform.service.MetadataTransformerItemService;
 import com.tapdata.tm.transform.service.MetadataTransformerService;
 import com.tapdata.tm.user.service.UserService;
 import com.tapdata.tm.userLog.constant.Modular;
 import com.tapdata.tm.userLog.service.UserLogService;
-import com.tapdata.tm.utils.FunctionUtils;
-import com.tapdata.tm.utils.GZIPUtil;
-import com.tapdata.tm.utils.Lists;
-import com.tapdata.tm.utils.MapUtils;
-import com.tapdata.tm.utils.MessageUtil;
-import com.tapdata.tm.utils.MongoUtils;
-import com.tapdata.tm.utils.NumberUtil;
-import com.tapdata.tm.utils.SpringContextHelper;
-import com.tapdata.tm.utils.TimeUtil;
-import com.tapdata.tm.utils.UUIDUtil;
+import com.tapdata.tm.utils.*;
 import com.tapdata.tm.worker.entity.Worker;
 import com.tapdata.tm.worker.service.WorkerService;
 import com.tapdata.tm.worker.vo.CalculationEngineVo;
 import com.tapdata.tm.ws.enums.MessageType;
 import io.tapdata.common.sample.request.Sample;
+import io.tapdata.entity.error.CoreException;
+
+import java.io.*;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import io.tapdata.entity.utils.InstanceFactory;
+import io.tapdata.entity.utils.JsonParser;
 import io.tapdata.exception.TapCodeException;
+import io.tapdata.modules.api.net.error.NetErrors;
+import io.tapdata.modules.api.net.service.EngineMessageExecutionService;
+import io.tapdata.pdk.apis.entity.message.ServiceCaller;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -174,25 +174,34 @@
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.util.IOUtils;
 import org.bson.Document;
 import org.bson.types.ObjectId;
 import org.jetbrains.annotations.NotNull;
 import org.quartz.CronScheduleBuilder;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.InputStreamResource;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.mongodb.core.aggregation.AggregationResults;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
+import org.springframework.web.context.request.RequestContextHolder;
+import org.springframework.web.context.request.ServletRequestAttributes;
 import org.springframework.web.multipart.MultipartFile;
 
+import javax.servlet.AsyncContext;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.text.MessageFormat;
@@ -200,20 +209,7 @@
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -230,6 +226,8 @@
 import static com.tapdata.tm.inspect.constant.InspectStatusEnum.SCHEDULING;
 import static com.tapdata.tm.inspect.constant.InspectStatusEnum.STOPPING;
 import static com.tapdata.tm.inspect.constant.InspectStatusEnum.WAITING;
+import static io.tapdata.entity.simplify.TapSimplify.fromJson;
+import static io.tapdata.entity.simplify.TapSimplify.toJson;
 import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
 import static org.springframework.data.mongodb.core.aggregation.Aggregation.match;
 
@@ -322,6 +320,7 @@ private static String getTableName() {
     private MetaDataHistoryService historyService;
     private WorkerService workerService;
     private FileService fileService1;
+    private TaskRecordService taskRecordService;
     private UserLogService userLogService;
     private MessageQueueServiceImpl messageQueueService;
     private UserService userService;
@@ -353,7 +352,6 @@ private static String getTableName() {
     private ExternalStorageService externalStorageService;
 
 
-    private CustomSqlService customSqlService;
 
 
     private LockControlService lockControlService;
@@ -367,12 +365,14 @@ private static String getTableName() {
     private SettingsServiceImpl settingsService;
 
     private TaskNodeService taskNodeService;
+    private final AsyncContextManager asyncContextManager = new AsyncContextManager();
 
     private AgentGroupService agentGroupService;
     private DataSourceDefinitionService dataSourceDefinitionService;
     private BatchUpChecker batchUpChecker;
     private ChartViewService chartViewService;
     private UserDataReportService userDataReportService;
+    private BatchService batchService;
 
     public TaskServiceImpl(@NonNull TaskRepository repository) {
         super(repository);
@@ -424,7 +424,6 @@ public TaskDto create(TaskDto taskDto, UserDetail user) {
 
         checkTaskName(taskDto.getName(), user, taskDto.getId());
 
-        customSqlService.checkCustomSqlTask(taskDto, user);
         dateNodeService.checkTaskDateNode(taskDto, user);
 
         boolean rename = false;
@@ -640,7 +639,6 @@ public TaskDto updateById(TaskDto taskDto, UserDetail user) {
         }
 
 
-        customSqlService.checkCustomSqlTask(taskDto, user);
         dateNodeService.checkTaskDateNode(taskDto, user);
 
         boolean agentReq = isAgentReq();
@@ -669,8 +667,13 @@ public TaskDto updateById(TaskDto taskDto, UserDetail user) {
                 if (CollectionUtils.isNotEmpty(dag.getSourceNode())) {
                     transformSchemaAsyncService.transformSchema(dag, user, taskDto.getId());
                 }
-            } else {
-                transformSchemaService.transformSchema(dag, user, taskDto.getId());
+            }else{
+                if(workerService.checkEngineVersion(user)){
+                    transformSchemaAsyncService.transformSchema(dag, user, taskDto.getId());
+                }else{
+                    transformSchemaService.transformSchema(dag, user, taskDto.getId());
+                }
+
             }
         }
         log.debug("check task dag complete, task id =- {}", taskDto.getId());
@@ -2854,10 +2857,8 @@ public static class Char1Group {
         private long count;
     }
 
-
-    public void batchLoadTask(HttpServletResponse response, List<String> taskIds, UserDetail user) {
+    public String exportTask(List<String> taskIds, UserDetail user) {
         List<TaskUpAndLoadDto> jsonList = new ArrayList<>();
-
         List<TaskDto> tasks = findAllTasksByIds(taskIds);
         Map<String, TaskDto> taskDtoMap = tasks.stream().collect(Collectors.toMap(t -> t.getId().toHexString(), Function.identity(), (e1, e2) -> e1));
         for (String taskId : taskIds) {
@@ -2874,7 +2875,7 @@ public void batchLoadTask(HttpServletResponse response, List<String> taskIds, Us
                 taskDto.setStatus(TaskDto.STATUS_EDIT);
                 taskDto.setSyncStatus(SyncStatus.NORMAL);
                 taskDto.setStatuses(new ArrayList<>());
-							taskDto.setAttrs(new HashMap<>()); // 导出任务不保留运行时信息
+                taskDto.setAttrs(new HashMap<>()); // 导出任务不保留运行时信息
                 jsonList.add(new TaskUpAndLoadDto("Task", JsonUtil.toJsonUseJackson(taskDto)));
                 DAG dag = taskDto.getDag();
                 List<Node> nodes = dag.getNodes();
@@ -2921,7 +2922,479 @@ public void batchLoadTask(HttpServletResponse response, List<String> taskIds, Us
                 }
             }
         }
-        String json = JsonUtil.toJsonUseJackson(jsonList);
+        return JsonUtil.toJsonUseJackson(jsonList);
+    }
+
+    private void addContentToTar(TarArchiveOutputStream taos, Map<String, byte[]> contents) throws IOException {
+        for (Map.Entry<String, byte[]> entry : contents.entrySet()) {
+            byte[] contentBytes = entry.getValue();
+            TarArchiveEntry entry1 = new TarArchiveEntry(entry.getKey());
+            entry1.setSize(contentBytes.length);
+            taos.putArchiveEntry(entry1);
+            try (ByteArrayInputStream bais = new ByteArrayInputStream(contentBytes)) {
+                IOUtils.copy(bais, taos);
+            }
+            taos.closeArchiveEntry();
+        }
+    }
+
+    public byte[] callEngineRpc(String engineId, String cls, String method, Object[] args) {
+        HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+        Mockito.when(request.isAsyncSupported()).thenReturn(true);
+        // 模拟 AsyncContext
+        AsyncContext asyncContext = Mockito.mock(AsyncContext.class);
+        Mockito.when(request.startAsync()).thenReturn(asyncContext);
+        ServiceCaller serviceCaller = new ServiceCaller()
+                .className(cls)
+                .method(method)
+                .args(args);
+        if (engineId != null) {
+            serviceCaller.subscribeIds("processId_" + engineId);
+        }
+        serviceCaller.setReturnClass(Object.class.getName());
+        String callId = UUID.randomUUID().toString().replace("-", "");
+        serviceCaller.setId(callId);
+        final Map<String, Object> response = new HashMap<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        asyncContextManager.registerAsyncJob(callId, request, (result, error) -> {
+            if (error != null) {
+                if (error instanceof CoreException) {
+                    response.put("code", ((CoreException) error).getCode());
+                    response.put("data", error.getMessage());
+                }
+            } else {
+                response.put("code", 0);
+                response.put("data", result);
+            }
+            latch.countDown();
+        });
+        EngineMessageExecutionService engineMessageExecutionService = InstanceFactory.instance(EngineMessageExecutionService.class, true);
+        try {
+            engineMessageExecutionService.call(serviceCaller, (result, throwable) -> {
+                asyncContextManager.applyAsyncJobResult(callId, result, throwable);
+            });
+        } catch(Throwable throwable) {
+            asyncContextManager.applyAsyncJobResult(callId, null, throwable);
+        }
+        try {
+            if (!latch.await(30, TimeUnit.SECONDS)) {
+                response.put("code", 1);
+                response.put("data", "call timeout");
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+
+        if (!response.get("code").equals(0)) {
+            return response.get("data").toString().getBytes();
+        }
+
+        String data = response.get("data").toString();
+        Map<String, Object> map = fromJson(data, Map.class);
+        return Base64.getDecoder().decode(map.get("content").toString());
+    }
+
+
+    public ResponseEntity<InputStreamResource> analyzeTask(HttpServletRequest request, HttpServletResponse response, String taskId, UserDetail user) throws IOException {
+        String tarFileName = "analyze-" + taskId + "-" + System.currentTimeMillis() + ".tar";
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        TarArchiveOutputStream taos = new TarArchiveOutputStream(baos);
+
+        Map<String, byte[]> contents = new HashMap<>();
+
+        TaskDto taskDto = findByTaskId(new ObjectId(taskId), AGENT_ID);
+        String agentId = taskDto.getAgentId();
+
+        // 导出任务信息与表结构信息
+        String taskFileName = "task.json.gz";
+        try {
+            List<String> taskIds = new ArrayList<>();
+            taskIds.add(taskId);
+            String taskJson = exportTask(taskIds, user);
+            byte[] gzipFile = GZIPUtil.gzip(taskJson.getBytes(StandardCharsets.UTF_8));
+            contents.put(taskFileName, gzipFile);
+        } catch (Exception e) {
+            contents.put(taskFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出任务运行历史记录
+        String runId = null;
+        String taskRecordFileName = "task_records.txt";
+        try {
+            Page<TaskRecordListVo> recordsPage = taskRecordService.queryRecords(new TaskRecordDto(taskId, 1, 100));
+            List<TaskRecordListVo> records = recordsPage.getItems();
+            if (recordsPage.getTotal() > 0) {
+                runId = records.get(0).getTaskRecordId();
+            }
+            contents.put(taskRecordFileName, toJson(records).toString().getBytes(StandardCharsets.UTF_8));
+        } catch (Exception e) {
+            contents.put(taskRecordFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+
+        // 导出任务日志信息
+        String logFileName = "log.txt";
+        try {
+            MonitoringLogQueryParam queryParam = new MonitoringLogQueryParam();
+            queryParam.setTaskId(taskId);
+            queryParam.setPageSize(100000L);
+            queryParam.setPage(1L);
+            if (runId != null) {
+                queryParam.setTaskRecordId(runId);
+                queryParam.setOrder("desc");
+                queryParam.setLevels(Lists.of("ERROR", "WARN", "INFO", "DEBUG"));
+                queryParam.setStart(0L);
+                queryParam.setEnd(System.currentTimeMillis());
+                try {
+                    Page<MonitoringLogsDto> logsDtoPage = monitoringLogsService.query(queryParam);
+                    contents.put(logFileName, JsonUtil.toJsonUseJackson(logsDtoPage).getBytes(StandardCharsets.UTF_8));
+                } catch (Exception e) {
+                    contents.put(logFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+                }
+            } else {
+                contents.put(logFileName, "[]".getBytes(StandardCharsets.UTF_8));
+            }
+        } catch (Exception e) {
+            contents.put(logFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出任务指标信息
+        String metricFileName = "metric.txt";
+        try {
+            String metricContent = "{\n" +
+                    "  \"verifyTotals\": {\n" +
+                    "    \"uri\": \"/api/task/auto-inspect-totals\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"id\": \"" + taskId + "\"\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"alarmData\": {\n" +
+                    "    \"uri\": \"/api/alarm/list_task\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"taskId\": \"" + taskId + "\"\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"logTotals\": {\n" +
+                    "    \"uri\": \"/api/MonitoringLogs/count\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"taskId\": \"" + taskId + "\",\n" +
+                    "      \"taskRecordId\": \"" + runId + "\"\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"totalData\": {\n" +
+                    "    \"uri\": \"/api/measurement/query/v2\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"startAt\": " + (System.currentTimeMillis() - 6 * 86400 * 1000) + ",\n" +
+                    "      \"endAt\": " + System.currentTimeMillis() + ",\n" +
+                    "      \"samples\": {\n" +
+                    "        \"data\": {\n" +
+                    "          \"tags\": {\n" +
+                    "            \"type\": \"task\",\n" +
+                    "            \"taskId\": \"" + taskId + "\",\n" +
+                    "            \"taskRecordId\": \"" + runId + "\"\n" +
+                    "          },\n" +
+                    "          \"endAt\": " + System.currentTimeMillis() + ",\n" +
+                    "          \"fields\": [\n" +
+                    "            \"inputInsertTotal\",\n" +
+                    "            \"inputUpdateTotal\",\n" +
+                    "            \"inputDeleteTotal\",\n" +
+                    "            \"inputDdlTotal\",\n" +
+                    "            \"inputOthersTotal\",\n" +
+                    "            \"outputInsertTotal\",\n" +
+                    "            \"outputUpdateTotal\",\n" +
+                    "            \"outputDeleteTotal\",\n" +
+                    "            \"outputDdlTotal\",\n" +
+                    "            \"outputOthersTotal\",\n" +
+                    "            \"tableTotal\",\n" +
+                    "            \"createTableTotal\",\n" +
+                    "            \"snapshotTableTotal\",\n" +
+                    "            \"initialCompleteTime\",\n" +
+                    "            \"sourceConnection\",\n" +
+                    "            \"targetConnection\",\n" +
+                    "            \"snapshotDoneAt\",\n" +
+                    "            \"snapshotRowTotal\",\n" +
+                    "            \"snapshotInsertRowTotal\",\n" +
+                    "            \"outputQps\",\n" +
+                    "            \"currentSnapshotTableRowTotal\",\n" +
+                    "            \"currentSnapshotTableInsertRowTotal\",\n" +
+                    "            \"replicateLag\",\n" +
+                    "            \"snapshotStartAt\",\n" +
+                    "            \"currentEventTimestamp\",\n" +
+                    "            \"snapshotDoneCost\",\n" +
+                    "            \"outputQpsMax\",\n" +
+                    "            \"outputQpsAvg\"\n" +
+                    "          ],\n" +
+                    "          \"type\": \"instant\"\n" +
+                    "        }\n" +
+                    "      }\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"barChartData\": {\n" +
+                    "    \"uri\": \"/api/measurement/query/v2\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"startAt\": " + (System.currentTimeMillis() - 6 * 86400 * 1000) + ",\n" +
+                    "       \"endAt\": " + System.currentTimeMillis() + ",\n" +
+                    "      \"samples\": {\n" +
+                    "        \"data\": {\n" +
+                    "          \"tags\": {\n" +
+                    "            \"type\": \"task\",\n" +
+                    "            \"taskId\": \"" + taskId + "\",\n" +
+                    "            \"taskRecordId\": \"" + runId + "\"\n" +
+                    "          },\n" +
+                    "          \"fields\": [\n" +
+                    "            \"inputInsertTotal\",\n" +
+                    "            \"inputUpdateTotal\",\n" +
+                    "            \"inputDeleteTotal\",\n" +
+                    "            \"inputDdlTotal\",\n" +
+                    "            \"inputOthersTotal\",\n" +
+                    "            \"outputInsertTotal\",\n" +
+                    "            \"outputUpdateTotal\",\n" +
+                    "            \"outputDeleteTotal\",\n" +
+                    "            \"outputDdlTotal\",\n" +
+                    "            \"outputOthersTotal\"\n" +
+                    "          ],\n" +
+                    "          \"type\": \"difference\"\n" +
+                    "        }\n" +
+                    "      }\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"lineChartData\": {\n" +
+                    "    \"uri\": \"/api/measurement/query/v2\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"startAt\": " + (System.currentTimeMillis() - 6 * 86400 * 1000) + ",\n" +
+                    "       \"endAt\": " + System.currentTimeMillis() + ",\n" +
+                    "      \"samples\": {\n" +
+                    "        \"data\": {\n" +
+                    "          \"tags\": {\n" +
+                    "            \"type\": \"task\",\n" +
+                    "            \"taskId\": \"" + taskId + "\",\n" +
+                    "            \"taskRecordId\": \"" + runId + "\"\n" +
+                    "          },\n" +
+                    "          \"fields\": [\n" +
+                    "            \"inputQps\",\n" +
+                    "            \"outputQps\",\n" +
+                    "            \"timeCostAvg\",\n" +
+                    "            \"replicateLag\",\n" +
+                    "            \"inputSizeQps\",\n" +
+                    "            \"outputSizeQps\",\n" +
+                    "            \"qpsType\"\n" +
+                    "          ],\n" +
+                    "          \"type\": \"continuous\"\n" +
+                    "        }\n" +
+                    "      }\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"dagData\": {\n" +
+                    "    \"uri\": \"/api/measurement/query/v2\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"startAt\": " + (System.currentTimeMillis() - 6 * 86400 * 1000) + ",\n" +
+                    "       \"endAt\": " + System.currentTimeMillis() + ",\n" +
+                    "      \"samples\": {\n" +
+                    "        \"data\": {\n" +
+                    "          \"tags\": {\n" +
+                    "            \"type\": \"node\",\n" +
+                    "            \"taskId\": \"" + taskId + "\",\n" +
+                    "            \"taskRecordId\": \"" + runId + "\"\n" +
+                    "          },\n" +
+                    "          \"fields\": [\n" +
+                    "            \"inputInsertTotal\",\n" +
+                    "            \"inputUpdateTotal\",\n" +
+                    "            \"inputDeleteTotal\",\n" +
+                    "            \"inputDdlTotal\",\n" +
+                    "            \"inputOthersTotal\",\n" +
+                    "            \"outputInsertTotal\",\n" +
+                    "            \"outputUpdateTotal\",\n" +
+                    "            \"outputDeleteTotal\",\n" +
+                    "            \"outputDdlTotal\",\n" +
+                    "            \"outputOthersTotal\",\n" +
+                    "            \"qps\",\n" +
+                    "            \"timeCostAvg\",\n" +
+                    "            \"currentEventTimestamp\",\n" +
+                    "            \"tcpPing\",\n" +
+                    "            \"connectPing\",\n" +
+                    "            \"inputTotal\",\n" +
+                    "            \"outputTotal\",\n" +
+                    "            \"inputQps\",\n" +
+                    "            \"outputQps\",\n" +
+                    "            \"snapshotRowTotal\",\n" +
+                    "            \"snapshotInsertRowTotal\",\n" +
+                    "            \"snapshotTableTotal\",\n" +
+                    "            \"tableTotal\",\n" +
+                    "            \"snapshotSourceReadTimeCostAvg\",\n" +
+                    "            \"incrementalSourceReadTimeCostAvg\",\n" +
+                    "            \"targetWriteTimeCostAvg\",\n" +
+                    "            \"snapshotStartAt\",\n" +
+                    "            \"snapshotDoneAt\",\n" +
+                    "            \"replicateLag\"\n" +
+                    "          ],\n" +
+                    "          \"type\": \"instant\"\n" +
+                    "        }\n" +
+                    "      }\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"agentData\": {\n" +
+                    "    \"uri\": \"/api/measurement/query/v2\",\n" +
+                    "    \"param\": {\n" +
+                    "     \"startAt\": " + (System.currentTimeMillis() - 6 * 86400 * 1000) + ",\n" +
+                    "       \"endAt\": " + System.currentTimeMillis() + ",\n" +
+                    "      \"samples\": {\n" +
+                    "        \"data\": {\n" +
+                    "          \"tags\": {\n" +
+                    "            \"type\": \"engine\",\n" +
+                    "            \"engineId\": \"" + agentId + "\"\n" +
+                    "          },\n" +
+                    "         \"endAt\": " + System.currentTimeMillis() + ",\n" +
+                    "          \"fields\": [\n" +
+                    "            \"memoryRate\",\n" +
+                    "            \"cpuUsage\",\n" +
+                    "            \"gcRate\"\n" +
+                    "          ],\n" +
+                    "          \"type\": \"instant\"\n" +
+                    "        }\n" +
+                    "      }\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"taskRecord\": {\n" +
+                    "    \"uri\": \"/api/task/records\",\n" +
+                    "    \"param\": {\n" +
+                    "      \"taskId\": \"" + taskId + "\",\n" +
+                    "      \"size\": 200,\n" +
+                    "      \"page\": 1\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "}";
+            BatchRequestDto batchRequestDto = fromJson(metricContent, BatchRequestDto.class);
+            Map<String, Object> data = batchService.batch(batchRequestDto, request.getLocale());
+            contents.put(metricFileName, toJson(data).toString().getBytes(StandardCharsets.UTF_8));
+        } catch (Exception e) {
+            contents.put(metricFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出引擎的 cpu 火焰图
+        String cpuFlameGraphFileName = "engine_cpu.html";
+        try {
+            byte[] cpuFlameGraph = callEngineRpc(agentId, "FlameGraphService", "cpu", new Object[]{});
+            contents.put(cpuFlameGraphFileName, cpuFlameGraph);
+        } catch (Exception e) {
+            contents.put(cpuFlameGraphFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 寻找 async-profiler 路径, 可能在当前路径, 也可能在父路径
+        String asyncProfilerPath = "./async-profiler";
+        if (!new File(asyncProfilerPath).exists()) {
+            asyncProfilerPath = "../async-profiler";
+        }
+
+        // 导出管理端的 cpu 火焰图
+        String tmCpuFlameGraphFileName = "tm_cpu.html";
+        try {
+            FileUtils.deleteQuietly(new File("tm_cpu.html"));
+            String name = ManagementFactory.getRuntimeMXBean().getName();
+            // 提取 PID
+            String pid = name.split("@")[0];
+            // 1. 执行命令:
+            String command = asyncProfilerPath + "/bin/asprof -e cpu -d " + 15 + " -f ./tm_cpu.html " + pid;
+            Process process = Runtime.getRuntime().exec(command);
+
+            byte[] content = null;
+            process.waitFor(20, TimeUnit.SECONDS);
+            content = FileUtils.readFileToByteArray(new File("tm_cpu.html"));
+            contents.put(tmCpuFlameGraphFileName, content);
+        } catch (Exception e) {
+            contents.put(tmCpuFlameGraphFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出引擎的 内存分配 火焰图
+        String memoryFlameGraphFileName = "engine_memory.html";
+        try {
+            byte[] memoryFlameGraph = callEngineRpc(agentId, "FlameGraphService", "memory", new Object[]{});
+            contents.put(memoryFlameGraphFileName, memoryFlameGraph);
+        } catch (Exception e) {
+            contents.put(memoryFlameGraphFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出管理端的 内存分配 火焰图
+        String tmMemoryFlameGraphFileName = "tm_memory.html";
+        try {
+            FileUtils.deleteQuietly(new File("tm_memory.html"));
+            String name = ManagementFactory.getRuntimeMXBean().getName();
+            // 提取 PID
+            String pid = name.split("@")[0];
+            // 1. 执行命令:
+            String command = asyncProfilerPath + "/bin/asprof -e alloc -d " + 15 + " -f ./tm_memory.html " + pid;
+            Process process = Runtime.getRuntime().exec(command);
+
+            byte[] content = null;
+            process.waitFor(20, TimeUnit.SECONDS);
+            content = FileUtils.readFileToByteArray(new File("tm_memory.html"));
+            contents.put(tmMemoryFlameGraphFileName, content);
+        } catch (Exception e) {
+            contents.put(tmMemoryFlameGraphFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出引擎的 jstack 信息
+        String threadFileName = "engine_jstack.txt";
+        try {
+            byte[] threadDump = callEngineRpc(agentId, "FlameGraphService", "jstack", new Object[]{});
+            contents.put(threadFileName, threadDump);
+        } catch (Exception e) {
+            contents.put(threadFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出管理端的 jstack 信息
+        String tmThreadFileName = "tm_jstack.txt";
+        try {
+            ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+            boolean lockedMonitors = false;
+            boolean lockedSynchronizers = false;
+            ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(lockedMonitors, lockedSynchronizers);
+            String threadDump = "";
+            for (ThreadInfo threadInfo : threadInfos) {
+                threadDump += threadInfo.toString();
+                threadDump += "\n";
+            }
+            contents.put(tmThreadFileName, threadDump.getBytes(StandardCharsets.UTF_8));
+        } catch (Exception e) {
+            contents.put(tmThreadFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出引擎的一些内存信息
+        String engineMemoryFileName = "engine_memory.txt";
+        try {
+            byte[] engineMemory = callEngineRpc(agentId, "MemoryService", "memory", new Object[]{});
+            contents.put(engineMemoryFileName, engineMemory);
+        } catch (Exception e) {
+            contents.put(engineMemoryFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        // 导出管理端的一些内存信息
+        String tmMemoryFileName = "tm_memory.txt";
+        try {
+            byte[] tmMemory = PDKIntegration.outputMemoryFetchers(null, null, "Detail").getBytes(StandardCharsets.UTF_8);
+            contents.put(tmMemoryFileName, tmMemory);
+        } catch (Exception e) {
+            contents.put(tmMemoryFileName, e.toString().getBytes(StandardCharsets.UTF_8));
+        }
+
+        addContentToTar(taos, contents);
+
+        InputStream inputStream = new ByteArrayInputStream(baos.toByteArray());
+        InputStreamResource resource = new  InputStreamResource(inputStream);
+        HttpHeaders headers = new HttpHeaders();
+        headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + tarFileName);
+        headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
+
+        return ResponseEntity.ok()
+                .headers(headers)
+                .body(resource);
+    }
+
+
+    public void batchLoadTask(HttpServletResponse response, List<String> taskIds, UserDetail user) {
+        List<TaskUpAndLoadDto> jsonList = new ArrayList<>();
+        List<TaskDto> tasks = findAllTasksByIds(taskIds);
+        Map<String, TaskDto> taskDtoMap = tasks.stream().collect(Collectors.toMap(t -> t.getId().toHexString(), Function.identity(), (e1, e2) -> e1));
+        String json = exportTask(taskIds, user);
 
         AtomicReference<String> fileName = new AtomicReference<>("");
         String yyyymmdd = DateUtil.today().replace("-", "");
@@ -3919,6 +4392,11 @@ public void updateSyncProgress(ObjectId taskId, Document document) {
             update(new Query(criteria), update);
         });
     }
+    public void updateDelayTime(ObjectId taskId, long delayTime) {
+        Criteria criteria = Criteria.where("_id").is(taskId);
+        Update update = new Update().set("delayTime", delayTime);
+        update(new Query(criteria), update);
+    }
 
 
     public void increaseClear(ObjectId taskId, String srcNode, String tgtNode, UserDetail user) {
diff --git a/manager/tm/src/main/java/com/tapdata/tm/task/service/TransformSchemaService.java b/manager/tm/src/main/java/com/tapdata/tm/task/service/TransformSchemaService.java
index 5bcb18921b..d96ec41b9a 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/task/service/TransformSchemaService.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/task/service/TransformSchemaService.java
@@ -5,6 +5,8 @@
 import com.tapdata.manager.common.utils.StringUtils;
 import com.tapdata.tm.agent.service.AgentGroupService;
 import com.tapdata.tm.commons.dag.*;
+import com.tapdata.tm.commons.dag.logCollector.LogCollecotrConnConfig;
+import com.tapdata.tm.commons.dag.logCollector.LogCollectorNode;
 import com.tapdata.tm.commons.dag.nodes.DataParentNode;
 import com.tapdata.tm.commons.dag.nodes.DatabaseNode;
 import com.tapdata.tm.commons.dag.nodes.TableNode;
@@ -71,6 +73,7 @@ public class TransformSchemaService {
     private WorkerService workerService;
     private TaskDagCheckLogService taskDagCheckLogService;
     private LdpService ldpService;
+    private static final String IS_DELETED = "is_deleted";
     @Autowired
     private AgentGroupService agentGroupService;
 
@@ -279,15 +282,14 @@ public List<SchemaTransformerResult> getSchemaTransformResult(String nodeId) {
             }
         } else {
             Criteria criteria = Criteria.where("taskId").is(taskDto.getId().toHexString())
-                    .and("is_deleted").ne(true)
+                    .and(IS_DELETED).ne(true)
                     .and("sourceType").is(SourceTypeEnum.VIRTUAL.name());
             Query query1 = new Query(criteria);
             query1.fields().exclude("histories");
             metadataList = metadataInstancesService.findAllDto(query1, user);
 
         }
-
-
+        getLogCollectorMetadataInstancesDto(dag,user,metadataList);
         List<MetadataInstancesDto> databaseSchemes = metadataInstancesService.findDatabaseSchemeNoHistory(connectionIds, user);
         metadataList.addAll(databaseSchemes);
 
@@ -302,6 +304,36 @@ public List<SchemaTransformerResult> getSchemaTransformResult(String nodeId) {
 
     }
 
+    protected void getLogCollectorMetadataInstancesDto(DAG dag,UserDetail user,List<MetadataInstancesDto> metadataInstancesDtoList) {
+        dag.getNodes().forEach(node -> {
+            if(node instanceof LogCollectorNode){
+                LogCollectorNode logNode = (LogCollectorNode) node;
+                Criteria criteriaTable = Criteria.where("meta_type").in("table", "collection", "view");
+                Query queryMetadata = new Query();
+                Map<String, LogCollecotrConnConfig> connConfigs = logNode.getLogCollectorConnConfigs();
+                if (null != connConfigs && !connConfigs.isEmpty()) {
+                    List<Criteria> criteriaList = new ArrayList<>();
+                    for (LogCollecotrConnConfig config : connConfigs.values()) {
+                        criteriaList.add(Criteria.where("source._id").is(config.getConnectionId())
+                                .and("originalName").in(config.getTableNames()));
+                    }
+                    criteriaTable.and(IS_DELETED).ne(true).orOperator(criteriaList);
+                    queryMetadata.addCriteria(criteriaTable);
+                    metadataInstancesDtoList.addAll(metadataInstancesService.findAllDto(queryMetadata, user));
+                } else {
+                    List<String> logConnectionIds = logNode.getConnectionIds();
+                    if (CollectionUtils.isNotEmpty(logConnectionIds)) {
+                        String connectionId = logConnectionIds.get(0);
+                        queryMetadata.addCriteria(criteriaTable);
+                        criteriaTable.and("source._id").is(connectionId)
+                                .and("originalName").in(logNode.getTableNames()).and(IS_DELETED).ne(true);
+                        metadataInstancesDtoList.addAll(metadataInstancesService.findAllDto(queryMetadata, user));
+                    }
+                }
+            }
+        });
+    }
+
     public void transformSchema(TaskDto taskDto, UserDetail user) {
         transformSchema(taskDto, user, true);
     }
@@ -578,4 +610,5 @@ public void transformSchemaAndUpdateTask(TaskDto taskDto, UserDetail user) {
             taskService.updateById(taskId, update, user);
         }
     }
+
 }
diff --git a/manager/tm/src/main/java/com/tapdata/tm/task/service/impl/CustomSqlServiceImpl.java b/manager/tm/src/main/java/com/tapdata/tm/task/service/impl/CustomSqlServiceImpl.java
deleted file mode 100644
index d6389863bc..0000000000
--- a/manager/tm/src/main/java/com/tapdata/tm/task/service/impl/CustomSqlServiceImpl.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package com.tapdata.tm.task.service.impl;
-
-import com.tapdata.tm.base.exception.BizException;
-import com.tapdata.tm.commons.dag.DAG;
-import com.tapdata.tm.commons.dag.Node;
-import com.tapdata.tm.commons.dag.nodes.TableNode;
-import com.tapdata.tm.commons.schema.DataSourceConnectionDto;
-import com.tapdata.tm.commons.task.dto.TaskDto;
-import com.tapdata.tm.config.security.UserDetail;
-import com.tapdata.tm.ds.service.impl.DataSourceDefinitionService;
-import com.tapdata.tm.ds.service.impl.DataSourceService;
-import com.tapdata.tm.task.service.CustomSqlService;
-import com.tapdata.tm.utils.MongoUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.bson.types.ObjectId;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.data.mongodb.core.query.Query;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-
-@Service
-@Slf4j
-public class CustomSqlServiceImpl implements CustomSqlService {
-
-    @Autowired
-    private DataSourceService dataSourceService;
-
-    @Autowired
-    private DataSourceDefinitionService definitionService;
-
-    @Override
-    public void checkCustomSqlTask(TaskDto taskDto, UserDetail user) {
-        //判断如果不是自定义sql，直接返回
-        DAG dag = taskDto.getDag();
-        if (dag == null || dag.getEdges().isEmpty()) {
-            return;
-        }
-        boolean isCustomCommand = false;
-        List<Node> sources = dag.getSources();
-        if (CollectionUtils.isEmpty(sources)) {
-            return;
-        }
-
-        for (Node source : sources) {
-            if (source instanceof TableNode) {
-                boolean customCommand = ((TableNode) source).isEnableCustomCommand();
-                if (customCommand) {
-                    isCustomCommand = true;
-                    break;
-                }
-
-            }
-        }
-
-        if (!isCustomCommand) {
-            return;
-        }
-
-        //只有弱schema类型的数据源才可能作为目标
-
-        List<Node> targets = dag.getTargets();
-        if (CollectionUtils.isNotEmpty(targets)) {
-
-            List<String> targetConIds = targets.stream().filter(s -> s instanceof TableNode).map(n ->  ((TableNode) n).getConnectionId()).collect(Collectors.toList());
-
-            if (CollectionUtils.isNotEmpty(targetConIds)) {
-                List<ObjectId> targetConObjIds = targetConIds.stream().map(MongoUtils::toObjectId).collect(Collectors.toList());
-                Criteria criteria = Criteria.where("_id").in(targetConObjIds);
-                Query query = new Query(criteria);
-                query.fields().include("database_type");
-                List<DataSourceConnectionDto> connectionDtos = dataSourceService.findAllDto(query, user);
-                List<String> databaseTypes = connectionDtos.stream().map(DataSourceConnectionDto::getDatabase_type).collect(Collectors.toList());
-
-								if (!databaseTypes.isEmpty() && !definitionService.isAllTypeSchemaFree(databaseTypes, user)) {
-									throw new BizException("CustomSql.TargetNotSchemaFree");
-								}
-            }
-        }
-
-        //只支持js处理器节点
-        //由于css的需求，需要放开所有的处理节点。
-//        List<Node> nodes = dag.getNodes();
-//        for (Node node : nodes) {
-//            if (node instanceof ProcessorNode && !(node instanceof JsProcessorNode)) {
-//                throw new BizException("CustomSql.ProcessorNodeNotJs");
-//            }
-//        }
-    }
-}
diff --git a/manager/tm/src/main/java/com/tapdata/tm/task/service/impl/LdpServiceImpl.java b/manager/tm/src/main/java/com/tapdata/tm/task/service/impl/LdpServiceImpl.java
index 779e889c46..fbe9a041ad 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/task/service/impl/LdpServiceImpl.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/task/service/impl/LdpServiceImpl.java
@@ -64,7 +64,6 @@
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.stereotype.Service;
-import sun.management.Agent;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
diff --git a/manager/tm/src/main/java/com/tapdata/tm/utils/EngineVersionUtil.java b/manager/tm/src/main/java/com/tapdata/tm/utils/EngineVersionUtil.java
new file mode 100644
index 0000000000..baffcdfcdd
--- /dev/null
+++ b/manager/tm/src/main/java/com/tapdata/tm/utils/EngineVersionUtil.java
@@ -0,0 +1,47 @@
+package com.tapdata.tm.utils;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class EngineVersionUtil {
+    protected EngineVersionUtil() {
+        throw new IllegalStateException("Utility class");
+    }
+    public static final String VERSION_3_8_0 = "3.8.0";
+
+    public static boolean checkEngineTransFormSchema(String version) {
+        try{
+            if(StringUtils.isNotEmpty(version)){
+                String[] versions =  version.split("-",2);
+                return compareVersions(versions[0],VERSION_3_8_0);
+            }
+        }catch (Exception e){
+            return false;
+        }
+        return false;
+    }
+
+    public static Boolean compareVersions(String version,String targetVersion) {
+        if (version.startsWith("v")) {
+            version = version.substring(1);
+        }
+
+        String[] levels = version.split("\\.");
+        String[] targetLevels = targetVersion.split("\\.");
+
+        int length = Math.max(levels.length, targetLevels.length);
+
+        for (int i = 0; i < length; i++) {
+            int v1 = i < levels.length ? Integer.parseInt(levels[i]) : 0;
+            int v2 = i < targetLevels.length ? Integer.parseInt(targetLevels[i]) : 0;
+
+            if (v1 < v2) {
+                return false;
+            }
+            if (v1 > v2) {
+                return true;
+            }
+        }
+        return true;
+    }
+
+}
diff --git a/manager/tm/src/main/java/com/tapdata/tm/worker/service/WorkerServiceImpl.java b/manager/tm/src/main/java/com/tapdata/tm/worker/service/WorkerServiceImpl.java
index c468b4d64a..e80a0da023 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/worker/service/WorkerServiceImpl.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/worker/service/WorkerServiceImpl.java
@@ -37,13 +37,11 @@
 import com.tapdata.tm.userLog.constant.Modular;
 import com.tapdata.tm.userLog.constant.Operation;
 import com.tapdata.tm.userLog.service.UserLogService;
+import com.tapdata.tm.utils.EngineVersionUtil;
 import com.tapdata.tm.utils.FunctionUtils;
 import com.tapdata.tm.utils.MongoUtils;
 import com.tapdata.tm.worker.WorkerSingletonLock;
-import com.tapdata.tm.worker.dto.WorkSchedule;
-import com.tapdata.tm.worker.dto.WorkerDto;
-import com.tapdata.tm.worker.dto.WorkerExpireDto;
-import com.tapdata.tm.worker.dto.WorkerProcessInfoDto;
+import com.tapdata.tm.worker.dto.*;
 import com.tapdata.tm.worker.entity.Worker;
 import com.tapdata.tm.worker.entity.WorkerExpire;
 import com.tapdata.tm.worker.repository.WorkerRepository;
@@ -889,4 +887,17 @@ public String getWorkerCurrentTime(UserDetail userDetail){
         }
         return simpleDateFormat.format(new Date());
     }
+
+    public Boolean checkEngineVersion(UserDetail userDetail){
+        boolean isCloud = settingsService.isCloud();
+        if(isCloud){
+            List<Worker> workers = findAvailableAgent(userDetail);
+            if(CollectionUtils.isEmpty(workers)) return false;
+            List<Worker> oldWorkers = workers.stream().filter(worker -> StringUtils.isBlank(worker.getVersion())
+                    || !EngineVersionUtil.checkEngineTransFormSchema(worker.getVersion())).collect(Collectors.toList());
+            return CollectionUtils.isEmpty(oldWorkers);
+        }else{
+            return true;
+        }
+    }
 }
diff --git a/manager/tm/src/main/java/com/tapdata/tm/ws/endpoint/WebSocketServer.java b/manager/tm/src/main/java/com/tapdata/tm/ws/endpoint/WebSocketServer.java
index d86d147f9d..8094799c44 100644
--- a/manager/tm/src/main/java/com/tapdata/tm/ws/endpoint/WebSocketServer.java
+++ b/manager/tm/src/main/java/com/tapdata/tm/ws/endpoint/WebSocketServer.java
@@ -36,11 +36,11 @@
 import org.springframework.stereotype.Component;
 import org.springframework.web.socket.*;
 import org.springframework.web.socket.handler.TextWebSocketHandler;
-import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl;
 
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.net.InetSocketAddress;
 import java.util.*;
@@ -324,7 +324,7 @@ public void handlerResult(MessageInfoV1 messageInfo) {
 
 			ReturnCallback<?> callback = WebSocketManager.getAndRemoveResultCallback(messageInfo.getReqId());
 			if (callback != null) {
-				Type[] type = ((ParameterizedTypeImpl) callback.getClass().getGenericSuperclass()).getActualTypeArguments();
+				Type[] type = ((ParameterizedType) callback.getClass().getGenericSuperclass()).getActualTypeArguments();
 				try {
 					if (ResultWrap.OK.equals(result.getCode())) {
 						Object argument = null;
diff --git a/manager/tm/src/main/resources/init/dfs/version b/manager/tm/src/main/resources/init/dfs/version
index 283318df94..bd42c8eba0 100644
--- a/manager/tm/src/main/resources/init/dfs/version
+++ b/manager/tm/src/main/resources/init/dfs/version
@@ -1,1 +1,1 @@
-3.5-4
\ No newline at end of file
+3.8-2
\ No newline at end of file
diff --git a/manager/tm/src/main/resources/init/idaas/version b/manager/tm/src/main/resources/init/idaas/version
index a172ed6375..52c2864109 100644
--- a/manager/tm/src/main/resources/init/idaas/version
+++ b/manager/tm/src/main/resources/init/idaas/version
@@ -1,1 +1,1 @@
-3.5-16
\ No newline at end of file
+3.8-1
\ No newline at end of file
diff --git a/manager/tm/src/main/resources/messages.properties b/manager/tm/src/main/resources/messages.properties
index c3d0c1c690..7a94383ca7 100644
--- a/manager/tm/src/main/resources/messages.properties
+++ b/manager/tm/src/main/resources/messages.properties
@@ -147,7 +147,6 @@ Ldp.FdmSourceTableTaskExist=The currently selected table already exists in the F
 Ldp.DeleteMdmTableFailed=Deleting model failed and model does not belong to MDM.
 Ldp.TaskTypeError=The sync type supported by the current data source is {0}, and the current setting type is {1}.
 
-CustomSql.TargetNotSchemaFree=The target node of the custom sql filter mode must be a data source without schema
 CustomSql.ProcessorNodeNotJs=The intermediate processing node of the custom sql filter mode only allows js nodes
 
 ApiApp.NewTagNotFound=The target application was not found
diff --git a/manager/tm/src/main/resources/messages_zh_CN.properties b/manager/tm/src/main/resources/messages_zh_CN.properties
index 4d2b55c2ea..3f2c4f3bf6 100644
--- a/manager/tm/src/main/resources/messages_zh_CN.properties
+++ b/manager/tm/src/main/resources/messages_zh_CN.properties
@@ -179,7 +179,6 @@ Ldp.FdmSourceTableTaskExist=\u5F53\u524D\u9009\u62E9\u7684\u8868\u5DF2\u7ECF\u5B
 Ldp.DeleteMdmTableFailed=\u5220\u9664\u6A21\u578B\u5931\u8D25\uFF0C\u6A21\u578B\u4E0D\u5C5E\u4E8EMDM.
 Ldp.TaskTypeError=\u5F53\u524D\u6570\u636E\u6E90\u652F\u6301\u7684\u540C\u6B65\u7C7B\u578B\u4E3A{0}\uFF0C \u5F53\u524D\u8BBE\u7F6E\u7684\u7C7B\u578B\u4E3A{1}.
 
-CustomSql.TargetNotSchemaFree=\u81EA\u5B9A\u4E49sql\u8FC7\u6EE4\u6A21\u5F0F\u7684\u76EE\u6807\u8282\u70B9\u5FC5\u987B\u662F\u6CA1\u6709\u6A21\u578B\u7EA6\u675F\u7C7B\u578B\u7684\u6570\u636E\u6E90
 CustomSql.ProcessorNodeNotJs=\u81EA\u5B9A\u4E49sql\u8FC7\u6EE4\u6A21\u5F0F\u7684\u4E2D\u95F4\u5904\u7406\u8282\u70B9\u53EA\u5141\u8BB8js\u8282\u70B9
 
 ApiApp.NewTagNotFound=\u76EE\u6807\u5E94\u7528\u4E0D\u5B58\u5728
diff --git a/pom.xml b/pom.xml
index 7adfb71861..a1ccb09d67 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,9 @@
             iengine/iengine-common/src/main/java/com/tapdata/exception/**,
             manager/tm-common/src/main/java/com/tapdata/tm/commons/task/dto/**,
             manager/tm/src/main/java/com/tapdata/tm/init/patches/daas/*,
-            manager/tm/src/main/java/com/tapdata/tm/init/patches/dfs/*
+            manager/tm/src/main/java/com/tapdata/tm/init/patches/dfs/*,
+            **/*Entity.java,
+            **/*VO.java,
         </sonar.coverage.exclusions>
         <sonar.exclusions>
             manager/**/dto/*,
diff --git a/pre-release b/pre-release
deleted file mode 100644
index fa27fcadca..0000000000
--- a/pre-release
+++ /dev/null
@@ -1,1 +0,0 @@
-v3.5.15
\ No newline at end of file
diff --git a/tapdata-cli/src/main/java/io/tapdata/pdk/cli/RegisterMain.java b/tapdata-cli/src/main/java/io/tapdata/pdk/cli/RegisterMain.java
index 38991188ad..c5e8341f2c 100644
--- a/tapdata-cli/src/main/java/io/tapdata/pdk/cli/RegisterMain.java
+++ b/tapdata-cli/src/main/java/io/tapdata/pdk/cli/RegisterMain.java
@@ -88,8 +88,10 @@ private enum ConnectorEnums {
 		YashanDB(BASE_PATH + "connectors/dist/yashandb-connector-v1.0-SNAPSHOT.jar", "all", "yashandb", "basic", "jdbc"),
 		Ali1688(BASE_PATH + "connectors/dist/ali1688-connector-v1.0-SNAPSHOT.jar", "all", "ali1688"),
         Temu(BASE_PATH + "connectors/dist/temu-connector-v1.0-SNAPSHOT.jar", "all", "temu"),
-        GreenPlum(BASE_PATH + "connectors/dist/greenplum-connector-v1.0-SNAPSHOT.jar", "all", "greenplum", "basic", "jdbc"),
-        ;
+
+		GreenPlum(BASE_PATH + "connectors/dist/greenplum-connector-v1.0-SNAPSHOT.jar", "all", "greenplum", "basic", "jdbc"),
+
+		;
 
 		private final String path;
 		private final Set<String> tags = new HashSet<>();


Test patch:
diff --git a/iengine/iengine-app/src/test/java/base/BaseTaskTest.java b/iengine/iengine-app/src/test/java/base/BaseTaskTest.java
index be5066911d..0f5cb64fee 100644
--- a/iengine/iengine-app/src/test/java/base/BaseTaskTest.java
+++ b/iengine/iengine-app/src/test/java/base/BaseTaskTest.java
@@ -3,6 +3,7 @@
 import com.tapdata.entity.task.context.DataProcessorContext;
 import com.tapdata.entity.task.context.ProcessorBaseContext;
 import com.tapdata.tm.commons.dag.Node;
+import com.tapdata.tm.commons.dag.nodes.DatabaseNode;
 import com.tapdata.tm.commons.dag.nodes.TableNode;
 import com.tapdata.tm.commons.task.dto.TaskDto;
 import io.tapdata.MockTaskUtil;
@@ -20,6 +21,7 @@ public abstract class BaseTaskTest extends BaseTest {
 	protected DataProcessorContext dataProcessorContext;
 	protected TableNode tableNode;
 	protected TaskDto taskDto;
+	protected DatabaseNode databaseNode;
 
 	protected void allSetup() {
 		setupTaskAndNode();
@@ -45,4 +47,20 @@ protected void setupContext() {
 		when(dataProcessorContext.getEdges()).thenReturn(taskDto.getDag().getEdges());
 		when(dataProcessorContext.getNodes()).thenReturn(taskDto.getDag().getNodes());
 	}
+	protected void setUpDatabaseNode() {
+		taskDto = MockTaskUtil.setUpTaskDtoByJsonFile("migrationdummy2dummy.json");
+		databaseNode = (DatabaseNode) taskDto.getDag().getNodes().get(1);
+
+		processorBaseContext = mock(ProcessorBaseContext.class);
+		when(processorBaseContext.getTaskDto()).thenReturn(taskDto);
+		when(processorBaseContext.getNode()).thenReturn((Node) databaseNode);
+		when(processorBaseContext.getEdges()).thenReturn(taskDto.getDag().getEdges());
+		when(processorBaseContext.getNodes()).thenReturn(taskDto.getDag().getNodes());
+
+		dataProcessorContext = mock(DataProcessorContext.class);
+		when(dataProcessorContext.getTaskDto()).thenReturn(taskDto);
+		when(dataProcessorContext.getNode()).thenReturn((Node) databaseNode);
+		when(dataProcessorContext.getEdges()).thenReturn(taskDto.getDag().getEdges());
+		when(dataProcessorContext.getNodes()).thenReturn(taskDto.getDag().getNodes());
+	}
 }
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/common/DAGDataEngineServiceImplTest.java b/iengine/iengine-app/src/test/java/io/tapdata/common/DAGDataEngineServiceImplTest.java
new file mode 100644
index 0000000000..3105d886d4
--- /dev/null
+++ b/iengine/iengine-app/src/test/java/io/tapdata/common/DAGDataEngineServiceImplTest.java
@@ -0,0 +1,593 @@
+package io.tapdata.common;
+
+import com.alibaba.fastjson.JSON;
+import com.tapdata.constant.BeanUtil;
+import com.tapdata.constant.ConfigurationCenter;
+import com.tapdata.entity.Connections;
+import com.tapdata.entity.schema.SchemaApplyResult;
+import com.tapdata.mongo.HttpClientMongoOperator;
+import com.tapdata.tm.commons.dag.DAG;
+import com.tapdata.tm.commons.dag.Node;
+import com.tapdata.tm.commons.dag.logCollector.LogCollecotrConnConfig;
+import com.tapdata.tm.commons.dag.logCollector.LogCollectorNode;
+import com.tapdata.tm.commons.dag.nodes.DatabaseNode;
+import com.tapdata.tm.commons.dag.nodes.TableNode;
+import com.tapdata.tm.commons.dag.process.MergeTableNode;
+import com.tapdata.tm.commons.dag.vo.MigrateJsResultVo;
+import com.tapdata.tm.commons.schema.*;
+import com.tapdata.tm.commons.schema.bean.SourceDto;
+import com.tapdata.tm.commons.schema.bean.SourceTypeEnum;
+import com.tapdata.tm.commons.task.dto.TaskDto;
+import io.tapdata.entity.conversion.PossibleDataTypes;
+import io.tapdata.entity.schema.TapField;
+import io.tapdata.entity.schema.TapIndex;
+import io.tapdata.entity.schema.TapTable;
+import io.tapdata.entity.schema.type.TapRaw;
+import io.tapdata.entity.schema.type.TapString;
+import io.tapdata.entity.schema.type.TapType;
+import io.tapdata.flow.engine.V2.node.hazelcast.data.HazelcastSchemaTargetNode;
+import io.tapdata.flow.engine.V2.task.impl.HazelcastTaskClient;
+import io.tapdata.flow.engine.V2.task.impl.HazelcastTaskService;
+import io.tapdata.observable.logging.ObsLogger;
+import io.tapdata.observable.logging.ObsLoggerFactory;
+import io.tapdata.schema.TapTableMap;
+import org.bson.types.ObjectId;
+import org.junit.jupiter.api.*;
+import org.mockito.MockedStatic;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.mockito.Mockito.*;
+
+public class DAGDataEngineServiceImplTest {
+    DAGDataEngineServiceImpl dagDataEngineService;
+    TransformerWsMessageDto transformerWsMessageDto;
+    HazelcastTaskService taskService;
+    HttpClientMongoOperator clientMongoOperator;
+    @BeforeEach
+    void setUp() {
+        transformerWsMessageDto = new TransformerWsMessageDto();
+        transformerWsMessageDto.setMetadataInstancesDtoList(new ArrayList<>());
+        transformerWsMessageDto.setUserId("test");
+        DAG.Options options = new DAG.Options();
+        options.setUuid("test");
+        transformerWsMessageDto.setOptions(options);
+        transformerWsMessageDto.setDefinitionDtoMap(new HashMap<>());
+        transformerWsMessageDto.setDataSourceMap(new HashMap<>());
+        transformerWsMessageDto.setTransformerDtoMap(new HashMap<>());
+        taskService = mock(HazelcastTaskService.class);
+        clientMongoOperator = mock(HttpClientMongoOperator.class);
+    }
+    @Nested
+    class loadTapTableTest{
+        @Test
+        @DisplayName("test loadTapTable main process ")
+        void testLoadTapTable(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<HazelcastSchemaTargetNode> hazelcastSchemaTargetNodeMockedStatic = mockStatic(HazelcastSchemaTargetNode.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                when(obsLogger.isDebugEnabled()).thenReturn(true);
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new ConcurrentHashMap<>(),clientMongoOperator);
+                when(taskService.startTestTask(any(TaskDto.class))).thenReturn(mock(HazelcastTaskClient.class));
+                TapTable tapTable = mock(TapTable.class);
+                hazelcastSchemaTargetNodeMockedStatic.when(()->HazelcastSchemaTargetNode.getTapTable(any())).thenReturn(tapTable);
+                TapTable result = dagDataEngineService.loadTapTable("test","test",taskDto);
+                Assertions.assertEquals(tapTable,result);
+            }
+        }
+
+        @Test
+        @DisplayName("test loadTapTable getTapTable error ")
+        void testLoadTapTableError(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new ConcurrentHashMap<>(),clientMongoOperator);
+                TapTable result = dagDataEngineService.loadTapTable("test","test",taskDto);
+                Assertions.assertNull(result);
+            }
+        }
+    }
+    @Nested
+    class getJsResultTest{
+        @Test
+        @DisplayName("Get JsResult main process ")
+        void GetJsResultTest(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<HazelcastSchemaTargetNode> hazelcastSchemaTargetNodeMockedStatic = mockStatic(HazelcastSchemaTargetNode.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                when(obsLogger.isDebugEnabled()).thenReturn(true);
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new ConcurrentHashMap<>(),clientMongoOperator);
+                when(taskService.startTestTask(any(TaskDto.class))).thenReturn(mock(HazelcastTaskClient.class));
+                List<SchemaApplyResult> schemaApplyResults = new ArrayList<>();
+                SchemaApplyResult schemaApplyResult = new SchemaApplyResult("test","test",new TapField("test","String"));
+                schemaApplyResult.setTapIndex(new TapIndex());
+                schemaApplyResults.add(schemaApplyResult);
+                hazelcastSchemaTargetNodeMockedStatic.when(()->HazelcastSchemaTargetNode.getSchemaApplyResultList(any())).thenReturn(schemaApplyResults);
+                List<MigrateJsResultVo> result = dagDataEngineService.getJsResult("test","test",taskDto);
+                Assertions.assertEquals(1,result.size());
+            }
+        }
+
+        @Test
+        @DisplayName("Get JsResult error")
+        void GetJsResultTestError(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new ConcurrentHashMap<>(),clientMongoOperator);
+                List<MigrateJsResultVo> result = dagDataEngineService.getJsResult("test","test",taskDto);
+                Assertions.assertTrue(result.isEmpty());
+            }
+        }
+    }
+    @Nested
+    class convertTapTableTest{
+        MetadataInstancesDto metadataInstancesDto;
+
+        @Test
+        @DisplayName("convert TapTable Test main process ")
+        void testConvertTapTable(){
+            metadataInstancesDto = new MetadataInstancesDto();
+            List<Field> fields = new ArrayList<>();
+            Field field1 = new Field();
+            field1.setFieldName("test1");
+            field1.setTapType("{\"type\":8}");
+            field1.setDeleted(false);
+            Field field2 = new Field();
+            field2.setFieldName("test2");
+            field2.setTapType("{\"type\":8}");
+            field2.setDeleted(false);
+            Field field3 = new Field();
+            field3.setFieldName("test3");
+            field3.setTapType("{\"type\":8}");
+            field3.setDeleted(false);
+            fields.add(field1);
+            fields.add(field2);
+            fields.add(field3);
+            metadataInstancesDto.setFields(fields);
+            List<TableIndex> indices = new ArrayList<>();
+            TableIndex tableIndex = new TableIndex();
+            List<TableIndexColumn> columns = new ArrayList<>();
+            TableIndexColumn column1 = new TableIndexColumn();
+            column1.setColumnName("test1");
+            TableIndexColumn column2 = new TableIndexColumn();
+            column2.setColumnName("test2");
+            columns.add(column1);
+            columns.add(column2);
+            tableIndex.setColumns(columns);
+            indices.add(tableIndex);
+            metadataInstancesDto.setIndices(indices);
+            Map<String, PossibleDataTypes> dataTypes = new HashMap<>();
+            dataTypes.put("test1",new PossibleDataTypes().dataType("String"));
+            dataTypes.put("test3",new PossibleDataTypes());
+            metadataInstancesDto.setFindPossibleDataTypes(dataTypes);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<JSON> jsonMockedStatic = mockStatic(JSON.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                jsonMockedStatic.when(()->JSON.parseObject("{\"type\":8}",TapType.class)).thenReturn(new TapString()).thenReturn(new TapRaw()).thenReturn(new TapString());
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new ConcurrentHashMap<>(),clientMongoOperator);
+                TapTable result = dagDataEngineService.convertTapTable(metadataInstancesDto);
+                Assertions.assertTrue(result.getNameFieldMap().containsKey("test1"));
+            }
+
+        }
+
+        @Test
+        @DisplayName("convert TapTable Test DataTypes Is Null ")
+        void testConvertTapTableDataTypesIsNull(){
+            metadataInstancesDto = new MetadataInstancesDto();
+            List<Field> fields = new ArrayList<>();
+            Field field1 = new Field();
+            field1.setFieldName("test1");
+            field1.setTapType("{\"type\":8}");
+            field1.setDeleted(false);
+            Field field2 = new Field();
+            field2.setFieldName("test2");
+            field2.setTapType("{\"type\":8}");
+            field2.setDeleted(false);
+            Field field3 = new Field();
+            field3.setFieldName("test3");
+            field3.setTapType("{\"type\":8}");
+            field3.setDeleted(false);
+            fields.add(field1);
+            fields.add(field2);
+            fields.add(field3);
+            metadataInstancesDto.setFields(fields);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<JSON> jsonMockedStatic = mockStatic(JSON.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                jsonMockedStatic.when(()->JSON.parseObject("{\"type\":8}",TapType.class)).thenReturn(new TapString()).thenReturn(new TapRaw()).thenReturn(new TapString());
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new ConcurrentHashMap<>(),clientMongoOperator);
+                TapTable result = dagDataEngineService.convertTapTable(metadataInstancesDto);
+                Assertions.assertTrue(result.getNameFieldMap().containsKey("test1"));
+                Assertions.assertTrue(result.getNameFieldMap().containsKey("test2"));
+                Assertions.assertTrue(result.getNameFieldMap().containsKey("test3"));
+            }
+
+        }
+
+        @Test
+        @DisplayName("convert TapTable Test Fields is null  ")
+        void testConvertTapTableFieldsIsNull(){
+            metadataInstancesDto = new MetadataInstancesDto();
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<JSON> jsonMockedStatic = mockStatic(JSON.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                jsonMockedStatic.when(()->JSON.parseObject("{\"type\":8}",TapType.class)).thenReturn(new TapString()).thenReturn(new TapRaw()).thenReturn(new TapString());
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new ConcurrentHashMap<>(),clientMongoOperator);
+                TapTable result = dagDataEngineService.convertTapTable(metadataInstancesDto);
+                Assertions.assertTrue(result.getNameFieldMap().isEmpty());
+            }
+        }
+    }
+    @Nested
+    class initializeModelTest{
+        Map<String, MetadataInstancesDto> batchMetadataUpdateMap = new LinkedHashMap<>();
+        List<MetadataInstancesDto> batchInsertMetaDataList = new ArrayList<>();
+        Map<String, MetadataInstancesDto> metadataMap = new HashMap<>();
+        @Test
+        @DisplayName("InitializeModel Test main process")
+        void testInitializeModel(){
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("node1");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+            metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto2.setNodeId("node2");
+            metadataInstancesDto2.setOriginalName("name2");
+            metadataInstancesDto2.setQualifiedName("test2");
+            metadataInstancesDto2.setMetaType("table");
+            batchMetadataUpdateMap.put("node1",metadataInstancesDto1);
+            batchMetadataUpdateMap.put("node2",metadataInstancesDto2);
+            MetadataInstancesDto metadataInstancesDto3 = new MetadataInstancesDto();
+            metadataInstancesDto3.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto3.setNodeId("node2");
+            metadataInstancesDto3.setOriginalName("name3");
+            metadataInstancesDto3.setQualifiedName("test3");
+            metadataInstancesDto3.setMetaType("processor_node");
+            batchInsertMetaDataList.add(metadataInstancesDto3);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)) {
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,tapTableMapHashMap,clientMongoOperator);
+                ReflectionTestUtils.setField(dagDataEngineService,"batchMetadataUpdateMap",batchMetadataUpdateMap);
+                ReflectionTestUtils.setField(dagDataEngineService,"batchInsertMetaDataList",batchInsertMetaDataList);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                dagDataEngineService.initializeModel(true);
+                Assertions.assertTrue(tapTableMapHashMap.get("node1").containsKey("name1"));
+                Assertions.assertTrue(tapTableMapHashMap.get("node2").containsKey("name2"));
+                Assertions.assertTrue(tapTableMapHashMap.get("node2").containsKey("node2"));
+            }
+        }
+
+        @Test
+        @DisplayName("InitializeModel Test getBatchMetadataUpdateMap and getBatchInsertMetaDataList is null")
+        void testInitializeModeMetadataInstancesDtoIsNull(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)) {
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,tapTableMapHashMap,clientMongoOperator);
+                ReflectionTestUtils.setField(dagDataEngineService,"batchMetadataUpdateMap",batchMetadataUpdateMap);
+                ReflectionTestUtils.setField(dagDataEngineService,"batchInsertMetaDataList",batchInsertMetaDataList);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                dagDataEngineService.initializeModel(false);
+                Assertions.assertTrue(tapTableMapHashMap.isEmpty());
+            }
+        }
+
+        @Test
+        @DisplayName("InitializeModel Test main process")
+        void testInitializeModelMergeNode(){
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("node1");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+            metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto2.setNodeId("node2");
+            metadataInstancesDto2.setOriginalName("name2");
+            metadataInstancesDto2.setQualifiedName("test2");
+            metadataInstancesDto2.setMetaType("table");
+            batchMetadataUpdateMap.put("node1",metadataInstancesDto1);
+            batchMetadataUpdateMap.put("node2",metadataInstancesDto2);
+            MetadataInstancesDto metadataInstancesDto3 = new MetadataInstancesDto();
+            metadataInstancesDto3.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto3.setNodeId("mergeNode");
+            metadataInstancesDto3.setOriginalName("name3");
+            metadataInstancesDto3.setQualifiedName("test3");
+            metadataInstancesDto3.setMetaType("processor_node");
+            batchInsertMetaDataList.add(metadataInstancesDto3);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG mocDag = mock(DAG.class);
+            taskDto.setDag(mocDag);
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)) {
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,tapTableMapHashMap,clientMongoOperator);
+                ReflectionTestUtils.setField(dagDataEngineService,"batchMetadataUpdateMap",batchMetadataUpdateMap);
+                ReflectionTestUtils.setField(dagDataEngineService,"batchInsertMetaDataList",batchInsertMetaDataList);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                List<Node> nodes = new ArrayList<>();
+                MergeTableNode mergeTableNode = new MergeTableNode();
+                mergeTableNode.setId("mergeNode");
+                nodes.add(mergeTableNode);
+                when(mocDag.getNodes()).thenReturn(nodes);
+                List<Node> predecessors = new ArrayList<>();
+                TableNode tableNode1 = new TableNode();
+                tableNode1.setId("node1");
+                TableNode tableNode2 = new TableNode();
+                tableNode2.setId("node2");
+                predecessors.add(tableNode1);
+                predecessors.add(tableNode2);
+                when(mocDag.predecessors("mergeNode")).thenReturn(predecessors);
+                dagDataEngineService.initializeModel(true);
+                Assertions.assertEquals(3,tapTableMapHashMap.get("mergeNode").size());
+            }
+        }
+
+        @Test
+        void testInitializeModelLogCollectorNode(){
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("LogCollectorNode");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+            metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto2.setNodeId("test");
+            metadataInstancesDto2.setOriginalName("name2");
+            metadataInstancesDto2.setQualifiedName("test2");
+            metadataInstancesDto2.setMetaType("table");
+            metadataMap.put("node1",metadataInstancesDto1);
+            metadataMap.put("node2",metadataInstancesDto2);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG mocDag = mock(DAG.class);
+            taskDto.setDag(mocDag);
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)) {
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,tapTableMapHashMap,clientMongoOperator);
+                ReflectionTestUtils.setField(dagDataEngineService,"metadataMap",metadataMap);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                List<Node> nodes = new ArrayList<>();
+                LogCollectorNode logCollectorNode = new LogCollectorNode();
+                logCollectorNode.setId("LogCollectorNode");
+                nodes.add(logCollectorNode);
+                when(mocDag.getNodes()).thenReturn(nodes);
+                dagDataEngineService.initializeModel(true);
+                Assertions.assertEquals(2,tapTableMapHashMap.get("LogCollectorNode").size());
+            }
+        }
+
+        @Test
+        void testInitializeModelLogCollectorNodeConnConfigsIsNotNull(){
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("LogCollectorNode");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            SourceDto sourceDto1 = new SourceDto();
+            sourceDto1.set_id("s_1");
+            metadataInstancesDto1.setSource(sourceDto1);
+            MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+            metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto2.setNodeId("test");
+            metadataInstancesDto2.setOriginalName("name2");
+            metadataInstancesDto2.setQualifiedName("test2");
+            metadataInstancesDto2.setMetaType("table");
+            SourceDto sourceDto2 = new SourceDto();
+            sourceDto2.set_id("s_2");
+            metadataInstancesDto2.setSource(sourceDto2);
+            MetadataInstancesDto metadataInstancesDto3 = new MetadataInstancesDto();
+            metadataInstancesDto3.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto3.setNodeId("test3");
+            metadataInstancesDto3.setOriginalName("name3");
+            metadataInstancesDto3.setQualifiedName("test3");
+            metadataInstancesDto3.setMetaType("table");
+            SourceDto sourceDto3 = new SourceDto();
+            sourceDto3.set_id("s_3");
+            metadataInstancesDto3.setSource(sourceDto3);
+            metadataMap.put("node1",metadataInstancesDto1);
+            metadataMap.put("node2",metadataInstancesDto2);
+            metadataMap.put("node3",metadataInstancesDto3);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG mocDag = mock(DAG.class);
+            taskDto.setDag(mocDag);
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)) {
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,tapTableMapHashMap,clientMongoOperator);
+                ReflectionTestUtils.setField(dagDataEngineService,"metadataMap",metadataMap);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                List<Node> nodes = new ArrayList<>();
+                LogCollectorNode logCollectorNode = new LogCollectorNode();
+                logCollectorNode.setId("LogCollectorNode");
+                Map<String, LogCollecotrConnConfig> connConfigs = new HashMap<>();
+                connConfigs.put("test1",new LogCollecotrConnConfig());
+                logCollectorNode.setLogCollectorConnConfigs(connConfigs);
+                Connections connections1 = new Connections();
+                connections1.setId("s_1");
+                connections1.setNamespace(Arrays.asList("C##Test1"));
+                Connections connections2 = new Connections();
+                connections2.setId("s_2");
+                connections2.setNamespace(Arrays.asList("C##Test2"));
+                connections2.setNamespace(Arrays.asList("C##Test2"));
+                List<Connections>connections = new ArrayList<>();
+                connections.add(connections1);
+                connections.add(connections2);
+                nodes.add(logCollectorNode);
+                when(clientMongoOperator.find(any(Query.class),any(),any())).thenReturn(new ArrayList<>(connections));
+                when(mocDag.getNodes()).thenReturn(nodes);
+                dagDataEngineService.initializeModel(true);
+                Assertions.assertEquals(2,tapTableMapHashMap.get("LogCollectorNode").size());
+            }
+        }
+        @Test
+        void testInitializeModelLogCollectorNodeConnConfigsIsEmpt(){
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("LogCollectorNode");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+            metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto2.setNodeId("test");
+            metadataInstancesDto2.setOriginalName("name2");
+            metadataInstancesDto2.setQualifiedName("test2");
+            metadataInstancesDto2.setMetaType("table");
+            metadataMap.put("node1",metadataInstancesDto1);
+            metadataMap.put("node2",metadataInstancesDto2);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG mocDag = mock(DAG.class);
+            taskDto.setDag(mocDag);
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)) {
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,tapTableMapHashMap,clientMongoOperator);
+                ReflectionTestUtils.setField(dagDataEngineService,"metadataMap",metadataMap);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                List<Node> nodes = new ArrayList<>();
+                LogCollectorNode logCollectorNode = new LogCollectorNode();
+                logCollectorNode.setId("LogCollectorNode");
+                logCollectorNode.setLogCollectorConnConfigs(new HashMap<>());
+                nodes.add(logCollectorNode);
+                when(mocDag.getNodes()).thenReturn(nodes);
+                dagDataEngineService.initializeModel(true);
+                Assertions.assertEquals(2,tapTableMapHashMap.get("LogCollectorNode").size());
+            }
+        }
+
+
+    }
+    @Nested
+    class uploadModelTest{
+        @Test
+        @DisplayName("UploadModel Test main process")
+        void testUploadModel(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG dag = mock(DAG.class);
+            taskDto.setDag(dag);
+            transformerWsMessageDto.setTaskDto(taskDto);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)) {
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                List<Node> nodes = new ArrayList<>();
+                nodes.add(new DatabaseNode());
+                when(dag.getNodes()).thenReturn(nodes);
+                dagDataEngineService = new DAGDataEngineServiceImpl(transformerWsMessageDto,taskService,new HashMap<>(),clientMongoOperator);
+                dagDataEngineService.uploadModel(new HashMap<>());
+                verify(clientMongoOperator,times(1)).insertOne(any(),any());
+            }
+        }
+    }
+}
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNodeTest.java b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNodeTest.java
index 262ade7cb5..ae9bfa0232 100644
--- a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNodeTest.java
+++ b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/HazelcastBaseNodeTest.java
@@ -48,9 +48,7 @@
 import io.tapdata.entity.utils.InstanceFactory;
 import io.tapdata.error.TapProcessorUnknownException;
 import io.tapdata.error.TaskProcessorExCode_11;
-import io.tapdata.exception.ManagementException;
-import io.tapdata.exception.MongodbException;
-import io.tapdata.exception.TapCodeException;
+import io.tapdata.exception.*;
 import io.tapdata.flow.engine.V2.exception.ErrorHandleException;
 import io.tapdata.flow.engine.V2.monitor.Monitor;
 import io.tapdata.flow.engine.V2.monitor.MonitorManager;
@@ -62,6 +60,7 @@
 import io.tapdata.flow.engine.V2.util.SyncTypeEnum;
 import io.tapdata.observable.logging.ObsLogger;
 import io.tapdata.observable.logging.ObsLoggerFactory;
+import io.tapdata.pdk.core.error.TapPdkRunnerUnknownException;
 import io.tapdata.pdk.core.utils.CommonUtils;
 import io.tapdata.schema.TapTableMap;
 import io.tapdata.task.skipError.SkipErrorStrategy;
@@ -75,6 +74,7 @@
 import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
 
 import static org.junit.jupiter.api.Assertions.*;
 import static org.mockito.Mockito.*;
@@ -440,7 +440,7 @@ void beforeEach() {
 		void testSetThreadName() {
 			hazelcastBaseNode.setThreadName();
 			String actual = Thread.currentThread().getName();
-			assertEquals("HazelcastBaseNode-dummy2dummy(6555b257407e2d16ae88c5ad)-dummy_test(2cbc1a4d-906d-4b32-9cf4-6596ed4bd0e4)", actual);
+			assertEquals(String.format("HazelcastBaseNode-dummy2dummy(6555b257407e2d16ae88c5ad)-dummy_test(%s)",tableNode.getId()), actual);
 		}
 	}
 
@@ -1936,4 +1936,27 @@ void test4(){
 		}
 
 	}
+	@Nested
+	class BuildErrorConsumerTest{
+		private String tableName = "table";
+		@Test
+		@DisplayName("test buildErrorConsumer method for TapPdkBaseException")
+		void test1(){
+			TapPdkWriteMissingPrivilegesEx ex = new TapPdkWriteMissingPrivilegesEx("mysql","createTable",null,new RuntimeException("test exception"));
+			assertThrows(TapPdkBaseException.class,()->hazelcastBaseNode.buildErrorConsumer(tableName).accept(ex));
+			assertEquals(tableName,ex.getTableName());
+		}
+		@Test
+		@DisplayName("test buildErrorConsumer method for TapPdkRunnerUnknownException")
+		void test2(){
+			TapPdkRunnerUnknownException ex = new TapPdkRunnerUnknownException(new RuntimeException("test exception"));
+			assertThrows(TapPdkRunnerUnknownException.class,()->hazelcastBaseNode.buildErrorConsumer(tableName).accept(ex));
+			assertEquals(tableName,ex.getTableName());
+		}
+		@Test
+		@DisplayName("test buildErrorConsumer method for RuntimeException")
+		void test3(){
+			assertThrows(RuntimeException.class,()->hazelcastBaseNode.buildErrorConsumer(tableName).accept(mock(RuntimeException.class)));
+		}
+	}
 }
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNodeTest.java b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNodeTest.java
index 3b3940c3da..d5f0e33002 100644
--- a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNodeTest.java
+++ b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastSourcePdkDataNodeTest.java
@@ -1,10 +1,7 @@
 package io.tapdata.flow.engine.V2.node.hazelcast.data.pdk;
 
 import base.hazelcast.BaseHazelcastNodeTest;
-import com.tapdata.entity.DatabaseTypeEnum;
-import com.tapdata.entity.SyncStage;
-import com.tapdata.entity.TapdataCompleteSnapshotEvent;
-import com.tapdata.entity.TapdataCompleteTableSnapshotEvent;
+import com.tapdata.entity.*;
 import com.tapdata.entity.dataflow.SyncProgress;
 import com.tapdata.entity.dataflow.TableBatchReadStatus;
 import com.tapdata.entity.dataflow.batch.BatchOffsetUtil;
@@ -12,12 +9,14 @@
 import com.tapdata.entity.task.config.TaskRetryConfig;
 import com.tapdata.entity.task.context.DataProcessorContext;
 import com.tapdata.entity.task.context.ProcessorBaseContext;
+import com.tapdata.mongo.ClientMongoOperator;
 import com.tapdata.tm.commons.dag.Node;
 import com.tapdata.tm.commons.dag.nodes.DatabaseNode;
 import com.tapdata.tm.commons.dag.nodes.TableNode;
 import io.tapdata.aspect.BatchReadFuncAspect;
 import io.tapdata.aspect.SourceStateAspect;
 import io.tapdata.aspect.taskmilestones.*;
+import io.tapdata.common.TapInterfaceUtil;
 import io.tapdata.entity.aspect.AspectInterceptResult;
 import io.tapdata.entity.codec.filter.TapCodecsFilterManager;
 import io.tapdata.entity.event.TapEvent;
@@ -47,6 +46,7 @@
 import io.tapdata.pdk.apis.functions.connector.source.BatchReadFunction;
 import io.tapdata.pdk.apis.functions.connector.source.CountByPartitionFilterFunction;
 import io.tapdata.pdk.apis.functions.connector.source.ExecuteCommandFunction;
+import io.tapdata.pdk.apis.functions.connector.target.CreateIndexFunction;
 import io.tapdata.pdk.apis.functions.connector.target.QueryByAdvanceFilterFunction;
 import io.tapdata.pdk.apis.spec.TapNodeSpecification;
 import io.tapdata.pdk.core.api.ConnectorNode;
@@ -54,12 +54,14 @@
 import io.tapdata.pdk.core.monitor.PDKInvocationMonitor;
 import io.tapdata.pdk.core.tapnode.TapNodeInfo;
 import io.tapdata.pdk.core.utils.CommonUtils;
+import io.tapdata.schema.SchemaProxy;
 import io.tapdata.schema.TapTableMap;
 import lombok.SneakyThrows;
 import org.junit.Assert;
 import org.junit.jupiter.api.*;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.mockito.internal.verification.Times;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import java.time.LocalDateTime;
@@ -1592,4 +1594,50 @@ void testFilterBatchCountNotSupport() {
 		}
 
 	}
+
+	@Nested
+	class createTargetIndexTest{
+		private List<String> updateConditionFields;
+		private boolean createUnique;
+		private String tableId;
+		private TapTable tapTable;
+		private ClientMongoOperator clientMongoOperator;
+		@BeforeEach
+		void beforeEach(){
+			updateConditionFields = new ArrayList<>();
+			updateConditionFields.add("field");
+			createUnique = true;
+			tableId = "test";
+			tapTable = mock(TapTable.class);
+			ReflectionTestUtils.setField(hazelcastSourcePdkDataNode,"obsLogger",mockObsLogger);
+			clientMongoOperator = mock(ClientMongoOperator.class);
+			ReflectionTestUtils.setField(hazelcastSourcePdkDataNode,"clientMongoOperator",clientMongoOperator);
+		}
+		@Test
+		@DisplayName("test createTargetIndex method for build error consumer")
+		void test1(){
+			try (MockedStatic<TapInterfaceUtil> mb = Mockito
+					.mockStatic(TapInterfaceUtil.class)) {
+				Connections connections = mock(Connections.class);
+				when(dataProcessorContext.getConnections()).thenReturn(connections);
+				when(connections.getDatabase_type()).thenReturn("test");
+				mb.when(()->TapInterfaceUtil.getTapInterface("test",null)).thenReturn(null);
+				ConnectorNode connectorNode = mock(ConnectorNode.class);
+				when(hazelcastSourcePdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+				ConnectorFunctions functions = mock(ConnectorFunctions.class);
+				when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+				when(functions.getCreateIndexFunction()).thenReturn(mock(CreateIndexFunction.class));
+				ArrayList<String> pks = new ArrayList<>();
+				when(tapTable.primaryKeys()).thenReturn(pks);
+				when(hazelcastSourcePdkDataNode.usePkAsUpdateConditions(updateConditionFields,pks)).thenReturn(false);
+				doCallRealMethod().when(hazelcastSourcePdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+				try (MockedStatic<SchemaProxy> schemaProxyMockedStatic = Mockito
+						.mockStatic(SchemaProxy.class)) {
+					schemaProxyMockedStatic.when(SchemaProxy::getSchemaProxy).thenReturn(mock(SchemaProxy.class));
+					hazelcastSourcePdkDataNode.createTargetIndex(updateConditionFields,createUnique,tableId,tapTable);
+					verify(hazelcastSourcePdkDataNode,new Times(1)).buildErrorConsumer(tableId);
+				}
+			}
+		}
+	}
 }
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNodeTest.java b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNodeTest.java
index 0108422b58..a6c284a428 100644
--- a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNodeTest.java
+++ b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkBaseNodeTest.java
@@ -24,6 +24,9 @@
 import io.tapdata.entity.event.dml.TapUpdateRecordEvent;
 import io.tapdata.entity.schema.TapField;
 import io.tapdata.entity.schema.TapTable;
+import io.tapdata.flow.engine.V2.exactlyonce.ExactlyOnceUtil;
+import io.tapdata.flow.engine.V2.exactlyonce.write.CheckExactlyOnceWriteEnableResult;
+import io.tapdata.flow.engine.V2.exactlyonce.write.ExactlyOnceWriteCleanerEntity;
 import io.tapdata.metric.collector.ISyncMetricCollector;
 import io.tapdata.observable.logging.ObsLogger;
 import io.tapdata.pdk.apis.entity.Capability;
@@ -31,15 +34,22 @@
 import io.tapdata.pdk.apis.entity.merge.MergeInfo;
 import io.tapdata.pdk.apis.entity.merge.MergeLookupResult;
 import io.tapdata.pdk.apis.functions.ConnectorFunctions;
+import io.tapdata.pdk.apis.functions.connector.target.CreateIndexFunction;
+import io.tapdata.pdk.apis.functions.connector.target.CreateTableFunction;
 import io.tapdata.pdk.apis.functions.connector.target.CreateTableV2Function;
 import io.tapdata.pdk.core.api.ConnectorNode;
 import io.tapdata.pdk.core.async.AsyncUtils;
 import io.tapdata.pdk.core.async.ThreadPoolExecutorEx;
+import io.tapdata.pdk.core.utils.CommonUtils;
 import org.junit.jupiter.api.*;
 import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.Times;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -274,6 +284,46 @@ void testIsNotUnwindProcess() {
 			boolean result = hazelcastTargetPdkBaseNode.createTable(tapTable, atomicBoolean);
 			Assertions.assertTrue(result);
 		}
+		@Test
+		void testCreateTableIsNull(){
+			ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"unwindProcess",false);
+			TapTable tapTable = new TapTable();
+			tapTable.setId("test");
+			AtomicBoolean succeed = new AtomicBoolean(true);
+			Node node = mock(Node.class);
+			when(hazelcastTargetPdkBaseNode.getNode()).thenReturn(node);
+			when(node.disabledNode()).thenReturn(false);
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkBaseNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getCreateTableFunction()).thenReturn(null);
+			when(functions.getCreateTableV2Function()).thenReturn(null);
+			when(dataProcessorContext.getTargetConn()).thenReturn(mock(Connections.class));
+			doCallRealMethod().when(hazelcastTargetPdkBaseNode).createTable(tapTable,succeed);
+			hazelcastTargetPdkBaseNode.createTable(tapTable, succeed);
+			verify(hazelcastTargetPdkBaseNode,new Times(0)).buildErrorConsumer("test");
+		}
+		@Test
+		void testCreateTableForBuildErrorConsumer(){
+			ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"unwindProcess",false);
+			TapTable tapTable = new TapTable();
+			tapTable.setId("test");
+			AtomicBoolean succeed = new AtomicBoolean(true);
+			Node node = mock(Node.class);
+			when(hazelcastTargetPdkBaseNode.getNode()).thenReturn(node);
+			when(node.disabledNode()).thenReturn(false);
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkBaseNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getCreateTableFunction()).thenReturn(mock(CreateTableFunction.class));
+			when(dataProcessorContext.getTargetConn()).thenReturn(mock(Connections.class));
+			doCallRealMethod().when(hazelcastTargetPdkBaseNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+			doCallRealMethod().when(hazelcastTargetPdkBaseNode).createTable(tapTable,succeed);
+			hazelcastTargetPdkBaseNode.createTable(tapTable, succeed);
+			verify(hazelcastTargetPdkBaseNode,new Times(1)).buildErrorConsumer("test");
+		}
 	}
 
 	@Nested
@@ -663,5 +713,95 @@ void testMainProcess() {
 				verify(hazelcastTargetPdkBaseNode).checkUnwindConfiguration();
 			}
 		}
+		@Nested
+		class handleTapTablePrimaryKeysTest{
+			private TapTable tapTable;
+			private ConcurrentHashMap<String, Boolean> everHandleTapTablePrimaryKeysMap;
+			private String writeStrategy = "updateOrInsert";
+			protected Map<String, List<String>> updateConditionFieldsMap;
+			private List<String> updateConditionFields;
+			@BeforeEach
+			void beforeEach(){
+				everHandleTapTablePrimaryKeysMap = new ConcurrentHashMap<>();
+				ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"everHandleTapTablePrimaryKeysMap",everHandleTapTablePrimaryKeysMap);
+				ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"writeStrategy",writeStrategy);
+				updateConditionFieldsMap = new HashMap<>();
+				tapTable = new TapTable();
+				tapTable.setId("test");
+				LinkedHashMap<String, TapField> nameFieldMap = new LinkedHashMap<>();
+				TapField primary = new TapField();
+				primary.setPrimaryKey(true);
+				nameFieldMap.put("primary",primary);
+				tapTable.setNameFieldMap(nameFieldMap);
+				updateConditionFields = new ArrayList<>();
+				updateConditionFields.add("field");
+				updateConditionFieldsMap.put("test",updateConditionFields);
+				ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"updateConditionFieldsMap",updateConditionFieldsMap);
+
+			}
+			@Test
+			@DisplayName("test handleTapTablePrimaryKeys method when everHandleTapTablePrimaryKeysMap not contains tapTable")
+			void test1(){
+				try (MockedStatic<HazelcastTargetPdkBaseNode> mb = Mockito
+						.mockStatic(HazelcastTargetPdkBaseNode.class)) {
+					mb.when(()->HazelcastTargetPdkBaseNode.ignorePksAndIndices(tapTable,updateConditionFields)).thenAnswer(invocationOnMock -> {return null;});
+					doCallRealMethod().when(hazelcastTargetPdkBaseNode).handleTapTablePrimaryKeys(tapTable);
+					hazelcastTargetPdkBaseNode.handleTapTablePrimaryKeys(tapTable);
+					assertTrue(everHandleTapTablePrimaryKeysMap.containsKey("test"));
+					assertTrue(everHandleTapTablePrimaryKeysMap.get("test"));
+					mb.verify(() -> HazelcastTargetPdkBaseNode.ignorePksAndIndices(tapTable,updateConditionFields),new Times(1));
+				}
+			}
+			@Test
+			@DisplayName("test handleTapTablePrimaryKeys method when everHandleTapTablePrimaryKeysMap contains tapTable")
+			void test2(){
+				try (MockedStatic<HazelcastTargetPdkBaseNode> mb = Mockito
+						.mockStatic(HazelcastTargetPdkBaseNode.class)) {
+					mb.when(()->HazelcastTargetPdkBaseNode.ignorePksAndIndices(tapTable,updateConditionFields)).thenAnswer(invocationOnMock -> {return null;});
+					everHandleTapTablePrimaryKeysMap.put("test",true);
+					doCallRealMethod().when(hazelcastTargetPdkBaseNode).handleTapTablePrimaryKeys(tapTable);
+					hazelcastTargetPdkBaseNode.handleTapTablePrimaryKeys(tapTable);
+					assertTrue(everHandleTapTablePrimaryKeysMap.containsKey("test"));
+					assertTrue(everHandleTapTablePrimaryKeysMap.get("test"));
+					mb.verify(() -> HazelcastTargetPdkBaseNode.ignorePksAndIndices(tapTable,updateConditionFields),new Times(0));
+				}
+			}
+		}
+	}
+	@Nested
+	class InitExactlyOnceWriteIfNeedTest{
+		@BeforeEach
+		void beforeEach(){
+			List<String> exactlyOnceWriteTables = new ArrayList<>();
+			ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"exactlyOnceWriteTables",exactlyOnceWriteTables);
+			List<ExactlyOnceWriteCleanerEntity> exactlyOnceWriteCleanerEntities = new ArrayList<>();
+			ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"exactlyOnceWriteCleanerEntities",exactlyOnceWriteCleanerEntities);
+			dataProcessorContext = mock(DataProcessorContext.class);
+			ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"dataProcessorContext",dataProcessorContext);
+			ReflectionTestUtils.setField(hazelcastTargetPdkBaseNode,"obsLogger",mockObsLogger);
+		}
+		@Test
+		@DisplayName("test initExactlyOnceWriteIfNeed method for createTable is true")
+		void test1(){
+			try (MockedStatic<ExactlyOnceUtil> mb = Mockito
+					.mockStatic(ExactlyOnceUtil.class)) {
+				TapTable exactlyOnceTable = mock(TapTable.class);
+				when(exactlyOnceTable.getId()).thenReturn("test");
+				ConnectorNode connectorNode = mock(ConnectorNode.class);
+				when(hazelcastTargetPdkBaseNode.getConnectorNode()).thenReturn(connectorNode);
+				mb.when(()->ExactlyOnceUtil.generateExactlyOnceTable(connectorNode)).thenReturn(exactlyOnceTable);
+				CheckExactlyOnceWriteEnableResult checkExactlyOnceWriteEnableResult = mock(CheckExactlyOnceWriteEnableResult.class);
+				when(hazelcastTargetPdkBaseNode.enableExactlyOnceWrite()).thenReturn(checkExactlyOnceWriteEnableResult);
+				when(checkExactlyOnceWriteEnableResult.getEnable()).thenReturn(true);
+				ConnectorFunctions functions = mock(ConnectorFunctions.class);
+				when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+				when(hazelcastTargetPdkBaseNode.createTable(any(TapTable.class), any(AtomicBoolean.class))).thenReturn(true);
+				when(functions.getCreateIndexFunction()).thenReturn(mock(CreateIndexFunction.class));
+				when(hazelcastTargetPdkBaseNode.getNode()).thenReturn((Node) mock(TableNode.class));
+				doCallRealMethod().when(hazelcastTargetPdkBaseNode).initExactlyOnceWriteIfNeed();
+				hazelcastTargetPdkBaseNode.initExactlyOnceWriteIfNeed();
+				verify(hazelcastTargetPdkBaseNode, new Times(1)).buildErrorConsumer("test");
+			}
+		}
 	}
 }
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNodeTest.java b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNodeTest.java
index 15f2088a5c..10c59614c9 100644
--- a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNodeTest.java
+++ b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/data/pdk/HazelcastTargetPdkDataNodeTest.java
@@ -1,25 +1,36 @@
 package io.tapdata.flow.engine.V2.node.hazelcast.data.pdk;
 
 import base.BaseTaskTest;
+import com.tapdata.constant.ConnectorContext;
+import com.tapdata.entity.TapdataEvent;
 import com.tapdata.entity.task.ExistsDataProcessEnum;
+import com.tapdata.entity.task.context.DataProcessorContext;
+import com.tapdata.entity.task.context.ProcessorBaseContext;
 import com.tapdata.tm.commons.dag.Node;
 import com.tapdata.tm.commons.dag.nodes.DataParentNode;
 import com.tapdata.tm.commons.dag.nodes.DatabaseNode;
 import com.tapdata.tm.commons.dag.nodes.TableNode;
+import com.tapdata.tm.commons.task.dto.TaskDto;
+import io.tapdata.MockTaskUtil;
 import io.tapdata.aspect.TableInitFuncAspect;
 import io.tapdata.entity.event.TapEvent;
+import io.tapdata.entity.event.ddl.index.TapCreateIndexEvent;
+import io.tapdata.entity.event.ddl.table.*;
 import io.tapdata.entity.event.dml.TapInsertRecordEvent;
+import io.tapdata.entity.schema.TapField;
 import io.tapdata.entity.schema.TapIndex;
 import io.tapdata.entity.schema.TapIndexField;
 import io.tapdata.entity.schema.TapTable;
+import io.tapdata.error.TapEventException;
+import io.tapdata.error.TaskTargetProcessorExCode_15;
 import io.tapdata.exception.TapCodeException;
+import io.tapdata.flow.engine.V2.util.SyncTypeEnum;
 import io.tapdata.observable.logging.ObsLogger;
 import io.tapdata.pdk.apis.context.TapConnectorContext;
 import io.tapdata.pdk.apis.functions.ConnectorFunctions;
 import io.tapdata.pdk.apis.functions.connection.GetTableInfoFunction;
 import io.tapdata.pdk.apis.functions.connection.TableInfo;
-import io.tapdata.pdk.apis.functions.connector.target.CreateIndexFunction;
-import io.tapdata.pdk.apis.functions.connector.target.QueryIndexesFunction;
+import io.tapdata.pdk.apis.functions.connector.target.*;
 import io.tapdata.pdk.core.api.ConnectorNode;
 import io.tapdata.pdk.core.utils.CommonUtils;
 import io.tapdata.schema.TapTableMap;
@@ -29,6 +40,7 @@
 import org.mockito.internal.verification.Times;
 import org.springframework.test.util.ReflectionTestUtils;
 
+import java.sql.Ref;
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -149,6 +161,7 @@ class SyncIndexTest{
 		private GetTableInfoFunction getTableInfoFunction;
 		private QueryIndexesFunction queryIndexesFunction;
 		@BeforeEach
+		@SneakyThrows
 		void beforeEach(){
 			tableId = "test";
 			tapTable = mock(TapTable.class);
@@ -162,6 +175,7 @@ void beforeEach(){
 			createIndexFunction = mock(CreateIndexFunction.class);
 			getTableInfoFunction = mock(GetTableInfoFunction.class);
 			queryIndexesFunction = mock(QueryIndexesFunction.class);
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).queryExistsIndexes(any(), any());
 		}
 		@Test
 		@DisplayName("test sync method when sync index switch is off")
@@ -425,4 +439,372 @@ void testCheckSyncIndexOpen4(){
 			assertEquals(false, actual);
 		}
 	}
+
+	@Nested
+	class executeCreateIndexFunctionTest {
+
+		private TapTableMap<String, TapTable> tapTableMap;
+
+		@BeforeEach
+		void setUp() {
+			when(hazelcastTargetPdkDataNode.executeCreateIndexFunction(any())).thenCallRealMethod();
+			tapTableMap = mock(TapTableMap.class);
+			DataProcessorContext dataProcessorContext = mock(DataProcessorContext.class);
+			when(dataProcessorContext.getTapTableMap()).thenReturn(tapTableMap);
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode, "dataProcessorContext", dataProcessorContext);
+		}
+
+		@Test
+		@DisplayName("test main process")
+		@SneakyThrows
+		void test1() {
+			List<TapIndex> tapIndices = new ArrayList<>();
+			tapIndices.add(new TapIndex()
+					.indexField(new TapIndexField().name("test_id").fieldAsc(true))
+					.indexField(new TapIndexField().name("test_id1").fieldAsc(false)));
+			tapIndices.add(new TapIndex()
+					.name("dup_index")
+					.indexField(new TapIndexField().name("dup_id").fieldAsc(true)));
+			TapCreateIndexEvent tapCreateIndexEvent = new TapCreateIndexEvent()
+					.indexList(tapIndices);
+			tapCreateIndexEvent.setTableId("test");
+			TapTable tapTable = new TapTable("test");
+			when(tapTableMap.get("test")).thenReturn(tapTable);
+			CreateIndexFunction createIndexFunction = mock(CreateIndexFunction.class);
+			ConnectorFunctions connectorFunctions = mock(ConnectorFunctions.class);
+			when(connectorFunctions.getCreateIndexFunction()).thenReturn(createIndexFunction);
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(connectorFunctions);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			List<TapIndex> existsIndexes = new ArrayList<>();
+			existsIndexes.add(tapIndices.get(1));
+			when(hazelcastTargetPdkDataNode.queryExistsIndexes(tapTable, tapIndices)).thenReturn(existsIndexes);
+			TapConnectorContext connectorContext = mock(TapConnectorContext.class);
+			when(connectorNode.getConnectorContext()).thenReturn(connectorContext);
+			doCallRealMethod().when(connectorNode).applyClassLoaderContext(any());
+			doAnswer(invocationOnMock -> {
+				Object argument3 = invocationOnMock.getArgument(2);
+				((CommonUtils.AnyErrorConsumer<?>) argument3).accept(null);
+				return null;
+			}).when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(), any(), any());
+			boolean result = hazelcastTargetPdkDataNode.executeCreateIndexFunction(tapCreateIndexEvent);
+			assertTrue(result);
+			assertEquals(1, tapCreateIndexEvent.getIndexList().size());
+			verify(hazelcastTargetPdkDataNode).queryExistsIndexes(tapTable, tapIndices);
+			verify(createIndexFunction).createIndex(connectorContext, tapTable, tapCreateIndexEvent);
+		}
+
+		@Test
+		@DisplayName("test event's tableId is empty")
+		void test2() {
+			TapCreateIndexEvent tapCreateIndexEvent = new TapCreateIndexEvent();
+			TapEventException tapEventException = assertThrows(TapEventException.class, () -> hazelcastTargetPdkDataNode.executeCreateIndexFunction(tapCreateIndexEvent));
+			assertEquals(TaskTargetProcessorExCode_15.CREATE_INDEX_EVENT_TABLE_ID_EMPTY, tapEventException.getCode());
+			assertNotNull(tapEventException.getEvents());
+			assertEquals(1, tapEventException.getEvents().size());
+			assertEquals(tapCreateIndexEvent, tapEventException.getEvents().get(0));
+		}
+
+		@Test
+		@DisplayName("test cannot get TapTable by tableId")
+		void test3() {
+			TapCreateIndexEvent tapCreateIndexEvent = new TapCreateIndexEvent();
+			tapCreateIndexEvent.setTableId("test");
+			when(tapTableMap.get("test")).thenReturn(null);
+			TapEventException tapEventException = assertThrows(TapEventException.class, () -> hazelcastTargetPdkDataNode.executeCreateIndexFunction(tapCreateIndexEvent));
+			assertEquals(TaskTargetProcessorExCode_15.CREATE_INDEX_TABLE_NOT_FOUND, tapEventException.getCode());
+			assertNotNull(tapEventException.getEvents());
+			assertEquals(1, tapEventException.getEvents().size());
+			assertEquals(tapCreateIndexEvent, tapEventException.getEvents().get(0));
+		}
+
+		@Test
+		@DisplayName("test not support CreateIndexFunction")
+		void test4() {
+			TapCreateIndexEvent tapCreateIndexEvent = new TapCreateIndexEvent();
+			tapCreateIndexEvent.setTableId("test");
+			TapTable tapTable = new TapTable("test");
+			when(tapTableMap.get("test")).thenReturn(tapTable);
+			ConnectorFunctions connectorFunctions = mock(ConnectorFunctions.class);
+			when(connectorFunctions.getCreateIndexFunction()).thenReturn(null);
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(connectorFunctions);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			assertFalse(hazelcastTargetPdkDataNode.executeCreateIndexFunction(tapCreateIndexEvent));
+		}
+
+		@Test
+		@DisplayName("test queryExistsIndexes error")
+		@SneakyThrows
+		void test5() {
+			List<TapIndex> tapIndices = new ArrayList<>();
+			tapIndices.add(new TapIndex()
+					.indexField(new TapIndexField().name("test_id").fieldAsc(true))
+					.indexField(new TapIndexField().name("test_id1").fieldAsc(false)));
+			tapIndices.add(new TapIndex()
+					.name("dup_index")
+					.indexField(new TapIndexField().name("dup_id").fieldAsc(true)));
+			TapCreateIndexEvent tapCreateIndexEvent = new TapCreateIndexEvent()
+					.indexList(tapIndices);
+			tapCreateIndexEvent.setTableId("test");
+			TapTable tapTable = new TapTable("test");
+			when(tapTableMap.get("test")).thenReturn(tapTable);
+			CreateIndexFunction createIndexFunction = mock(CreateIndexFunction.class);
+			ConnectorFunctions connectorFunctions = mock(ConnectorFunctions.class);
+			when(connectorFunctions.getCreateIndexFunction()).thenReturn(createIndexFunction);
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(connectorFunctions);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			when(hazelcastTargetPdkDataNode.queryExistsIndexes(tapTable, tapIndices)).thenThrow(new RuntimeException("test"));
+
+			TapEventException tapEventException = assertThrows(TapEventException.class, () -> hazelcastTargetPdkDataNode.executeCreateIndexFunction(tapCreateIndexEvent));
+			assertEquals(TaskTargetProcessorExCode_15.CREATE_INDEX_QUERY_EXISTS_INDEX_FAILED, tapEventException.getCode());
+			assertNotNull(tapEventException.getEvents());
+			assertEquals(1, tapEventException.getEvents().size());
+			assertEquals(tapCreateIndexEvent, tapEventException.getEvents().get(0));
+		}
+	}
+	@Nested
+	class createTargetIndexTest{
+		private List<String> updateConditionFields;
+		private boolean createUnique;
+		private String tableId;
+		private TapTable tapTable;
+		private boolean createdTable;
+		@BeforeEach
+		void beforeEach(){
+			updateConditionFields = new ArrayList<>();
+			updateConditionFields.add("field");
+			tableId = "table";
+			tapTable = mock(TapTable.class);
+			ArrayList<String> pks = new ArrayList<>();
+			when(tapTable.primaryKeys()).thenReturn(pks);
+			when(hazelcastTargetPdkDataNode.usePkAsUpdateConditions(updateConditionFields,pks)).thenReturn(false);
+			createdTable = true;
+			String writeStrategy = "updateOrInsert";
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"writeStrategy",writeStrategy);
+			Boolean unwindProcess = false;
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"unwindProcess",unwindProcess);
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+		}
+		@Test
+		@DisplayName("test createTargetIndex method for build error consumer")
+		void test1(){
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getCreateIndexFunction()).thenReturn(mock(CreateIndexFunction.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).createTargetIndex(updateConditionFields,createUnique,tableId,tapTable,createdTable);
+			hazelcastTargetPdkDataNode.createTargetIndex(updateConditionFields,createUnique,tableId,tapTable,createdTable);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).buildErrorConsumer(tableId);
+		}
+	}
+	@Nested
+	class clearDataTest{
+		private ExistsDataProcessEnum existsDataProcessEnum;
+		private String tableId = "test";
+		@Test
+		@DisplayName("test clearData method for build error consumer")
+		void test1(){
+			SyncTypeEnum syncType = SyncTypeEnum.INITIAL_SYNC;
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"syncType",syncType);
+			existsDataProcessEnum = ExistsDataProcessEnum.REMOVE_DATE;
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getClearTableFunction()).thenReturn(mock(ClearTableFunction.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).clearData(existsDataProcessEnum,tableId);
+			hazelcastTargetPdkDataNode.clearData(existsDataProcessEnum,tableId);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).buildErrorConsumer(tableId);
+		}
+	}
+	@Nested
+	class dropTableTest{
+		private ExistsDataProcessEnum existsDataProcessEnum;
+		private String tableId = "test";
+		@Test
+		@DisplayName("test dropTable method for build error consumer")
+		void test1(){
+			SyncTypeEnum syncType = SyncTypeEnum.INITIAL_SYNC;
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"syncType",syncType);
+			existsDataProcessEnum = ExistsDataProcessEnum.DROP_TABLE;
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getDropTableFunction()).thenReturn(mock(DropTableFunction.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).dropTable(existsDataProcessEnum,tableId);
+			hazelcastTargetPdkDataNode.dropTable(existsDataProcessEnum,tableId);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).buildErrorConsumer(tableId);
+		}
+	}
+	@Nested
+	class executeNewFieldFunctionTest{
+		private TapNewFieldEvent tapNewFieldEvent;
+		private TapTable tapTable;
+		@BeforeEach
+		void beforeEach(){
+			tapNewFieldEvent = new TapNewFieldEvent();
+			tapNewFieldEvent.setTableId("test");
+			List<TapField> fields = new ArrayList<>();
+			TapField newField = new TapField();
+			newField.setName("newField");
+			fields.add(newField);
+			tapNewFieldEvent.setNewFields(fields);
+			dataProcessorContext = mock(DataProcessorContext.class);
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"dataProcessorContext",dataProcessorContext);
+			TapTableMap tapTableMap = mock(TapTableMap.class);
+			when(dataProcessorContext.getTapTableMap()).thenReturn(tapTableMap);
+			tapTable = new TapTable();
+			tapTable.setId("table");
+			LinkedHashMap<String, TapField> nameFieldMap = new LinkedHashMap<>();
+			nameFieldMap.put("field1",mock(TapField.class));
+			nameFieldMap.put("newField",newField);
+			tapTable.setNameFieldMap(nameFieldMap);
+			when(tapTableMap.get("test")).thenReturn(tapTable);
+		}
+		@Test
+		@DisplayName("test executeNewFieldFunction method for build error consumer")
+		void test1(){
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getNewFieldFunction()).thenReturn(mock(NewFieldFunction.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeNewFieldFunction(tapNewFieldEvent);
+			hazelcastTargetPdkDataNode.executeNewFieldFunction(tapNewFieldEvent);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).buildErrorConsumer("table");
+		}
+	}
+	@Nested
+	class executeAlterFieldNameFunctionTest{
+		private TapAlterFieldNameEvent tapAlterFieldNameEvent;
+		@Test
+		@DisplayName("test executeAlterFieldNameFunction method for build error consumer")
+		void test1(){
+			tapAlterFieldNameEvent = mock(TapAlterFieldNameEvent.class);
+			when(tapAlterFieldNameEvent.getTableId()).thenReturn("test");
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getAlterFieldNameFunction()).thenReturn(mock(AlterFieldNameFunction.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeAlterFieldNameFunction(tapAlterFieldNameEvent);
+			hazelcastTargetPdkDataNode.executeAlterFieldNameFunction(tapAlterFieldNameEvent);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).buildErrorConsumer("test");
+		}
+	}
+	@Nested
+	class executeAlterFieldAttrFunctionTest{
+		private TapAlterFieldAttributesEvent tapAlterFieldAttributesEvent;
+		private TapTable tapTable;
+		@BeforeEach
+		void beforeEach(){
+			tapAlterFieldAttributesEvent = mock(TapAlterFieldAttributesEvent.class);
+			when(tapAlterFieldAttributesEvent.getTableId()).thenReturn("test");
+			dataProcessorContext = mock(DataProcessorContext.class);
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"dataProcessorContext",dataProcessorContext);
+			TapTableMap tapTableMap = mock(TapTableMap.class);
+			when(dataProcessorContext.getTapTableMap()).thenReturn(tapTableMap);
+			tapTable = new TapTable();
+			tapTable.setId("table");
+			LinkedHashMap<String, TapField> nameFieldMap = new LinkedHashMap<>();
+			nameFieldMap.put("field1",mock(TapField.class));
+			tapTable.setNameFieldMap(nameFieldMap);
+			when(tapTableMap.get("test")).thenReturn(tapTable);
+		}
+		@Test
+		@DisplayName("test executeAlterFieldAttrFunction method for build error consumer")
+		void test1(){
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getAlterFieldAttributesFunction()).thenReturn(mock(AlterFieldAttributesFunction.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeAlterFieldAttrFunction(tapAlterFieldAttributesEvent);
+			hazelcastTargetPdkDataNode.executeAlterFieldAttrFunction(tapAlterFieldAttributesEvent);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).buildErrorConsumer("table");
+		}
+	}
+	@Nested
+	class executeDropFieldFunctionTest{
+		private TapDropFieldEvent tapDropFieldEvent;
+		@Test
+		@DisplayName("test executeDropFieldFunction method for build error consumer")
+		void test1(){
+			tapDropFieldEvent = mock(TapDropFieldEvent.class);
+			when(tapDropFieldEvent.getTableId()).thenReturn("test");
+			ConnectorNode connectorNode = mock(ConnectorNode.class);
+			when(hazelcastTargetPdkDataNode.getConnectorNode()).thenReturn(connectorNode);
+			ConnectorFunctions functions = mock(ConnectorFunctions.class);
+			when(connectorNode.getConnectorFunctions()).thenReturn(functions);
+			when(functions.getDropFieldFunction()).thenReturn(mock(DropFieldFunction.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDataFuncAspect(any(Class.class),any(Callable.class),any(CommonUtils.AnyErrorConsumer.class));
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).executeDropFieldFunction(tapDropFieldEvent);
+			hazelcastTargetPdkDataNode.executeDropFieldFunction(tapDropFieldEvent);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).buildErrorConsumer("test");
+		}
+	}
+	@Nested
+	class updateDagTest{
+		@DisplayName("test update Dag when create table Event")
+		@Test
+		void test1(){
+			TapCreateTableEvent tapCreateTableEvent = new TapCreateTableEvent();
+			TapdataEvent tapdataEvent = new TapdataEvent();
+			tapdataEvent.setTapEvent(tapCreateTableEvent);
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).updateDAG(tapdataEvent);
+			hazelcastTargetPdkDataNode.updateDAG(tapdataEvent);
+			verify(hazelcastTargetPdkDataNode,new Times(1)).updateDAG(tapdataEvent);
+		}
+		@DisplayName("test update Dag when tapAlterFieldNameEvent Event when Data Transformation")
+		@Test
+		void test2(){
+			allSetup();
+			Map<String, List<String>> updateConditionFieldsMap = new HashMap<>();
+			List<String> updateConditionFields = new ArrayList<>();
+			updateConditionFields.add("id");
+			updateConditionFieldsMap.put("dummy_test",updateConditionFields);
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"updateConditionFieldsMap",updateConditionFieldsMap);
+			TapAlterFieldNameEvent tapAlterFieldNameEvent=new TapAlterFieldNameEvent();
+			TapdataEvent tapdataEvent = new TapdataEvent();
+			tapdataEvent.setTapEvent(tapAlterFieldNameEvent);
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).updateDAG(tapdataEvent);
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).getNode();
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode, "dataProcessorContext", dataProcessorContext);
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"processorBaseContext",processorBaseContext);
+			hazelcastTargetPdkDataNode.updateDAG(tapdataEvent);
+			TableNode node = (TableNode) processorBaseContext.getNode();
+			List<String> updateConditionFields1 = node.getUpdateConditionFields();
+			assertEquals(updateConditionFields,updateConditionFields1);
+		}
+		@DisplayName("test update Dag when tapAlterFieldNameEvent Event when Data Replications")
+		@Test
+		void test3(){
+			setUpDatabaseNode();
+			Map<String, List<String>> concurrentWritePartitionMap =new HashMap<>();
+			List<String> list=new ArrayList<>();
+			list.add("partition1");
+			concurrentWritePartitionMap.put("dummyTest",list);
+			TapAlterFieldNameEvent tapAlterFieldNameEvent=new TapAlterFieldNameEvent();
+			tapAlterFieldNameEvent.setTableId("dummyTest");
+			TapdataEvent tapdataEvent = new TapdataEvent();
+			tapdataEvent.setTapEvent(tapAlterFieldNameEvent);
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).updateDAG(tapdataEvent);
+			doCallRealMethod().when(hazelcastTargetPdkDataNode).getNode();
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode, "dataProcessorContext", dataProcessorContext);
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"processorBaseContext",processorBaseContext);
+			ReflectionTestUtils.setField(hazelcastTargetPdkDataNode,"concurrentWritePartitionMap",concurrentWritePartitionMap);
+			hazelcastTargetPdkDataNode.updateDAG(tapdataEvent);
+			DatabaseNode node = (DatabaseNode) processorBaseContext.getNode();
+			assertEquals(node.getConcurrentWritePartitionMap().size(),concurrentWritePartitionMap.size());
+		}
+	}
 }
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNodeTest.java b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNodeTest.java
index 83ff5c7c51..bcc9f33b4f 100644
--- a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNodeTest.java
+++ b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/node/hazelcast/processor/HazelcastMergeNodeTest.java
@@ -4,6 +4,8 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.hazelcast.core.HazelcastInstance;
 import com.hazelcast.jet.core.Processor;
+import io.tapdata.entity.schema.value.TapArrayValue;
+import io.tapdata.entity.schema.value.TapMapValue;
 import io.tapdata.utils.AppType;
 import com.tapdata.entity.Connections;
 import com.tapdata.entity.SyncStage;
@@ -1556,6 +1558,12 @@ void testMainProcess() {
 			Map<String, Object> data = new HashMap<String, Object>() {{
 				put("_id", id);
 				put("name", "test");
+				put("subDoc", new Document("subId", id));
+				put("subList", new ArrayList<Object>() {{
+					add(new HashMap<String, Object>() {{
+						put("sub_id", id);
+					}});
+				}});
 			}};
 			mergeLookupResult.setData(data);
 			io.tapdata.pdk.apis.entity.merge.MergeTableProperties mergeTableProperties = new io.tapdata.pdk.apis.entity.merge.MergeTableProperties();
@@ -1590,6 +1598,8 @@ void testMainProcess() {
 			assertTrue(tapType.getFixed());
 			assertEquals(BsonType.OBJECT_ID.name(), tapStringValue.getOriginType());
 			assertNull(tapStringValue.getOriginValue());
+			assertInstanceOf(TapMapValue.class, data.get("subDoc"));
+			assertInstanceOf(TapArrayValue.class, data.get("subList"));
 		}
 
 		@Test
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskServiceTest.java b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskServiceTest.java
index d28fc563b1..622df47ad6 100644
--- a/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskServiceTest.java
+++ b/iengine/iengine-app/src/test/java/io/tapdata/flow/engine/V2/task/impl/HazelcastTaskServiceTest.java
@@ -1,7 +1,9 @@
 package io.tapdata.flow.engine.V2.task.impl;
 
 import com.hazelcast.core.HazelcastInstance;
+import com.mongodb.MongoClientException;
 import com.tapdata.constant.ConfigurationCenter;
+import com.tapdata.constant.ConnectionUtil;
 import com.tapdata.constant.ConnectorConstant;
 import com.tapdata.constant.HazelcastUtil;
 import com.tapdata.entity.Connections;
@@ -10,16 +12,21 @@
 import com.tapdata.entity.task.config.TaskRetryConfig;
 import com.tapdata.mongo.ClientMongoOperator;
 import com.tapdata.mongo.HttpClientMongoOperator;
+import com.tapdata.tm.commons.dag.DAG;
 import com.tapdata.tm.commons.dag.Edge;
 import com.tapdata.tm.commons.dag.Node;
 import com.tapdata.tm.commons.dag.nodes.*;
 import com.tapdata.tm.commons.dag.process.*;
 import com.tapdata.tm.commons.dag.vo.ReadPartitionOptions;
 import com.tapdata.tm.commons.externalStorage.ExternalStorageDto;
+import com.tapdata.tm.commons.schema.TransformerWsMessageDto;
 import com.tapdata.tm.commons.task.dto.ErrorEvent;
 import com.tapdata.tm.commons.task.dto.TaskDto;
+import io.tapdata.MockTaskUtil;
+import io.tapdata.aspect.utils.AspectUtils;
 import io.tapdata.common.SettingService;
 import io.tapdata.entity.schema.TapTable;
+import io.tapdata.exception.TapCodeException;
 import io.tapdata.flow.engine.V2.node.hazelcast.HazelcastBaseNode;
 import io.tapdata.flow.engine.V2.node.hazelcast.data.*;
 import io.tapdata.flow.engine.V2.node.hazelcast.data.pdk.*;
@@ -28,22 +35,25 @@
 import io.tapdata.flow.engine.V2.util.ExternalStorageUtil;
 import io.tapdata.flow.engine.util.TaskDtoUtil;
 import io.tapdata.observable.logging.ObsLogger;
+import io.tapdata.observable.logging.ObsLoggerFactory;
 import io.tapdata.schema.TapTableMap;
+import io.tapdata.schema.TapTableUtil;
 import lombok.SneakyThrows;
 import org.bson.types.ObjectId;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.Nested;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.*;
 
+
 public class HazelcastTaskServiceTest {
     @Nested
     class GetTaskRetryConfigTest{
@@ -439,4 +449,311 @@ void testEmpty() {
             }
         }
     }
+    @Nested
+    class Task2HazelcastDAGTest{
+        HazelcastTaskService hazelcastTaskService;
+        ClientMongoOperator clientMongoOperator;
+        ConfigurationCenter configurationCenter;
+        @BeforeEach
+        void init(){
+            clientMongoOperator = mock(ClientMongoOperator.class);
+            hazelcastTaskService = mock(HazelcastTaskService.class);
+            configurationCenter = mock(ConfigurationCenter.class);
+            ReflectionTestUtils.setField(hazelcastTaskService, "clientMongoOperator", clientMongoOperator);
+            ReflectionTestUtils.setField(hazelcastTaskService, "configurationCenter", configurationCenter);
+        }
+        @DisplayName("test task2HazelcastDAG method when node is tableNode")
+        @Test
+        void test1() {
+            try (MockedStatic<HazelcastTaskService> hazelcastTaskServiceMockedStatic = mockStatic(HazelcastTaskService.class);
+                 MockedStatic<ConnectionUtil> connectionUtilMockedStatic = mockStatic(ConnectionUtil.class)) {
+
+                TaskDto taskDto = MockTaskUtil.setUpTaskDtoByJsonFile();
+                doCallRealMethod().when(hazelcastTaskService).task2HazelcastDAG(taskDto,true);
+
+                when(hazelcastTaskService.getTaskConfig(any())).thenReturn(mock(TaskConfig.class));
+                Connections connections = new Connections();
+                connections.setPdkHash("dummy");
+                when(hazelcastTaskService.getConnection(anyString())).thenReturn(connections);
+                DatabaseTypeEnum.DatabaseType databaseType = mock(DatabaseTypeEnum.DatabaseType.class);
+                connectionUtilMockedStatic.when(() -> ConnectionUtil.getDatabaseType(any(), any())).thenReturn(databaseType);
+
+                HashMap<String, String> stringStringHashMap = new HashMap<>();
+                stringStringHashMap.put("testNodeId", "testQualifiedName");
+                HashMap<String, String> tableNameAndQualifiedNameMap = new HashMap();
+                tableNameAndQualifiedNameMap.put("testNodeId", "1234");
+                TapTableMap<String, TapTable> tapTableMap = TapTableMap.create("testNodeId", tableNameAndQualifiedNameMap);
+                tapTableMap.put("testNodeId", new TapTable());
+
+                when(hazelcastTaskService.getTapTableMap(any(), any(), any(),any())).thenReturn(tapTableMap);
+                hazelcastTaskService.task2HazelcastDAG(taskDto,true);
+                verify(hazelcastTaskService, times(2)).singleTaskFilterEventDataIfNeed(eq(connections), any(), any());
+            }
+        }
+
+        @DisplayName("test task2HazelcastDAG method when node is tableNode")
+        @Test
+        void test2() {
+            try (MockedStatic<HazelcastTaskService> hazelcastTaskServiceMockedStatic = mockStatic(HazelcastTaskService.class);
+                 MockedStatic<ConnectionUtil> connectionUtilMockedStatic = mockStatic(ConnectionUtil.class)) {
+
+                TaskDto taskDto = MockTaskUtil.setUpTaskDtoByJsonFile();
+                doCallRealMethod().when(hazelcastTaskService).task2HazelcastDAG(taskDto,false);
+
+                when(hazelcastTaskService.getTaskConfig(any())).thenReturn(mock(TaskConfig.class));
+                Connections connections = new Connections();
+                connections.setPdkHash("dummy");
+                when(hazelcastTaskService.getConnection(anyString())).thenReturn(connections);
+                DatabaseTypeEnum.DatabaseType databaseType = mock(DatabaseTypeEnum.DatabaseType.class);
+                connectionUtilMockedStatic.when(() -> ConnectionUtil.getDatabaseType(any(), any())).thenReturn(databaseType);
+
+                HashMap<String, String> stringStringHashMap = new HashMap<>();
+                stringStringHashMap.put("testNodeId", "testQualifiedName");
+                HashMap<String, String> tableNameAndQualifiedNameMap = new HashMap();
+                tableNameAndQualifiedNameMap.put("testNodeId", "1234");
+                TapTableMap<String, TapTable> tapTableMap = TapTableMap.create("testNodeId", tableNameAndQualifiedNameMap);
+                tapTableMap.put("testNodeId", new TapTable());
+
+                when(hazelcastTaskService.getTapTableMap(any(), any(), any(),any())).thenReturn(tapTableMap);
+                hazelcastTaskService.task2HazelcastDAG(taskDto,false);
+                verify(hazelcastTaskService, times(2)).singleTaskFilterEventDataIfNeed(eq(connections), any(), any());
+            }
+        }
+    }
+    @Nested
+    class SingleTaskFilterEventDataIfNeedTest{
+        HazelcastTaskService hazelcastTaskService;
+        @BeforeEach
+        void init(){
+            hazelcastTaskService = mock(HazelcastTaskService.class);
+        }
+
+        @DisplayName("test SingleTaskFilterEventDataIfNeed when connections is null")
+        @Test
+        void test1() {
+            AtomicBoolean needFilterEvent = new AtomicBoolean(true);
+            TableNode tableNode = new TableNode();
+            doCallRealMethod().when(hazelcastTaskService).singleTaskFilterEventDataIfNeed(null,needFilterEvent,tableNode);
+            hazelcastTaskService.singleTaskFilterEventDataIfNeed(null, needFilterEvent, tableNode);
+            assertEquals(true, needFilterEvent.get());
+        }
+
+        @DisplayName("test SingleTaskFilterEventDataIfNeed when needFilterEvent is not null")
+        @Test
+        void test2() {
+            Connections connections = new Connections();
+            doCallRealMethod().when(hazelcastTaskService).singleTaskFilterEventDataIfNeed(connections,null,null);
+            assertDoesNotThrow(() -> hazelcastTaskService.singleTaskFilterEventDataIfNeed(connections, null, null));
+        }
+
+        @DisplayName("test SingleTaskFilterEventDataIfNeed when connections is schema-free")
+        @Test
+        void test3() {
+            Connections connections = new Connections();
+            List<String> definitionTags = new ArrayList<>();
+            definitionTags.add("schema-free");
+            connections.setDefinitionTags(definitionTags);
+            AtomicBoolean needFilterEvent = new AtomicBoolean(true);
+            doCallRealMethod().when(hazelcastTaskService).singleTaskFilterEventDataIfNeed(connections,needFilterEvent,null);
+            hazelcastTaskService.singleTaskFilterEventDataIfNeed(connections, needFilterEvent, null);
+            assertEquals(false, needFilterEvent.get());
+        }
+        @DisplayName("test SingleTaskFilterEventDataIfNeed when connections is not schema-free")
+        @Test
+        void test4(){
+            Connections connections = new Connections();
+            List<String> definitionTags = new ArrayList<>();
+            definitionTags.add("Database");
+            definitionTags.add("ssl");
+            definitionTags.add("doubleActive");
+            connections.setDefinitionTags(definitionTags);
+            AtomicBoolean needFilterEvent = new AtomicBoolean(true);
+            doCallRealMethod().when(hazelcastTaskService).singleTaskFilterEventDataIfNeed(connections,needFilterEvent,null);
+            hazelcastTaskService.singleTaskFilterEventDataIfNeed(connections, needFilterEvent, null);
+            assertEquals(true, needFilterEvent.get());
+        }
+        @DisplayName("test SingleTaskFilterEventDataIfNeed when connections is not schema-free and tableNode is enableCustomSql")
+        @Test
+        void test5(){
+            Connections connections = new Connections();
+            List<String> definitionTags = new ArrayList<>();
+            definitionTags.add("Database");
+            definitionTags.add("ssl");
+            definitionTags.add("doubleActive");
+            connections.setDefinitionTags(definitionTags);
+            AtomicBoolean needFilterEvent = new AtomicBoolean(true);
+            TableNode tableNode=new TableNode();
+            tableNode.setEnableCustomCommand(true);
+            doCallRealMethod().when(hazelcastTaskService).singleTaskFilterEventDataIfNeed(connections,needFilterEvent,tableNode);
+            hazelcastTaskService.singleTaskFilterEventDataIfNeed(connections, needFilterEvent, tableNode);
+            assertEquals(false, needFilterEvent.get());
+        }
+        @DisplayName("test SingleTaskFilterEventDataIfNeed when connections tags is null")
+        @Test
+        void test6(){
+            Connections connections = new Connections();
+            connections.setDefinitionTags(null);
+            AtomicBoolean needFilterEvent = new AtomicBoolean(true);
+            doCallRealMethod().when(hazelcastTaskService).singleTaskFilterEventDataIfNeed(connections,needFilterEvent,null);
+            hazelcastTaskService.singleTaskFilterEventDataIfNeed(connections, needFilterEvent, null);
+            assertEquals(true, needFilterEvent.get());
+        }
+
+
+    }
+    @Nested
+    class EngineTransformSchemaTest{
+        HazelcastTaskService hazelcastTaskService;
+        ClientMongoOperator clientMongoOperator;
+        @BeforeEach
+        void init(){
+            clientMongoOperator = mock(ClientMongoOperator.class);
+            hazelcastTaskService = new HazelcastTaskService(clientMongoOperator);
+        }
+
+        @DisplayName("test initializeModel main process")
+        @Test
+        void test1() {
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG dag = mock(DAG.class);
+            taskDto.setDag(dag);
+            taskDto.setSyncType("sync");
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> obsLoggerFactoryMockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<AspectUtils> mockedStatic = mockStatic(AspectUtils.class)){
+                obsLoggerFactoryMockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                TransformerWsMessageDto transformerWsMessageDto = new TransformerWsMessageDto();
+                transformerWsMessageDto.setMetadataInstancesDtoList(new ArrayList<>());
+                transformerWsMessageDto.setUserId("test");
+                DAG.Options options = new DAG.Options();
+                options.setUuid("test");
+                transformerWsMessageDto.setOptions(options);
+                transformerWsMessageDto.setDefinitionDtoMap(new HashMap<>());
+                transformerWsMessageDto.setDataSourceMap(new HashMap<>());
+                transformerWsMessageDto.setTransformerDtoMap(new HashMap<>());
+                transformerWsMessageDto.setTaskDto(taskDto);
+                mockedStatic.when(() -> AspectUtils.executeAspect(any(), any())).thenReturn(null);
+                when(clientMongoOperator.findOne(any(Query.class),any(),any())).thenReturn(transformerWsMessageDto);
+                DAG cloneDag = mock(DAG.class);
+
+                when(dag.clone()).thenReturn(cloneDag);
+                when(cloneDag.transformSchema(any(),any(),any())).thenReturn(new HashMap<>());
+                Map<String,TapTableMap<String, TapTable>> result = hazelcastTaskService.engineTransformSchema(taskDto);
+                Assertions.assertNotNull(result);
+            } catch (CloneNotSupportedException e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+
+
+        @DisplayName("test initializeModel Deduction error")
+        @Test
+        void test2() {
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG dag = mock(DAG.class);
+            taskDto.setDag(dag);
+            taskDto.setSyncType("migrate");
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> obsLoggerFactoryMockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<AspectUtils> mockedStatic = mockStatic(AspectUtils.class)){
+                obsLoggerFactoryMockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                TransformerWsMessageDto transformerWsMessageDto = new TransformerWsMessageDto();
+                transformerWsMessageDto.setMetadataInstancesDtoList(new ArrayList<>());
+                transformerWsMessageDto.setUserId("test");
+                DAG.Options options = new DAG.Options();
+                options.setUuid("test");
+                transformerWsMessageDto.setOptions(options);
+                transformerWsMessageDto.setDefinitionDtoMap(new HashMap<>());
+                transformerWsMessageDto.setDataSourceMap(new HashMap<>());
+                transformerWsMessageDto.setTransformerDtoMap(new HashMap<>());
+                transformerWsMessageDto.setTaskDto(taskDto);
+                mockedStatic.when(() -> AspectUtils.executeAspect(any(), any())).thenReturn(null);
+                when(clientMongoOperator.findOne(any(Query.class),any(),any())).thenReturn(transformerWsMessageDto);
+                DAG cloneDag = mock(DAG.class);
+                when(dag.clone()).thenReturn(cloneDag);
+                doThrow(new MongoClientException("")).when(cloneDag).transformSchema(any(),any(),any(),any());
+                Assertions.assertThrows(TapCodeException.class,()->{
+                    hazelcastTaskService.engineTransformSchema(taskDto);
+                });
+            } catch (CloneNotSupportedException e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+
+        @DisplayName("test initializeModel get TransformerWsMessageDto error")
+        @Test
+        void test3() {
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            DAG dag = mock(DAG.class);
+            taskDto.setDag(dag);
+            taskDto.setSyncType("migrate");
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> obsLoggerFactoryMockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<AspectUtils> mockedStatic = mockStatic(AspectUtils.class)){
+                mockedStatic.when(() -> AspectUtils.executeAspect(any(), any())).thenReturn(null);
+                obsLoggerFactoryMockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                Assertions.assertThrows(TapCodeException.class,()->{
+                    hazelcastTaskService.engineTransformSchema(taskDto);
+                });
+            }
+
+        }
+    }
+
+    @Nested
+    class GetTapTableMapTest{
+        @Test
+        void testTaskTypeIsTestRun(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setSyncType("testRun");
+            DatabaseNode databaseNode = new DatabaseNode();
+            databaseNode.setId("databaseNode");
+            try(MockedStatic<TapTableUtil> tapTableUtilMockedStatic = mockStatic(TapTableUtil.class)){
+                tapTableUtilMockedStatic.when(()->TapTableUtil.getTapTableMapByNodeId(anyString(),any())).thenAnswer(invocationOnMock -> {
+                    Assertions.assertEquals("databaseNode",invocationOnMock.getArgument(0));
+                    return null;
+                });
+                HazelcastTaskService.getTapTableMap(taskDto,1L,databaseNode,new HashMap<>());
+            }
+        }
+
+        @Test
+        void testNormalTask(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setSyncType("sync");
+            DatabaseNode databaseNode = new DatabaseNode();
+            databaseNode.setId("databaseNode");
+            try(MockedStatic<TapTableMap> tableMapMockedStatic = mockStatic(TapTableMap.class)){
+                tableMapMockedStatic.when(()->TapTableMap.create(anyString())).thenAnswer(invocationOnMock -> {
+                    Assertions.assertEquals("databaseNode",invocationOnMock.getArgument(0));
+                    return null;
+                });
+                HazelcastTaskService.getTapTableMap(taskDto,1L,databaseNode,new HashMap<>());
+            }
+        }
+
+        @Test
+        void testNormalTaskTapTableMapHashMapIsNotNull(){
+            TaskDto taskDto = new TaskDto();
+            taskDto.setSyncType("sync");
+            DatabaseNode databaseNode = new DatabaseNode();
+            databaseNode.setId("databaseNode");
+            Map<String, TapTableMap<String, TapTable>> tapTableMapHashMap = new HashMap<>();
+            TapTableMap<String, TapTable> except =  TapTableMap.create("databaseNode");
+            tapTableMapHashMap.put("databaseNode",except);
+            TapTableMap<String, TapTable> result = HazelcastTaskService.getTapTableMap(taskDto,1L,databaseNode,tapTableMapHashMap);
+            Assertions.assertEquals(except,result);
+        }
+    }
 }
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/services/FlameGraphServiceTest.java b/iengine/iengine-app/src/test/java/io/tapdata/services/FlameGraphServiceTest.java
new file mode 100644
index 0000000000..66c803a145
--- /dev/null
+++ b/iengine/iengine-app/src/test/java/io/tapdata/services/FlameGraphServiceTest.java
@@ -0,0 +1,41 @@
+package io.tapdata.services;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class FlameGraphServiceTest {
+    @Test
+    public void testFlameCPU() throws Throwable {
+        // Given
+        try {
+            FlameGraphService flameGraphService = new FlameGraphService();
+            flameGraphService.cpu();
+        } catch (Throwable e) {
+            Assert.fail("FlameGraphService.cpu() should not throw any exceptions");
+        }
+        assert true;
+    }
+
+    @Test
+    public void testFlameMemory() throws Throwable {
+        // Given
+        try {
+            FlameGraphService flameGraphService = new FlameGraphService();
+            flameGraphService.memory();
+        } catch (Throwable e) {
+            Assert.fail("FlameGraphService.memory() should not throw any exceptions");
+        }
+        assert true;
+    }
+
+    @Test
+    public void testFlameJstack() throws Throwable {
+        // Given
+        try {
+            FlameGraphService flameGraphService = new FlameGraphService();
+            flameGraphService.jstack();
+        } catch (Throwable e) {
+            Assert.fail("FlameGraphService.jstack() should not throw any exceptions");
+        }
+        assert true;
+    }
+}
\ No newline at end of file
diff --git a/iengine/iengine-app/src/test/java/io/tapdata/websocket/handler/DeduceSchemaHandlerTest.java b/iengine/iengine-app/src/test/java/io/tapdata/websocket/handler/DeduceSchemaHandlerTest.java
new file mode 100644
index 0000000000..f486b3f8f1
--- /dev/null
+++ b/iengine/iengine-app/src/test/java/io/tapdata/websocket/handler/DeduceSchemaHandlerTest.java
@@ -0,0 +1,102 @@
+package io.tapdata.websocket.handler;
+
+import com.tapdata.constant.BeanUtil;
+import com.tapdata.constant.ConfigurationCenter;
+import com.tapdata.entity.ResponseBody;
+import com.tapdata.mongo.ClientMongoOperator;
+import com.tapdata.tm.autoinspect.utils.GZIPUtil;
+import com.tapdata.tm.commons.dag.DAG;
+import com.tapdata.tm.commons.dag.DmlPolicy;
+import com.tapdata.tm.commons.dag.Edge;
+import com.tapdata.tm.commons.dag.Node;
+import com.tapdata.tm.commons.dag.logCollector.LogCollectorNode;
+import com.tapdata.tm.commons.dag.nodes.TableNode;
+import com.tapdata.tm.commons.task.dto.Dag;
+import com.tapdata.tm.commons.task.dto.TaskDto;
+import com.tapdata.tm.commons.util.JsonUtil;
+import io.tapdata.observable.logging.ObsLogger;
+import io.tapdata.observable.logging.ObsLoggerFactory;
+import io.tapdata.websocket.WebSocketEventResult;
+import org.bson.types.ObjectId;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+public class DeduceSchemaHandlerTest {
+
+    @Nested
+    class HandleTest {
+        DeduceSchemaHandler deduceSchemaHandler;
+        ResponseBody responseBody;
+        Map<String, Object> event;
+        ClientMongoOperator clientMongoOperator;
+        @BeforeEach
+        void before() {
+            deduceSchemaHandler = new DeduceSchemaHandler();
+            responseBody = new ResponseBody();
+            event = new HashMap<>();
+            clientMongoOperator = mock(ClientMongoOperator.class);
+            ReflectionTestUtils.setField(deduceSchemaHandler, "clientMongoOperator", clientMongoOperator);
+        }
+        @Test
+        void test(){
+            DeduceSchemaHandler.DeduceSchemaRequest request = new DeduceSchemaHandler.DeduceSchemaRequest();
+            TaskDto taskDto = new TaskDto();
+            taskDto.setId(new ObjectId());
+            List<Node> nodeList = new ArrayList<>();
+            LogCollectorNode logCollectorNode = new LogCollectorNode();
+            List<String> ids = new ArrayList<>();
+            ids.add("test");
+            logCollectorNode.setId("source123");
+            logCollectorNode.setConnectionIds(ids);
+            TableNode tableNode2 = new TableNode();
+            tableNode2.setId("target123");
+            tableNode2.setDmlPolicy(new DmlPolicy());
+            nodeList.add(tableNode2);
+            nodeList.add(logCollectorNode);
+            Dag dag = new Dag();
+            Edge edge=new Edge("source123","target123");
+            List<Edge> edges = Arrays.asList(edge);
+            dag.setEdges(edges);
+            dag.setNodes(nodeList);
+            DAG mockDag =  DAG.build(dag);
+            taskDto.setDag(mockDag);
+            request.setTaskDto(taskDto);
+            request.setMetadataInstancesDtoList(new ArrayList<>());
+            request.setOptions(new DAG.Options());
+            request.setDataSourceMap(new HashMap<>());
+            request.setDefinitionDtoMap(new HashMap<>());
+            request.setTransformerDtoMap(new HashMap<>());
+            request.setUserId("userId");
+            request.setUserName("name");
+            String jsonResult = JsonUtil.toJsonUseJackson(request);
+            byte[] gzip = GZIPUtil.gzip(jsonResult.getBytes());
+            byte[] encode = Base64.getEncoder().encode(gzip);
+            String dataString = new String(encode, StandardCharsets.UTF_8);
+            event.put("data",dataString);
+            ObsLoggerFactory obsLoggerFactory = mock(ObsLoggerFactory.class);
+            try(MockedStatic<ObsLoggerFactory> mockedStatic = mockStatic(ObsLoggerFactory.class);
+                MockedStatic<BeanUtil> beanUtilMockedStatic = mockStatic(BeanUtil.class)){
+                mockedStatic.when(ObsLoggerFactory::getInstance).thenReturn(obsLoggerFactory);
+                ObsLogger obsLogger = mock(ObsLogger.class);
+                when(obsLoggerFactory.getObsLogger(any(TaskDto.class))).thenReturn(obsLogger);
+                ConfigurationCenter configurationCenter = mock(ConfigurationCenter.class);
+                beanUtilMockedStatic.when(()->BeanUtil.getBean(any())).thenReturn(configurationCenter);
+                when(configurationCenter.getConfig(any())).thenReturn("agentId");
+                WebSocketEventResult result = deduceSchemaHandler.handle(event);
+                Assertions.assertTrue((Boolean) result.getResult());
+            }
+
+
+        }
+    }
+}
diff --git a/iengine/iengine-app/src/test/resources/task/json/migrationdummy2dummy.json b/iengine/iengine-app/src/test/resources/task/json/migrationdummy2dummy.json
new file mode 100644
index 0000000000..d0104ee666
--- /dev/null
+++ b/iengine/iengine-app/src/test/resources/task/json/migrationdummy2dummy.json
@@ -0,0 +1,397 @@
+{
+  "id": "666c1bf713485e58d071fa69",
+  "attrs": {},
+  "crontabExpressionFlag": false,
+  "deduplicWriteMode": "intelligent",
+  "desc": "",
+  "increHysteresis": false,
+  "increOperationMode": false,
+  "increSyncConcurrency": false,
+  "processorThreadNum": 1,
+  "increaseReadSize": 1,
+  "readBatchSize": 500,
+  "writeBatchSize": 0,
+  "writeBatchWaitMs": 0,
+  "isAutoCreateIndex": true,
+  "isFilter": false,
+  "isOpenAutoDDL": false,
+  "isSchedule": false,
+  "isStopOnError": true,
+  "name": "Duplicate-dummy-dummy",
+  "shareCdcEnable": false,
+  "statuses": [],
+  "status": "edit",
+  "syncStatus": "",
+  "type": "initial_sync+cdc",
+  "writeThreadSize": 8,
+  "editVersion": "1718361095826",
+  "syncPoints": [
+    {
+      "nodeId": "0f1bf113-ba08-4afe-a6c4-8054816d0166",
+      "nodeName": "dummy",
+      "connectionId": "659d126ab39d4c6d9e9a2eaa",
+      "connectionName": "dummy",
+      "pointType": "current",
+      "timeZone": "+8"
+    }
+  ],
+  "syncType": "migrate",
+  "transformProcess": 0.0,
+  "planStartDateFlag": false,
+  "accessNodeType": "AUTOMATIC_PLATFORM_ALLOCATION",
+  "accessNodeProcessIdList": [],
+  "transformUuid": "1718361095829",
+  "transformed": true,
+  "transformDagHash": 0,
+  "enforceShareCdc": true,
+  "pageVersion": "1718361057362",
+  "dag": {
+    "edges": [
+      {
+        "disabled": false,
+        "source": "0f1bf113-ba08-4afe-a6c4-8054816d0166",
+        "target": "403c670a-8cc3-4009-a8b7-c0f5e63374ab"
+      }
+    ],
+    "nodes": [
+      {
+        "existDataProcessMode": "keepData",
+        "syncObjects": [
+          {
+            "type": "table",
+            "objectNames": [
+              "dummy_test"
+            ],
+            "tableNameRelation": {
+              "dummy_test": "dummy_test"
+            }
+          }
+        ],
+        "migrateTableSelectType": "custom",
+        "nodeConfig": {},
+        "updateConditionFieldMap": {},
+        "connectionId": "666c1bf313485e58d071fa62",
+        "databaseType": "Dummy",
+        "dmlPolicy": {
+          "insertPolicy": "update_on_exists",
+          "updatePolicy": "ignore_on_nonexists"
+        },
+        "readBatchSize": 500,
+        "increaseReadSize": 1,
+        "writeBatchSize": 100,
+        "writeBatchWaitMs": 500,
+        "writeStrategy": "updateOrInsert",
+        "type": "database",
+        "catalog": "data",
+        "isTransformed": false,
+        "alarmSettings": [
+          {
+            "type": "DATANODE",
+            "open": true,
+            "key": "DATANODE_AVERAGE_HANDLE_CONSUME",
+            "sort": 4,
+            "notify": [
+              "SYSTEM",
+              "EMAIL"
+            ],
+            "interval": 300,
+            "unit": "SECOND"
+          }
+        ],
+        "alarmRules": [
+          {
+            "key": "DATANODE_AVERAGE_HANDLE_CONSUME",
+            "point": 12,
+            "equalsFlag": 1,
+            "ms": 5000
+          }
+        ],
+        "id": "403c670a-8cc3-4009-a8b7-c0f5e63374ab",
+        "name": "targetDummy",
+        "elementType": "Node",
+        "attrs": {
+          "position": [
+            109,
+            23
+          ],
+          "connectionName": "targetDummy",
+          "connectionType": "source_and_target",
+          "accessNodeProcessId": "",
+          "pdkType": "pdk",
+          "pdkHash": "f45ed8744261175abce5f902473472866aa1ffc40cccec0a0d2abceef4a338c2",
+          "capabilities": [
+            {
+              "type": 11,
+              "id": "batch_read_function"
+            },
+            {
+              "type": 11,
+              "id": "stream_read_function"
+            },
+            {
+              "type": 11,
+              "id": "batch_count_function"
+            },
+            {
+              "type": 11,
+              "id": "timestamp_to_stream_offset_function"
+            },
+            {
+              "type": 11,
+              "id": "write_record_function"
+            },
+            {
+              "type": 11,
+              "id": "query_by_advance_filter_function"
+            },
+            {
+              "type": 11,
+              "id": "clear_table_function"
+            },
+            {
+              "type": 11,
+              "id": "drop_table_function"
+            },
+            {
+              "type": 11,
+              "id": "alter_field_attributes_function"
+            },
+            {
+              "type": 11,
+              "id": "alter_field_name_function"
+            },
+            {
+              "type": 11,
+              "id": "drop_field_function"
+            },
+            {
+              "type": 11,
+              "id": "new_field_function"
+            },
+            {
+              "type": 11,
+              "id": "get_table_names_function"
+            },
+            {
+              "type": 11,
+              "id": "error_handle_function"
+            },
+            {
+              "type": 20,
+              "id": "master_slave_merge"
+            },
+            {
+              "type": 20,
+              "id": "dynamic_schema"
+            }
+          ],
+          "accessNodeType": "AUTOMATIC_PLATFORM_ALLOCATION"
+        },
+        "disabled": false
+      },
+      {
+        "existDataProcessMode": "keepData",
+        "tableNames": [
+          "dummy_test"
+        ],
+        "migrateTableSelectType": "custom",
+        "tableExpression": ".*",
+        "noPrimaryKeyTableSelectType": "All",
+        "nodeConfig": {},
+        "connectionId": "659d126ab39d4c6d9e9a2eaa",
+        "databaseType": "Dummy",
+        "ddlConfiguration": "FILTER",
+        "readBatchSize": 100,
+        "increaseReadSize": 1,
+        "writeStrategy": "updateOrInsert",
+        "type": "database",
+        "catalog": "data",
+        "isTransformed": false,
+        "alarmSettings": [
+          {
+            "type": "DATANODE",
+            "open": true,
+            "key": "DATANODE_AVERAGE_HANDLE_CONSUME",
+            "sort": 4,
+            "notify": [
+              "SYSTEM",
+              "EMAIL"
+            ],
+            "interval": 300,
+            "unit": "SECOND"
+          }
+        ],
+        "alarmRules": [
+          {
+            "key": "DATANODE_AVERAGE_HANDLE_CONSUME",
+            "point": 12,
+            "equalsFlag": 1,
+            "ms": 5000
+          }
+        ],
+        "id": "0f1bf113-ba08-4afe-a6c4-8054816d0166",
+        "name": "dummy",
+        "elementType": "Node",
+        "attrs": {
+          "position": [
+            -487,
+            34
+          ],
+          "connectionName": "dummy",
+          "connectionType": "source_and_target",
+          "accessNodeProcessId": "",
+          "pdkType": "pdk",
+          "pdkHash": "f45ed8744261175abce5f902473472866aa1ffc40cccec0a0d2abceef4a338c2",
+          "capabilities": [
+            {
+              "type": 11,
+              "id": "batch_read_function"
+            },
+            {
+              "type": 11,
+              "id": "stream_read_function"
+            },
+            {
+              "type": 11,
+              "id": "batch_count_function"
+            },
+            {
+              "type": 11,
+              "id": "timestamp_to_stream_offset_function"
+            },
+            {
+              "type": 11,
+              "id": "write_record_function"
+            },
+            {
+              "type": 11,
+              "id": "query_by_advance_filter_function"
+            },
+            {
+              "type": 11,
+              "id": "clear_table_function"
+            },
+            {
+              "type": 11,
+              "id": "drop_table_function"
+            },
+            {
+              "type": 11,
+              "id": "alter_field_attributes_function"
+            },
+            {
+              "type": 11,
+              "id": "alter_field_name_function"
+            },
+            {
+              "type": 11,
+              "id": "drop_field_function"
+            },
+            {
+              "type": 11,
+              "id": "new_field_function"
+            },
+            {
+              "type": 11,
+              "id": "get_table_names_function"
+            },
+            {
+              "type": 11,
+              "id": "error_handle_function"
+            },
+            {
+              "type": 20,
+              "id": "master_slave_merge"
+            },
+            {
+              "type": 20,
+              "id": "dynamic_schema"
+            }
+          ],
+          "accessNodeType": "AUTOMATIC_PLATFORM_ALLOCATION"
+        },
+        "disabled": false
+      }
+    ]
+  },
+  "shareCache": false,
+  "canOpenInspect": false,
+  "isAutoInspect": false,
+  "skipErrorEvent": {
+    "errorMode": "Disable",
+    "limitMode": "SkipByLimit",
+    "limitModeEnum": "SkipByLimit",
+    "errorModeEnum": "Disable"
+  },
+  "showInspectTips": false,
+  "alarmSettings": [
+    {
+      "type": "TASK",
+      "open": true,
+      "key": "TASK_STATUS_ERROR",
+      "sort": 1,
+      "notify": [
+        "SYSTEM",
+        "EMAIL"
+      ],
+      "interval": 1,
+      "unit": "SECOND"
+    },
+    {
+      "type": "TASK",
+      "open": true,
+      "key": "TASK_FULL_COMPLETE",
+      "sort": 3,
+      "notify": [
+        "SYSTEM"
+      ],
+      "interval": 1,
+      "unit": "SECOND"
+    },
+    {
+      "type": "TASK",
+      "open": true,
+      "key": "TASK_INCREMENT_START",
+      "sort": 4,
+      "notify": [
+        "SYSTEM",
+        "EMAIL"
+      ],
+      "interval": 300,
+      "unit": "SECOND"
+    },
+    {
+      "type": "TASK",
+      "open": true,
+      "key": "TASK_INCREMENT_DELAY",
+      "sort": 6,
+      "notify": [
+        "SYSTEM",
+        "EMAIL"
+      ],
+      "interval": 300,
+      "unit": "SECOND"
+    }
+  ],
+  "alarmRules": [
+    {
+      "key": "TASK_INCREMENT_DELAY",
+      "point": 60,
+      "equalsFlag": 1,
+      "ms": 60000
+    }
+  ],
+  "testTaskId": "666c1bf713485e58d071fa67",
+  "transformTaskId": "666c1bf713485e58d071fa68",
+  "stopRetryTimes": 0,
+  "dynamicAdjustMemoryUsage": false,
+  "doubleActive": false,
+  "autoInspect": false,
+  "snapShotInterrupt": false,
+  "testTask": false,
+  "deduceSchemaTask": false,
+  "normalTask": true,
+  "_deleted": false,
+  "createTime": 1718361079551,
+  "last_updated": 1718361177988
+}
\ No newline at end of file
diff --git a/iengine/iengine-common/src/test/java/com/tapdata/processor/ScriptUtilTest.java b/iengine/iengine-common/src/test/java/com/tapdata/processor/ScriptUtilTest.java
index 6655fc12dd..4c4f0efb96 100644
--- a/iengine/iengine-common/src/test/java/com/tapdata/processor/ScriptUtilTest.java
+++ b/iengine/iengine-common/src/test/java/com/tapdata/processor/ScriptUtilTest.java
@@ -5,14 +5,17 @@
 import com.tapdata.entity.Job;
 import com.tapdata.entity.MessageEntity;
 import com.tapdata.mongo.ClientMongoOperator;
+import com.tapdata.processor.constant.JSEngineEnum;
 import io.tapdata.entity.logger.Log;
 import io.tapdata.exception.TapCodeException;
+import lombok.SneakyThrows;
 import org.apache.logging.log4j.core.Logger;
 import org.junit.Assert;
-import org.junit.Test;
 import org.junit.jupiter.api.Assertions;
-import org.junit.runner.RunWith;
-import org.mockito.junit.MockitoJUnitRunner;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.mockito.internal.verification.Times;
 
 import javax.script.Invocable;
 import java.net.URL;
@@ -21,9 +24,8 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.*;
 
-@RunWith(MockitoJUnitRunner.class)
 public class ScriptUtilTest {
 
     @Test
@@ -40,7 +42,7 @@ public void testGetScriptEngine3() throws Exception {
     public void testInvokeScript()  {
         Map<String,Object> input = new HashMap<>();
         Assertions.assertThrows(TapCodeException.class,()-> ScriptUtil.invokeScript(null,"functionName", mock(MessageEntity.class)
-                , mock(Connections.class), mock(Connections.class), mock(Job.class),input, mock(Logger.class)));
+                , mock(Connections.class), mock(Connections.class), mock(Job.class),input, mock(Logger.class), null));
     }
     @Test
     public void testUrlClassLoader(){
@@ -50,5 +52,63 @@ public void testUrlClassLoader(){
         ScriptUtil.urlClassLoader(urlClassLoader -> externalClassLoader[0] = urlClassLoader,urlList);
         Assert.assertNotNull(externalClassLoader[0]);
     }
+    @Nested
+    public class InvokeScriptWithTagTest{
+        private Map<String,Object> input;
+        private Invocable engine;
+        private MessageEntity message;
+        @BeforeEach
+        @SneakyThrows
+        void beforeEach(){
+            input = new HashMap<>();
+            engine = spy(ScriptUtil.getScriptEngine(
+                    JSEngineEnum.GRAALVM_JS.getEngineName(),
+                    "",
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    true));
+            message = mock(MessageEntity.class);
 
+        }
+        @Test
+        @SneakyThrows
+        public void testInvokeScriptWithBefore(){
+            when(message.getBefore()).thenReturn(mock(HashMap.class));
+            when(message.getAfter()).thenReturn(mock(HashMap.class));
+            ScriptUtil.invokeScript(engine,"process", message
+                    , mock(Connections.class), mock(Connections.class), mock(Job.class),input, mock(Logger.class), "before");
+            verify(message,new Times(2)).getAfter();
+            verify(message,new Times(3)).getBefore();
+        }
+        @Test
+        @SneakyThrows
+        public void testInvokeScriptWithoutBefore(){
+            when(message.getAfter()).thenReturn(mock(HashMap.class));
+            ScriptUtil.invokeScript(engine,"process", message
+                    , mock(Connections.class), mock(Connections.class), mock(Job.class),input, mock(Logger.class), "before");
+            verify(message,new Times(3)).getAfter();
+            verify(message,new Times(2)).getBefore();
+        }
+        @Test
+        @SneakyThrows
+        public void testInvokeScriptWithAfter(){
+            ScriptUtil.invokeScript(engine,"process", message
+                    , mock(Connections.class), mock(Connections.class), mock(Job.class),input, mock(Logger.class), "after");
+            verify(message,new Times(2)).getBefore();
+            verify(message,new Times(1)).getAfter();
+        }
+        @Test
+        @SneakyThrows
+        public void testInvokeScriptWithNull(){
+            ScriptUtil.invokeScript(engine,"process", message
+                    , mock(Connections.class), mock(Connections.class), mock(Job.class),input, mock(Logger.class), null);
+            verify(message,new Times(2)).getBefore();
+            verify(message,new Times(1)).getAfter();
+        }
+
+    }
 }
diff --git a/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/FieldDataFlowProcessorTest.java b/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/FieldDataFlowProcessorTest.java
new file mode 100644
index 0000000000..b9d8ff296e
--- /dev/null
+++ b/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/FieldDataFlowProcessorTest.java
@@ -0,0 +1,134 @@
+package com.tapdata.processor.dataflow;
+
+import com.tapdata.entity.FieldProcess;
+import com.tapdata.entity.MessageEntity;
+import com.tapdata.processor.ScriptUtil;
+import com.tapdata.processor.constant.JSEngineEnum;
+import lombok.SneakyThrows;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.Times;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import javax.script.Invocable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+public class FieldDataFlowProcessorTest {
+    private FieldDataFlowProcessor fieldDataFlowProcessor;
+    @BeforeEach
+    void beforeEach(){
+        fieldDataFlowProcessor = mock(FieldDataFlowProcessor.class);
+    }
+    @Nested
+    class processTest{
+        private MessageEntity message;
+        private List<FieldProcess> fieldProcesses;
+        @BeforeEach
+        void beforeEach() {
+            message = mock(MessageEntity.class);
+            fieldProcesses = new ArrayList<>();
+            ReflectionTestUtils.setField(fieldDataFlowProcessor, "fieldProcesses", fieldProcesses);
+        }
+        @Test
+        @SneakyThrows
+        void testSimple(){
+            when(message.getOp()).thenReturn("u");
+            Map<String, Object> before = mock(HashMap.class);
+            Map<String, Object> after = mock(HashMap.class);
+            when(message.getBefore()).thenReturn(before);
+            when(message.getAfter()).thenReturn(after);
+            doNothing().when(fieldDataFlowProcessor).fieldScript(any(MessageEntity.class),any(Map.class),anyString());
+            doCallRealMethod().when(fieldDataFlowProcessor).process(message);
+            fieldDataFlowProcessor.process(message);
+            verify(fieldDataFlowProcessor, new Times(1)).fieldScript(message,before,"before");
+            verify(fieldDataFlowProcessor, new Times(1)).fieldScript(message,after,"after");
+        }
+        @Test
+        @SneakyThrows
+        void testSimpleWithoutBefore(){
+            when(message.getOp()).thenReturn("u");
+            Map<String, Object> before = mock(HashMap.class);
+            Map<String, Object> after = mock(HashMap.class);
+            when(message.getBefore()).thenReturn(null);
+            when(message.getAfter()).thenReturn(after);
+            doNothing().when(fieldDataFlowProcessor).fieldScript(any(MessageEntity.class),any(Map.class),anyString());
+            doCallRealMethod().when(fieldDataFlowProcessor).process(message);
+            fieldDataFlowProcessor.process(message);
+            verify(fieldDataFlowProcessor, new Times(0)).fieldScript(message,before,"before");
+            verify(fieldDataFlowProcessor, new Times(1)).fieldScript(message,after,"after");
+        }
+        @Test
+        @SneakyThrows
+        void testSimpleWithoutAfter(){
+            when(message.getOp()).thenReturn("u");
+            Map<String, Object> before = mock(HashMap.class);
+            Map<String, Object> after = mock(HashMap.class);
+            when(message.getBefore()).thenReturn(before);
+            when(message.getAfter()).thenReturn(null);
+            doNothing().when(fieldDataFlowProcessor).fieldScript(any(MessageEntity.class),any(Map.class),anyString());
+            doCallRealMethod().when(fieldDataFlowProcessor).process(message);
+            fieldDataFlowProcessor.process(message);
+            verify(fieldDataFlowProcessor, new Times(1)).fieldScript(message,before,"before");
+            verify(fieldDataFlowProcessor, new Times(0)).fieldScript(message,after,"after");
+        }
+    }
+    @Nested
+    class fieldScriptTest{
+        private MessageEntity message;
+        private Map<String, Object> record;
+        private String tag;
+        private Map<String, Invocable> fieldScriptEngine;
+        private ProcessorContext context;
+        private Map<String, Object> processContext;
+        private Invocable engine;
+        private Logger logger;
+        @BeforeEach
+        @SneakyThrows
+        void beforeEach(){
+            message = mock(MessageEntity.class);
+            record = mock(HashMap.class);
+            tag = null;
+            fieldScriptEngine = new HashMap<>();
+            engine = spy(ScriptUtil.getScriptEngine(
+                    JSEngineEnum.GRAALVM_JS.getEngineName(),
+                    "",
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    true));
+            fieldScriptEngine.put("doc_id", engine);
+            ReflectionTestUtils.setField(fieldDataFlowProcessor,"fieldScriptEngine",fieldScriptEngine);
+            context = mock(ProcessorContext.class);
+            ReflectionTestUtils.setField(fieldDataFlowProcessor,"context",context);
+            processContext = mock(HashMap.class);
+            ReflectionTestUtils.setField(fieldDataFlowProcessor,"processContext",processContext);
+            logger = mock(Logger.class);
+            ReflectionTestUtils.setField(fieldDataFlowProcessor,"logger",logger);
+        }
+        @Test
+        void testSimple(){
+            try (MockedStatic<ScriptUtil> mb = Mockito
+                    .mockStatic(ScriptUtil.class)) {
+                mb.when(()->ScriptUtil.invokeScript(engine,ScriptUtil.FUNCTION_NAME, message, context.getSourceConn(),
+                        context.getTargetConn(), context.getJob(), processContext, logger, tag)).thenReturn(null);
+                doCallRealMethod().when(fieldDataFlowProcessor).fieldScript(message,record,tag);
+                fieldDataFlowProcessor.fieldScript(message,record,tag);
+                mb.verify(() -> ScriptUtil.invokeScript(engine,ScriptUtil.FUNCTION_NAME, message, context.getSourceConn(),
+                        context.getTargetConn(), context.getJob(), processContext, logger, tag),new Times(1));
+            }
+        }
+    }
+}
diff --git a/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/RowFilterProcessorTest.java b/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/RowFilterProcessorTest.java
new file mode 100644
index 0000000000..24d3481bae
--- /dev/null
+++ b/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/RowFilterProcessorTest.java
@@ -0,0 +1,76 @@
+package com.tapdata.processor.dataflow;
+
+import com.tapdata.entity.MessageEntity;
+import com.tapdata.processor.ScriptUtil;
+import com.tapdata.processor.constant.JSEngineEnum;
+import lombok.SneakyThrows;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.Times;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import javax.script.Invocable;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+public class RowFilterProcessorTest {
+    private RowFilterProcessor rowFilterProcessor;
+    @BeforeEach
+    void beforeEach(){
+        rowFilterProcessor = mock(RowFilterProcessor.class);
+    }
+    @Nested
+    class processTest{
+        private MessageEntity message;
+        private ProcessorContext context;
+        private Map<String, Object> processContext;
+        private Invocable engine;
+        private Logger logger;
+        @BeforeEach
+        @SneakyThrows
+        void beforeEach() {
+            message = mock(MessageEntity.class);
+            engine = spy(ScriptUtil.getScriptEngine(
+                    JSEngineEnum.GRAALVM_JS.getEngineName(),
+                    "",
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    true));
+            ReflectionTestUtils.setField(rowFilterProcessor,"engine",engine);
+            context = mock(ProcessorContext.class);
+            ReflectionTestUtils.setField(rowFilterProcessor,"context",context);
+            processContext = mock(HashMap.class);
+            ReflectionTestUtils.setField(rowFilterProcessor,"processContext",processContext);
+            logger = mock(Logger.class);
+            ReflectionTestUtils.setField(rowFilterProcessor,"logger",logger);
+        }
+        @Test
+        @SneakyThrows
+        void testSimple(){
+            try (MockedStatic<ScriptUtil> mb = Mockito
+                    .mockStatic(ScriptUtil.class)) {
+                mb.when(()->ScriptUtil.invokeScript(engine, "filter", message, context.getSourceConn(),
+                        context.getTargetConn(), context.getJob(), processContext, logger, null)).thenReturn(null);
+                when(message.getOp()).thenReturn("u");
+                Map<String, Object> before = mock(HashMap.class);
+                Map<String, Object> after = mock(HashMap.class);
+                when(message.getBefore()).thenReturn(before);
+                when(message.getAfter()).thenReturn(after);
+                doCallRealMethod().when(rowFilterProcessor).process(message);
+                rowFilterProcessor.process(message);
+                mb.verify(() -> ScriptUtil.invokeScript(engine, "filter", message, context.getSourceConn(),
+                        context.getTargetConn(), context.getJob(), processContext, logger, null),new Times(1));
+            }
+        }
+    }
+}
diff --git a/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/ScriptDataFlowProcessorTest.java b/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/ScriptDataFlowProcessorTest.java
new file mode 100644
index 0000000000..798e8fb108
--- /dev/null
+++ b/iengine/iengine-common/src/test/java/com/tapdata/processor/dataflow/ScriptDataFlowProcessorTest.java
@@ -0,0 +1,77 @@
+package com.tapdata.processor.dataflow;
+
+import com.tapdata.entity.MessageEntity;
+import com.tapdata.processor.ScriptUtil;
+import com.tapdata.processor.constant.JSEngineEnum;
+import lombok.SneakyThrows;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.Times;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import javax.script.Invocable;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+public class ScriptDataFlowProcessorTest {
+    private ScriptDataFlowProcessor scriptDataFlowProcessor;
+    @BeforeEach
+    void beforeEach(){
+        scriptDataFlowProcessor = mock(ScriptDataFlowProcessor.class);
+    }
+    @Nested
+    class processTest{
+        private MessageEntity message;
+        private ProcessorContext context;
+        private Map<String, Object> processContext;
+        private Invocable engine;
+        private Logger logger;
+        @BeforeEach
+        @SneakyThrows
+        void beforeEach() {
+            message = mock(MessageEntity.class);
+            engine = spy(ScriptUtil.getScriptEngine(
+                    JSEngineEnum.GRAALVM_JS.getEngineName(),
+                    "",
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    true));
+            ReflectionTestUtils.setField(scriptDataFlowProcessor,"engine",engine);
+            context = mock(ProcessorContext.class);
+            ReflectionTestUtils.setField(scriptDataFlowProcessor,"context",context);
+            processContext = mock(HashMap.class);
+            ReflectionTestUtils.setField(scriptDataFlowProcessor,"processContext",processContext);
+            logger = mock(Logger.class);
+            ReflectionTestUtils.setField(scriptDataFlowProcessor,"logger",logger);
+        }
+        @Test
+        @SneakyThrows
+        void testSimple(){
+            try (MockedStatic<ScriptUtil> mb = Mockito
+                    .mockStatic(ScriptUtil.class)) {
+                mb.when(()->ScriptUtil.invokeScript(engine, "process", message, context.getSourceConn(),
+                        context.getTargetConn(), context.getJob(), processContext, logger, null)).thenReturn(null);
+                when(message.getOp()).thenReturn("u");
+                Map<String, Object> before = mock(HashMap.class);
+                Map<String, Object> after = mock(HashMap.class);
+                when(message.getBefore()).thenReturn(before);
+                when(message.getAfter()).thenReturn(after);
+                doCallRealMethod().when(scriptDataFlowProcessor).process(message);
+                scriptDataFlowProcessor.process(message);
+                mb.verify(() -> ScriptUtil.invokeScript(engine, "process", message, context.getSourceConn(),
+                        context.getTargetConn(), context.getJob(), processContext, logger, null),new Times(1));
+            }
+        }
+    }
+}
+
diff --git a/iengine/iengine-oss/src/test/java/io/tapdata/inspect/InspectServiceImplTest.java b/iengine/iengine-oss/src/test/java/io/tapdata/inspect/InspectServiceImplTest.java
new file mode 100644
index 0000000000..91b0039f27
--- /dev/null
+++ b/iengine/iengine-oss/src/test/java/io/tapdata/inspect/InspectServiceImplTest.java
@@ -0,0 +1,21 @@
+package io.tapdata.inspect;
+
+import com.tapdata.entity.inspect.Inspect;
+import com.tapdata.entity.inspect.InspectResult;
+import io.tapdata.exception.TapOssNonsupportFunctionException;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class InspectServiceImplTest {
+   @Test
+    void testHandleDiffInspect(){
+       InspectServiceImpl inspectService = new InspectServiceImpl();
+       try {
+           inspectService.handleDiffInspect(new Inspect(), 10,new InspectResult());
+       }catch (Exception e) {
+           Assertions.assertTrue(e instanceof TapOssNonsupportFunctionException);
+       }
+   }
+
+
+}
diff --git a/iengine/modules/milestone-module/src/test/java/io/tapdata/milestone/MilestoneAspectTaskTest.java b/iengine/modules/milestone-module/src/test/java/io/tapdata/milestone/MilestoneAspectTaskTest.java
index 9564af70cb..22793b4a2f 100644
--- a/iengine/modules/milestone-module/src/test/java/io/tapdata/milestone/MilestoneAspectTaskTest.java
+++ b/iengine/modules/milestone-module/src/test/java/io/tapdata/milestone/MilestoneAspectTaskTest.java
@@ -20,18 +20,7 @@
 import io.tapdata.aspect.TaskStartAspect;
 import io.tapdata.aspect.TaskStopAspect;
 import io.tapdata.aspect.task.AspectTask;
-import io.tapdata.aspect.taskmilestones.CDCReadBeginAspect;
-import io.tapdata.aspect.taskmilestones.CDCReadErrorAspect;
-import io.tapdata.aspect.taskmilestones.CDCReadStartedAspect;
-import io.tapdata.aspect.taskmilestones.CDCWriteBeginAspect;
-import io.tapdata.aspect.taskmilestones.Snapshot2CDCAspect;
-import io.tapdata.aspect.taskmilestones.SnapshotReadBeginAspect;
-import io.tapdata.aspect.taskmilestones.SnapshotReadEndAspect;
-import io.tapdata.aspect.taskmilestones.SnapshotReadErrorAspect;
-import io.tapdata.aspect.taskmilestones.SnapshotReadTableEndAspect;
-import io.tapdata.aspect.taskmilestones.SnapshotWriteBeginAspect;
-import io.tapdata.aspect.taskmilestones.SnapshotWriteEndAspect;
-import io.tapdata.aspect.taskmilestones.WriteErrorAspect;
+import io.tapdata.aspect.taskmilestones.*;
 import io.tapdata.entity.aspect.Aspect;
 import io.tapdata.entity.aspect.AspectInterceptResult;
 import io.tapdata.entity.logger.Log;
@@ -69,6 +58,7 @@
 import static io.tapdata.aspect.DataFunctionAspect.STATE_END;
 import static io.tapdata.aspect.DataFunctionAspect.STATE_START;
 import static io.tapdata.aspect.TableInitFuncAspect.STATE_PROCESS;
+import static io.tapdata.aspect.taskmilestones.EngineDeductionAspect.*;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyMap;
@@ -1335,5 +1325,34 @@ void testCDCFINISH() {
 			Assertions.assertEquals(MilestoneAspectTask.KPI_CDC, milestoneAspectTask.getTaskSyncStatus());
 		}
 	}
+	@Nested
+	class handleEngineDeductionTest{
+		EngineDeductionAspect aspect;
+		MilestoneAspectTask milestoneAspectTask = new MilestoneAspectTask();
+		@BeforeEach
+		void init() {
+			aspect = mock(EngineDeductionAspect.class);
+			when(aspect.getState()).thenReturn(DEDUCTION_START);
+			when(milestoneAspectTask.handleEngineDeduction(aspect)).thenCallRealMethod();
+		}
+
+
+		@Test
+		void testDEDUCTION_START() {
+			Assertions.assertNull(milestoneAspectTask.handleEngineDeduction(aspect));
+		}
+
+		@Test
+		void testDEDUCTION_END() {
+			when(aspect.getState()).thenReturn(DEDUCTION_END);
+			Assertions.assertNull(milestoneAspectTask.handleEngineDeduction(aspect));
+		}
+
+		@Test
+		void testDEDUCTION_ERROR() {
+			when(aspect.getState()).thenReturn(DEDUCTION_ERROR);
+			Assertions.assertNull(milestoneAspectTask.handleEngineDeduction(aspect));
+		}
+	}
 
 }
diff --git a/manager/tm-api/src/test/java/com/tapdata/tm/utils/QueryUtilTest.java b/manager/tm-api/src/test/java/com/tapdata/tm/utils/QueryUtilTest.java
index ad4828627b..a3d194bab4 100644
--- a/manager/tm-api/src/test/java/com/tapdata/tm/utils/QueryUtilTest.java
+++ b/manager/tm-api/src/test/java/com/tapdata/tm/utils/QueryUtilTest.java
@@ -1,9 +1,15 @@
 package com.tapdata.tm.utils;
 
+import com.tapdata.tm.base.dto.Filter;
 import com.tapdata.tm.base.dto.Where;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.query.Query;
+
+import java.util.List;
 
 class QueryUtilTest {
     @Nested
@@ -13,4 +19,31 @@ void test() {
             Assertions.assertNotNull(QueryUtil.parseWhereToCriteria(new Where()));
         }
     }
+
+    @Nested
+    class parsePageParam {
+        @Test
+        void testNormal() {
+            Filter filter = new Filter();
+            Query query = new Query();
+            Assertions.assertDoesNotThrow(() -> QueryUtil.parsePageParam(filter, query));
+        }
+    }
+
+    @Nested
+    class parseOrder {
+        Filter filter;
+        @BeforeEach
+        void init() {
+            filter = new Filter();
+            filter.setSort(Lists.newArrayList("add DESC", "edd ASC", "err", "kk cc"));
+        }
+
+        @Test
+        void testNormal() {
+            List<Sort> sorts = QueryUtil.parseOrder(filter);
+            Assertions.assertNotNull(sorts);
+            Assertions.assertEquals(3, sorts.size());
+        }
+    }
 }
\ No newline at end of file
diff --git a/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGDataServiceImplTest.java b/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGDataServiceImplTest.java
index 59673f9467..3920bc4450 100644
--- a/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGDataServiceImplTest.java
+++ b/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGDataServiceImplTest.java
@@ -3,29 +3,22 @@
 import com.tapdata.tm.commons.dag.deduction.rule.ChangeRuleStage;
 import com.tapdata.tm.commons.dag.vo.FieldChangeRule;
 import com.tapdata.tm.commons.dag.vo.FieldChangeRuleGroup;
-import com.tapdata.tm.commons.schema.DataSourceConnectionDto;
-import com.tapdata.tm.commons.schema.Field;
-import com.tapdata.tm.commons.schema.MetadataInstancesDto;
-import com.tapdata.tm.commons.schema.Schema;
+import com.tapdata.tm.commons.schema.*;
+import com.tapdata.tm.commons.schema.bean.SourceTypeEnum;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.mockito.MockedStatic;
+import org.mockito.internal.stubbing.answers.CallsRealMethods;
+import org.springframework.test.util.ReflectionTestUtils;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+
+import java.util.*;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
 
 
 class DAGDataServiceImplTest {
@@ -92,5 +85,73 @@ void testModelDeductionNotIsomorphismTask() {
             }
         }
     }
+    @Nested
+    class clearTransformerTest{
+        DAGDataServiceImpl dagDataService = new DAGDataServiceImpl(mock(TransformerWsMessageDto.class));
+        @Test
+        void test(){
+            dagDataService.clearTransformer();
+            Assertions.assertTrue(dagDataService.getBatchMetadataUpdateMap().isEmpty());
+            Assertions.assertTrue(dagDataService.getBatchInsertMetaDataList().isEmpty());
+            Assertions.assertTrue(dagDataService.getUpsertItems().isEmpty());
+        }
+    }
+    @Nested
+    class logCollectorMetadataInstancesDtoTest{
+        DAGDataServiceImpl dagDataService = new DAGDataServiceImpl(mock(TransformerWsMessageDto.class));
+        Map<String, MetadataInstancesDto> metadataMap = new HashMap<>();
+        @Test
+        void test(){
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("LogCollectorNode");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+            metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto2.setNodeId("test");
+            metadataInstancesDto2.setOriginalName("name2");
+            metadataInstancesDto2.setQualifiedName("test2");
+            metadataInstancesDto2.setMetaType("table");
+            metadataMap.put("node1",metadataInstancesDto1);
+            metadataMap.put("node2",metadataInstancesDto2);
+            ReflectionTestUtils.setField(dagDataService,"metadataMap",metadataMap);
+            Assertions.assertEquals(2,dagDataService.getLogCollectorMetadataInstancesDto().size());
+        }
+
+        @Test
+        void testMetaDataMapIsNull(){
+            ReflectionTestUtils.setField(dagDataService,"metadataMap",metadataMap);
+            Assertions.assertEquals(0,dagDataService.getLogCollectorMetadataInstancesDto().size());
+        }
+
+        @Test
+        void testQualifiedNameIsNullOne(){
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("LogCollectorNode");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+            metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto2.setNodeId("test");
+            metadataInstancesDto2.setOriginalName("name2");
+            metadataInstancesDto2.setQualifiedName("test2");
+            metadataInstancesDto2.setMetaType("table");
+            MetadataInstancesDto metadataInstancesDto3 = new MetadataInstancesDto();
+            metadataInstancesDto3.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto3.setNodeId("test");
+            metadataInstancesDto3.setOriginalName("name2");
+            metadataInstancesDto3.setQualifiedName("test2");
+            metadataInstancesDto3.setMetaType("table");
+            metadataMap.put("node1",metadataInstancesDto1);
+            metadataMap.put("node2",metadataInstancesDto2);
+            metadataMap.put("node3",metadataInstancesDto3);
+            ReflectionTestUtils.setField(dagDataService,"metadataMap",metadataMap);
+            Assertions.assertEquals(2,dagDataService.getLogCollectorMetadataInstancesDto().size());
+        }
+    }
 
 }
\ No newline at end of file
diff --git a/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGTest.java b/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGTest.java
index 2654567c26..b55a52186c 100644
--- a/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGTest.java
+++ b/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/DAGTest.java
@@ -2,10 +2,14 @@
 
 import com.tapdata.tm.commons.dag.nodes.DataParentNode;
 import com.tapdata.tm.commons.dag.process.JsProcessorNode;
+import com.tapdata.tm.commons.task.dto.Message;
+import org.bson.types.ObjectId;
 import org.junit.jupiter.api.*;
+import org.springframework.test.util.ReflectionTestUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
@@ -254,4 +258,25 @@ void testSetIsomorphismValueToOptionsWithNullOptions() {
         }
 
     }
+    @Nested
+    class TransformSchemaTest{
+        @BeforeEach
+        void setUp(){
+            ReflectionTestUtils.setField(dag,"taskId",new ObjectId());
+        }
+        @Test
+        void testConsumerIsNotNull(){
+            doCallRealMethod().when(dag).transformSchema(any(),any(DAGDataService.class),any(DAG.Options.class),any());
+            Assertions.assertThrows(RuntimeException.class,()->{
+                dag.transformSchema("test",mock(DAGDataServiceImpl.class),mock(DAG.Options.class),(e) -> {throw new RuntimeException(e);});
+            });
+        }
+
+        @Test
+        void testConsumerIsNull(){
+            doCallRealMethod().when(dag).transformSchema(any(),any(DAGDataService.class),any(DAG.Options.class),any());
+            Map<String,List<Message>> result = dag.transformSchema("test",mock(DAGDataServiceImpl.class),mock(DAG.Options.class),null);
+            Assertions.assertEquals(1,result.size());
+        }
+    }
 }
\ No newline at end of file
diff --git a/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/nodes/DatabaseNodeTest.java b/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/nodes/DatabaseNodeTest.java
new file mode 100644
index 0000000000..f232f6337c
--- /dev/null
+++ b/manager/tm-common/src/test/java/com/tapdata/tm/commons/dag/nodes/DatabaseNodeTest.java
@@ -0,0 +1,64 @@
+package com.tapdata.tm.commons.dag.nodes;
+
+import com.tapdata.tm.commons.dag.DAG;
+import com.tapdata.tm.commons.dag.DAGDataServiceImpl;
+import io.github.openlg.graphlib.Graph;
+import org.junit.jupiter.api.*;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.*;
+
+public class DatabaseNodeTest {
+    DatabaseNode databaseNode;
+
+    @BeforeEach
+    void init() {
+        databaseNode = mock(DatabaseNode.class);
+    }
+    @Nested
+    class transformSchemaTest {
+        @DisplayName("test transformSchema main process ")
+        @Test
+        void test() {
+            DAGDataServiceImpl dagDataService = mock(DAGDataServiceImpl.class);
+            ReflectionTestUtils.setField(databaseNode, "tableNames", new ArrayList<>());
+            ReflectionTestUtils.setField(databaseNode, "service", dagDataService);
+            DAG.Options options = new DAG.Options();
+            options.setBatchNum(1);
+            doCallRealMethod().when(databaseNode).transformSchema(options);
+            List<String> tableNames = new ArrayList<>();
+            tableNames.add("test1");
+            Graph graph = mock(Graph.class);
+            when(databaseNode.getGraph()).thenReturn(graph);
+            when(graph.predecessors(any())).thenReturn(new ArrayList<>());
+            when(databaseNode.getSourceNodeTableNames(any())).thenReturn(tableNames);
+            databaseNode.transformSchema(options);
+            verify(dagDataService, times(0)).initializeModel(false);
+
+        }
+
+        @DisplayName("test transformSchema initializeModel ")
+        @Test
+        void test1() {
+            DAGDataServiceImpl dagDataService = mock(DAGDataServiceImpl.class);
+            ReflectionTestUtils.setField(databaseNode, "tableNames", new ArrayList<>());
+            ReflectionTestUtils.setField(databaseNode, "service", dagDataService);
+            DAG.Options options = new DAG.Options();
+            options.setBatchNum(1);
+            doCallRealMethod().when(databaseNode).transformSchema(options);
+            List<String> tableNames = new ArrayList<>();
+            tableNames.add("test1");
+            Graph graph = mock(Graph.class);
+            when(databaseNode.getGraph()).thenReturn(graph);
+            when(graph.predecessors(any())).thenReturn(new ArrayList<>());
+            when(databaseNode.getSourceNodeTableNames(any())).thenReturn(tableNames);
+            when(dagDataService.whetherEngineDeduction()).thenReturn(true);
+            databaseNode.transformSchema(options);
+            verify(dagDataService, times(1)).initializeModel(false);
+
+        }
+    }
+}
diff --git a/manager/tm-common/src/test/java/com/tapdata/tm/commons/util/FilterMetadataInstanceUtilTest.java b/manager/tm-common/src/test/java/com/tapdata/tm/commons/util/FilterMetadataInstanceUtilTest.java
new file mode 100644
index 0000000000..c3b6e4eca5
--- /dev/null
+++ b/manager/tm-common/src/test/java/com/tapdata/tm/commons/util/FilterMetadataInstanceUtilTest.java
@@ -0,0 +1,108 @@
+package com.tapdata.tm.commons.util;
+
+import com.alibaba.fastjson.JSON;
+import com.tapdata.tm.commons.schema.Field;
+import com.tapdata.tm.commons.schema.MetadataInstancesDto;
+import com.tapdata.tm.commons.schema.TableIndex;
+import com.tapdata.tm.commons.schema.TableIndexColumn;
+import io.tapdata.entity.conversion.PossibleDataTypes;
+import io.tapdata.entity.schema.type.TapRaw;
+import io.tapdata.entity.schema.type.TapString;
+import io.tapdata.entity.schema.type.TapType;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+public class FilterMetadataInstanceUtilTest {
+    @Test
+    void testFilterTypeRawAndPossibleDataTypeIsNull(){
+        MetadataInstancesDto metadataInstancesDto = new MetadataInstancesDto();
+        List<Field> fields = new ArrayList<>();
+        Field field1 = new Field();
+        field1.setFieldName("test1");
+        field1.setTapType("{\"type\":8}");
+        field1.setDeleted(false);
+        Field field2 = new Field();
+        field2.setFieldName("test2");
+        field2.setTapType("{\"type\":8}");
+        field2.setDeleted(false);
+        Field field3 = new Field();
+        field3.setFieldName("test3");
+        field3.setTapType("{\"type\":8}");
+        field3.setDeleted(false);
+        fields.add(field1);
+        fields.add(field2);
+        fields.add(field3);
+        metadataInstancesDto.setFields(fields);
+        List<TableIndex> indices = new ArrayList<>();
+        TableIndex tableIndex = new TableIndex();
+        List<TableIndexColumn> columns = new ArrayList<>();
+        TableIndexColumn column1 = new TableIndexColumn();
+        column1.setColumnName("test1");
+        TableIndexColumn column2 = new TableIndexColumn();
+        column2.setColumnName("test2");
+        columns.add(column1);
+        columns.add(column2);
+        tableIndex.setColumns(columns);
+        indices.add(tableIndex);
+        metadataInstancesDto.setIndices(indices);
+        Map<String, PossibleDataTypes> dataTypes = new HashMap<>();
+        dataTypes.put("test1",new PossibleDataTypes().dataType("String"));
+        dataTypes.put("test3",new PossibleDataTypes());
+        metadataInstancesDto.setFindPossibleDataTypes(dataTypes);
+        try(MockedStatic<JSON> jsonMockedStatic = mockStatic(JSON.class)){
+            jsonMockedStatic.when(()->JSON.parseObject("{\"type\":8}", TapType.class)).thenReturn(new TapString()).thenReturn(new TapRaw()).thenReturn(new TapString());
+            FilterMetadataInstanceUtil.filterMetadataInstancesFields(metadataInstancesDto);
+            Assertions.assertTrue(metadataInstancesDto.getFields().contains(field1));
+        }
+
+    }
+
+    @Test
+    void testDataTypesIsNull(){
+        MetadataInstancesDto metadataInstancesDto = new MetadataInstancesDto();
+        List<Field> fields = new ArrayList<>();
+        Field field1 = new Field();
+        field1.setFieldName("test1");
+        field1.setTapType("{\"type\":8}");
+        field1.setDeleted(false);
+        Field field2 = new Field();
+        field2.setFieldName("test2");
+        field2.setTapType("{\"type\":8}");
+        field2.setDeleted(false);
+        Field field3 = new Field();
+        field3.setFieldName("test3");
+        field3.setTapType("{\"type\":8}");
+        field3.setDeleted(false);
+        fields.add(field1);
+        fields.add(field2);
+        fields.add(field3);
+        metadataInstancesDto.setFields(fields);
+        try(MockedStatic<JSON> jsonMockedStatic = mockStatic(JSON.class)){
+            jsonMockedStatic.when(()->JSON.parseObject("{\"type\":8}",TapType.class)).thenReturn(new TapString()).thenReturn(new TapRaw()).thenReturn(new TapString());
+            Assertions.assertTrue(metadataInstancesDto.getFields().contains(field1));
+            Assertions.assertTrue(metadataInstancesDto.getFields().contains(field2));
+            Assertions.assertTrue(metadataInstancesDto.getFields().contains(field3));
+        }
+    }
+
+    @Test
+    @DisplayName("convert TapTable Test Fields is null  ")
+    void testFieldsIsNull(){
+        MetadataInstancesDto metadataInstancesDto = new MetadataInstancesDto();
+        metadataInstancesDto.setFields(new ArrayList<>());
+        try(MockedStatic<JSON> jsonMockedStatic = mockStatic(JSON.class)){
+            jsonMockedStatic.when(()->JSON.parseObject("{\"type\":8}",TapType.class)).thenReturn(new TapString()).thenReturn(new TapRaw()).thenReturn(new TapString());
+            Assertions.assertTrue(metadataInstancesDto.getFields().isEmpty());
+        }
+
+    }
+}
diff --git a/manager/tm-oss/src/test/java/com/tapdata/tm/inspect/service/InspectServiceImplTest.java b/manager/tm-oss/src/test/java/com/tapdata/tm/inspect/service/InspectServiceImplTest.java
index 8dbd96a15b..27c3528c8d 100644
--- a/manager/tm-oss/src/test/java/com/tapdata/tm/inspect/service/InspectServiceImplTest.java
+++ b/manager/tm-oss/src/test/java/com/tapdata/tm/inspect/service/InspectServiceImplTest.java
@@ -2,6 +2,7 @@
 
 import com.tapdata.tm.base.dto.Field;
 import com.tapdata.tm.base.dto.Filter;
+import com.tapdata.tm.base.dto.Page;
 import com.tapdata.tm.base.dto.Where;
 import com.tapdata.tm.base.exception.BizException;
 import com.tapdata.tm.commons.task.dto.TaskDto;
@@ -18,7 +19,9 @@
 import org.junit.jupiter.api.Test;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.springframework.data.mongodb.core.query.Query;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.mockito.ArgumentMatchers.any;
@@ -28,6 +31,7 @@
 import static org.mockito.Mockito.doCallRealMethod;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class InspectServiceImplTest {
@@ -90,14 +94,10 @@ void list() {
         when(impl.list(any(Filter.class), any(UserDetail.class))).thenCallRealMethod();
         try(MockedStatic<MessageUtil> messageUtilMockedStatic = Mockito.mockStatic(MessageUtil.class)) {
             messageUtilMockedStatic.when(() -> MessageUtil.getMessage(anyString())).thenReturn("error");
-            Assertions.assertThrows(BizException.class, () -> {
-                try {
-                    impl.list(mock(Filter.class), mock(UserDetail.class));
-                } catch (BizException e) {
-                    Assertions.assertEquals(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION, e.getErrorCode());
-                    throw e;
-                }
-            });
+            Page<InspectDto> list = impl.list(mock(Filter.class), mock(UserDetail.class));
+            Assertions.assertEquals(0L, list.getTotal());
+            Assertions.assertNotNull(list.getItems());
+            Assertions.assertEquals(0, list.getItems().size());
         }
     }
 
@@ -151,17 +151,12 @@ void createCheckByTask() {
 
     @Test
     void findByTaskIdList() {
+        when(impl.findAll(any(Query.class))).thenReturn(new ArrayList<>());
         when(impl.findByTaskIdList(anyList())).thenCallRealMethod();
         try(MockedStatic<MessageUtil> messageUtilMockedStatic = Mockito.mockStatic(MessageUtil.class)) {
             messageUtilMockedStatic.when(() -> MessageUtil.getMessage(anyString())).thenReturn("error");
-            Assertions.assertThrows(BizException.class, () -> {
-                try {
-                    impl.findByTaskIdList(mock(List.class));
-                } catch (BizException e) {
-                    Assertions.assertEquals(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION, e.getErrorCode());
-                    throw e;
-                }
-            });
+            Assertions.assertDoesNotThrow(() -> impl.findByTaskIdList(mock(List.class)));
+            verify(impl).findAll(any(Query.class));
         }
     }
 
@@ -170,14 +165,7 @@ void deleteByTaskId() {
         when(impl.deleteByTaskId(anyString())).thenCallRealMethod();
         try(MockedStatic<MessageUtil> messageUtilMockedStatic = Mockito.mockStatic(MessageUtil.class)) {
             messageUtilMockedStatic.when(() -> MessageUtil.getMessage(anyString())).thenReturn("error");
-            Assertions.assertThrows(BizException.class, () -> {
-                try {
-                    impl.deleteByTaskId("id");
-                } catch (BizException e) {
-                    Assertions.assertEquals(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION, e.getErrorCode());
-                    throw e;
-                }
-            });
+            Assertions.assertDoesNotThrow(() -> impl.deleteByTaskId("id"));
         }
     }
 
@@ -250,14 +238,7 @@ void setRepeatInspectTask() {
         doCallRealMethod().when(impl).setRepeatInspectTask();
         try(MockedStatic<MessageUtil> messageUtilMockedStatic = Mockito.mockStatic(MessageUtil.class)) {
             messageUtilMockedStatic.when(() -> MessageUtil.getMessage(anyString())).thenReturn("error");
-            Assertions.assertThrows(BizException.class, () -> {
-                try {
-                    impl.setRepeatInspectTask();
-                } catch (BizException e) {
-                    Assertions.assertEquals(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION, e.getErrorCode());
-                    throw e;
-                }
-            });
+            Assertions.assertDoesNotThrow(impl::setRepeatInspectTask);
         }
     }
 
@@ -346,14 +327,7 @@ void cleanDeadInspect() {
         doCallRealMethod().when(impl).cleanDeadInspect();
         try(MockedStatic<MessageUtil> messageUtilMockedStatic = Mockito.mockStatic(MessageUtil.class)) {
             messageUtilMockedStatic.when(() -> MessageUtil.getMessage(anyString())).thenReturn("error");
-            Assertions.assertThrows(BizException.class, () -> {
-                try {
-                    impl.cleanDeadInspect();
-                } catch (BizException e) {
-                    Assertions.assertEquals(ConstVariable.TA_OSS_NON_SUPPORT_FUNCTION_EXCEPTION, e.getErrorCode());
-                    throw e;
-                }
-            });
+            Assertions.assertDoesNotThrow(impl::cleanDeadInspect);
         }
     }
 
diff --git a/manager/tm/src/test/java/com/tapdata/tm/measurementServiceV2/controller/MeasureControllerTest.java b/manager/tm/src/test/java/com/tapdata/tm/measurementServiceV2/controller/MeasureControllerTest.java
new file mode 100644
index 0000000000..05d6c61d89
--- /dev/null
+++ b/manager/tm/src/test/java/com/tapdata/tm/measurementServiceV2/controller/MeasureControllerTest.java
@@ -0,0 +1,32 @@
+package com.tapdata.tm.measurementServiceV2.controller;
+
+import com.tapdata.tm.base.dto.ResponseMessage;
+import com.tapdata.tm.monitor.controller.MeasureController;
+import com.tapdata.tm.monitor.entity.MeasurementEntity;
+import com.tapdata.tm.monitor.service.MeasurementServiceV2;
+import com.tapdata.tm.monitor.service.MeasurementServiceV2Impl;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MeasureControllerTest {
+
+    @Test
+    void testFindLastMinuteByTaskId(){
+
+        MeasureController measureController = new MeasureController();
+        MeasurementServiceV2 measurementServiceV2 = mock(MeasurementServiceV2Impl.class);
+        String taskId = "66616f30c7a14d46120ef0b1";
+          ReflectionTestUtils.setField(measureController, "measurementServiceV2", measurementServiceV2);
+        MeasurementEntity measurementEntity = new MeasurementEntity();
+        String taskIdEntity = "66616f30c7a14d46120ef0b2";
+        measurementEntity.setId(taskIdEntity);
+        when(measurementServiceV2.findLastMinuteByTaskId(taskId)).thenReturn(measurementEntity);
+        ResponseMessage<MeasurementEntity> responseMessage = measureController.findLastMinuteByTaskId("{\"where\":{\"taskId\":\"66616f30c7a14d46120ef0b1\"}}");
+        Assertions.assertTrue(taskIdEntity.equals(responseMessage.getData().getId()));
+
+    }
+}
diff --git a/manager/tm/src/test/java/com/tapdata/tm/monitor/service/MeasurementServiceV2ImplTest.java b/manager/tm/src/test/java/com/tapdata/tm/monitor/service/MeasurementServiceV2ImplTest.java
index 014574a65c..538ad8a58f 100644
--- a/manager/tm/src/test/java/com/tapdata/tm/monitor/service/MeasurementServiceV2ImplTest.java
+++ b/manager/tm/src/test/java/com/tapdata/tm/monitor/service/MeasurementServiceV2ImplTest.java
@@ -5,22 +5,23 @@
 import com.tapdata.tm.monitor.constant.KeyWords;
 import com.tapdata.tm.monitor.entity.MeasurementEntity;
 import com.tapdata.tm.monitor.vo.TableSyncStaticVo;
+import com.tapdata.tm.task.bean.TableStatusInfoDto;
+import com.tapdata.tm.task.service.TaskService;
+import io.tapdata.common.sample.request.Sample;
+import io.tapdata.common.sample.request.SampleRequest;
 import org.bson.types.ObjectId;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Nested;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
+import org.mockito.internal.verification.Times;
+import org.springframework.data.mongodb.core.BulkOperations;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyList;
-import static org.mockito.ArgumentMatchers.anyString;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.*;
 import static org.mockito.Mockito.doCallRealMethod;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.mock;
@@ -36,9 +37,9 @@ void init() {
     }
     @Test
     void testParams() {
-        Assertions.assertEquals("initial_sync", KeyWords.INITIAL_SYNC);
-        Assertions.assertEquals("DONE", KeyWords.DONE);
-        Assertions.assertEquals("complete", KeyWords.COMPLETE);
+        assertEquals("initial_sync", KeyWords.INITIAL_SYNC);
+        assertEquals("DONE", KeyWords.DONE);
+        assertEquals("complete", KeyWords.COMPLETE);
     }
 
     @Nested
@@ -117,4 +118,155 @@ void testOneHasResult() {
             verify(mongoOperations).findOne(any(Query.class), any(Class.class), anyString());
         }
     }
+    @Nested
+    class AddBulkAgentMeasurementTest{
+        private MongoTemplate mongoOperations;
+        private TaskService taskService;
+        private Map<String, Long> taskDelayTimeMap;
+        private SampleRequest sampleRequest;
+        private List<SampleRequest> samples;
+        @BeforeEach
+        void beforeEach(){
+            mongoOperations = mock(MongoTemplate.class);
+            ReflectionTestUtils.setField(measurementServiceV2,"mongoOperations",mongoOperations);
+            when(mongoOperations.bulkOps(BulkOperations.BulkMode.UNORDERED, MeasurementEntity.class, MeasurementEntity.COLLECTION_NAME)).thenReturn(mock(BulkOperations.class));
+            taskService = mock(TaskService.class);
+            ReflectionTestUtils.setField(measurementServiceV2,"taskService",taskService);
+            taskDelayTimeMap = new HashMap<>();
+            ReflectionTestUtils.setField(measurementServiceV2,"taskDelayTimeMap",taskDelayTimeMap);
+            samples = new ArrayList<>();
+            sampleRequest = mock(SampleRequest.class);
+            samples.add(sampleRequest);
+            Map<String, String> tags = new HashMap<>();
+            String id = "665d2b9b889245e73373cf49";
+            tags.put("type","task");
+            tags.put("taskId",id);
+            when(sampleRequest.getTags()).thenReturn(tags);
+        }
+        @Test
+        @DisplayName("test addBulkAgentMeasurement method normal")
+        void test1(){
+            Sample sample = new Sample();
+            Date date = new Date();
+            sample.setDate(date);
+            Map vs = new HashMap();
+            vs.put("replicateLag",111);
+            sample.setVs(vs);
+            when(sampleRequest.getSample()).thenReturn(sample);
+            doCallRealMethod().when(measurementServiceV2).addAgentMeasurement(samples);
+            measurementServiceV2.addAgentMeasurement(samples);
+            verify(taskService).updateDelayTime(new ObjectId("665d2b9b889245e73373cf49"), 111L);
+        }
+        @Test
+        @DisplayName("test addBulkAgentMeasurement method when replicateLag is null")
+        void test2(){
+            Sample sample = new Sample();
+            Date date = new Date();
+            sample.setDate(date);
+            Map vs = new HashMap();
+            vs.put("replicateLag",null);
+            sample.setVs(vs);
+            when(sampleRequest.getSample()).thenReturn(sample);
+            doCallRealMethod().when(measurementServiceV2).addAgentMeasurement(samples);
+            measurementServiceV2.addAgentMeasurement(samples);
+            verify(taskService).updateDelayTime(new ObjectId("665d2b9b889245e73373cf49"), 0L);
+        }
+        @Test
+        @DisplayName("test addBulkAgentMeasurement method when taskDelayTimeMap contains taskId")
+        void test3(){
+            Sample sample = new Sample();
+            Date date = new Date();
+            sample.setDate(date);
+            Map vs = new HashMap();
+            vs.put("replicateLag",null);
+            sample.setVs(vs);
+            when(sampleRequest.getSample()).thenReturn(sample);
+            samples.add(sampleRequest);
+            doCallRealMethod().when(measurementServiceV2).addAgentMeasurement(samples);
+            measurementServiceV2.addAgentMeasurement(samples);
+            verify(taskService, new Times(1)).updateDelayTime(new ObjectId("665d2b9b889245e73373cf49"), 0L);
+        }
+        @Test
+        @DisplayName("test addBulkAgentMeasurement method when taskDelayTimeMap not contains taskId")
+        void test4(){
+            Sample sample = new Sample();
+            Date date = new Date();
+            sample.setDate(date);
+            Map vs = new HashMap();
+            vs.put("replicateLag",null);
+            sample.setVs(vs);
+            when(sampleRequest.getSample()).thenReturn(sample);
+            SampleRequest sampleRequest1 = mock(SampleRequest.class);
+            Map<String, String> tags = new HashMap<>();
+            String id = "665d2b9b889245e73373cf50";
+            tags.put("type","task");
+            tags.put("taskId",id);
+            when(sampleRequest1.getTags()).thenReturn(tags);
+            Sample sample1 = new Sample();
+            Date date1 = new Date();
+            sample1.setDate(date1);
+            Map vs1 = new HashMap();
+            vs1.put("replicateLag",111);
+            sample1.setVs(vs1);
+            when(sampleRequest1.getSample()).thenReturn(sample1);
+            samples.add(sampleRequest1);
+            SampleRequest sampleRequest2 = mock(SampleRequest.class);
+            when(sampleRequest2.getTags()).thenReturn(null);
+            samples.add(sampleRequest2);
+            doCallRealMethod().when(measurementServiceV2).addAgentMeasurement(samples);
+            measurementServiceV2.addAgentMeasurement(samples);
+            verify(taskService, new Times(1)).updateDelayTime(new ObjectId("665d2b9b889245e73373cf49"), 0L);
+            verify(taskService, new Times(1)).updateDelayTime(new ObjectId("665d2b9b889245e73373cf50"), 111L);
+            verify(taskService, new Times(2)).updateDelayTime(any(ObjectId.class),anyLong());
+        }
+        @Test
+        @DisplayName("test addBulkAgentMeasurement method when type is table")
+        void test5(){
+            Map<String, String> tags = new HashMap<>();
+            String id = "665d2b9b889245e73373cf49";
+            tags.put("type","table");
+            tags.put("taskId",id);
+            when(sampleRequest.getTags()).thenReturn(tags);
+            Sample sample = new Sample();
+            Date date = new Date();
+            sample.setDate(date);
+            Map vs = new HashMap();
+            vs.put("replicateLag",null);
+            sample.setVs(vs);
+            when(sampleRequest.getSample()).thenReturn(sample);
+            doCallRealMethod().when(measurementServiceV2).addAgentMeasurement(samples);
+            measurementServiceV2.addAgentMeasurement(samples);
+            verify(taskService,new Times(0)).updateDelayTime(new ObjectId("665d2b9b889245e73373cf49"), 0L);
+        }
+    }
+    @Nested
+    class QueryTableMeasurementTest{
+        private String taskId;
+        private TableStatusInfoDto tableStatusInfoDto;
+        private MongoTemplate mongoOperations;
+        @BeforeEach
+        void beforeEach(){
+            taskId = "111";
+            tableStatusInfoDto = new TableStatusInfoDto();
+            mongoOperations = mock(MongoTemplate.class);
+            ReflectionTestUtils.setField(measurementServiceV2,"mongoOperations",mongoOperations);
+        }
+        @Test
+        void testQueryTableMeasurement(){
+            MeasurementEntity measurementEntity = new MeasurementEntity();
+            List<Sample> samples = new ArrayList<>();
+            Sample sample = new Sample();
+            Map<String, Number> vs = new HashMap<>();
+            vs.put("replicateLag",111);
+            vs.put("currentEventTimestamp",1717413352);
+            sample.setVs(vs);
+            samples.add(sample);
+            measurementEntity.setSamples(samples);
+            when(mongoOperations.findOne(any(Query.class),any(Class.class),anyString())).thenReturn(measurementEntity);
+            doCallRealMethod().when(measurementServiceV2).queryTableMeasurement(taskId,tableStatusInfoDto);
+            measurementServiceV2.queryTableMeasurement(taskId,tableStatusInfoDto);
+            assertEquals(111,tableStatusInfoDto.getCdcDelayTime());
+            assertEquals(new Date(1717413352L),tableStatusInfoDto.getLastDataChangeTime());
+        }
+    }
 }
\ No newline at end of file
diff --git a/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceImplTest.java b/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceImplTest.java
index fb3b5bb766..ae3f7eb29c 100644
--- a/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceImplTest.java
+++ b/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceImplTest.java
@@ -49,6 +49,7 @@
 import com.tapdata.tm.metadatainstance.service.MetadataInstancesServiceImpl;
 import com.tapdata.tm.monitor.entity.MeasurementEntity;
 import com.tapdata.tm.monitor.param.IdParam;
+import com.tapdata.tm.monitor.service.BatchService;
 import com.tapdata.tm.monitor.service.MeasurementServiceV2;
 import com.tapdata.tm.monitoringlogs.service.MonitoringLogsService;
 import com.tapdata.tm.permissions.DataPermissionHelper;
@@ -74,10 +75,7 @@
 import com.tapdata.tm.task.service.batchin.entity.ParseParam;
 import com.tapdata.tm.task.service.chart.ChartViewService;
 import com.tapdata.tm.task.service.utils.TaskServiceUtil;
-import com.tapdata.tm.task.vo.ShareCacheDetailVo;
-import com.tapdata.tm.task.vo.ShareCacheVo;
-import com.tapdata.tm.task.vo.TaskDetailVo;
-import com.tapdata.tm.task.vo.TaskStatsDto;
+import com.tapdata.tm.task.vo.*;
 import com.tapdata.tm.transform.service.MetadataTransformerService;
 import com.tapdata.tm.user.service.UserService;
 import com.tapdata.tm.userLog.service.UserLogService;
@@ -99,11 +97,13 @@
 import org.mockito.internal.verification.Times;
 import org.quartz.CronScheduleBuilder;
 import org.springframework.beans.BeanUtils;
+import org.springframework.core.io.InputStreamResource;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.aggregation.AggregationResults;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.http.ResponseEntity;
 import org.springframework.mock.web.MockMultipartFile;
 import org.springframework.test.util.ReflectionTestUtils;
 import org.springframework.web.context.request.RequestContextHolder;
@@ -118,10 +118,12 @@
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static com.tapdata.tm.task.service.TaskServiceImpl.AGENT_ID;
 import static org.junit.jupiter.api.Assertions.*;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.mockito.Mockito.*;
@@ -130,6 +132,9 @@
 
 class TaskServiceImplTest {
     TaskServiceImpl taskService;
+    TaskRecordService taskRecordService;
+    BatchService batchService;
+    MonitoringLogsService monitoringLogsService;
     AgentGroupService agentGroupService;
     WorkerService workerService;
     TaskDto taskDto;
@@ -138,6 +143,9 @@ class TaskServiceImplTest {
     @BeforeEach
     void init() {
         taskService = mock(TaskServiceImpl.class);
+        taskRecordService = mock(TaskRecordService.class);
+        monitoringLogsService = mock(MonitoringLogsService.class);
+        batchService = mock(BatchService.class);
         agentGroupService = mock(AgentGroupService.class);
         ReflectionTestUtils.setField(taskService, "agentGroupService", agentGroupService);
         workerService = mock(WorkerService.class);
@@ -562,17 +570,14 @@ void testDataPermissionFindByIdNormal(){
     }
     @Nested
     class CreateTest{
-        private CustomSqlService customSqlService;
         private DateNodeService dateNodeService;
         private TransformSchemaService transformSchemaService;
         @BeforeEach
         void setUp(){
             taskDto = mock(TaskDto.class);
             user = mock(UserDetail.class);
-            customSqlService = mock(CustomSqlService.class);
             dateNodeService = mock(DateNodeService.class);
             transformSchemaService = mock(TransformSchemaService.class);
-            ReflectionTestUtils.setField(taskService, "customSqlService",customSqlService);
             ReflectionTestUtils.setField(taskService, "dateNodeService",dateNodeService);
             ReflectionTestUtils.setField(taskService, "transformSchemaService",transformSchemaService);
         }
@@ -590,7 +595,6 @@ void test1(){
             ObjectId taskId = mock(ObjectId.class);
             when(taskDto.getId()).thenReturn(taskId);
             doNothing().when(taskService).checkTaskName("test", user, taskId);
-            doNothing().when(customSqlService).checkCustomSqlTask(taskDto, user);
             doNothing().when(dateNodeService).checkTaskDateNode(taskDto, user);
             when(taskService.save(taskDto,user)).thenReturn(taskDto);
             doNothing().when(transformSchemaService).transformSchema(dag, user, taskId);
@@ -676,24 +680,27 @@ void test4(){
     @Nested
     class UpdateByIdTest{
         private MeasurementServiceV2 measurementServiceV2;
-        private CustomSqlService customSqlService;
         private DateNodeService dateNodeService;
         private TaskSaveService taskSaveService;
         private TransformSchemaAsyncService transformSchemaAsyncService;
+        private TransformSchemaService transformSchemaService;
+        private WorkerService workerService;
         @BeforeEach
         void setUp() {
             taskDto = mock(TaskDto.class);
             user = mock(UserDetail.class);
             measurementServiceV2 = mock(MeasurementServiceV2.class);
-            customSqlService = mock(CustomSqlService.class);
             dateNodeService = mock(DateNodeService.class);
             taskSaveService = mock(TaskSaveService.class);
             transformSchemaAsyncService = mock(TransformSchemaAsyncService.class);
-            ReflectionTestUtils.setField(taskService, "customSqlService",customSqlService);
+            transformSchemaService = mock(TransformSchemaService.class);
+            workerService = mock(WorkerService.class);
             ReflectionTestUtils.setField(taskService, "dateNodeService",dateNodeService);
             ReflectionTestUtils.setField(taskService, "measurementServiceV2", measurementServiceV2);
             ReflectionTestUtils.setField(taskService, "taskSaveService", taskSaveService);
+            ReflectionTestUtils.setField(taskService, "transformSchemaService", transformSchemaService);
             ReflectionTestUtils.setField(taskService, "transformSchemaAsyncService", transformSchemaAsyncService);
+            ReflectionTestUtils.setField(taskService, "workerService", workerService);
         }
         @Test
         @DisplayName("test updateById method when cron expression occur error")
@@ -760,6 +767,59 @@ void test4(){
             taskService.updateById(taskDto,user);
             verify(taskService, new Times(1)).create(taskDto, user);
         }
+
+        @Test
+        @DisplayName("test Old version engine ")
+        void test5(){
+            doNothing().when(taskService).checkTaskInspectFlag(taskDto);
+            ObjectId taskId = mock(ObjectId.class);
+            when(taskDto.getId()).thenReturn(taskId);
+            TaskDto oldTask = mock(TaskDto.class);
+            when(taskService.findById(taskId)).thenReturn(oldTask);
+            when(taskDto.getSyncType()).thenReturn("sync");
+            when(taskDto.getType()).thenReturn("initial_sync");
+            DAG newDag = mock(DAG.class);
+            when(taskDto.getDag()).thenReturn(newDag);
+            when(oldTask.getTaskRecordId()).thenReturn("111");
+            List<String> runTables = new ArrayList<>();
+            runTables.add("test_table");
+            when(workerService.checkEngineVersion(any())).thenReturn(false);
+            doNothing().when(transformSchemaService).transformSchema(any(), any(), any());
+            when(measurementServiceV2.findRunTable(taskId.toHexString(),"111")).thenReturn(runTables);
+            LinkedList<DatabaseNode> newSourceNode = new LinkedList();
+            newSourceNode.add(mock(DatabaseNode.class));
+            when(newDag.getSourceNode()).thenReturn(newSourceNode);
+            doCallRealMethod().when(taskService).updateById(taskDto, user);
+            taskService.updateById(taskDto,user);
+            verify(taskService, new Times(1)).save(taskDto, user);
+            verify(transformSchemaService,times(1)).transformSchema(any(),any(),any());
+        }
+        @Test
+        @DisplayName("test new version engine ")
+        void test6(){
+            doNothing().when(taskService).checkTaskInspectFlag(taskDto);
+            ObjectId taskId = mock(ObjectId.class);
+            when(taskDto.getId()).thenReturn(taskId);
+            TaskDto oldTask = mock(TaskDto.class);
+            when(taskService.findById(taskId)).thenReturn(oldTask);
+            when(taskDto.getSyncType()).thenReturn("sync");
+            when(taskDto.getType()).thenReturn("initial_sync");
+            DAG newDag = mock(DAG.class);
+            when(taskDto.getDag()).thenReturn(newDag);
+            when(oldTask.getTaskRecordId()).thenReturn("111");
+            List<String> runTables = new ArrayList<>();
+            runTables.add("test_table");
+            when(workerService.checkEngineVersion(any())).thenReturn(true);
+            doNothing().when(transformSchemaAsyncService).transformSchema(any(DAG.class), any(), any());
+            when(measurementServiceV2.findRunTable(taskId.toHexString(),"111")).thenReturn(runTables);
+            LinkedList<DatabaseNode> newSourceNode = new LinkedList();
+            newSourceNode.add(mock(DatabaseNode.class));
+            when(newDag.getSourceNode()).thenReturn(newSourceNode);
+            doCallRealMethod().when(taskService).updateById(taskDto, user);
+            taskService.updateById(taskDto,user);
+            verify(taskService, new Times(1)).save(taskDto, user);
+            verify(transformSchemaAsyncService,times(1)).transformSchema(any(DAG.class),any(),any());
+        }
     }
     @Nested
     class UpdateShareCacheTaskTest{
@@ -2824,6 +2884,16 @@ void test(){
             verify(taskService).update(any(Query.class),any(Update.class));
         }
     }
+    @Nested
+    class UpdateDelayTimeTest{
+        @Test
+        void testUpdateDelayTimeNormal(){
+            ObjectId id = mock(ObjectId.class);
+            doCallRealMethod().when(taskService).updateDelayTime(id, 0);
+            taskService.updateDelayTime(id, 0);
+            verify(taskService,new Times(1)).update(any(Query.class), any(Update.class));
+        }
+    }
 
     @Test
     void testFindRunningTasksByAgentIdWithoutId() {
@@ -3716,6 +3786,31 @@ void testFindByConIdTest(){
             taskService.findByConId(sourceConnectionId,targetConnectionId,syncType,status,where,user);
         }
     }
+
+    @Nested
+    class downloadAnalyzeTest {
+        @Test
+        void testDownloadAnalyze() throws ExecutionException, InterruptedException, IOException {
+            TaskRepository repository = mock(TaskRepository.class);
+            taskService = spy(new TaskServiceImpl(repository));
+            HttpServletRequest request = mock(HttpServletRequest.class);
+            HttpServletResponse response = mock(HttpServletResponse.class);
+            TaskDto taskDto = new TaskDto();
+            taskDto.setAgentId("362b27e0-0679-4e50-b418-389fbe70d7df");
+            String taskId = "665f28cb65481e5a8ce96849";
+            doReturn(taskDto).when(taskService).findByTaskId(new ObjectId(taskId), AGENT_ID);
+            doReturn("mock export task json").when(taskService).exportTask(anyList(), any());
+            Page<TaskRecordListVo> recordsPage = new Page<>(0, Collections.emptyList());
+            when(taskRecordService.queryRecords(any())).thenReturn(recordsPage);
+            when(monitoringLogsService.query(any())).thenReturn(new Page<>(0, Collections.emptyList()));
+            when(batchService.batch(any(), any())).thenReturn(new HashMap<>());
+            doReturn(new byte[]{}).when(taskService).callEngineRpc(any(), any(), any(), any());
+            ResponseEntity<InputStreamResource> res = taskService.analyzeTask(request, response, taskId, user);
+            assert (res.getStatusCode().is2xxSuccessful());
+        }
+    }
+
+
     @Nested
     class importRmProjectTest {
         ParseRelMigFile parseRelMigFile;
@@ -3889,8 +3984,6 @@ public List<TaskDto> parse() {
 
         @Test
         void testImportRmProject() throws IOException {
-            CustomSqlService customSqlService = mock(CustomSqlService.class);
-            taskService.setCustomSqlService(customSqlService);
             DateNodeService dataNodeService = mock(DateNodeService.class);
             taskService.setDateNodeService(dataNodeService);
             ParseParam param = new ParseParam()
diff --git a/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceTest.java b/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceTest.java
index 4e31202b2b..c1a2ccfa43 100644
--- a/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceTest.java
+++ b/manager/tm/src/test/java/com/tapdata/tm/task/service/TaskServiceTest.java
@@ -843,8 +843,6 @@ public List<TaskDto> parse() {
 
         @Test
         void testImportRmProject() throws IOException {
-            CustomSqlService customSqlService = mock(CustomSqlService.class);
-            taskService.setCustomSqlService(customSqlService);
             DateNodeService dataNodeService = mock(DateNodeService.class);
             taskService.setDateNodeService(dataNodeService);
             ParseParam param = new ParseParam()
diff --git a/manager/tm/src/test/java/com/tapdata/tm/task/service/TransformSchemaServiceTest.java b/manager/tm/src/test/java/com/tapdata/tm/task/service/TransformSchemaServiceTest.java
index 48fedac5a0..b9841c2cda 100644
--- a/manager/tm/src/test/java/com/tapdata/tm/task/service/TransformSchemaServiceTest.java
+++ b/manager/tm/src/test/java/com/tapdata/tm/task/service/TransformSchemaServiceTest.java
@@ -6,13 +6,17 @@
 import com.tapdata.tm.commons.dag.DAG;
 import com.tapdata.tm.commons.dag.Node;
 import com.tapdata.tm.commons.dag.NodeEnum;
+import com.tapdata.tm.commons.dag.logCollector.LogCollecotrConnConfig;
+import com.tapdata.tm.commons.dag.logCollector.LogCollectorNode;
 import com.tapdata.tm.commons.dag.nodes.CacheNode;
 import com.tapdata.tm.commons.dag.nodes.DataParentNode;
 import com.tapdata.tm.commons.dag.nodes.DatabaseNode;
 import com.tapdata.tm.commons.dag.nodes.TableNode;
 import com.tapdata.tm.commons.schema.DataSourceConnectionDto;
 import com.tapdata.tm.commons.schema.DataSourceDefinitionDto;
+import com.tapdata.tm.commons.schema.MetadataInstancesDto;
 import com.tapdata.tm.commons.schema.TransformerWsMessageDto;
+import com.tapdata.tm.commons.schema.bean.SourceTypeEnum;
 import com.tapdata.tm.commons.task.dto.TaskDto;
 import com.tapdata.tm.commons.util.JsonUtil;
 import com.tapdata.tm.config.security.UserDetail;
@@ -37,11 +41,7 @@
 import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.test.util.ReflectionTestUtils;
 
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import static org.mockito.Mockito.*;
@@ -413,5 +413,95 @@ void testContainsJs() {
                 verify(taskService).updateById(any(ObjectId.class), any(Update.class), any(UserDetail.class));
             }
         }
+        @Nested
+        class getLogCollectorMetadataInstancesDtoTest{
+            @Test
+            void testConnConfigMapIsNotNull(){
+                DAG dag1 = mock(DAG.class);
+                List<Node> nodes = new ArrayList<>();
+                LogCollectorNode logCollectorNode = new LogCollectorNode();
+                logCollectorNode.setId("LogCollectorNode");
+                Map<String, LogCollecotrConnConfig> connConfigMap = new HashMap<>();
+                LogCollecotrConnConfig logCollecotrConnConfig1 = new LogCollecotrConnConfig();
+                logCollecotrConnConfig1.setConnectionId("c_1");
+                logCollecotrConnConfig1.setTableNames(Arrays.asList("test1"));
+                LogCollecotrConnConfig logCollecotrConnConfig2 = new LogCollecotrConnConfig();
+                logCollecotrConnConfig2.setConnectionId("c_2");
+                logCollecotrConnConfig2.setTableNames(Arrays.asList("test2"));
+                connConfigMap.put("c_1",logCollecotrConnConfig1);
+                connConfigMap.put("c_2",logCollecotrConnConfig2);
+                logCollectorNode.setLogCollectorConnConfigs(connConfigMap);
+                nodes.add(logCollectorNode);
+                when(dag1.getNodes()).thenReturn(nodes);
+                MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+                metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+                metadataInstancesDto1.setNodeId("LogCollectorNode");
+                metadataInstancesDto1.setOriginalName("name1");
+                metadataInstancesDto1.setQualifiedName("test1");
+                metadataInstancesDto1.setMetaType("table");
+                MetadataInstancesDto metadataInstancesDto2 = new MetadataInstancesDto();
+                metadataInstancesDto2.setSourceType(SourceTypeEnum.VIRTUAL.name());
+                metadataInstancesDto2.setNodeId("test");
+                metadataInstancesDto2.setOriginalName("name2");
+                metadataInstancesDto2.setQualifiedName("test2");
+                metadataInstancesDto2.setMetaType("table");
+                when(metadataInstancesService.findAllDto(any(),any())).thenReturn(Arrays.asList(metadataInstancesDto1,metadataInstancesDto2));
+                List<MetadataInstancesDto> result = new ArrayList<>();
+                UserDetail userDetail = mock(UserDetail.class);
+                doCallRealMethod().when(transformSchemaService).getLogCollectorMetadataInstancesDto(dag1,userDetail,result);
+                transformSchemaService.getLogCollectorMetadataInstancesDto(dag1,userDetail,result);
+                Assertions.assertEquals(2,result.size());
+            }
+        }
+
+        @Test
+        void testConnConfigMapIsNull(){
+            DAG dag1 = mock(DAG.class);
+            List<Node> nodes = new ArrayList<>();
+            LogCollectorNode logCollectorNode = new LogCollectorNode();
+            logCollectorNode.setId("LogCollectorNode");
+            logCollectorNode.setConnectionIds(Arrays.asList("c_1"));
+            logCollectorNode.setTableNames(Arrays.asList("test1"));
+            nodes.add(logCollectorNode);
+            when(dag1.getNodes()).thenReturn(nodes);
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("LogCollectorNode");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            when(metadataInstancesService.findAllDto(any(),any())).thenReturn(Arrays.asList(metadataInstancesDto1));
+            List<MetadataInstancesDto> result = new ArrayList<>();
+            UserDetail userDetail = mock(UserDetail.class);
+            doCallRealMethod().when(transformSchemaService).getLogCollectorMetadataInstancesDto(dag1,userDetail,result);
+            transformSchemaService.getLogCollectorMetadataInstancesDto(dag1,userDetail,result);
+            Assertions.assertEquals(1,result.size());
+        }
+        @Test
+        void testConnConfigMapIsEmpty(){
+            DAG dag1 = mock(DAG.class);
+            List<Node> nodes = new ArrayList<>();
+            LogCollectorNode logCollectorNode = new LogCollectorNode();
+            logCollectorNode.setId("LogCollectorNode");
+            List<String> connectionIds = new ArrayList<>();
+            logCollectorNode.setConnectionIds(connectionIds);
+            logCollectorNode.setTableNames(Arrays.asList("test1"));
+            logCollectorNode.setLogCollectorConnConfigs(new HashMap<>());
+            nodes.add(logCollectorNode);
+            when(dag1.getNodes()).thenReturn(nodes);
+            MetadataInstancesDto metadataInstancesDto1 = new MetadataInstancesDto();
+            metadataInstancesDto1.setSourceType(SourceTypeEnum.VIRTUAL.name());
+            metadataInstancesDto1.setNodeId("LogCollectorNode");
+            metadataInstancesDto1.setOriginalName("name1");
+            metadataInstancesDto1.setQualifiedName("test1");
+            metadataInstancesDto1.setMetaType("table");
+            when(metadataInstancesService.findAllDto(any(),any())).thenReturn(Arrays.asList(metadataInstancesDto1));
+            List<MetadataInstancesDto> result = new ArrayList<>();
+            UserDetail userDetail = mock(UserDetail.class);
+            doCallRealMethod().when(transformSchemaService).getLogCollectorMetadataInstancesDto(dag1,userDetail,result);
+            transformSchemaService.getLogCollectorMetadataInstancesDto(dag1,userDetail,result);
+            Assertions.assertEquals(0,result.size());
+        }
     }
+
 }
\ No newline at end of file
diff --git a/manager/tm/src/test/java/com/tapdata/tm/utils/EngineVersionUtilTest.java b/manager/tm/src/test/java/com/tapdata/tm/utils/EngineVersionUtilTest.java
new file mode 100644
index 0000000000..42c2eabe89
--- /dev/null
+++ b/manager/tm/src/test/java/com/tapdata/tm/utils/EngineVersionUtilTest.java
@@ -0,0 +1,67 @@
+package com.tapdata.tm.utils;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class EngineVersionUtilTest {
+    @Test
+    @DisplayName("Should return false when version is empty")
+    public void testCheckEngineTransformSchemaWithEmptyVersion() {
+        assertFalse(EngineVersionUtil.checkEngineTransFormSchema(""), "Expected false for empty version");
+    }
+
+    @Test
+    @DisplayName("Should return false when version is -")
+    public void testCheckEngineTransformSchemaWithSymbol() {
+        assertFalse(EngineVersionUtil.checkEngineTransFormSchema("-"), "Expected false for empty version");
+    }
+
+    @Test
+    @DisplayName("Should return false when version is null")
+    public void testCheckEngineTransformSchemaWithNullVersion() {
+        assertFalse(EngineVersionUtil.checkEngineTransFormSchema(null), "Expected false for null version");
+    }
+
+    @Test
+    @DisplayName("Should return false when version is less than 3.8.0")
+    public void testCheckEngineTransformSchemaWithLessThan380() {
+        assertFalse(EngineVersionUtil.checkEngineTransFormSchema("v3.6.0-test"), "Expected false for version less than 3.8.0");
+    }
+
+    @Test
+    public void testCheckEngineTransformSchemaVersionError() {
+        assertFalse(EngineVersionUtil.checkEngineTransFormSchema("v3.2-224-gb5d94f4d2-1683360843"), "Expected false for version less than 3.8.0");
+    }
+
+    @Test
+    @DisplayName("Should return true when version is equal to 3.8.0")
+    public void testCheckEngineTransformSchemaWithEqualTo380() {
+        assertTrue(EngineVersionUtil.checkEngineTransFormSchema("v3.8.0-test"), "Expected true for version 3.8.0");
+    }
+
+    @Test
+    @DisplayName("Should return true when version is greater than 3.8.0")
+    public void testCheckEngineTransformSchemaWithGreaterThan380() {
+        assertTrue(EngineVersionUtil.checkEngineTransFormSchema("v3.8.1-test"), "Expected true for version greater than 3.8.0");
+    }
+
+    @Test
+    @DisplayName("Should return true when version is greater than 4.0.0")
+    public void testCheckEngineTransformSchemaWithGreaterThan400() {
+        assertTrue(EngineVersionUtil.checkEngineTransFormSchema("v4.0.0-test"), "Expected true for version greater than 3.8.0");
+    }
+
+    @Test
+    @DisplayName("Should return true when version starts with 3.8.0 but has additional identifiers")
+    public void testCheckEngineTransformSchemaWithStartsWith380AdditionalIdentifiers() {
+        assertTrue(EngineVersionUtil.checkEngineTransFormSchema("v3.8.0-alpha1"), "Expected true for version starting with 3.8.0 and having additional identifiers");
+        assertFalse(EngineVersionUtil.checkEngineTransFormSchema("v3.8.0+build.123"), "Expected true for version 3.8.0 with build metadata");
+    }
+
+    @Test
+    public void testEngineVersionUtilConstructor() {
+        assertThrows(IllegalStateException.class, EngineVersionUtil::new);
+    }
+}
diff --git a/manager/tm/src/test/java/com/tapdata/tm/worker/service/WorkerServiceTest.java b/manager/tm/src/test/java/com/tapdata/tm/worker/service/WorkerServiceTest.java
index dec422c10a..29277bdd0c 100644
--- a/manager/tm/src/test/java/com/tapdata/tm/worker/service/WorkerServiceTest.java
+++ b/manager/tm/src/test/java/com/tapdata/tm/worker/service/WorkerServiceTest.java
@@ -307,4 +307,76 @@ void test1(){
         }
 
     }
+    @Nested
+    class checkEngineVersionTest{
+        @DisplayName("Test main process")
+        @Test
+        void test(){
+            when(settingsService.isCloud()).thenReturn(true);
+            List<Worker> workers = new ArrayList<>();
+            Worker worker1 = new Worker();
+            worker1.setVersion("v3.7.0-65db776b");
+            Worker worker2 = new Worker();
+            worker2.setVersion("v3.5.11-65db776b");
+            workers.add(worker1);
+            workers.add(worker2);
+            doReturn(workers).when(workerService).findAvailableAgent(any());
+            Assertions.assertFalse(workerService.checkEngineVersion(mock(UserDetail.class)));
+        }
+
+        @DisplayName("Test version matching")
+        @Test
+        void testEngineVersionIsCloud(){
+            when(settingsService.isCloud()).thenReturn(true);
+            List<Worker> workers = new ArrayList<>();
+            Worker worker1 = new Worker();
+            worker1.setVersion("v3.8.0-65db776b");
+            workers.add(worker1);
+            doReturn(workers).when(workerService).findAvailableAgent(any());
+            Assertions.assertTrue(workerService.checkEngineVersion(mock(UserDetail.class)));
+        }
+
+        @DisplayName("Test version mismatch")
+        @Test
+        void testEngineVersionMismatch(){
+            when(settingsService.isCloud()).thenReturn(true);
+            List<Worker> workers = new ArrayList<>();
+            Worker worker1 = new Worker();
+            worker1.setVersion("v3.8.0-65db776b");
+            Worker worker2 = new Worker();
+            worker2.setVersion("v3.5.11-65db776b");
+            workers.add(worker1);
+            workers.add(worker2);
+            doReturn(workers).when(workerService).findAvailableAgent(any());
+            Assertions.assertFalse(workerService.checkEngineVersion(mock(UserDetail.class)));
+        }
+
+        @DisplayName("Test version Worker is Null")
+        @Test
+        void testWorkerIsNull(){
+            when(settingsService.isCloud()).thenReturn(true);
+            List<Worker> workers = new ArrayList<>();
+            doReturn(workers).when(workerService).findAvailableAgent(any());
+            Assertions.assertFalse(workerService.checkEngineVersion(mock(UserDetail.class)));
+        }
+
+        @DisplayName("Test version Worker version is null")
+        @Test
+        void testWorkerVersionIsNull(){
+            when(settingsService.isCloud()).thenReturn(true);
+            List<Worker> workers = new ArrayList<>();
+            Worker worker1 = new Worker();
+            worker1.setVersion(null);
+            workers.add(worker1);
+            doReturn(workers).when(workerService).findAvailableAgent(any());
+            Assertions.assertFalse(workerService.checkEngineVersion(mock(UserDetail.class)));
+        }
+
+        @DisplayName("Test version is DASS")
+        @Test
+        void testEngineVersionDASS(){
+            when(settingsService.isCloud()).thenReturn(false);
+            Assertions.assertTrue(workerService.checkEngineVersion(mock(UserDetail.class)));
+        }
+    }
 }
diff --git a/manager/tm/src/test/java/com/tapdata/tm/ws/endpoint/WebSocketServerTest.java b/manager/tm/src/test/java/com/tapdata/tm/ws/endpoint/WebSocketServerTest.java
new file mode 100644
index 0000000000..81c902eccc
--- /dev/null
+++ b/manager/tm/src/test/java/com/tapdata/tm/ws/endpoint/WebSocketServerTest.java
@@ -0,0 +1,55 @@
+package com.tapdata.tm.ws.endpoint;
+
+import com.tapdata.tm.commons.util.JsonUtil;
+import com.tapdata.tm.commons.websocket.ReturnCallback;
+import com.tapdata.tm.commons.websocket.v1.MessageInfoV1;
+import com.tapdata.tm.commons.websocket.v1.ResultWrap;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.Times;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+public class WebSocketServerTest {
+    private WebSocketServer webSocketServer;
+    @BeforeEach
+    void beforeEach(){
+        webSocketServer = mock(WebSocketServer.class);
+    }
+    @Nested
+    class HandlerResultTest{
+        private MessageInfoV1 messageInfo;
+        @Test
+        void testHandlerResultNormal(){
+            try (MockedStatic<WebSocketManager> mb = Mockito
+                    .mockStatic(WebSocketManager.class)) {
+                messageInfo = mock(MessageInfoV1.class);
+                when(messageInfo.getType()).thenReturn("return");
+                when(messageInfo.getReqId()).thenReturn("111");
+                when(messageInfo.getBody()).thenReturn("test");
+                ReturnCallback<String> callback = new ReturnCallback<String>() {
+                    @Override
+                    public void success(String s) {
+                    }
+                    @Override
+                    public void error(String code, String message) {
+                    }
+                };
+                mb.when(()->WebSocketManager.getAndRemoveResultCallback("111")).thenReturn(callback);
+                try (MockedStatic<JsonUtil> jsonUtilMockedStatic = Mockito
+                        .mockStatic(JsonUtil.class)) {
+                    jsonUtilMockedStatic.when(()->JsonUtil.parseJsonUseJackson("test", ResultWrap.class)).thenReturn(mock(ResultWrap.class));
+                    doCallRealMethod().when(webSocketServer).handlerResult(messageInfo);
+                    webSocketServer.handlerResult(messageInfo);
+                    jsonUtilMockedStatic.verify(()->JsonUtil.parseJsonUseJackson(anyString(), any(Class.class)),new Times(1));
+                }
+            }
+        }
+    }
+}