Code review comment: yes, some PlanFragment profit from pipeline parallelization

Code under review:
private void computeFragmentHosts() throws Exception {
        
        
        

        boolean dopAdaptionEnabled = ConnectContext.get() != null &&
                ConnectContext.get().getSessionVariable().isPipelineDopAdaptionEnabled();
        int degreeOfParallelism = ConnectContext.get().getSessionVariable().getDegreeOfParallelism();
        for (int i = fragments.size() - 1; i >= 0; --i) {
            PlanFragment fragment = fragments.get(i);
            FragmentExecParams params = fragmentExecParamsMap.get(fragment.getFragmentId());

            if (fragment.getDataPartition() == DataPartition.UNPARTITIONED) {
                Reference<Long> backendIdRef = new Reference<>();
                TNetworkAddress execHostport = SimpleScheduler.getHost(this.idToBackend, backendIdRef);
                if (execHostport == null) {
                    LOG.warn("DataPartition UNPARTITIONED, no scanNode Backend");
                    throw new UserException("there is no scanNode Backend");
                }
                this.addressToBackendID.put(execHostport, backendIdRef.getRef());
                FInstanceExecParam instanceParam = new FInstanceExecParam(null, execHostport,
                        0, params);
                params.instanceExecParams.add(instanceParam);
                continue;
            }

            PlanNode leftMostNode = findLeftmostNode(fragment.getPlanRoot());

            /*
             * Case A:
             *      if the left most is ScanNode, which means there is no child fragment,
             *      we should assign fragment instances on every scan node hosts.
             * Case B:
             *      if not, there should be exchange nodes to collect all data from child fragments(input fragments),
             *      so we should assign fragment instances corresponding to the child fragments' host
             */
            if (!(leftMostNode instanceof ScanNode)) {
                
                
                
                int inputFragmentIndex = 0;
                int maxParallelism = 0;
                for (int j = 0; j < fragment.getChildren().size(); j++) {
                    int currentChildFragmentParallelism =
                            fragmentExecParamsMap.get(fragment.getChild(j).getFragmentId()).instanceExecParams.size();
                    
                    
                    if (dopAdaptionEnabled) {
                        currentChildFragmentParallelism *= fragment.getChild(j).getPipelineDop();
                    }
                    if (currentChildFragmentParallelism > maxParallelism) {
                        maxParallelism = currentChildFragmentParallelism;
                        inputFragmentIndex = j;
                    }
                }

                PlanFragmentId inputFragmentId = fragment.getChild(inputFragmentIndex).getFragmentId();
                FragmentExecParams maxParallelismFragmentExecParams = fragmentExecParamsMap.get(inputFragmentId);

                boolean hasShuffleHashBucketJoinInFragment = hasShuffleHashBucketJoin(fragment.getPlanRoot());
                if (hasShuffleHashBucketJoinInFragment) {
                    FragmentExecParams execParams = getMaxParallelismScanFragmentExecParams();
                    if (execParams != null) {
                        maxParallelism = execParams.instanceExecParams.size();
                        
                        
                        if (dopAdaptionEnabled) {
                            maxParallelism *= execParams.fragment.getPipelineDop();
                        }
                        maxParallelismFragmentExecParams = execParams;
                    }
                }

                
                
                
                Set<TNetworkAddress> hostSet = Sets.newHashSet();
                for (FInstanceExecParam execParams : maxParallelismFragmentExecParams.instanceExecParams) {
                    hostSet.add(execParams.host);
                }

                if (dopAdaptionEnabled) {
                    Preconditions.checkArgument(leftMostNode instanceof ExchangeNode);
                    DataSink sink = getDataStreamSink(maxParallelismFragmentExecParams.fragment, fragment);
                    Preconditions.checkArgument(sink != null);
                    
                    
                    
                    if (!sink.getOutputPartition().isPartitioned()) {
                        
                        maxParallelism = Math.min(hostSet.size() * degreeOfParallelism, maxParallelism);
                        fragment.setPipelineDop(1);
                    } else {
                        
                        maxParallelism = hostSet.size();
                        fragment.setPipelineDop(degreeOfParallelism);
                    }
                }

                
                int exchangeInstances = -1;
                if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
                    exchangeInstances = ConnectContext.get().getSessionVariable().getExchangeInstanceParallel();
                }
                if (exchangeInstances > 0 && maxParallelism > exchangeInstances) {
                    
                    
                    List<TNetworkAddress> hosts = Lists.newArrayList(hostSet);
                    if (!hasShuffleHashBucketJoinInFragment) {
                        Collections.shuffle(hosts, instanceRandom);
                    }
                    for (int index = 0; index < exchangeInstances; index++) {
                        FInstanceExecParam instanceParam =
                                new FInstanceExecParam(null, hosts.get(index % hosts.size()), 0, params);
                        params.instanceExecParams.add(instanceParam);
                    }
                } else {
                    List<TNetworkAddress> hosts = Lists.newArrayList(hostSet);
                    for (int index = 0; index < maxParallelism; ++index) {
                        TNetworkAddress host = hosts.get(index % hosts.size());
                        FInstanceExecParam instanceParam = new FInstanceExecParam(null, host, 0, params);
                        params.instanceExecParams.add(instanceParam);
                    }
                }

                
                
                
                if (!hasShuffleHashBucketJoinInFragment) {
                    Collections.shuffle(params.instanceExecParams, instanceRandom);
                }

                
                
                continue;
            }

            int parallelExecInstanceNum = fragment.getParallelExecNum();
            boolean hasColocate = (isColocateFragment(fragment.getPlanRoot()) &&
                    fragmentIdToSeqToAddressMap.containsKey(fragment.getFragmentId())
                    && fragmentIdToSeqToAddressMap.get(fragment.getFragmentId()).size() > 0);
            boolean hasBucketShuffle = isBucketShuffleJoin(fragment.getFragmentId().asInt());

            if (hasColocate || hasBucketShuffle) {
                computeColocatedJoinInstanceParam(fragment.getFragmentId(), parallelExecInstanceNum, params);
            } else {
                for (Map.Entry<TNetworkAddress, Map<Integer, List<TScanRangeParams>>> tNetworkAddressMapEntry :
                        fragmentExecParamsMap.get(fragment.getFragmentId()).scanRangeAssignment.entrySet()) {
                    TNetworkAddress key = tNetworkAddressMapEntry.getKey();
                    Map<Integer, List<TScanRangeParams>> value = tNetworkAddressMapEntry.getValue();

                    
                    for (Integer planNodeId : value.keySet()) {
                        if (replicateScanIds.contains(planNodeId)) {
                            continue;
                        }
                        List<TScanRangeParams> perNodeScanRanges = value.get(planNodeId);
                        int expectedInstanceNum = 1;
                        if (parallelExecInstanceNum > 1) {
                            
                            expectedInstanceNum = Math.min(perNodeScanRanges.size(), parallelExecInstanceNum);
                        }
                        List<List<TScanRangeParams>> perInstanceScanRanges = ListUtil.splitBySize(perNodeScanRanges,
                                expectedInstanceNum);

                        for (List<TScanRangeParams> scanRangeParams : perInstanceScanRanges) {
                            FInstanceExecParam instanceParam = new FInstanceExecParam(null, key, 0, params);
                            instanceParam.perNodeScanRanges.put(planNodeId, scanRangeParams);
                            params.instanceExecParams.add(instanceParam);
                        }
                    }

                    
                    boolean isReplicated = isReplicatedFragment(fragment.getPlanRoot());
                    if (isReplicated) {
                        for (Integer planNodeId : value.keySet()) {
                            if (!replicateScanIds.contains(planNodeId)) {
                                continue;
                            }
                            List<TScanRangeParams> perNodeScanRanges = value.get(planNodeId);
                            for (FInstanceExecParam instanceParam : params.instanceExecParams) {
                                instanceParam.perNodeScanRanges.put(planNodeId, perNodeScanRanges);
                            }
                        }
                    }
                }
                
                if (dopAdaptionEnabled) {
                    FragmentExecParams param = fragmentExecParamsMap.get(fragment.getFragmentId());
                    int numBackends = param.scanRangeAssignment.size();
                    int numInstances = param.instanceExecParams.size();
                    int pipelineDop = Math.max(1, degreeOfParallelism / Math.max(1, numInstances / numBackends));
                    param.fragment.setPipelineDop(pipelineDop);
                }
            }

            if (params.instanceExecParams.isEmpty()) {
                Reference<Long> backendIdRef = new Reference<>();
                TNetworkAddress execHostport = SimpleScheduler.getHost(this.idToBackend, backendIdRef);
                if (execHostport == null) {
                    throw new UserException("there is no scanNode Backend");
                }
                this.addressToBackendID.put(execHostport, backendIdRef.getRef());
                FInstanceExecParam instanceParam = new FInstanceExecParam(null, execHostport,
                        0, params);
                params.instanceExecParams.add(instanceParam);
            }
        }
    }

Line commented on: fragment.setPipelineDop(degreeOfParallelism);

Revised code:
private void computeFragmentHosts() throws Exception {
        
        
        

        boolean dopAdaptionEnabled = ConnectContext.get() != null &&
                ConnectContext.get().getSessionVariable().isPipelineDopAdaptionEnabled();
        int degreeOfParallelism = ConnectContext.get().getSessionVariable().getDegreeOfParallelism();
        for (int i = fragments.size() - 1; i >= 0; --i) {
            PlanFragment fragment = fragments.get(i);
            FragmentExecParams params = fragmentExecParamsMap.get(fragment.getFragmentId());

            if (fragment.getDataPartition() == DataPartition.UNPARTITIONED) {
                Reference<Long> backendIdRef = new Reference<>();
                TNetworkAddress execHostport = SimpleScheduler.getHost(this.idToBackend, backendIdRef);
                if (execHostport == null) {
                    LOG.warn("DataPartition UNPARTITIONED, no scanNode Backend");
                    throw new UserException("there is no scanNode Backend");
                }
                this.addressToBackendID.put(execHostport, backendIdRef.getRef());
                FInstanceExecParam instanceParam = new FInstanceExecParam(null, execHostport,
                        0, params);
                params.instanceExecParams.add(instanceParam);
                continue;
            }

            PlanNode leftMostNode = findLeftmostNode(fragment.getPlanRoot());

            /*
             * Case A:
             *      if the left most is ScanNode, which means there is no child fragment,
             *      we should assign fragment instances on every scan node hosts.
             * Case B:
             *      if not, there should be exchange nodes to collect all data from child fragments(input fragments),
             *      so we should assign fragment instances corresponding to the child fragments' host
             */
            if (!(leftMostNode instanceof ScanNode)) {
                
                
                
                int inputFragmentIndex = 0;
                int maxParallelism = 0;
                for (int j = 0; j < fragment.getChildren().size(); j++) {
                    int currentChildFragmentParallelism =
                            fragmentExecParamsMap.get(fragment.getChild(j).getFragmentId()).instanceExecParams.size();
                    
                    
                    if (dopAdaptionEnabled) {
                        currentChildFragmentParallelism *= fragment.getChild(j).getPipelineDop();
                    }
                    if (currentChildFragmentParallelism > maxParallelism) {
                        maxParallelism = currentChildFragmentParallelism;
                        inputFragmentIndex = j;
                    }
                }

                PlanFragmentId inputFragmentId = fragment.getChild(inputFragmentIndex).getFragmentId();
                FragmentExecParams maxParallelismFragmentExecParams = fragmentExecParamsMap.get(inputFragmentId);

                boolean hasShuffleHashBucketJoinInFragment = hasShuffleHashBucketJoin(fragment.getPlanRoot());
                if (hasShuffleHashBucketJoinInFragment) {
                    FragmentExecParams execParams = getMaxParallelismScanFragmentExecParams();
                    if (execParams != null) {
                        maxParallelism = execParams.instanceExecParams.size();
                        
                        
                        if (dopAdaptionEnabled) {
                            maxParallelism *= execParams.fragment.getPipelineDop();
                        }
                        maxParallelismFragmentExecParams = execParams;
                    }
                }

                
                
                
                Set<TNetworkAddress> hostSet = Sets.newHashSet();
                for (FInstanceExecParam execParams : maxParallelismFragmentExecParams.instanceExecParams) {
                    hostSet.add(execParams.host);
                }

                if (dopAdaptionEnabled) {
                    Preconditions.checkArgument(leftMostNode instanceof ExchangeNode);
                    DataSink sink = getDataStreamSink(maxParallelismFragmentExecParams.fragment, fragment);
                    Preconditions.checkArgument(sink != null);
                    
                    
                    
                    if (!sink.getOutputPartition().isPartitioned()) {
                        
                        maxParallelism = Math.min(hostSet.size() * degreeOfParallelism, maxParallelism);
                        fragment.setPipelineDop(1);
                    } else {
                        
                        maxParallelism = hostSet.size();
                        fragment.setPipelineDop(degreeOfParallelism);
                    }
                }

                
                int exchangeInstances = -1;
                if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
                    exchangeInstances = ConnectContext.get().getSessionVariable().getExchangeInstanceParallel();
                }
                if (exchangeInstances > 0 && maxParallelism > exchangeInstances) {
                    
                    
                    List<TNetworkAddress> hosts = Lists.newArrayList(hostSet);
                    if (!hasShuffleHashBucketJoinInFragment) {
                        Collections.shuffle(hosts, instanceRandom);
                    }
                    for (int index = 0; index < exchangeInstances; index++) {
                        FInstanceExecParam instanceParam =
                                new FInstanceExecParam(null, hosts.get(index % hosts.size()), 0, params);
                        params.instanceExecParams.add(instanceParam);
                    }
                } else {
                    List<TNetworkAddress> hosts = Lists.newArrayList(hostSet);
                    for (int index = 0; index < maxParallelism; ++index) {
                        TNetworkAddress host = hosts.get(index % hosts.size());
                        FInstanceExecParam instanceParam = new FInstanceExecParam(null, host, 0, params);
                        params.instanceExecParams.add(instanceParam);
                    }
                }

                
                
                
                if (!hasShuffleHashBucketJoinInFragment) {
                    Collections.shuffle(params.instanceExecParams, instanceRandom);
                }

                
                
                continue;
            }

            int parallelExecInstanceNum = fragment.getParallelExecNum();
            boolean hasColocate = (isColocateFragment(fragment.getPlanRoot()) &&
                    fragmentIdToSeqToAddressMap.containsKey(fragment.getFragmentId())
                    && fragmentIdToSeqToAddressMap.get(fragment.getFragmentId()).size() > 0);
            boolean hasBucketShuffle = isBucketShuffleJoin(fragment.getFragmentId().asInt());

            if (hasColocate || hasBucketShuffle) {
                computeColocatedJoinInstanceParam(fragment.getFragmentId(), parallelExecInstanceNum, params);
            } else {
                for (Map.Entry<TNetworkAddress, Map<Integer, List<TScanRangeParams>>> tNetworkAddressMapEntry :
                        fragmentExecParamsMap.get(fragment.getFragmentId()).scanRangeAssignment.entrySet()) {
                    TNetworkAddress key = tNetworkAddressMapEntry.getKey();
                    Map<Integer, List<TScanRangeParams>> value = tNetworkAddressMapEntry.getValue();

                    
                    for (Integer planNodeId : value.keySet()) {
                        if (replicateScanIds.contains(planNodeId)) {
                            continue;
                        }
                        List<TScanRangeParams> perNodeScanRanges = value.get(planNodeId);
                        int expectedInstanceNum = 1;
                        if (parallelExecInstanceNum > 1) {
                            
                            expectedInstanceNum = Math.min(perNodeScanRanges.size(), parallelExecInstanceNum);
                        }
                        List<List<TScanRangeParams>> perInstanceScanRanges = ListUtil.splitBySize(perNodeScanRanges,
                                expectedInstanceNum);

                        for (List<TScanRangeParams> scanRangeParams : perInstanceScanRanges) {
                            FInstanceExecParam instanceParam = new FInstanceExecParam(null, key, 0, params);
                            instanceParam.perNodeScanRanges.put(planNodeId, scanRangeParams);
                            params.instanceExecParams.add(instanceParam);
                        }
                    }

                    
                    boolean isReplicated = isReplicatedFragment(fragment.getPlanRoot());
                    if (isReplicated) {
                        for (Integer planNodeId : value.keySet()) {
                            if (!replicateScanIds.contains(planNodeId)) {
                                continue;
                            }
                            List<TScanRangeParams> perNodeScanRanges = value.get(planNodeId);
                            for (FInstanceExecParam instanceParam : params.instanceExecParams) {
                                instanceParam.perNodeScanRanges.put(planNodeId, perNodeScanRanges);
                            }
                        }
                    }
                }
                
                if (dopAdaptionEnabled) {
                    FragmentExecParams param = fragmentExecParamsMap.get(fragment.getFragmentId());
                    int numBackends = param.scanRangeAssignment.size();
                    int numInstances = param.instanceExecParams.size();
                    int pipelineDop = Math.max(1, degreeOfParallelism / Math.max(1, numInstances / numBackends));
                    param.fragment.setPipelineDop(pipelineDop);
                }
            }

            if (params.instanceExecParams.isEmpty()) {
                Reference<Long> backendIdRef = new Reference<>();
                TNetworkAddress execHostport = SimpleScheduler.getHost(this.idToBackend, backendIdRef);
                if (execHostport == null) {
                    throw new UserException("there is no scanNode Backend");
                }
                this.addressToBackendID.put(execHostport, backendIdRef.getRef());
                FInstanceExecParam instanceParam = new FInstanceExecParam(null, execHostport,
                        0, params);
                params.instanceExecParams.add(instanceParam);
            }
        }
    }