Program: 31735
Project: 31735
Buggy source:
protected int runDistributedTrain() throws IOException, InterruptedException, ClassNotFoundException {
  LOG.info("String_Node_Str",isDryTrain ? "String_Node_Str" : "String_Node_Str");
  int status=0;
  Configuration conf=new Configuration();
  SourceType sourceType=super.getModelConfig().getDataSet().getSource();
  final List<String> args=new ArrayList<String>();
  GridSearch gs=new GridSearch(modelConfig.getTrain().getParams());
  prepareCommonParams(gs.hasHyperParam(),args,sourceType);
  String alg=super.getModelConfig().getTrain().getAlgorithm();
  FileSystem fileSystem=ShifuFileUtils.getFileSystemBySourceType(sourceType);
  Path tmpModelsPath=fileSystem.makeQualified(new Path(super.getPathFinder().getPathBySourceType(new Path(Constants.TMP,Constants.DEFAULT_MODELS_TMP_FOLDER),sourceType)));
  args.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_TMP_MODELS_FOLDER,tmpModelsPath.toString()));
  int baggingNum=isForVarSelect ? 1 : super.getModelConfig().getBaggingNum();
  if (modelConfig.isClassification() && modelConfig.getTrain().isOneVsAll()) {
    baggingNum=modelConfig.getTags().size();
    if (baggingNum != super.getModelConfig().getBaggingNum()) {
      LOG.warn("String_Node_Str",baggingNum);
    }
  }
  boolean isKFoldCV=false;
  Integer kCrossValidation=this.modelConfig.getTrain().getNumKFold();
  if (kCrossValidation != null && kCrossValidation > 0) {
    isKFoldCV=true;
    baggingNum=modelConfig.getTrain().getNumKFold();
    if (baggingNum != super.getModelConfig().getBaggingNum()) {
      LOG.warn("String_Node_Str",baggingNum);
    }
  }
  long start=System.currentTimeMillis();
  LOG.info("String_Node_Str",baggingNum);
  boolean isParallel=Boolean.valueOf(Environment.getProperty(Constants.SHIFU_DTRAIN_PARALLEL,SHIFU_DEFAULT_DTRAIN_PARALLEL)).booleanValue();
  GuaguaMapReduceClient guaguaClient;
  int[] inputOutputIndex=DTrainUtils.getInputOutputCandidateCounts(this.columnConfigList);
  int inputNodeCount=inputOutputIndex[0] == 0 ? inputOutputIndex[2] : inputOutputIndex[0];
  int candidateCount=inputOutputIndex[2];
  boolean isAfterVarSelect=(inputOutputIndex[0] != 0);
  List<Integer> allFeatures=CommonUtils.getAllFeatureList(this.columnConfigList,isAfterVarSelect);
  if (modelConfig.getNormalize().getIsParquet()) {
    guaguaClient=new GuaguaParquetMapReduceClient();
    RequiredFieldList requiredFieldList=new RequiredFieldList();
    for (    ColumnConfig columnConfig : super.columnConfigList) {
      if (columnConfig.isTarget()) {
        requiredFieldList.add(new RequiredField(columnConfig.getColumnName(),columnConfig.getColumnNum(),null,DataType.FLOAT));
      }
 else {
        if (inputNodeCount == candidateCount) {
          if (!columnConfig.isMeta() && !columnConfig.isTarget() && CommonUtils.isGoodCandidate(columnConfig)) {
            requiredFieldList.add(new RequiredField(columnConfig.getColumnName(),columnConfig.getColumnNum(),null,DataType.FLOAT));
          }
        }
 else {
          if (!columnConfig.isMeta() && !columnConfig.isTarget() && columnConfig.isFinalSelect()) {
            requiredFieldList.add(new RequiredField(columnConfig.getColumnName(),columnConfig.getColumnNum(),null,DataType.FLOAT));
          }
        }
      }
    }
    requiredFieldList.add(new RequiredField("String_Node_Str",columnConfigList.size(),null,DataType.DOUBLE));
    args.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,"String_Node_Str",serializeRequiredFieldList(requiredFieldList)));
    args.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,"String_Node_Str","String_Node_Str"));
  }
 else {
    guaguaClient=new GuaguaMapReduceClient();
  }
  int parallelNum=Integer.parseInt(Environment.getProperty(CommonConstants.SHIFU_TRAIN_BAGGING_INPARALLEL,"String_Node_Str"));
  int parallelGroups=1;
  if (gs.hasHyperParam()) {
    parallelGroups=(gs.getFlattenParams().size() % parallelNum == 0 ? gs.getFlattenParams().size() / parallelNum : gs.getFlattenParams().size() / parallelNum + 1);
  }
 else {
    parallelGroups=baggingNum % parallelNum == 0 ? baggingNum / parallelNum : baggingNum / parallelNum + 1;
  }
  List<String> progressLogList=new ArrayList<String>(baggingNum);
  boolean isOneJobNotContinuous=false;
  for (int j=0; j < parallelGroups; j++) {
    int currBags=baggingNum;
    if (gs.hasHyperParam()) {
      if (j == parallelGroups - 1) {
        currBags=gs.getFlattenParams().size() % parallelNum == 0 ? parallelNum : gs.getFlattenParams().size() % parallelNum;
      }
 else {
        currBags=parallelNum;
      }
    }
 else {
      if (j == parallelGroups - 1) {
        currBags=baggingNum % parallelNum == 0 ? parallelNum : baggingNum % parallelNum;
      }
 else {
        currBags=parallelNum;
      }
    }
    for (int k=0; k < currBags; k++) {
      int i=j * parallelNum + k;
      if (gs.hasHyperParam()) {
        LOG.info("String_Node_Str",i,gs.getParams(i));
      }
 else       if (isKFoldCV) {
        LOG.info("String_Node_Str",i);
      }
      List<String> localArgs=new ArrayList<String>(args);
      localArgs.add("String_Node_Str");
      localArgs.add(String.format("String_Node_Str",alg,super.getModelConfig().getModelSetName(),i));
      LOG.info("String_Node_Str",i);
      String modelName=getModelName(i);
      Path modelPath=fileSystem.makeQualified(new Path(super.getPathFinder().getModelsPath(sourceType),modelName));
      boolean isContinous=false;
      if (gs.hasHyperParam()) {
        isContinous=false;
      }
 else {
        int intContinuous=checkContinuousTraining(fileSystem,localArgs,modelPath,modelConfig.getTrain().getParams());
        if (intContinuous == -1) {
          LOG.warn("String_Node_Str",i);
          continue;
        }
 else {
          isContinous=(intContinuous == 1);
        }
      }
      if (gs.hasHyperParam() || isKFoldCV) {
        isContinous=false;
      }
      if (!isContinous && !isOneJobNotContinuous) {
        isOneJobNotContinuous=true;
        String srcModelPath=super.getPathFinder().getModelsPath(sourceType);
        String mvModelPath=srcModelPath + "String_Node_Str" + System.currentTimeMillis();
        LOG.info("String_Node_Str",mvModelPath);
        fileSystem.rename(new Path(srcModelPath),new Path(mvModelPath));
        fileSystem.mkdirs(new Path(srcModelPath));
        FileSystem.getLocal(conf).delete(new Path(super.getPathFinder().getModelsPath(SourceType.LOCAL)),true);
      }
      if (NNConstants.NN_ALG_NAME.equalsIgnoreCase(alg)) {
        Map<String,Object> params=gs.hasHyperParam() ? gs.getParams(i) : this.modelConfig.getTrain().getParams();
        Object fssObj=params.get("String_Node_Str");
        FeatureSubsetStrategy featureSubsetStrategy=null;
        double featureSubsetRate=0d;
        if (fssObj != null) {
          try {
            featureSubsetRate=Double.parseDouble(fssObj.toString());
            featureSubsetStrategy=null;
          }
 catch (          NumberFormatException ee) {
            featureSubsetStrategy=FeatureSubsetStrategy.of(fssObj.toString());
          }
        }
 else {
          LOG.warn("String_Node_Str");
          featureSubsetStrategy=FeatureSubsetStrategy.ALL;
          featureSubsetRate=0;
        }
        Set<Integer> subFeatures=null;
        if (isContinous) {
          BasicFloatNetwork existingModel=(BasicFloatNetwork)CommonUtils.loadModel(modelConfig,modelPath,ShifuFileUtils.getFileSystemBySourceType(this.modelConfig.getDataSet().getSource()));
          if (existingModel == null) {
            subFeatures=new HashSet<Integer>(getSubsamplingFeatures(allFeatures,featureSubsetStrategy,featureSubsetRate,inputNodeCount));
          }
 else {
            subFeatures=existingModel.getFeatureSet();
          }
        }
 else {
          subFeatures=new HashSet<Integer>(getSubsamplingFeatures(allFeatures,featureSubsetStrategy,featureSubsetRate,inputNodeCount));
        }
        if (subFeatures == null || subFeatures.size() == 0) {
          localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_NN_FEATURE_SUBSET,"String_Node_Str"));
        }
 else {
          localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_NN_FEATURE_SUBSET,StringUtils.join(subFeatures,',')));
          LOG.debug("String_Node_Str",subFeatures.size(),StringUtils.join(subFeatures,','));
        }
      }
      localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.GUAGUA_OUTPUT,modelPath.toString()));
      if (gs.hasHyperParam() || isKFoldCV) {
        Path valErrPath=fileSystem.makeQualified(new Path(super.getPathFinder().getValErrorPath(sourceType),"String_Node_Str" + i));
        localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.GS_VALIDATION_ERROR,valErrPath.toString()));
      }
      localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_TRAINER_ID,String.valueOf(i)));
      final String progressLogFile=getProgressLogFile(i);
      progressLogList.add(progressLogFile);
      localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_DTRAIN_PROGRESS_FILE,progressLogFile));
      String hdpVersion=HDPUtils.getHdpVersionForHDP224();
      if (StringUtils.isNotBlank(hdpVersion)) {
        localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,"String_Node_Str",hdpVersion));
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
      }
      if (isParallel) {
        guaguaClient.addJob(localArgs.toArray(new String[0]));
      }
 else {
        TailThread tailThread=startTailThread(new String[]{progressLogFile});
        guaguaClient.createJob(localArgs.toArray(new String[0])).waitForCompletion(true);
        stopTailThread(tailThread);
      }
    }
    if (isParallel) {
      TailThread tailThread=startTailThread(progressLogList.toArray(new String[0]));
      guaguaClient.run();
      stopTailThread(tailThread);
    }
  }
  if (isKFoldCV) {
    List<Double> valErrs=readAllValidationErrors(sourceType,fileSystem,kCrossValidation);
    double sum=0d;
    for (    Double err : valErrs) {
      sum+=err;
    }
    LOG.info("String_Node_Str",sum / valErrs.size());
    LOG.info("String_Node_Str",System.currentTimeMillis() - start);
  }
 else   if (gs.hasHyperParam()) {
    LOG.info("String_Node_Str",modelConfig.getParams());
    Map<String,Object> params=findBestParams(sourceType,fileSystem,gs);
    for (    Entry<String,Object> entry : params.entrySet()) {
      modelConfig.getParams().put(entry.getKey(),entry.getValue());
    }
    super.pathFinder.getModelConfigPath(SourceType.LOCAL);
    JSONUtils.writeValue(new File(super.pathFinder.getModelConfigPath(SourceType.LOCAL)),modelConfig);
    LOG.info("String_Node_Str",System.currentTimeMillis() - start);
  }
 else {
    if (!gs.hasHyperParam()) {
      for (int i=0; i < baggingNum; i++) {
        String modelName=getModelName(i);
        Path modelPath=fileSystem.makeQualified(new Path(super.getPathFinder().getModelsPath(sourceType),modelName));
        if (ShifuFileUtils.getFileSystemBySourceType(sourceType).exists(modelPath)) {
          copyModelToLocal(modelName,modelPath,sourceType);
        }
 else {
          LOG.warn("String_Node_Str",modelPath.toString());
          status=1;
        }
      }
      boolean copyTmpModelsToLocal=Boolean.TRUE.toString().equalsIgnoreCase(Environment.getProperty(Constants.SHIFU_TMPMODEL_COPYTOLOCAL,"String_Node_Str"));
      if (CommonUtils.isTreeModel(modelConfig.getAlgorithm())) {
        copyTmpModelsToLocal=Boolean.TRUE.toString().equalsIgnoreCase(Environment.getProperty(Constants.SHIFU_TMPMODEL_COPYTOLOCAL,"String_Node_Str"));
        List<BasicML> models=CommonUtils.loadBasicModels(this.modelConfig,this.columnConfigList,null);
        Map<Integer,MutablePair<String,Double>> featureImportances=CommonUtils.computeTreeModelFeatureImportance(models);
        String localFsFolder=pathFinder.getLocalFeatureImportanceFolder();
        processRollupForFIFiles(localFsFolder);
        CommonUtils.writeFeatureImportance(pathFinder.getLocalFeatureImportancePath(),featureImportances);
      }
      if (copyTmpModelsToLocal) {
        copyTmpModelsToLocal(tmpModelsPath,sourceType);
      }
 else {
        LOG.info("String_Node_Str",tmpModelsPath);
      }
      LOG.info("String_Node_Str",System.currentTimeMillis() - start);
    }
  }
  if (status != 0) {
    LOG.error("String_Node_Str");
  }
  return status;
}

Fixed version (ground truth):
protected int runDistributedTrain() throws IOException, InterruptedException, ClassNotFoundException {
  LOG.info("String_Node_Str",isDryTrain ? "String_Node_Str" : "String_Node_Str");
  int status=0;
  Configuration conf=new Configuration();
  SourceType sourceType=super.getModelConfig().getDataSet().getSource();
  final List<String> args=new ArrayList<String>();
  GridSearch gs=new GridSearch(modelConfig.getTrain().getParams());
  prepareCommonParams(gs.hasHyperParam(),args,sourceType);
  String alg=super.getModelConfig().getTrain().getAlgorithm();
  FileSystem fileSystem=ShifuFileUtils.getFileSystemBySourceType(sourceType);
  Path tmpModelsPath=fileSystem.makeQualified(new Path(super.getPathFinder().getPathBySourceType(new Path(Constants.TMP,Constants.DEFAULT_MODELS_TMP_FOLDER),sourceType)));
  args.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_TMP_MODELS_FOLDER,tmpModelsPath.toString()));
  int baggingNum=isForVarSelect ? 1 : super.getModelConfig().getBaggingNum();
  if (modelConfig.isClassification() && modelConfig.getTrain().isOneVsAll()) {
    baggingNum=modelConfig.getTags().size();
    if (baggingNum != super.getModelConfig().getBaggingNum()) {
      LOG.warn("String_Node_Str",baggingNum);
    }
  }
  boolean isKFoldCV=false;
  Integer kCrossValidation=this.modelConfig.getTrain().getNumKFold();
  if (kCrossValidation != null && kCrossValidation > 0) {
    isKFoldCV=true;
    baggingNum=modelConfig.getTrain().getNumKFold();
    if (baggingNum != super.getModelConfig().getBaggingNum()) {
      LOG.warn("String_Node_Str",baggingNum);
    }
  }
  long start=System.currentTimeMillis();
  LOG.info("String_Node_Str",baggingNum);
  boolean isParallel=Boolean.valueOf(Environment.getProperty(Constants.SHIFU_DTRAIN_PARALLEL,SHIFU_DEFAULT_DTRAIN_PARALLEL)).booleanValue();
  GuaguaMapReduceClient guaguaClient;
  int[] inputOutputIndex=DTrainUtils.getInputOutputCandidateCounts(this.columnConfigList);
  int inputNodeCount=inputOutputIndex[0] == 0 ? inputOutputIndex[2] : inputOutputIndex[0];
  int candidateCount=inputOutputIndex[2];
  boolean isAfterVarSelect=(inputOutputIndex[0] != 0);
  List<Integer> allFeatures=CommonUtils.getAllFeatureList(this.columnConfigList,isAfterVarSelect);
  if (modelConfig.getNormalize().getIsParquet()) {
    guaguaClient=new GuaguaParquetMapReduceClient();
    RequiredFieldList requiredFieldList=new RequiredFieldList();
    for (    ColumnConfig columnConfig : super.columnConfigList) {
      if (columnConfig.isTarget()) {
        requiredFieldList.add(new RequiredField(columnConfig.getColumnName(),columnConfig.getColumnNum(),null,DataType.FLOAT));
      }
 else {
        if (inputNodeCount == candidateCount) {
          if (!columnConfig.isMeta() && !columnConfig.isTarget() && CommonUtils.isGoodCandidate(columnConfig)) {
            requiredFieldList.add(new RequiredField(columnConfig.getColumnName(),columnConfig.getColumnNum(),null,DataType.FLOAT));
          }
        }
 else {
          if (!columnConfig.isMeta() && !columnConfig.isTarget() && columnConfig.isFinalSelect()) {
            requiredFieldList.add(new RequiredField(columnConfig.getColumnName(),columnConfig.getColumnNum(),null,DataType.FLOAT));
          }
        }
      }
    }
    requiredFieldList.add(new RequiredField("String_Node_Str",columnConfigList.size(),null,DataType.DOUBLE));
    args.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,"String_Node_Str",serializeRequiredFieldList(requiredFieldList)));
    args.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,"String_Node_Str","String_Node_Str"));
  }
 else {
    guaguaClient=new GuaguaMapReduceClient();
  }
  int parallelNum=Integer.parseInt(Environment.getProperty(CommonConstants.SHIFU_TRAIN_BAGGING_INPARALLEL,"String_Node_Str"));
  int parallelGroups=1;
  if (gs.hasHyperParam()) {
    parallelGroups=(gs.getFlattenParams().size() % parallelNum == 0 ? gs.getFlattenParams().size() / parallelNum : gs.getFlattenParams().size() / parallelNum + 1);
  }
 else {
    parallelGroups=baggingNum % parallelNum == 0 ? baggingNum / parallelNum : baggingNum / parallelNum + 1;
  }
  List<String> progressLogList=new ArrayList<String>(baggingNum);
  boolean isOneJobNotContinuous=false;
  for (int j=0; j < parallelGroups; j++) {
    int currBags=baggingNum;
    if (gs.hasHyperParam()) {
      if (j == parallelGroups - 1) {
        currBags=gs.getFlattenParams().size() % parallelNum == 0 ? parallelNum : gs.getFlattenParams().size() % parallelNum;
      }
 else {
        currBags=parallelNum;
      }
    }
 else {
      if (j == parallelGroups - 1) {
        currBags=baggingNum % parallelNum == 0 ? parallelNum : baggingNum % parallelNum;
      }
 else {
        currBags=parallelNum;
      }
    }
    for (int k=0; k < currBags; k++) {
      int i=j * parallelNum + k;
      if (gs.hasHyperParam()) {
        LOG.info("String_Node_Str",i,gs.getParams(i));
      }
 else       if (isKFoldCV) {
        LOG.info("String_Node_Str",i);
      }
      List<String> localArgs=new ArrayList<String>(args);
      localArgs.add("String_Node_Str");
      localArgs.add(String.format("String_Node_Str",alg,super.getModelConfig().getModelSetName(),i));
      LOG.info("String_Node_Str",i);
      String modelName=getModelName(i);
      Path modelPath=fileSystem.makeQualified(new Path(super.getPathFinder().getModelsPath(sourceType),modelName));
      boolean isContinous=false;
      if (gs.hasHyperParam()) {
        isContinous=false;
      }
 else {
        int intContinuous=checkContinuousTraining(fileSystem,localArgs,modelPath,modelConfig.getTrain().getParams());
        if (intContinuous == -1) {
          LOG.warn("String_Node_Str",i);
          continue;
        }
 else {
          isContinous=(intContinuous == 1);
        }
      }
      if (gs.hasHyperParam() || isKFoldCV) {
        isContinous=false;
      }
      if (!isContinous && !isOneJobNotContinuous) {
        isOneJobNotContinuous=true;
        String srcModelPath=super.getPathFinder().getModelsPath(sourceType);
        String mvModelPath=srcModelPath + "String_Node_Str" + System.currentTimeMillis();
        LOG.info("String_Node_Str",mvModelPath);
        fileSystem.rename(new Path(srcModelPath),new Path(mvModelPath));
        fileSystem.mkdirs(new Path(srcModelPath));
        FileSystem.getLocal(conf).delete(new Path(super.getPathFinder().getModelsPath(SourceType.LOCAL)),true);
      }
      if (NNConstants.NN_ALG_NAME.equalsIgnoreCase(alg)) {
        Map<String,Object> params=gs.hasHyperParam() ? gs.getParams(i) : this.modelConfig.getTrain().getParams();
        Object fssObj=params.get("String_Node_Str");
        FeatureSubsetStrategy featureSubsetStrategy=null;
        double featureSubsetRate=0d;
        if (fssObj != null) {
          try {
            featureSubsetRate=Double.parseDouble(fssObj.toString());
            featureSubsetStrategy=null;
          }
 catch (          NumberFormatException ee) {
            featureSubsetStrategy=FeatureSubsetStrategy.of(fssObj.toString());
          }
        }
 else {
          LOG.warn("String_Node_Str");
          featureSubsetStrategy=FeatureSubsetStrategy.ALL;
          featureSubsetRate=0;
        }
        Set<Integer> subFeatures=null;
        if (isContinous) {
          BasicFloatNetwork existingModel=(BasicFloatNetwork)CommonUtils.loadModel(modelConfig,modelPath,ShifuFileUtils.getFileSystemBySourceType(this.modelConfig.getDataSet().getSource()));
          if (existingModel == null) {
            subFeatures=new HashSet<Integer>(getSubsamplingFeatures(allFeatures,featureSubsetStrategy,featureSubsetRate,inputNodeCount));
          }
 else {
            subFeatures=existingModel.getFeatureSet();
          }
        }
 else {
          subFeatures=new HashSet<Integer>(getSubsamplingFeatures(allFeatures,featureSubsetStrategy,featureSubsetRate,inputNodeCount));
        }
        if (subFeatures == null || subFeatures.size() == 0) {
          localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_NN_FEATURE_SUBSET,"String_Node_Str"));
        }
 else {
          localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_NN_FEATURE_SUBSET,StringUtils.join(subFeatures,',')));
          LOG.debug("String_Node_Str",subFeatures.size(),StringUtils.join(subFeatures,','));
        }
      }
      localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.GUAGUA_OUTPUT,modelPath.toString()));
      if (gs.hasHyperParam() || isKFoldCV) {
        Path valErrPath=fileSystem.makeQualified(new Path(super.getPathFinder().getValErrorPath(sourceType),"String_Node_Str" + i));
        localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.GS_VALIDATION_ERROR,valErrPath.toString()));
      }
      localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_TRAINER_ID,String.valueOf(i)));
      final String progressLogFile=getProgressLogFile(i);
      progressLogList.add(progressLogFile);
      localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,CommonConstants.SHIFU_DTRAIN_PROGRESS_FILE,progressLogFile));
      String hdpVersion=HDPUtils.getHdpVersionForHDP224();
      if (StringUtils.isNotBlank(hdpVersion)) {
        localArgs.add(String.format(CommonConstants.MAPREDUCE_PARAM_FORMAT,"String_Node_Str",hdpVersion));
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
        HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("String_Node_Str"),conf);
      }
      if (isParallel) {
        guaguaClient.addJob(localArgs.toArray(new String[0]));
      }
 else {
        TailThread tailThread=startTailThread(new String[]{progressLogFile});
        guaguaClient.createJob(localArgs.toArray(new String[0])).waitForCompletion(true);
        stopTailThread(tailThread);
      }
    }
    if (isParallel) {
      TailThread tailThread=startTailThread(progressLogList.toArray(new String[0]));
      guaguaClient.run();
      stopTailThread(tailThread);
    }
  }
  if (isKFoldCV) {
    List<Double> valErrs=readAllValidationErrors(sourceType,fileSystem,kCrossValidation);
    double sum=0d;
    for (    Double err : valErrs) {
      sum+=err;
    }
    LOG.info("String_Node_Str",sum / valErrs.size());
    LOG.info("String_Node_Str",System.currentTimeMillis() - start);
  }
 else   if (gs.hasHyperParam()) {
    LOG.info("String_Node_Str",modelConfig.getParams());
    Map<String,Object> params=findBestParams(sourceType,fileSystem,gs);
    for (    Entry<String,Object> entry : params.entrySet()) {
      modelConfig.getParams().put(entry.getKey(),entry.getValue());
    }
    super.pathFinder.getModelConfigPath(SourceType.LOCAL);
    JSONUtils.writeValue(new File(super.pathFinder.getModelConfigPath(SourceType.LOCAL)),modelConfig);
    LOG.info("String_Node_Str",System.currentTimeMillis() - start);
  }
 else {
    if (!gs.hasHyperParam()) {
      for (int i=0; i < baggingNum; i++) {
        String modelName=getModelName(i);
        Path modelPath=fileSystem.makeQualified(new Path(super.getPathFinder().getModelsPath(sourceType),modelName));
        if (ShifuFileUtils.getFileSystemBySourceType(sourceType).exists(modelPath)) {
          copyModelToLocal(modelName,modelPath,sourceType);
        }
 else {
          LOG.warn("String_Node_Str",modelPath.toString());
          status=1;
        }
      }
      boolean copyTmpModelsToLocal=Boolean.TRUE.toString().equalsIgnoreCase(Environment.getProperty(Constants.SHIFU_TMPMODEL_COPYTOLOCAL,"String_Node_Str"));
      if (CommonUtils.isTreeModel(modelConfig.getAlgorithm())) {
        copyTmpModelsToLocal=Boolean.TRUE.toString().equalsIgnoreCase(Environment.getProperty(Constants.SHIFU_TMPMODEL_COPYTOLOCAL,"String_Node_Str"));
        List<BasicML> models=CommonUtils.loadBasicModels(this.modelConfig,this.columnConfigList,null);
        Map<Integer,MutablePair<String,Double>> featureImportances=CommonUtils.computeTreeModelFeatureImportance(models);
        String localFsFolder=pathFinder.getLocalFeatureImportanceFolder();
        String localFIPath=pathFinder.getLocalFeatureImportancePath();
        processRollupForFIFiles(localFsFolder,localFIPath);
        CommonUtils.writeFeatureImportance(localFIPath,featureImportances);
      }
      if (copyTmpModelsToLocal) {
        copyTmpModelsToLocal(tmpModelsPath,sourceType);
      }
 else {
        LOG.info("String_Node_Str",tmpModelsPath);
      }
      LOG.info("String_Node_Str",System.currentTimeMillis() - start);
    }
  }
  if (status != 0) {
    LOG.error("String_Node_Str");
  }
  return status;
}