Program: 31732
Project: 31732
Buggy source:
@Override protected void reduce(IntWritable key,Iterable<BinningInfoWritable> values,Context context) throws IOException, InterruptedException {
  long start=System.currentTimeMillis();
  double sum=0d;
  double squaredSum=0d;
  double tripleSum=0d;
  double quarticSum=0d;
  long count=0L, missingCount=0L;
  double min=Double.MAX_VALUE, max=Double.MIN_VALUE;
  List<Double> binBoundaryList=null;
  List<String> binCategories=null;
  long[] binCountPos=null;
  long[] binCountNeg=null;
  double[] binWeightPos=null;
  double[] binWeightNeg=null;
  ColumnConfig columnConfig=this.columnConfigList.get(key.get());
  HyperLogLogPlus hyperLogLogPlus=null;
  Set<String> fis=new HashSet<String>();
  long totalCount=0, invalidCount=0, validNumCount=0;
  int binSize=0;
  for (  BinningInfoWritable info : values) {
    CountAndFrequentItemsWritable cfiw=info.getCfiw();
    totalCount+=cfiw.getCount();
    invalidCount+=cfiw.getInvalidCount();
    validNumCount+=cfiw.getValidNumCount();
    fis.addAll(cfiw.getFrequetItems());
    if (hyperLogLogPlus == null) {
      hyperLogLogPlus=HyperLogLogPlus.Builder.build(cfiw.getHyperBytes());
    }
 else {
      try {
        hyperLogLogPlus=(HyperLogLogPlus)hyperLogLogPlus.merge(HyperLogLogPlus.Builder.build(cfiw.getHyperBytes()));
      }
 catch (      CardinalityMergeException e) {
        throw new RuntimeException(e);
      }
    }
    if (info.isNumeric() && binBoundaryList == null) {
      binBoundaryList=info.getBinBoundaries();
      binSize=binBoundaryList.size();
      binCountPos=new long[binSize + 1];
      binCountNeg=new long[binSize + 1];
      binWeightPos=new double[binSize + 1];
      binWeightNeg=new double[binSize + 1];
    }
    if (!info.isNumeric() && binCategories == null) {
      binCategories=info.getBinCategories();
      binSize=binCategories.size();
      binCountPos=new long[binSize + 1];
      binCountNeg=new long[binSize + 1];
      binWeightPos=new double[binSize + 1];
      binWeightNeg=new double[binSize + 1];
    }
    count+=info.getTotalCount();
    missingCount+=info.getMissingCount();
    sum+=info.getSum();
    squaredSum+=info.getSquaredSum();
    tripleSum+=info.getTripleSum();
    quarticSum+=info.getQuarticSum();
    if (Double.compare(max,info.getMax()) < 0) {
      max=info.getMax();
    }
    if (Double.compare(min,info.getMin()) > 0) {
      min=info.getMin();
    }
    for (int i=0; i < (binSize + 1); i++) {
      binCountPos[i]+=info.getBinCountPos()[i];
      binCountNeg[i]+=info.getBinCountNeg()[i];
      binWeightPos[i]+=info.getBinWeightPos()[i];
      binWeightNeg[i]+=info.getBinWeightNeg()[i];
    }
  }
  if (columnConfig.isCategorical() && modelConfig.getStats().getCateMaxNumBin() > 0) {
    CateBinningStats cateBinningStats=rebinCategoricalValues(new CateBinningStats(binCategories,binCountPos,binCountNeg,binWeightPos,binWeightNeg));
    LOG.info("String_Node_Str",columnConfig.getColumnName(),binCategories.size(),cateBinningStats.binCategories.size());
    binCategories=cateBinningStats.binCategories;
    binCountPos=cateBinningStats.binCountPos;
    binCountNeg=cateBinningStats.binCountNeg;
    binWeightPos=cateBinningStats.binWeightPos;
    binWeightNeg=cateBinningStats.binWeightNeg;
  }
  double[] binPosRate;
  if (modelConfig.isRegression()) {
    binPosRate=computePosRate(binCountPos,binCountNeg);
  }
 else {
    binPosRate=computeRateForMultiClassfication(binCountPos);
  }
  String binBounString=null;
  if (columnConfig.isCategorical()) {
    if (binCategories.size() < 0 || binCategories.size() > MAX_CATEGORICAL_BINC_COUNT) {
      LOG.warn("String_Node_Str",key.get(),columnConfig.getColumnName(),binCategories.size());
      return;
    }
    binBounString=Base64Utils.base64Encode("String_Node_Str" + StringUtils.join(binCategories,CalculateStatsUDF.CATEGORY_VAL_SEPARATOR) + "String_Node_Str");
    min=Double.MAX_VALUE;
    max=Double.MIN_VALUE;
    sum=0d;
    squaredSum=0d;
    for (int i=0; i < binPosRate.length; i++) {
      if (!Double.isNaN(binPosRate[i])) {
        if (Double.compare(max,binPosRate[i]) < 0) {
          max=binPosRate[i];
        }
        if (Double.compare(min,binPosRate[i]) > 0) {
          min=binPosRate[i];
        }
        long binCount=binCountPos[i] + binCountNeg[i];
        sum+=binPosRate[i] * binCount;
        double squaredVal=binPosRate[i] * binPosRate[i];
        squaredSum+=squaredVal * binCount;
        tripleSum+=squaredVal * binPosRate[i] * binCount;
        quarticSum+=squaredVal * squaredVal * binCount;
      }
    }
  }
 else {
    if (binBoundaryList.size() == 0) {
      LOG.warn("String_Node_Str",key.get(),columnConfig.getColumnName(),binBoundaryList.size());
      return;
    }
    binBounString=binBoundaryList.toString();
  }
  ColumnMetrics columnCountMetrics=null;
  ColumnMetrics columnWeightMetrics=null;
  if (modelConfig.isRegression()) {
    columnCountMetrics=ColumnStatsCalculator.calculateColumnMetrics(binCountNeg,binCountPos);
    columnWeightMetrics=ColumnStatsCalculator.calculateColumnMetrics(binWeightNeg,binWeightPos);
  }
  long realCount=this.statsExcludeMissingValue ? (count - missingCount) : count;
  double mean=sum / realCount;
  double stdDev=Math.sqrt(Math.abs((squaredSum - (sum * sum) / realCount + EPS) / (realCount - 1)));
  double aStdDev=Math.sqrt(Math.abs((squaredSum - (sum * sum) / realCount + EPS) / realCount));
  double skewness=ColumnStatsCalculator.computeSkewness(realCount,mean,aStdDev,sum,squaredSum,tripleSum);
  double kurtosis=ColumnStatsCalculator.computeKurtosis(realCount,mean,aStdDev,sum,squaredSum,tripleSum,quarticSum);
  sb.append(key.get()).append(Constants.DEFAULT_DELIMITER).append(binBounString).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binCountNeg)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binCountPos)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(new double[0])).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binPosRate)).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "String_Node_Str" : df.format(columnCountMetrics.getKs())).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "String_Node_Str" : df.format(columnWeightMetrics.getIv())).append(Constants.DEFAULT_DELIMITER).append(df.format(max)).append(Constants.DEFAULT_DELIMITER).append(df.format(min)).append(Constants.DEFAULT_DELIMITER).append(df.format(mean)).append(Constants.DEFAULT_DELIMITER).append(df.format(stdDev)).append(Constants.DEFAULT_DELIMITER).append(columnConfig.isCategorical() ? "String_Node_Str" : "String_Node_Str").append(Constants.DEFAULT_DELIMITER).append(df.format(mean)).append(Constants.DEFAULT_DELIMITER).append(missingCount).append(Constants.DEFAULT_DELIMITER).append(count).append(Constants.DEFAULT_DELIMITER).append(missingCount * 1.0d / count).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binWeightNeg)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binWeightPos)).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "String_Node_Str" : columnCountMetrics.getWoe()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "String_Node_Str" : columnWeightMetrics.getWoe()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "String_Node_Str" : columnWeightMetrics.getKs()).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "String_Node_Str" : columnCountMetrics.getIv()).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? Arrays.toString(new double[binSize + 1]) : columnCountMetrics.getBinningWoe().toString()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? Arrays.toString(new double[binSize + 1]) : columnWeightMetrics.getBinningWoe().toString()).append(Constants.DEFAULT_DELIMITER).append(skewness).append(Constants.DEFAULT_DELIMITER).append(kurtosis).append(Constants.DEFAULT_DELIMITER).append(totalCount).append(Constants.DEFAULT_DELIMITER).append(invalidCount).append(Constants.DEFAULT_DELIMITER).append(validNumCount).append(Constants.DEFAULT_DELIMITER).append(hyperLogLogPlus.cardinality()).append(Constants.DEFAULT_DELIMITER).append(limitedFrequentItems(fis));
  outputValue.set(sb.toString());
  context.write(NullWritable.get(),outputValue);
  sb.delete(0,sb.length());
  LOG.debug("String_Node_Str",(System.currentTimeMillis() - start));
}

Fixed version (ground truth):
@Override protected void reduce(IntWritable key,Iterable<BinningInfoWritable> values,Context context) throws IOException, InterruptedException {
  long start=System.currentTimeMillis();
  double sum=0d;
  double squaredSum=0d;
  double tripleSum=0d;
  double quarticSum=0d;
  long count=0L, missingCount=0L;
  double min=Double.MAX_VALUE, max=Double.MIN_VALUE;
  List<Double> binBoundaryList=null;
  List<String> binCategories=null;
  long[] binCountPos=null;
  long[] binCountNeg=null;
  double[] binWeightPos=null;
  double[] binWeightNeg=null;
  ColumnConfig columnConfig=this.columnConfigList.get(key.get());
  HyperLogLogPlus hyperLogLogPlus=null;
  Set<String> fis=new HashSet<String>();
  long totalCount=0, invalidCount=0, validNumCount=0;
  int binSize=0;
  for (  BinningInfoWritable info : values) {
    CountAndFrequentItemsWritable cfiw=info.getCfiw();
    totalCount+=cfiw.getCount();
    invalidCount+=cfiw.getInvalidCount();
    validNumCount+=cfiw.getValidNumCount();
    fis.addAll(cfiw.getFrequetItems());
    if (hyperLogLogPlus == null) {
      hyperLogLogPlus=HyperLogLogPlus.Builder.build(cfiw.getHyperBytes());
    }
 else {
      try {
        hyperLogLogPlus=(HyperLogLogPlus)hyperLogLogPlus.merge(HyperLogLogPlus.Builder.build(cfiw.getHyperBytes()));
      }
 catch (      CardinalityMergeException e) {
        throw new RuntimeException(e);
      }
    }
    if (info.isNumeric() && binBoundaryList == null) {
      binBoundaryList=info.getBinBoundaries();
      binSize=binBoundaryList.size();
      binCountPos=new long[binSize + 1];
      binCountNeg=new long[binSize + 1];
      binWeightPos=new double[binSize + 1];
      binWeightNeg=new double[binSize + 1];
    }
    if (!info.isNumeric() && binCategories == null) {
      binCategories=info.getBinCategories();
      binSize=binCategories.size();
      binCountPos=new long[binSize + 1];
      binCountNeg=new long[binSize + 1];
      binWeightPos=new double[binSize + 1];
      binWeightNeg=new double[binSize + 1];
    }
    count+=info.getTotalCount();
    missingCount+=info.getMissingCount();
    sum+=info.getSum();
    squaredSum+=info.getSquaredSum();
    tripleSum+=info.getTripleSum();
    quarticSum+=info.getQuarticSum();
    if (Double.compare(max,info.getMax()) < 0) {
      max=info.getMax();
    }
    if (Double.compare(min,info.getMin()) > 0) {
      min=info.getMin();
    }
    for (int i=0; i < (binSize + 1); i++) {
      binCountPos[i]+=info.getBinCountPos()[i];
      binCountNeg[i]+=info.getBinCountNeg()[i];
      binWeightPos[i]+=info.getBinWeightPos()[i];
      binWeightNeg[i]+=info.getBinWeightNeg()[i];
    }
  }
  if (columnConfig.isCategorical() && modelConfig.getStats().getCateMaxNumBin() > 0) {
    CateBinningStats cateBinningStats=rebinCategoricalValues(new CateBinningStats(binCategories,binCountPos,binCountNeg,binWeightPos,binWeightNeg));
    LOG.info("String_Node_Str",columnConfig.getColumnName(),binCategories.size(),cateBinningStats.binCategories.size());
    binCategories=cateBinningStats.binCategories;
    binCountPos=cateBinningStats.binCountPos;
    binCountNeg=cateBinningStats.binCountNeg;
    binWeightPos=cateBinningStats.binWeightPos;
    binWeightNeg=cateBinningStats.binWeightNeg;
  }
  double[] binPosRate;
  if (modelConfig.isRegression()) {
    binPosRate=computePosRate(binCountPos,binCountNeg);
  }
 else {
    binPosRate=computeRateForMultiClassfication(binCountPos);
  }
  String binBounString=null;
  if (columnConfig.isCategorical()) {
    if (binCategories.size() < 0 || binCategories.size() > MAX_CATEGORICAL_BINC_COUNT) {
      LOG.warn("String_Node_Str",key.get(),columnConfig.getColumnName(),binCategories.size());
      return;
    }
    binBounString=Base64Utils.base64Encode("String_Node_Str" + StringUtils.join(binCategories,CalculateStatsUDF.CATEGORY_VAL_SEPARATOR) + "String_Node_Str");
    min=Double.MAX_VALUE;
    max=Double.MIN_VALUE;
    sum=0d;
    squaredSum=0d;
    for (int i=0; i < binPosRate.length; i++) {
      if (!Double.isNaN(binPosRate[i])) {
        if (Double.compare(max,binPosRate[i]) < 0) {
          max=binPosRate[i];
        }
        if (Double.compare(min,binPosRate[i]) > 0) {
          min=binPosRate[i];
        }
        long binCount=binCountPos[i] + binCountNeg[i];
        sum+=binPosRate[i] * binCount;
        double squaredVal=binPosRate[i] * binPosRate[i];
        squaredSum+=squaredVal * binCount;
        tripleSum+=squaredVal * binPosRate[i] * binCount;
        quarticSum+=squaredVal * squaredVal * binCount;
      }
    }
  }
 else {
    if (binBoundaryList.size() == 0) {
      LOG.warn("String_Node_Str",key.get(),columnConfig.getColumnName(),binBoundaryList.size());
      return;
    }
    binBounString=binBoundaryList.toString();
  }
  ColumnMetrics columnCountMetrics=null;
  ColumnMetrics columnWeightMetrics=null;
  if (modelConfig.isRegression()) {
    columnCountMetrics=ColumnStatsCalculator.calculateColumnMetrics(binCountNeg,binCountPos);
    columnWeightMetrics=ColumnStatsCalculator.calculateColumnMetrics(binWeightNeg,binWeightPos);
  }
  long realCount=this.statsExcludeMissingValue ? (count - missingCount) : count;
  double mean=sum / realCount;
  double stdDev=Math.sqrt(Math.abs((squaredSum - (sum * sum) / realCount + EPS) / (realCount - 1)));
  double aStdDev=Math.sqrt(Math.abs((squaredSum - (sum * sum) / realCount + EPS) / realCount));
  double skewness=ColumnStatsCalculator.computeSkewness(realCount,mean,aStdDev,sum,squaredSum,tripleSum);
  double kurtosis=ColumnStatsCalculator.computeKurtosis(realCount,mean,aStdDev,sum,squaredSum,tripleSum,quarticSum);
  sb.append(key.get()).append(Constants.DEFAULT_DELIMITER).append(binBounString).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binCountNeg)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binCountPos)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(new double[0])).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binPosRate)).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "String_Node_Str" : df.format(columnCountMetrics.getKs())).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "String_Node_Str" : df.format(columnCountMetrics.getIv())).append(Constants.DEFAULT_DELIMITER).append(df.format(max)).append(Constants.DEFAULT_DELIMITER).append(df.format(min)).append(Constants.DEFAULT_DELIMITER).append(df.format(mean)).append(Constants.DEFAULT_DELIMITER).append(df.format(stdDev)).append(Constants.DEFAULT_DELIMITER).append(columnConfig.isCategorical() ? "String_Node_Str" : "String_Node_Str").append(Constants.DEFAULT_DELIMITER).append(df.format(mean)).append(Constants.DEFAULT_DELIMITER).append(missingCount).append(Constants.DEFAULT_DELIMITER).append(count).append(Constants.DEFAULT_DELIMITER).append(missingCount * 1.0d / count).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binWeightNeg)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binWeightPos)).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "String_Node_Str" : columnCountMetrics.getWoe()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "String_Node_Str" : columnWeightMetrics.getWoe()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "String_Node_Str" : columnWeightMetrics.getKs()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "String_Node_Str" : columnWeightMetrics.getIv()).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? Arrays.toString(new double[binSize + 1]) : columnCountMetrics.getBinningWoe().toString()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? Arrays.toString(new double[binSize + 1]) : columnWeightMetrics.getBinningWoe().toString()).append(Constants.DEFAULT_DELIMITER).append(skewness).append(Constants.DEFAULT_DELIMITER).append(kurtosis).append(Constants.DEFAULT_DELIMITER).append(totalCount).append(Constants.DEFAULT_DELIMITER).append(invalidCount).append(Constants.DEFAULT_DELIMITER).append(validNumCount).append(Constants.DEFAULT_DELIMITER).append(hyperLogLogPlus.cardinality()).append(Constants.DEFAULT_DELIMITER).append(limitedFrequentItems(fis));
  outputValue.set(sb.toString());
  context.write(NullWritable.get(),outputValue);
  sb.delete(0,sb.length());
  LOG.debug("String_Node_Str",(System.currentTimeMillis() - start));
}