Source:
public AnalysisResultFuture run(AnalysisJob job) { assert job == _sparkJobContext.getAnalysisJob(); final AnalysisJob analysisJob = _sparkJobContext.getAnalysisJob(); final Datastore datastore = analysisJob.getDatastore(); final JavaRDD<InputRow> inputRowsRDD = openSourceDatastore(datastore); final JavaPairRDD<String, NamedAnalyzerResult> namedAnalyzerResultsRDD; if (isDistributable(job)) { logger.info("Running the job in distributed mode"); final JavaPairRDD<String, NamedAnalyzerResult> partialNamedAnalyzerResultsRDD = inputRowsRDD .mapPartitionsToPair(new RowProcessingFunction(_sparkJobContext)); namedAnalyzerResultsRDD = partialNamedAnalyzerResultsRDD .reduceByKey(new AnalyzerResultReduceFunction(_sparkJobContext)); } else { logger.warn("Running the job in non-distributed mode"); JavaRDD<InputRow> coalescedInputRowsRDD = inputRowsRDD.coalesce(1); namedAnalyzerResultsRDD = coalescedInputRowsRDD .mapPartitionsToPair(new RowProcessingFunction(_sparkJobContext)); } JavaPairRDD<String, AnalyzerResult> finalAnalyzerResultsRDD = namedAnalyzerResultsRDD .mapValues(new ExtractAnalyzerResultFunction()); finalAnalyzerResultsRDD = finalAnalyzerResultsRDD.persist(StorageLevel.MEMORY_AND_DISK()); final List<Tuple2<String, AnalyzerResult>> results = finalAnalyzerResultsRDD.collect(); logger.info("Finished! Number of AnalyzerResult objects: {}", results.size()); for (Tuple2<String, AnalyzerResult> analyzerResultTuple : results) { final String key = analyzerResultTuple._1; final AnalyzerResult result = analyzerResultTuple._2; logger.info("AnalyzerResult: " + key + "->" + result); } final Map<String, Accumulator<Integer>> accumulators = _sparkJobContext.getAccumulators(); for (Entry<String, Accumulator<Integer>> entry : accumulators.entrySet()) { final String name = entry.getKey(); final Accumulator<Integer> accumulator = entry.getValue(); logger.info("Accumulator: -> {}", name, accumulator.value()); } return new SparkAnalysisResultFuture(results); }

Target:
public AnalysisResultFuture run(AnalysisJob job) { assert job == _sparkJobContext.getAnalysisJob(); final AnalysisJob analysisJob = _sparkJobContext.getAnalysisJob(); final Datastore datastore = analysisJob.getDatastore(); final JavaRDD<InputRow> inputRowsRDD = openSourceDatastore(datastore); final JavaPairRDD<String, NamedAnalyzerResult> namedAnalyzerResultsRDD; if (isDistributable(job)) { logger.info("Running the job in distributed mode"); final JavaPairRDD<String, NamedAnalyzerResult> partialNamedAnalyzerResultsRDD = inputRowsRDD .mapPartitionsToPair(new RowProcessingFunction(_sparkJobContext)); namedAnalyzerResultsRDD = partialNamedAnalyzerResultsRDD.reduceByKey(new AnalyzerResultReduceFunction( _sparkJobContext)); } else { logger.warn("Running the job in non-distributed mode"); JavaRDD<InputRow> coalescedInputRowsRDD = inputRowsRDD.coalesce(1); namedAnalyzerResultsRDD = coalescedInputRowsRDD.mapPartitionsToPair(new RowProcessingFunction( _sparkJobContext)); } JavaPairRDD<String, AnalyzerResult> finalAnalyzerResultsRDD = namedAnalyzerResultsRDD .mapValues(new ExtractAnalyzerResultFunction()); final List<Tuple2<String, AnalyzerResult>> results = finalAnalyzerResultsRDD.collect(); logger.info("Finished! Number of AnalyzerResult objects: {}", results.size()); for (Tuple2<String, AnalyzerResult> analyzerResultTuple : results) { final String key = analyzerResultTuple._1; final AnalyzerResult result = analyzerResultTuple._2; logger.info("AnalyzerResult: " + key + "->" + result); } final Map<String, Accumulator<Integer>> accumulators = _sparkJobContext.getAccumulators(); for (Entry<String, Accumulator<Integer>> entry : accumulators.entrySet()) { final String name = entry.getKey(); final Accumulator<Integer> accumulator = entry.getValue(); logger.info("Accumulator: -> {}", name, accumulator.value()); } return new SparkAnalysisResultFuture(results); }