Source:
public void submit() throws Exception { Exception error = null; try { final JavaRDD<String> streamed = rdd.mapPartitionsWithIndex(new ResultStreamer(context, uuid, host, port, rdd.getNumPartitions(), clientBatches, clientBatchSize), true); int numPartitions = streamed.getNumPartitions(); int partitionsBatchSize = PARALLEL_PARTITIONS / 2; int partitionBatches = numPartitions / partitionsBatchSize; if (numPartitions % partitionsBatchSize > 0) partitionBatches++; if (LOG.isTraceEnabled()) LOG.trace("Num partitions " + numPartitions + " clientBatches " + partitionBatches); Properties properties = SpliceSpark.getContextUnsafe().sc().getLocalProperties(); submitBatch(0, partitionsBatchSize, numPartitions, streamed, properties); if (partitionBatches > 1) submitBatch(1, partitionsBatchSize, numPartitions, streamed, properties); int received = 0; int submitted = 2; while (received < partitionBatches && error == null) { try { Future<Object> resultFuture = futures.remove(); Object result = null; while (result == null) { if (jobStatus != null && !jobStatus.isRunning()) { throw new CancellationException("The olap job is no longer running, cancelling Spark job"); } try { result = resultFuture.get(10, TimeUnit.SECONDS);; } catch (TimeoutException ex) { } } received++; if ("STOP".equals(result)) { if (LOG.isTraceEnabled()) LOG.trace("Stopping after receiving " + received + " clientBatches of " + partitionBatches); break; } if (submitted < partitionBatches) { submitBatch(submitted, partitionsBatchSize, numPartitions, streamed, properties); submitted++; } } catch (Exception e) { error = e; } } } finally { executor.shutdown(); } if (error != null) { LOG.error(error); throw error; } }

Target:
public void submit() throws Exception { Exception error = null; try { final JavaRDD<String> streamed = rdd.mapPartitionsWithIndex(new ResultStreamer(context, uuid, host, port, rdd.getNumPartitions(), clientBatches, clientBatchSize), true); int numPartitions = streamed.getNumPartitions(); int partitionsBatchSize = PARALLEL_PARTITIONS / 2; int partitionBatches = numPartitions / partitionsBatchSize; if (numPartitions % partitionsBatchSize > 0) partitionBatches++; if (LOG.isTraceEnabled()) LOG.trace("Num partitions " + numPartitions + " clientBatches " + partitionBatches); Properties properties = SpliceSpark.getContextUnsafe().sc().getLocalProperties(); submitBatch(0, partitionsBatchSize, numPartitions, streamed, properties); if (partitionBatches > 1) submitBatch(1, partitionsBatchSize, numPartitions, streamed, properties); int received = 0; int submitted = 2; while (received < partitionBatches && error == null) { try { Future<Object> resultFuture = futures.remove(); Object result = null; while (result == null) { if (jobStatus != null && !jobStatus.isRunning()) { throw new CancellationException("The olap job is no longer running, cancelling Spark job"); } try { result = resultFuture.get(10, TimeUnit.SECONDS); } catch (TimeoutException ex) { } } received++; if ("STOP".equals(result)) { if (LOG.isTraceEnabled()) LOG.trace("Stopping after receiving " + received + " clientBatches of " + partitionBatches); break; } if (submitted < partitionBatches) { submitBatch(submitted, partitionsBatchSize, numPartitions, streamed, properties); submitted++; } } catch (Exception e) { error = e; } } } finally { executor.shutdown(); } if (error != null) { LOG.error(error); throw error; } }