Source:
<code>private void submitBatch(int batch, int batchSize, int numPartitions, final JavaRDD<String> streamed, final Properties properties) { final List<Integer> list = new ArrayList<>(); for (int j = batch*batchSize; j < numPartitions && j < (batch+1)*batchSize; j++) { list.add(j); } if (LOG.isTraceEnabled()) LOG.trace("Submitting batch " + batch + " with partitions " + list); final Seq objects = JavaConversions.asScalaBuffer(list).toList(); completionService.submit(new Callable<Object>() { @Override public Object call() throws Exception { SparkContext sc = SpliceSpark.getContextUnsafe().sc(); sc.setLocalProperties(properties); if (shufflePartitions != null) <START> sc.setLocalProperty(SQLConf.SHUFFLE_PARTITIONS().key(), shufflePartitions.toString()); <END> AtomicBoolean cont = new AtomicBoolean(true); SimpleFutureAction<Boolean> job = sc.submitJob(streamed.rdd(), new FunctionAdapter(), objects, new AbstractFunction2<Object, String, BoxedUnit>() { @Override public BoxedUnit apply(Object index, String result) { if ("STOP".equals(result)) cont.set(false); return BoxedUnit.UNIT; } }, new AbstractFunction0<Boolean>() { @Override public Boolean apply() { return cont.get(); } }); Boolean result = null; while (result == null) { try { result = Await.result(job, Duration.apply(1, TimeUnit.SECONDS)); } catch (InterruptedException e) { job.cancel(); throw new CancellationException("Interrupted"); } catch (TimeoutException e) { if (!jobStatus.isRunning()) { job.cancel(); throw new CancellationException("The olap job is no longer running, cancelling Spark job"); } } } return result ? "CONTINUE" : "STOP"; } }); }</code><technical_language>do need set shuffle partition property in QueryJob</technical_language>

Target:
private void submitBatch(int batch, int batchSize, int numPartitions, final JavaRDD<String> streamed, final Properties properties) { final List<Integer> list = new ArrayList<>(); for (int j = batch*batchSize; j < numPartitions && j < (batch+1)*batchSize; j++) { list.add(j); } if (LOG.isTraceEnabled()) LOG.trace("Submitting batch " + batch + " with partitions " + list); final Seq objects = JavaConversions.asScalaBuffer(list).toList(); completionService.submit(new Callable<Object>() { @Override public Object call() throws Exception { SparkContext sc = SpliceSpark.getContextUnsafe().sc(); sc.setLocalProperties(properties); AtomicBoolean cont = new AtomicBoolean(true); SimpleFutureAction<Boolean> job = sc.submitJob(streamed.rdd(), new FunctionAdapter(), objects, new AbstractFunction2<Object, String, BoxedUnit>() { @Override public BoxedUnit apply(Object index, String result) { if ("STOP".equals(result)) cont.set(false); return BoxedUnit.UNIT; } }, new AbstractFunction0<Boolean>() { @Override public Boolean apply() { return cont.get(); } }); Boolean result = null; while (result == null) { try { result = Await.result(job, Duration.apply(1, TimeUnit.SECONDS)); } catch (InterruptedException e) { job.cancel(); throw new CancellationException("Interrupted"); } catch (TimeoutException e) { if (!jobStatus.isRunning()) { job.cancel(); throw new CancellationException("The olap job is no longer running, cancelling Spark job"); } } } return result ? "CONTINUE" : "STOP"; } }); }