Source:
public DatasetStatus ingestDataset(final RelationKey relationKey, final Set<Integer> workersToIngest, final List<List<IndexRef>> indexes, final Operator source, final PartitionFunction partitionFunction) throws InterruptedException, DbException { Set<Integer> actualWorkers = workersToIngest; if (workersToIngest == null) { actualWorkers = getAliveWorkers(); } Preconditions.checkArgument(actualWorkers.size() > 0, "Must use > 0 workers"); int[] workersArray = MyriaUtils.integerCollectionToIntArray(actualWorkers); PartitionFunction partitionFn = partitionFunction; if (partitionFn == null) { partitionFn = new RoundRobinPartitionFunction(workersArray.length); } partitionFn.setNumPartitions(workersArray.length); ExchangePairID scatterId = ExchangePairID.newID(); GenericShuffleProducer scatter = new GenericShuffleProducer(source, scatterId, workersArray, partitionFn); GenericShuffleConsumer gather = new GenericShuffleConsumer(source.getSchema(), scatterId, new int[] { MyriaConstants.MASTER_ID }); DbInsert insert = new DbInsert(gather, relationKey, true, indexes); Map<Integer, SingleQueryPlanWithArgs> workerPlans = new HashMap<Integer, SingleQueryPlanWithArgs>(); for (Integer workerId : workersArray) { workerPlans.put(workerId, new SingleQueryPlanWithArgs(insert)); } try { QueryFuture qf = submitQuery("ingest " + relationKey.toString("sqlite"), "ingest " + relationKey.toString("sqlite"), "ingest " + relationKey.toString("sqlite"), new SingleQueryPlanWithArgs(scatter), workerPlans, false) .sync(); if (qf == null) { return null; } DatasetStatus status = new DatasetStatus(relationKey, source.getSchema(), -1, qf.getQuery().getQueryID(), qf.getQuery() .getExecutionStatistics().getEndTime()); catalog.updatePartitionFunction(relationKey, partitionFn); return status; } catch (CatalogException e) { throw new DbException(e); } }

Target:
public DatasetStatus ingestDataset(final RelationKey relationKey, final Set<Integer> workersToIngest, final List<List<IndexRef>> indexes, final Operator source, final PartitionFunction partitionFunction) throws InterruptedException, DbException { Set<Integer> actualWorkers = workersToIngest; if (workersToIngest == null) { actualWorkers = getAliveWorkers(); } Preconditions.checkArgument(actualWorkers.size() > 0, "Must use > 0 workers"); int[] workersArray = MyriaUtils.integerCollectionToIntArray(actualWorkers); PartitionFunction partitionFn = Objects.firstNonNull(partitionFunction, new RoundRobinPartitionFunction(workersArray.length)); partitionFn.setNumPartitions(workersArray.length); ExchangePairID scatterId = ExchangePairID.newID(); GenericShuffleProducer scatter = new GenericShuffleProducer(source, scatterId, workersArray, partitionFn); GenericShuffleConsumer gather = new GenericShuffleConsumer(source.getSchema(), scatterId, new int[] { MyriaConstants.MASTER_ID }); DbInsert insert = new DbInsert(gather, relationKey, true, indexes); Map<Integer, SubQueryPlan> workerPlans = new HashMap<>(); for (Integer workerId : workersArray) { workerPlans.put(workerId, new SubQueryPlan(insert)); } ListenableFuture<Query> qf; try { qf = submitQuery("ingest " + relationKey.toString(), "ingest " + relationKey.toString(), "ingest " + relationKey.toString(), new SubQueryPlan(scatter), workerPlans, false); } catch (CatalogException e) { throw new DbException("Error submitting query", e); } try { qf.get(); } catch (ExecutionException e) { throw new DbException("Error executing query", e.getCause()); } try { catalog.updatePartitionFunction(relationKey, partitionFn); } catch (CatalogException e) { throw new DbException("Error updating the partition function", e); } return getDatasetStatus(relationKey); }