Source:
public byte[] call(final byte[] memento) throws Exception { LOG.log(Level.INFO, "{0} starting...", taskId); if (addedEval) { LOG.log(Level.INFO, "This worker is added by EM. " + "Will skip initializing TrainingDataProvider and Trainer"); } else { trainingDataProvider.initialize(); trainer.initialize(); } synchronizer.globalBarrier(); workerClock.initialize(); final int initialClock = workerClock.getWorkerClock(); for (int iteration = initialClock; iteration < maxIterations; ++iteration) { LOG.log(Level.INFO, "Starting iteration {0}", iteration); final long epochStartTime = System.currentTimeMillis(); final int numEMBlocks = memoryStore.getNumBlocks(); trainingDataProvider.prepareDataForEpoch(); final Collection<V> epochData = new LinkedList<>(); int miniBatchIdx = 0; while (true) { final Collection<V> miniBatchData = trainingDataProvider.getNextTrainingData().values(); if (miniBatchData.isEmpty()) { break; } final MiniBatchInfo miniBatchInfo = MiniBatchInfo.getBuilder() .setEpochIdx(iteration) .setMiniBatchIdx(miniBatchIdx) .build(); trainer.runMiniBatch(miniBatchData, miniBatchInfo); epochData.addAll(miniBatchData); miniBatchIdx++; if (abortFlag.get()) { LOG.log(Level.INFO, "Stop task"); workerClock.recordClockNetworkWaitingTime(); return null; } } final EpochInfo epochInfo = EpochInfo.getBuilder() .setEpochIdx(iteration) .setNumMiniBatches(miniBatchIdx) .setNumEMBlocks(numEMBlocks) .setEpochStartTime(epochStartTime) .build(); trainer.onEpochFinished(epochData, epochInfo); workerClock.clock(); } synchronizer.globalBarrier(); trainer.cleanup(); workerClock.recordClockNetworkWaitingTime(); return null; }

Target:
public byte[] call(final byte[] memento) throws Exception { LOG.log(Level.INFO, "{0} starting...", taskId); if (addedEval) { LOG.log(Level.INFO, "This worker is added by EM. " + "Will skip initializing TrainingDataProvider and Trainer"); } else { trainingDataProvider.loadData(); trainer.initGlobalSettings(); } synchronizer.globalBarrier(); workerClock.initialize(); final int initialClock = workerClock.getWorkerClock(); for (int iteration = initialClock; iteration < maxIterations; ++iteration) { LOG.log(Level.INFO, "Starting iteration {0}", iteration); final long epochStartTime = System.currentTimeMillis(); final int numEMBlocks = memoryStore.getNumBlocks(); trainingDataProvider.prepareDataForEpoch(); final Collection<V> epochData = new LinkedList<>(); int miniBatchIdx = 0; while (true) { final Collection<V> miniBatchData = trainingDataProvider.getNextTrainingData().values(); if (miniBatchData.isEmpty()) { break; } final MiniBatchInfo miniBatchInfo = MiniBatchInfo.getBuilder() .setEpochIdx(iteration) .setMiniBatchIdx(miniBatchIdx) .build(); trainer.runMiniBatch(miniBatchData, miniBatchInfo); epochData.addAll(miniBatchData); miniBatchIdx++; if (abortFlag.get()) { LOG.log(Level.INFO, "Stop task"); workerClock.recordClockNetworkWaitingTime(); return null; } } final EpochInfo epochInfo = EpochInfo.getBuilder() .setEpochIdx(iteration) .setNumMiniBatches(miniBatchIdx) .setNumEMBlocks(numEMBlocks) .setEpochStartTime(epochStartTime) .build(); trainer.onEpochFinished(epochData, epochInfo); workerClock.clock(); } synchronizer.globalBarrier(); trainer.cleanup(); workerClock.recordClockNetworkWaitingTime(); return null; }