Source:
public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException { Preconditions.checkState( config.getGranularitySpec() instanceof UniformGranularitySpec, String.format( "Cannot use %s without %s", GranularUnprocessedPathSpec.class.getSimpleName(), UniformGranularitySpec.class.getSimpleName() ) ); final Path betaInput = new Path(getInputPath()); final FileSystem fs = betaInput.getFileSystem(job.getConfiguration()); final Granularity segmentGranularity = config.getGranularitySpec().getSegmentGranularity(); Map<Long, Long> inputModifiedTimes = new TreeMap<>(Ordering.natural()); for (FileStatus status : FSSpideringIterator.spiderIterable(fs, betaInput)) { final DateTime key = segmentGranularity.toDate(status.getPath().toString()); final Long currVal = inputModifiedTimes.get(key.getMillis()); final long mTime = status.getModificationTime(); inputModifiedTimes.put(key.getMillis(), currVal == null ? mTime : Math.max(currVal, mTime)); } Set<Interval> bucketsToRun = Sets.newTreeSet(Comparators.intervals()); for (Map.Entry<Long, Long> entry : inputModifiedTimes.entrySet()) { DateTime timeBucket = new DateTime(entry.getKey()); long mTime = entry.getValue(); String bucketOutput = String.format( "%s/%s", config.getSchema().getIOConfig().getSegmentOutputPath(), segmentGranularity.toPath(timeBucket) ); for (FileStatus fileStatus : FSSpideringIterator.spiderIterable(fs, new Path(bucketOutput))) { if (fileStatus.getModificationTime() > mTime) { bucketsToRun.add(new Interval(timeBucket, segmentGranularity.increment(timeBucket))); break; } } if (bucketsToRun.size() >= maxBuckets) { break; } } config.setGranularitySpec( new UniformGranularitySpec( segmentGranularity, config.getGranularitySpec().getQueryGranularity(), config.getGranularitySpec().isRollup(), Lists.newArrayList(bucketsToRun) ) ); return super.addInputPaths(config, job); }

Target:
public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException { Preconditions.checkState( config.getGranularitySpec() instanceof UniformGranularitySpec, String.format( "Cannot use %s without %s", GranularUnprocessedPathSpec.class.getSimpleName(), UniformGranularitySpec.class.getSimpleName() ) ); final Path betaInput = new Path(getInputPath()); final FileSystem fs = betaInput.getFileSystem(job.getConfiguration()); final Granularity segmentGranularity = config.getGranularitySpec().getSegmentGranularity(); Map<Long, Long> inputModifiedTimes = new TreeMap<>(Ordering.natural().reverse()); for (FileStatus status : FSSpideringIterator.spiderIterable(fs, betaInput)) { final DateTime key = segmentGranularity.toDate(status.getPath().toString()); final Long currVal = inputModifiedTimes.get(key.getMillis()); final long mTime = status.getModificationTime(); inputModifiedTimes.put(key.getMillis(), currVal == null ? mTime : Math.max(currVal, mTime)); } Set<Interval> bucketsToRun = Sets.newTreeSet(Comparators.intervals()); for (Map.Entry<Long, Long> entry : inputModifiedTimes.entrySet()) { DateTime timeBucket = new DateTime(entry.getKey()); long mTime = entry.getValue(); String bucketOutput = String.format( "%s/%s", config.getSchema().getIOConfig().getSegmentOutputPath(), segmentGranularity.toPath(timeBucket) ); for (FileStatus fileStatus : FSSpideringIterator.spiderIterable(fs, new Path(bucketOutput))) { if (fileStatus.getModificationTime() > mTime) { bucketsToRun.add(new Interval(timeBucket, segmentGranularity.increment(timeBucket))); break; } } if (bucketsToRun.size() >= maxBuckets) { break; } } config.setGranularitySpec( new UniformGranularitySpec( segmentGranularity, config.getGranularitySpec().getQueryGranularity(), config.getGranularitySpec().isRollup(), Lists.newArrayList(bucketsToRun) ) ); return super.addInputPaths(config, job); }