Source:
<code>@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException { Configuration configuration = jobContext.getConfiguration(); List<InputSplit> result = new ArrayList<>(); List<BulkoadInputWritable> bulkoadInputWritables = new ArrayList<>(); int depth = configuration.getInt(DEPTH, 2); if (depth != 2 && depth != 3) { throw new IllegalStateException("Invalid depth " + depth); } Set<String> validFamilies = getValidFamiliesSet(configuration); try { Path rootPath = new Path(configuration.get(ROOT_DIR)); FileSystem fs = rootPath.getFileSystem(configuration); for (FileStatus child : fs.listStatus(rootPath)) { if (!child.isDirectory()) { LOG.info("Skipping non dir " + child.getPath()); continue; } if (child.getPath().getName().startsWith("_")) { LOG.info("Skipping " + child.getPath()); continue; } if (depth == 2) { bulkoadInputWritables .addAll(createSplitsForFamilyDirectory(fs, child.getPath(), validFamilies)); } else { for (FileStatus family : fs.listStatus(child.getPath())) { bulkoadInputWritables .addAll(createSplitsForFamilyDirectory(fs, family.getPath(), validFamilies)); } } } <START> int maxMapTasks = configuration.getInt(MAX_MAP_TASK, 1); int filesPerMap = bulkoadInputWritables.size() / maxMapTasks; for (List<BulkoadInputWritable> partition : Lists .partition(bulkoadInputWritables, filesPerMap)) { BulkoadInputWritableArray bulkoadInputWritableArray = new BulkoadInputWritableArray(); bulkoadInputWritableArray .set(partition.toArray(new BulkoadInputWritable[partition.size()])); result.add(new BulkloadInputSplit(bulkoadInputWritableArray)); } <END> } catch (Exception e) { LOG.error("Error computing splits for bulkload M/R job", e); throw new RuntimeException("Failed to compute splits", e); } return result; }</code><technical_language>default a single mapper? this outperform non-distributed LoadIncrementalHFiles</technical_language>

Target:
@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException { Configuration configuration = jobContext.getConfiguration(); List<InputSplit> result = new ArrayList<>(); List<BulkoadInputWritable> bulkoadInputWritables = new ArrayList<>(); int depth = configuration.getInt(DEPTH, 2); if (depth != 2 && depth != 3) { throw new IllegalStateException("Invalid depth " + depth); } Set<String> validFamilies = getValidFamiliesSet(configuration); Path rootPath = new Path(configuration.get(ROOT_DIR)); FileSystem fs = rootPath.getFileSystem(configuration); for (FileStatus child : fs.listStatus(rootPath)) { if (!child.isDirectory()) { LOG.info("Skipping non dir " + child.getPath()); continue; } if (child.getPath().getName().startsWith("_")) { LOG.info("Skipping " + child.getPath()); continue; } if (depth == 2) { bulkoadInputWritables .addAll(createSplitsForFamilyDirectory(fs, child.getPath(), validFamilies)); } else { for (FileStatus family : fs.listStatus(child.getPath())) { bulkoadInputWritables .addAll(createSplitsForFamilyDirectory(fs, family.getPath(), validFamilies)); } } } int maxMapTasks = configuration.getInt(MAX_MAP_TASK, 1); int filesPerMap = bulkoadInputWritables.size() / maxMapTasks; for (List<BulkoadInputWritable> partition : Lists .partition(bulkoadInputWritables, filesPerMap)) { BulkoadInputWritableArray bulkoadInputWritableArray = new BulkoadInputWritableArray(); bulkoadInputWritableArray .set(partition.toArray(new BulkoadInputWritable[partition.size()])); result.add(new BulkloadInputSplit(bulkoadInputWritableArray)); } return result; }