Source:
<code>public void getStats(Path segment, final SegmentReaderStats stats) throws Exception { long cnt = 0L; Text key = new Text(); Text val = new Text(); FileSystem fs = segment.getFileSystem(getConf()); if (ge) { MapFile.Reader[] readers = MapFileOutputFormat.getReaders( new Path(segment, CrawlDatum.GENERATE_DIR_NAME), getConf()); for (int i = 0; i < readers.length; i++) { while (readers[i].next(key, val)) cnt++; readers[i].close(); <START> } <END> stats.generated = cnt; } if (fe) { Path fetchDir = new Path(segment, CrawlDatum.FETCH_DIR_NAME); if (fs.exists(fetchDir) && fs.getFileStatus(fetchDir).isDirectory()) { cnt = 0L; long start = Long.MAX_VALUE; long end = Long.MIN_VALUE; CrawlDatum value = new CrawlDatum(); MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(fetchDir, getConf()); for (int i = 0; i < mreaders.length; i++) { while (mreaders[i].next(key, value)) { cnt++; if (value.getFetchTime() < start) start = value.getFetchTime(); if (value.getFetchTime() > end) end = value.getFetchTime(); } mreaders[i].close(); } stats.start = start; stats.end = end; stats.fetched = cnt; } } if (pd) { Path parseDir = new Path(segment, ParseData.DIR_NAME); if (fs.exists(parseDir) && fs.getFileStatus(parseDir).isDirectory()) { cnt = 0L; long errors = 0L; ParseData value = new ParseData(); MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(parseDir, getConf()); for (int i = 0; i < mreaders.length; i++) { while (mreaders[i].next(key, value)) { cnt++; if (!value.getStatus().isSuccess()) errors++; } mreaders[i].close(); } stats.parsed = cnt; stats.parseErrors = errors; } } }</code><technical_language>crawl_generate is stored SequenceFile (and ordered key) - a SequenceFile reader</technical_language>

Target:
public void getStats(Path segment, final SegmentReaderStats stats) throws Exception { long cnt = 0L; Text key = new Text(); Text val = new Text(); FileSystem fs = segment.getFileSystem(getConf()); if (ge) { SequenceFile.Reader[] readers = SegmentReaderUtil.getReaders( new Path(segment, CrawlDatum.GENERATE_DIR_NAME), getConf()); for (int i = 0; i < readers.length; i++) { while (readers[i].next(key, val)) cnt++; readers[i].close(); } stats.generated = cnt; } if (fe) { Path fetchDir = new Path(segment, CrawlDatum.FETCH_DIR_NAME); if (fs.exists(fetchDir) && fs.getFileStatus(fetchDir).isDirectory()) { cnt = 0L; long start = Long.MAX_VALUE; long end = Long.MIN_VALUE; CrawlDatum value = new CrawlDatum(); MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(fetchDir, getConf()); for (int i = 0; i < mreaders.length; i++) { while (mreaders[i].next(key, value)) { cnt++; if (value.getFetchTime() < start) start = value.getFetchTime(); if (value.getFetchTime() > end) end = value.getFetchTime(); } mreaders[i].close(); } stats.start = start; stats.end = end; stats.fetched = cnt; } } if (pd) { Path parseDir = new Path(segment, ParseData.DIR_NAME); if (fs.exists(parseDir) && fs.getFileStatus(parseDir).isDirectory()) { cnt = 0L; long errors = 0L; ParseData value = new ParseData(); MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(parseDir, getConf()); for (int i = 0; i < mreaders.length; i++) { while (mreaders[i].next(key, value)) { cnt++; if (!value.getStatus().isSuccess()) errors++; } mreaders[i].close(); } stats.parsed = cnt; stats.parseErrors = errors; } } }