Source:
<code>public void map(String key, WebPage page, Context context) throws IOException, InterruptedException { ParseStatus pstatus = page.getParseStatus(); if (pstatus == null || !ParseStatusUtils.isSuccess(pstatus) || pstatus.getMinorCode() == ParseStatusCodes.SUCCESS_REDIRECT) { return; } Utf8 mark = Mark.UPDATEDB_MARK.checkMark(page); if (mark == null) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping " + TableUtil.unreverseUrl(key) + "; not updated on db yet"); } return; } if (deduplicate) { <START> Duplicate duplicate = duplicateStore.get( <END> new String(page.getSignature().array())); if (!indexUtil.isOriginal(TableUtil.unreverseUrl(key), duplicate.getUrls())) { return; } else { StringBuilder duplicates = new StringBuilder(""); for (CharSequence url : duplicate.getUrls()) { String duplicateKey = TableUtil.reverseUrl(url.toString()); if (!key.equals(duplicateKey)) { duplicates.append(url); duplicates.append(Nutch.DUPLICATE_URLS_SPLITTER); writers.delete(duplicateKey); } } page.getMetadata().put(Nutch.DUPLICATE_URLS_KEY, ByteBuffer.wrap(duplicates.toString().getBytes())); } } NutchDocument doc = indexUtil.index(key, page); if (doc == null) { return; } if (mark != null) { Mark.INDEX_MARK.putMark(page, Mark.UPDATEDB_MARK.checkMark(page)); store.put(key, page); } context.write(key, doc); context.getCounter("IndexerJob", "DocumentCount").increment(1); }</code><technical_language>@benmvachon, lines add a quadratic complexity number of duplicates? I if 10,000 URLs for a signature (maybe "Internal Server Error"), for 10,000 URLs list of 10,000 duplicates is looped determine URL is "original". This 100 million times a URL is parsed split count number of path segments. Please correct if wrong. if this is case, determine "original" URL in reducer of DeduplicatorJob save in duplicate store</technical_language>

Target:
public void map(String key, WebPage page, Context context) throws IOException, InterruptedException { ParseStatus pstatus = page.getParseStatus(); if (pstatus == null || !ParseStatusUtils.isSuccess(pstatus) || pstatus.getMinorCode() == ParseStatusCodes.SUCCESS_REDIRECT) { return; } Utf8 mark = Mark.UPDATEDB_MARK.checkMark(page); String url = TableUtil.unreverseUrl(key); if (mark == null) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping " + url + "; not updated on db yet"); } return; } if (deduplicate) { Duplicate duplicate = duplicateStore.get( new String(page.getSignature().array())); if (duplicate != null) { if (duplicate.getOriginal() != null && !duplicate.getOriginal().toString().equals(url)) { return; } else { StringBuilder duplicates = new StringBuilder(""); for (CharSequence duplicateUrl : duplicate.getUrls()) { String duplicateKey = TableUtil.reverseUrl(duplicateUrl.toString()); if (!key.equals(duplicateKey)) { duplicates.append(duplicateUrl); duplicates.append(Nutch.DUPLICATE_URLS_SPLITTER); writers.delete(duplicateKey); } } page.getMetadata().put(Nutch.DUPLICATE_URLS_KEY, ByteBuffer.wrap(duplicates.toString().getBytes())); } } } NutchDocument doc = indexUtil.index(key, page); if (doc == null) { return; } if (mark != null) { Mark.INDEX_MARK.putMark(page, Mark.UPDATEDB_MARK.checkMark(page)); store.put(key, page); } context.write(key, doc); context.getCounter("IndexerJob", "DocumentCount").increment(1); }