Source:
private ContentRetrieverResponse retrieveUrlContent(String currentUrl, int retryCount) throws MalformedURLException, IOException, InterruptedException { if (retryCount > maxRetriesCount) { String message = String.format("number of maximum retries exceeded: '%d' for url: %s", maxRetriesCount, currentUrl); log.error(message); return new ContentRetrieverResponse(new RetryLimitExceededException(message)); } try (CloseableHttpResponse httpResponse = httpClient.execute(new HttpGet(currentUrl))) { int statusCode = httpResponse.getStatusLine().getStatusCode(); switch (statusCode) { case HttpURLConnection.HTTP_OK: { return readPageContent(httpResponse.getEntity(), maxPageContentLength, currentUrl); } case HttpURLConnection.HTTP_NOT_FOUND: { return new ContentRetrieverResponse(new NoSuchElementException("unable to find page at: " + currentUrl)); } case HttpURLConnection.HTTP_MOVED_TEMP: case HttpURLConnection.HTTP_MOVED_PERM: case HttpURLConnection.HTTP_SEE_OTHER: { String redirectedUrl = getHeaderValue(httpResponse.getAllHeaders(), HEADER_LOCATION); if (StringUtils.isNotBlank(redirectedUrl)) { log.info(String.format("got %d response code, redirecting to %s, server response: %s", statusCode, redirectedUrl, EntityUtils.toString(httpResponse.getEntity()))); return retrieveUrlContent(redirectedUrl, ++retryCount); } else { return new ContentRetrieverResponse( new RuntimeException("resource was moved, missing redirect header for the url: " + currentUrl)); } } case 429: { log.warn(String.format("got %d response code, potential reason: rate limit reached. Delaying for %d ms, server response: %s", statusCode, throttleSleepTime, EntityUtils.toString(httpResponse.getEntity()))); Thread.sleep(throttleSleepTime); return retrieveUrlContent(currentUrl, ++retryCount); } default: { return new ContentRetrieverResponse(new RuntimeException(String.format( "got unsupported HTTP response code: %d when accessing page at url: %s", statusCode, currentUrl))); } } } }

Target:
private ContentRetrieverResponse retrieveUrlContent(String currentUrl, int retryCount) throws Exception { if (retryCount > maxRetriesCount) { String message = String.format("number of maximum retries exceeded: '%d' for url: %s", maxRetriesCount, currentUrl); log.error(message); return new ContentRetrieverResponse(new RetryLimitExceededException(message)); } try (CloseableHttpResponse httpResponse = httpClient.execute(new HttpGet(currentUrl))) { int statusCode = httpResponse.getStatusLine().getStatusCode(); switch (statusCode) { case HttpURLConnection.HTTP_OK: { return readPageContent(httpResponse.getEntity(), maxPageContentLength, currentUrl); } case HttpURLConnection.HTTP_NOT_FOUND: { return new ContentRetrieverResponse(new NoSuchElementException("unable to find page at: " + currentUrl)); } case HttpURLConnection.HTTP_MOVED_TEMP: case HttpURLConnection.HTTP_MOVED_PERM: case HttpURLConnection.HTTP_SEE_OTHER: { String redirectedUrl = getHeaderValue(httpResponse.getAllHeaders(), HEADER_LOCATION); if (StringUtils.isNotBlank(redirectedUrl)) { log.info("got response code, redirecting to {}, server response: {}", statusCode, redirectedUrl, EntityUtils.toString(httpResponse.getEntity())); return retrieveUrlContent(redirectedUrl, ++retryCount); } else { return new ContentRetrieverResponse( new RuntimeException("resource was moved, missing redirect header for the url: " + currentUrl)); } } case HTTP_TOO_MANY_REQUESTS: { log.warn("got response code, potential reason: rate limit reached. Delaying for ms, server response: {}", statusCode, throttleSleepTime, EntityUtils.toString(httpResponse.getEntity())); Thread.sleep(throttleSleepTime); return retrieveUrlContent(currentUrl, ++retryCount); } default: { return new ContentRetrieverResponse(new RuntimeException(String.format( "got unsupported HTTP response code: %d when accessing page at url: %s", statusCode, currentUrl))); } } } }