# Buggy code
def _extract_links(self, selector, response_url, response_encoding, base_url):
    # hacky way to get the underlying lxml parsed document
    for el, attr, attr_val in self._iter_links(selector._root):
        if self.scan_tag(el.tag) and self.scan_attr(attr):
            # pseudo _root.make_links_absolute(base_url)
            attr_val = urljoin(base_url, attr_val)
            url = self.process_attr(attr_val)
            if url is None:
                continue
            if isinstance(url, unicode):
                url = url.encode(response_encoding)
            # to fix relative links after process_value
            url = urljoin(response_url, url)
            link = Link(
                url,
                _collect_string_content(el) or "",
                nofollow=True if el.get("rel") == "nofollow" else False,
            )
            self.links.append(link)

    links = (
        unique_list(self.links, key=lambda link: link.url)
        if self.unique
        else self.links
    )

    return links


# Traceback (test_retry_dns_error)
$ trial scrapy.tests.test_crawl.CrawlTestCase
scrapy.tests.test_crawl
CrawlTestCase
test_delay ...                                                         [OK]
test_engine_status ...                                                 [OK]
test_follow_all ...                                                  [FAIL]
test_referer_header ...                                                [OK]
test_retry_503 ...                                                     [OK]
test_retry_conn_aborted ...                                            [OK]
test_retry_conn_failed ...                                             [OK]
test_retry_conn_lost ...                                               [OK]
test_retry_dns_error ...                                               [OK]
test_start_requests_bug_before_yield ...                               [OK]
test_start_requests_bug_yielding ...                                   [OK]
test_start_requests_dupes ...                                          [OK]
test_start_requests_lazyness ...                                       [OK]
test_timeout_failure ...                                               [OK]
test_timeout_success ...                                               [OK]
test_unbounded_response ...                                            [OK]

===============================================================================
[FAIL]
Traceback (most recent call last):
File "/home/daniel/envs/scrapy/lib/python2.7/site-packages/twisted/internet/defer.py", line 1099, in _inlineCallbacks
result = g.send(result)
File "/home/daniel/src/scrapy/scrapy/tests/test_crawl.py", line 26, in test_follow_all
self.assertEqual(len(spider.urls_visited), 11)  # 10 + start_url
File "/home/daniel/envs/scrapy/lib/python2.7/site-packages/twisted/trial/_synctest.py", line 356, in assertEqual
% (msg, pformat(first), pformat(second)))
twisted.trial.unittest.FailTest: not equal:
a = 31
b = 11


scrapy.tests.test_crawl.CrawlTestCase.test_follow_all
-------------------------------------------------------------------------------
Ran 16 tests in 90.150s

FAILED (failures=1, successes=15)

$ trial scrapy.tests.test_crawl.CrawlTestCase.test_follow_all
scrapy.tests.test_crawl
CrawlTestCase
test_follow_all ...                                                    [OK]

-------------------------------------------------------------------------------
Ran 1 tests in 0.874s

PASSED (successes=1)

# Fixed code
def _extract_links(self, selector, response_url, response_encoding, base_url):
    links = []
    # hacky way to get the underlying lxml parsed document
    for el, attr, attr_val in self._iter_links(selector._root):
        if self.scan_tag(el.tag) and self.scan_attr(attr):
            # pseudo _root.make_links_absolute(base_url)
            attr_val = urljoin(base_url, attr_val)
            url = self.process_attr(attr_val)
            if url is None:
                continue
            if isinstance(url, unicode):
                url = url.encode(response_encoding)
            # to fix relative links after process_value
            url = urljoin(response_url, url)
            link = Link(
                url,
                _collect_string_content(el) or "",
                nofollow=True if el.get("rel") == "nofollow" else False,
            )
            links.append(link)

    return unique_list(links, key=lambda link: link.url) if self.unique else links