Repository: scrapy/scrapy

Issue:
nofollow doesnt work correcly when there multiple values in rel attribute
According to specs rel can have multiple values: http://www.w3.org/TR/html401/struct/links.html#adef-rel

But scrapy ( LxmlParserLinkExtractor and SgmlLinkExtractor(but this one doesnt matter i guess since its deprecated)) just check if it strictly only follow.

``` python
link = Link(url, _collect_string_content(el) or u'',
                nofollow=True if el.get('rel') == 'nofollow' else False)
```

So the cases when links looks like this will not work correctly:

``` html
 <a href='http://blablabla.com/' rel='external nofollow'>bla bla</a>
```

And its not from  a vacuum, its from real world sites where i encountered that scrapy follows nofollow link. For example at this site: www.bruceclay.com/blog/secondary-keywords/



Discussion:

Resolving patch:
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -9,7 +9,7 @@
 
 from scrapy.selector import Selector
 from scrapy.link import Link
-from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.response import get_base_url
@@ -62,7 +62,7 @@ def _extract_links(self, selector, response_url, response_encoding, base_url):
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',
-                nofollow=True if el.get('rel') == 'nofollow' else False)
+                        nofollow=rel_has_nofollow(el.get('rel')))
             links.append(link)
 
         return unique_list(links, key=lambda link: link.url) \
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -9,7 +9,7 @@
 from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
-from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -80,7 +80,7 @@ def unknown_starttag(self, tag, attrs):
                 if self.scan_attr(attr):
                     url = self.process_value(value)
                     if url is not None:
-                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)
+                        link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
                         self.links.append(link)
                         self.current_link = link
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -112,3 +112,8 @@ def md5sum(file):
             break
         m.update(d)
     return m.hexdigest()
+
+def rel_has_nofollow(rel):
+    """Return True if link rel attribute has nofollow type"""
+    return True if rel is not None and 'nofollow' in rel.split() else False
+    


Test patch:
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -96,12 +96,14 @@ def test_link_nofollow(self):
         html = """
         <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
         <a href="about.html">About us</a>
+        <a href="http://google.com/something" rel="external nofollow">Something</a>
         """
         response = HtmlResponse("http://example.org/page.html", body=html)
         lx = SgmlLinkExtractor()
         self.assertEqual([link for link in lx.extract_links(response)], [
             Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
             Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
+            Link(url='http://google.com/something', text=u'Something', nofollow=True),
         ])
 
 
@@ -205,6 +207,9 @@ def test_nofollow(self):
         <div>
         <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
         </div>
+        <div>
+        <p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p>
+        </div>
         </body></html>"""
         response = HtmlResponse("http://example.org/somepage/index.html", body=html)
 
@@ -214,6 +219,7 @@ def test_nofollow(self):
             Link(url='http://example.org/follow.html', text=u'Follow this link'),
             Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
             Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
+            Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
         ])
 
     def test_matches(self):
@@ -467,6 +473,9 @@ def test_xhtml(self):
     <div>
     <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
     </div>
+    <div>
+    <p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p>
+    </div>
 </body>
 </html>
         """
@@ -478,7 +487,8 @@ def test_xhtml(self):
                          [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
                           Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
                           Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                          Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
                         )
 
         response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
@@ -488,7 +498,8 @@ def test_xhtml(self):
                          [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
                           Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
                           Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)]
+                          Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                          Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
                         )
 
     def test_link_wrong_href(self):