Repository: internetarchive/openlibrary

Issue:
Canonical link should only include page and sort parameters
Google search results include multiple copies of author pages because we're not setting the canonical link correctly. This is probably diluting the authority of the pages and hurting their search results placement.

### Evidence / Screenshot (if possible)

The search results for: https://www.google.com/search?q=%22the+art+of+nonfiction%22+openlibrary
include both: https://webcache.googleusercontent.com/search?q=cache:iydBXvzKMQUJ:https://openlibrary.org/authors/OL59188A/Ayn_Rand+&cd=2&hl=en&ct=clnk&gl=us
and
https://webcache.googleusercontent.com/search?q=cache:7fLLKIz0fpYJ:https://openlibrary.org/authors/OL59188A/Ayn_Rand%3Fsort%3Dnew+&cd=4&hl=en&ct=clnk&gl=us

in the second case the HTML header includes 

`    <link rel="canonical" href="https://openlibrary.org/authors/OL59188A/Ayn_Rand?sort=new" />
`

which should instead be

`    <link rel="canonical" href="https://openlibrary.org/authors/OL59188A/Ayn_Rand" />
`

ie the canonical URL shouldn't include any sort parameters.

### Stakeholders

@jdlrobson @mekarpeles 


Discussion:
The offending code is here: https://github.com/internetarchive/openlibrary/blob/2f1f2806c5bf3c0aeb65753a08614cab9086e874/openlibrary/plugins/upstream/utils.py#L730-L739

There may be some cases where we want to include the query parameters, both `sort=` definitely isn't one of them. I guess we could start by blacklisting it and then add more exceptions as we find them.


Resolving patch:
diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py
--- a/openlibrary/plugins/upstream/utils.py
+++ b/openlibrary/plugins/upstream/utils.py
@@ -14,6 +14,7 @@
 from six import PY3
 from six.moves import urllib
 from six.moves.collections_abc import MutableMapping
+from six.moves.urllib.parse import parse_qs, urlencode as parse_urlencode, urlparse, urlunparse
 
 from infogami import config
 from infogami.utils import view, delegate, stats
@@ -804,8 +805,22 @@ def canonical_url(self):
         readable_path = web.ctx.get('readable_path', web.ctx.path) or ''
         query = web.ctx.query or ''
         host = web.ctx.host or ''
-        url = (host + readable_path + query)
-        return ("https://" + url) if url else ''
+        url = host + readable_path + query
+        if url:
+            url = "https://" + url
+            parsed_url = urlparse(url)
+
+            parsed_query = parse_qs(parsed_url.query)
+            queries_to_exclude = ['sort', 'mode', 'v', 'type', 'debug']
+
+            canonical_query = {q: v for q, v in parsed_query.items() if q not in queries_to_exclude}
+            query = parse_urlencode(canonical_query, doseq=True)
+            parsed_url = parsed_url._replace(query=query)
+
+            url = urlunparse(parsed_url)
+
+            return url
+        return ''
 
 
 @public


Test patch:
diff --git a/openlibrary/plugins/upstream/tests/test_utils.py b/openlibrary/plugins/upstream/tests/test_utils.py
--- a/openlibrary/plugins/upstream/tests/test_utils.py
+++ b/openlibrary/plugins/upstream/tests/test_utils.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from .. import utils
+import web
 
 def test_url_quote():
     assert utils.url_quote('https://foo bar') == 'https%3A%2F%2Ffoo+bar'
@@ -62,4 +63,42 @@ def __init__(self):
 def test_item_image():
     assert utils.item_image('//foo') == 'https://foo'
     assert utils.item_image(None, 'bar') == 'bar'
-    assert utils.item_image(None) is None
+    assert utils.item_image(None) == None
+
+
+def test_canonical_url():
+    web.ctx.path = '/authors/Ayn_Rand'
+    web.ctx.query = ''
+    web.ctx.host = 'www.openlibrary.org'
+    request = utils.Request()
+
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand'
+    assert request.canonical_url == url
+
+    web.ctx.query = '?sort=newest'
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand'
+    assert request.canonical_url == url
+
+    web.ctx.query = '?page=2'
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand?page=2'
+    assert request.canonical_url == url
+
+    web.ctx.query = '?page=2&sort=newest'
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand?page=2'
+    assert request.canonical_url == url
+
+    web.ctx.query = '?sort=newest&page=2'
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand?page=2'
+    assert request.canonical_url == url
+
+    web.ctx.query = '?sort=newest&page=2&mode=e'
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand?page=2'
+    assert request.canonical_url == url
+
+    web.ctx.query = '?sort=newest&page=2&mode=e&test=query'
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand?page=2&test=query'
+    assert request.canonical_url == url
+
+    web.ctx.query = '?sort=new&mode=2'
+    url = 'https://www.openlibrary.org/authors/Ayn_Rand'
+    assert request.canonical_url == url