Repository: oduwsdl/MementoEmbed

Issue:
parsing multiple Link: response headers
Martin brought this to my attention.  Here's a sample URL:

https://scholarlyorphans.org/memento/20181009212548/https://ianmilligan.ca/

and it returns two different Link headers:

$ curl -IL https://scholarlyorphans.org/memento/20181009212548/https://ianmilligan.ca/
HTTP/1.1 200 OK
Server: nginx/1.12.1
Content-Type: text/html; charset=UTF-8
Connection: keep-alive
X-Archive-Orig-Server: nginx
Date: Tue, 09 Oct 2018 21:25:48 GMT
X-Archive-Orig-Transfer-Encoding: chunked
X-Archive-Orig-Connection: keep-alive
X-Archive-Orig-Strict-Transport-Security: max-age=86400
X-Archive-Orig-Vary: Accept-Encoding
X-Archive-Orig-Vary: Cookie
X-hacker: If you're reading this, you should visit automattic.com/jobs and apply to join the fun, mention this header.
Link: <https://wp.me/4cEB>; rel=shortlink
X-Archive-Orig-Content-Encoding: gzip
X-ac: 3.sea _bur
Memento-Datetime: Tue, 09 Oct 2018 21:25:48 GMT
Link: <https://ianmilligan.ca/>; rel="original", <https://scholarlyorphans.org/memento/https://ianmilligan.ca/>; rel="timegate", <https://scholarlyorphans.org/memento/timemap/link/https://ianmilligan.ca/>; rel="timemap"; type="application/link-format", <https://scholarlyorphans.org/memento/20181009212548/https://ianmilligan.ca/>; rel="memento"; datetime="Tue, 09 Oct 2018 21:25:48 GMT"; collection="memento"
Content-Location: https://scholarlyorphans.org/memento/20181009212548/https://ianmilligan.ca/
Content-Security-Policy: default-src 'unsafe-eval' 'unsafe-inline' 'self' data: blob: mediastream: ws: wss: ; form-action 'self'



Now, the first header is in error (it should be in X-Archive-orig-Link), but multiple Link headers *are* allowed as per RFC 2616 (and 7230).  Martin said MementoEmbed wasn't finding link rel="original", probably bc it occurs in the 2nd Link header and not the first.



Discussion:

Patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 00e7098..230f9e2 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -26,7 +26,7 @@ author = 'Shawn M. Jones'
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = '0.2018.10.11.153710'
+release = '0.2018.10.11.193540'
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/mementoembed/mementoresource.py b/mementoembed/mementoresource.py
index c444937..52f65c4 100644
--- a/mementoembed/mementoresource.py
+++ b/mementoembed/mementoresource.py
@@ -105,8 +105,9 @@ def get_timegate_from_response(response):
     urig = None
 
     try:
-        urig = aiu.convert_LinkTimeMap_to_dict(
-            response.headers['link'] )['timegate_uri']
+        # urig = aiu.convert_LinkTimeMap_to_dict(
+        #     response.headers['link'] )['timegate_uri']
+        urig = response.links['timegate']['url']
     except KeyError as e:
         raise NotAMementoError(
             "link header coult not be parsed for timegate URI",
@@ -119,8 +120,9 @@ def get_original_uri_from_response(response):
     urir = None
 
     try:
-        urir = aiu.convert_LinkTimeMap_to_dict(
-            response.headers['link'] )['original_uri']
+        # urir = aiu.convert_LinkTimeMap_to_dict(
+        #     response.headers['link'] )['original_uri']
+        urir = response.links['original']['url']
     except KeyError as e:
         raise NotAMementoError(
             "link header coult not be parsed for original URI",
diff --git a/mementoembed/version.py b/mementoembed/version.py
index fe6ad2b..74615bf 100644
--- a/mementoembed/version.py
+++ b/mementoembed/version.py
@@ -1,3 +1,3 @@
 __appname__ = "MementoEmbed"
-__appversion__ = '0.2018.10.11.153710'
+__appversion__ = '0.2018.10.11.193540'
 __useragent__ = "{}/{}".format(__appname__, __appversion__)


Test patch:
diff --git a/tests/test_mementoresource.py b/tests/test_mementoresource.py
index aaff116..e9a45e5 100644
--- a/tests/test_mementoresource.py
+++ b/tests/test_mementoresource.py
@@ -13,10 +13,11 @@ testdir = os.path.dirname(os.path.realpath(__file__))
 
 class mock_response:
 
-    def __init__(self, headers, text, status, url, content=None):
+    def __init__(self, headers, text, status, url, content=None, links={}):
         self.headers = headers
         self.text = text
         self.url = url
+        self.links = links
         
         if content is None:
 
@@ -82,7 +83,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_urig: # requests follows all redirects, so we present the result at the end of the chain
                 mock_response(
@@ -97,7 +106,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 )
         }
 
@@ -160,7 +177,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             raw_urim:
                 mock_response(
@@ -232,7 +257,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             raw_urim:
                 mock_response(
@@ -314,7 +347,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             zipurim:
                 mock_response(
@@ -476,7 +517,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             "http://archive.is/20130508132946id_/http://flexispy.com/":
                 mock_response(
@@ -552,7 +601,15 @@ class TestMementoResource(unittest.TestCase):
                     text = metaredirecthtml,
                     content = metaredirecthtml,
                     status = 200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             redirurim: 
                 mock_response(
@@ -568,7 +625,15 @@ class TestMementoResource(unittest.TestCase):
                     text = expected_content,
                     content = expected_content,
                     status = 200,
-                    url = redirurim
+                    url = redirurim,
+                    links = {
+                        "original": {
+                            "url": redir_expected_original_uri
+                        },
+                        "timegate": {
+                            "url": redir_expected_urig
+                        }
+                    }
                 ),
             redirurim_raw: 
                 mock_response(
@@ -624,7 +689,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_raw_uri:
                 mock_response(
@@ -639,7 +712,15 @@ class TestMementoResource(unittest.TestCase):
                     },
                     text = expected_raw_content,
                     status = 200,
-                    url = expected_raw_uri
+                    url = expected_raw_uri,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_urig: # requests follows all redirects, so we present the result at the end of the chain
                 mock_response(
@@ -654,7 +735,15 @@ class TestMementoResource(unittest.TestCase):
                      },
                     text = expected_content,
                     status = 200, # after following redirects
-                    url = expected_urim
+                    url = expected_urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_urim:
                 mock_response(
@@ -669,7 +758,15 @@ class TestMementoResource(unittest.TestCase):
                      },
                     text = expected_content,
                     status = 200, # after following redirects
-                    url = expected_urim
+                    url = expected_urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 )
         }
 
diff --git a/tests/test_originalresource.py b/tests/test_originalresource.py
index 6f2e84e..db1a790 100644
--- a/tests/test_originalresource.py
+++ b/tests/test_originalresource.py
@@ -5,7 +5,7 @@ from mementoembed.originalresource import OriginalResource
 
 class mock_response:
 
-    def __init__(self, headers, text, status, url, content=None):
+    def __init__(self, headers, text, status, url, content=None, links={}):
         self.headers = headers
         self.text = text
         if content is None:
@@ -16,6 +16,8 @@ class mock_response:
         self.status_code = status
         self.url = url
 
+        self.links = links
+
 class mock_httpcache:
     """
         rather than hitting the actual HTTP cache,
@@ -64,7 +66,15 @@ class TestOriginalResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_urig:
                 mock_response(
@@ -79,7 +89,15 @@ class TestOriginalResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_original_uri:
                 mock_response(
@@ -130,7 +148,15 @@ class TestOriginalResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_urig:
                 mock_response(
@@ -145,7 +171,15 @@ class TestOriginalResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_original_uri:
                 mock_response(
@@ -202,7 +236,15 @@ class TestOriginalResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_urig:
                 mock_response(
@@ -217,7 +259,15 @@ class TestOriginalResource(unittest.TestCase):
                     },
                     text = expected_content,
                     status=200,
-                    url = urim
+                    url = urim,
+                    links = {
+                        "original": {
+                            "url": expected_original_uri
+                        },
+                        "timegate": {
+                            "url": expected_urig
+                        }
+                    }
                 ),
             expected_original_uri:
                 mock_response(