Repository: ckan/ckan

Issue:
Replace repoze.who-friendlyform with a modern alternative
We use the [repoze.who-friendlyform](http://code.gustavonarea.net/repoze.who-friendlyform/) plugin during our authentication process. This plugin was last updated on 2010 and it's not py3 ready. More specifically we use it as [Indentifier](https://repoze.readthedocs.io/projects/repozewho/en/latest/plugins.html#identifier-plugins) (extract user name and password from the WSGI env) and as a [Challenger](https://repoze.readthedocs.io/projects/repozewho/en/latest/plugins.html#challenger-plugins) (present the user with a login/logout page). See the [who.ini](https://github.com/ckan/ckan/blob/master/ckan/config/who.ini) file.

The FriendlyForm plugin extends a default repoze.who one, [RedirectorPlugin](https://repoze.readthedocs.io/projects/repozewho/en/latest/plugins.html#repoze.who.plugins.redirector.RedirectorPlugin) with some extra functionality. The one that we use is

> Developers may define post-login and/or post-logout pages.



#### Approach

This is a pretty critical functionality so we need to come up with a good replacement strategy. It might be tempting to try to replace the whole repoze.who authenticating system with something like [Flask-login](https://flask-login.readthedocs.io/en/latest/), which has a much simpler and straight-forward integration and is better supported, but that is a big change, as we use some of the repoze.who assumptions at differents parts of the code (eg relying on the `REMOTE_USER` environ variable). So I'd suggest to plan that for a later stage and keep repoze.who in place for at least 2.9.

This means replacing or upgrading repoze.who-friendlyform. The plugin itself is not massive:

https://gist.github.com/amercader/0643f17f3d9641a49c3ed9f630d545df

So perhaps the best solution for now is to include this as part of the CKAN code base and upgrade it to run on Python 3


#### Story points

5



Discussion:

Resolving patch:
diff --git a/ckan/lib/repoze_plugins/__init__.py b/ckan/lib/repoze_plugins/__init__.py
new file mode 100644
diff --git a/ckan/lib/auth_tkt.py b/ckan/lib/repoze_plugins/auth_tkt.py
similarity index 79%
rename from ckan/lib/auth_tkt.py
rename to ckan/lib/repoze_plugins/auth_tkt.py
--- a/ckan/lib/auth_tkt.py
+++ b/ckan/lib/repoze_plugins/auth_tkt.py
@@ -19,7 +19,7 @@ def __init__(self, httponly, *args, **kwargs):
         self.httponly = httponly
 
     def _get_cookies(self, *args, **kwargs):
-        '''
+        u'''
         Override method in superclass to ensure HttpOnly is set appropriately.
         '''
         super_cookies = super(CkanAuthTktCookiePlugin, self). \
@@ -46,29 +46,29 @@ def make_plugin(secret=None,
 
     # ckan specifics:
     # Get secret from beaker setting if necessary
-    if secret is None or secret == 'somesecret':
-        secret = config['beaker.session.secret']
+    if secret is None or secret == u'somesecret':
+        secret = config[u'beaker.session.secret']
     # Session timeout and reissue time for auth cookie
-    if timeout is None and config.get('who.timeout'):
-        timeout = config.get('who.timeout')
-    if reissue_time is None and config.get('who.reissue_time'):
-        reissue_time = config.get('who.reissue_time')
+    if timeout is None and config.get(u'who.timeout'):
+        timeout = config.get(u'who.timeout')
+    if reissue_time is None and config.get(u'who.reissue_time'):
+        reissue_time = config.get(u'who.reissue_time')
     if timeout is not None and reissue_time is None:
         reissue_time = int(math.ceil(int(timeout) * 0.1))
     # Set httponly based on config value. Default is True
-    httponly = config.get('who.httponly', True)
+    httponly = config.get(u'who.httponly', True)
     # Set secure based on config value. Default is False
-    secure = config.get('who.secure', False)
+    secure = config.get(u'who.secure', False)
 
     # back to repoze boilerplate
     if (secret is None and secretfile is None):
-        raise ValueError("One of 'secret' or 'secretfile' must not be None.")
+        raise ValueError(u"One of 'secret' or 'secretfile' must not be None.")
     if (secret is not None and secretfile is not None):
-        raise ValueError("Specify only one of 'secret' or 'secretfile'.")
+        raise ValueError(u"Specify only one of 'secret' or 'secretfile'.")
     if secretfile:
         secretfile = os.path.abspath(os.path.expanduser(secretfile))
         if not os.path.exists(secretfile):
-            raise ValueError("No such 'secretfile': %s" % secretfile)
+            raise ValueError(u"No such 'secretfile': %s" % secretfile)
         secret = open(secretfile).read().strip()
     if timeout:
         timeout = int(timeout)
diff --git a/ckan/lib/repoze_plugins/friendly_form.py b/ckan/lib/repoze_plugins/friendly_form.py
new file mode 100644
--- /dev/null
+++ b/ckan/lib/repoze_plugins/friendly_form.py
@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+
+#
+# This is a modified version of repoze.who-friendlyform, written by
+# Gustavo Narea <me@gustavonarea.net>
+#
+# Modifications include:
+# * Python 3 support
+# * Replace usage of paster methods with webob ones
+#
+
+##############################################################################
+#
+# Copyright (c) 2009-2010, Gustavo Narea <me@gustavonarea.net> and
+# contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the BSD-like license at
+# http://www.repoze.org/LICENSE.txt.  A copy of the license should accompany
+# this distribution.  THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL
+# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND
+# FITNESS FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+u'''Collection of :mod:`repoze.who` friendly forms'''
+
+from six.moves.urllib.parse import urlparse, urlunparse, urlencode, parse_qs
+
+from webob import Request, UnicodeMultiDict
+from webob.exc import HTTPFound, HTTPUnauthorized
+from zope.interface import implements
+
+from repoze.who.interfaces import IChallenger, IIdentifier
+
+__all__ = [u'FriendlyFormPlugin']
+
+
+def construct_url(environ):
+    return Request(environ).url
+
+
+class FriendlyFormPlugin(object):
+    u'''
+    :class:`RedirectingFormPlugin
+    <repoze.who.plugins.form.RedirectingFormPlugin>`-like form plugin with
+    more features.
+
+    It is like ``RedirectingFormPlugin``, but provides us with the following
+    features:
+
+    * Users are not challenged on logout, unless the referrer URL is a
+      private one (but that's up to the application).
+    * Developers may define post-login and/or post-logout pages.
+    * In the login URL, the amount of failed logins is available in the
+      environ. It's also increased by one on every login try. This counter
+      will allow developers not using a post-login page to handle logins that
+      fail/succeed.
+
+    You should keep in mind that if you're using a post-login or a post-logout
+    page, that page will receive the referrer URL as a query string variable
+    whose name is 'came_from'.
+
+    Forms can be submitted with any encoding (non-ASCII credentials are
+    supported) and ISO-8859-1 (aka 'Latin-1') is the default one.
+
+    '''
+    implements(IChallenger, IIdentifier)
+
+    classifications = {
+        IIdentifier: [u'browser'],
+        IChallenger: [u'browser'],
+    }
+
+    def __init__(self, login_form_url, login_handler_path, post_login_url,
+                 logout_handler_path, post_logout_url, rememberer_name,
+                 login_counter_name=None, charset=u'iso-8859-1'):
+        u'''
+
+        :param login_form_url: The URL/path where the login form is located.
+        :type login_form_url: str
+        :param login_handler_path: The URL/path where the login form is
+            submitted to (where it is processed by this plugin).
+        :type login_handler_path: str
+        :param post_login_url: The URL/path where the user should be redirected
+            to after login (even if wrong credentials were provided).
+        :type post_login_url: str
+        :param logout_handler_path: The URL/path where the user is logged out.
+        :type logout_handler_path: str
+        :param post_logout_url: The URL/path where the user should be
+            redirected to after logout.
+        :type post_logout_url: str
+        :param rememberer_name: The name of the repoze.who identifier which
+            acts as rememberer.
+        :type rememberer_name: str
+        :param login_counter_name: The name of the query string variable which
+            will represent the login counter.
+        :type login_counter_name: str
+        :param charset: The character encoding to be assumed when the user
+            agent does not submit the form with an explicit charset.
+        :type charset: :class:`str`
+
+        The login counter variable's name will be set to ``__logins`` if
+        ``login_counter_name`` equals None.
+
+        .. versionchanged:: 1.0.1
+            Added the ``charset`` argument.
+
+        '''
+        self.login_form_url = login_form_url
+        self.login_handler_path = login_handler_path
+        self.post_login_url = post_login_url
+        self.logout_handler_path = logout_handler_path
+        self.post_logout_url = post_logout_url
+        self.rememberer_name = rememberer_name
+        self.login_counter_name = login_counter_name
+        if not login_counter_name:
+            self.login_counter_name = u'__logins'
+        self.charset = charset
+
+    # IIdentifier
+    def identify(self, environ):
+        u'''
+        Override the parent's identifier to introduce a login counter
+        (possibly along with a post-login page) and load the login counter into
+        the ``environ``.
+
+        '''
+        request = Request(environ, charset=self.charset)
+
+        path_info = environ[u'PATH_INFO']
+        script_name = environ.get(u'SCRIPT_NAME') or u'/'
+        query = request.GET
+
+        if path_info == self.login_handler_path:
+            # We are on the URL where repoze.who processes authentication. #
+            # Let's append the login counter to the query string of the
+            # 'came_from' URL. It will be used by the challenge below if
+            # authorization is denied for this request.
+            form = dict(request.POST)
+            form.update(query)
+            try:
+                login = form[u'login']
+                password = form[u'password']
+            except KeyError:
+                credentials = None
+            else:
+                if request.charset == u'us-ascii':
+                    credentials = {
+                        u'login': str(login),
+                        u'password': str(password),
+                    }
+                else:
+                    credentials = {u'login': login, u'password': password}
+
+            try:
+                credentials[u'max_age'] = form[u'remember']
+            except KeyError:
+                pass
+
+            referer = environ.get(u'HTTP_REFERER', script_name)
+            destination = form.get(u'came_from', referer)
+
+            if self.post_login_url:
+                # There's a post-login page, so we have to replace the
+                # destination with it.
+                destination = self._get_full_path(self.post_login_url,
+                                                  environ)
+                if u'came_from' in query:
+                    # There's a referrer URL defined, so we have to pass it to
+                    # the post-login page as a GET variable.
+                    destination = self._insert_qs_variable(destination,
+                                                           u'came_from',
+                                                           query[u'came_from'])
+            failed_logins = self._get_logins(environ, True)
+            new_dest = self._set_logins_in_url(destination, failed_logins)
+            environ[u'repoze.who.application'] = HTTPFound(location=new_dest)
+            return credentials
+
+        elif path_info == self.logout_handler_path:
+            #    We are on the URL where repoze.who logs the user out.    #
+            r = Request(environ)
+            params = dict(list(r.GET.items()) + list(r.POST.items()))
+            form = UnicodeMultiDict(params)
+            form.update(query)
+            referer = environ.get(u'HTTP_REFERER', script_name)
+            came_from = form.get(u'came_from', referer)
+            # set in environ for self.challenge() to find later
+            environ[u'came_from'] = came_from
+            environ[u'repoze.who.application'] = HTTPUnauthorized()
+            return None
+
+        elif path_info == self.login_form_url or self._get_logins(environ):
+            #  We are on the URL that displays the from OR any other page  #
+            #   where the login counter is included in the query string.   #
+            # So let's load the counter into the environ and then hide it from
+            # the query string (it will cause problems in frameworks like TG2,
+            # where this unexpected variable would be passed to the controller)
+            environ[u'repoze.who.logins'] = self._get_logins(environ, True)
+            # Hiding the GET variable in the environ:
+            if self.login_counter_name in query:
+                del query[self.login_counter_name]
+                environ[u'QUERY_STRING'] = urlencode(query, doseq=True)
+
+    # IChallenger
+    def challenge(self, environ, status, app_headers, forget_headers):
+        u'''
+        Override the parent's challenge to avoid challenging the user on
+        logout, introduce a post-logout page and/or pass the login counter
+        to the login form.
+
+        '''
+        url_parts = list(urlparse(self.login_form_url))
+        query = url_parts[4]
+        query_elements = parse_qs(query)
+        came_from = environ.get(u'came_from', construct_url(environ))
+        query_elements[u'came_from'] = came_from
+        url_parts[4] = urlencode(query_elements, doseq=True)
+        login_form_url = urlunparse(url_parts)
+        login_form_url = self._get_full_path(login_form_url, environ)
+        destination = login_form_url
+        # Configuring the headers to be set:
+        cookies = [
+            (h, v) for (h, v) in app_headers if h.lower() == u'set-cookie'
+        ]
+        headers = forget_headers + cookies
+
+        if environ[u'PATH_INFO'] == self.logout_handler_path:
+            # Let's log the user out without challenging.
+            came_from = environ.get(u'came_from')
+            if self.post_logout_url:
+                # Redirect to a predefined u'post logout' URL.
+                destination = self._get_full_path(self.post_logout_url,
+                                                  environ)
+                if came_from:
+                    destination = self._insert_qs_variable(
+                        destination, u'came_from', came_from)
+            else:
+                # Redirect to the referrer URL.
+                script_name = environ.get(u'SCRIPT_NAME', u'')
+                destination = came_from or script_name or u'/'
+
+        elif u'repoze.who.logins' in environ:
+            # Login failed! Let's redirect to the login form and include
+            # the login counter in the query string
+            environ[u'repoze.who.logins'] += 1
+            # Re-building the URL:
+            destination = self._set_logins_in_url(
+                destination, environ[u'repoze.who.logins'])
+
+        return HTTPFound(location=destination, headers=headers)
+
+    # IIdentifier
+    def remember(self, environ, identity):
+        rememberer = self._get_rememberer(environ)
+        return rememberer.remember(environ, identity)
+
+    # IIdentifier
+    def forget(self, environ, identity):
+        rememberer = self._get_rememberer(environ)
+        return rememberer.forget(environ, identity)
+
+    def _get_rememberer(self, environ):
+        rememberer = environ[u'repoze.who.plugins'][self.rememberer_name]
+        return rememberer
+
+    def _get_full_path(self, path, environ):
+        u'''
+        Return the full path to ``path`` by prepending the SCRIPT_NAME.
+
+        If ``path`` is a URL, do nothing.
+
+        '''
+        if path.startswith(u'/'):
+            path = environ.get(u'SCRIPT_NAME', u'') + path
+        return path
+
+    def _get_logins(self, environ, force_typecast=False):
+        u'''
+        Return the login counter from the query string in the ``environ``.
+
+        If it's not possible to convert it into an integer and
+        ``force_typecast`` is ``True``, it will be set to zero (int(0)).
+        Otherwise, it will be ``None`` or an string.
+
+        '''
+        variables = Request(environ).queryvars
+        failed_logins = variables.get(self.login_counter_name)
+        if force_typecast:
+            try:
+                failed_logins = int(failed_logins)
+            except (ValueError, TypeError):
+                failed_logins = 0
+        return failed_logins
+
+    def _set_logins_in_url(self, url, logins):
+        u'''
+        Insert the login counter variable with the ``logins`` value into
+        ``url`` and return the new URL.
+
+        '''
+        return self._insert_qs_variable(url, self.login_counter_name, logins)
+
+    def _insert_qs_variable(self, url, var_name, var_value):
+        u'''
+        Insert the variable ``var_name`` with value ``var_value`` in the query
+        string of ``url`` and return the new URL.
+
+        '''
+        url_parts = list(urlparse(url))
+        query_parts = parse_qs(url_parts[4])
+        query_parts[var_name] = var_value
+        url_parts[4] = urlencode(query_parts, doseq=True)
+        return urlunparse(url_parts)
+
+    def __repr__(self):
+        return u'<%s %s>' % (self.__class__.__name__, id(self))


Test patch:
diff --git a/ckan/tests/lib/test_auth_tkt.py b/ckan/tests/lib/test_auth_tkt.py
--- a/ckan/tests/lib/test_auth_tkt.py
+++ b/ckan/tests/lib/test_auth_tkt.py
@@ -3,7 +3,7 @@
 from nose import tools as nose_tools
 
 from ckan.tests import helpers
-from ckan.lib.auth_tkt import make_plugin
+from ckan.lib.repoze_plugins.auth_tkt import make_plugin
 
 
 class TestCkanAuthTktCookiePlugin(helpers.FunctionalTestBase):
diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py
--- a/ckan/tests/test_coding_standards.py
+++ b/ckan/tests/test_coding_standards.py
@@ -264,7 +264,6 @@ def find_unprefixed_string_literals(filename):
     u'ckan/lib/activity_streams.py',
     u'ckan/lib/activity_streams_session_extension.py',
     u'ckan/lib/app_globals.py',
-    u'ckan/lib/auth_tkt.py',
     u'ckan/lib/authenticator.py',
     u'ckan/lib/base.py',
     u'ckan/lib/captcha.py',
@@ -298,6 +297,7 @@ def find_unprefixed_string_literals(filename):
     u'ckan/lib/search/index.py',
     u'ckan/lib/search/query.py',
     u'ckan/lib/search/sql.py',
+    u'ckan/lib/repoze_plugins/auth_tkt.py',
     u'ckan/lib/uploader.py',
     u'ckan/logic/__init__.py',
     u'ckan/logic/action/__init__.py',