Repository: google/personfinder

Issue:
Use Accept-Language header as the default UI language
We should use Accept-Language header as the default UI language, instead of the first one in the language list for the repository.

We should use the language even if the language is not in the language list for the repository. We can still render most of the contents in the language, except for the repository title (disaster name).

Once the user manually choose a language from the language menu, we should store it in Cookie and stick to it as we do currently.


Discussion:

Resolving patch:
diff --git a/app/main.py b/app/main.py
--- a/app/main.py
+++ b/app/main.py
@@ -217,18 +217,31 @@ def select_charset(request):
 
 def select_lang(request, config=None):
     """Selects the best language to use for a given request.  The 'lang' query
-    parameter has priority, then the django_language cookie, then the first
-    language in the language menu, then the default setting."""
-    default_lang = (config and
-                    config.language_menu_options and
-                    config.language_menu_options[0])
+    parameter has priority, then the django_language cookie, then
+    'Accept-Language' HTTP header, then the first language in the language menu,
+    then the default setting."""
+    default_lang = (
+        (config and
+         config.language_menu_options and
+         config.language_menu_options[0]) or
+            django_setup.LANGUAGE_CODE)
     lang = (request.get('lang') or
             request.cookies.get('django_language', None) or
-            default_lang or
-            django_setup.LANGUAGE_CODE)
+            select_lang_from_header(request, default_lang=default_lang))
     lang = re.sub('[^A-Za-z0-9-]', '', lang)
     return const.LANGUAGE_SYNONYMS.get(lang, lang)
 
+def select_lang_from_header(request, default_lang):
+    """Selects the best language matching 'Accept-Language' HTTP header."""
+    # Either of the first item in the first argument or the default_match
+    # argument is used as the default depending on the situation. So we need to
+    # put the default language to both. See:
+    #   https://docs.pylonsproject.org/projects/webob/en/stable/api/webob.html#webob.acceptparse.AcceptLanguageValidHeader.best_match
+    #   https://docs.pylonsproject.org/projects/webob/en/stable/api/webob.html#webob.acceptparse.AcceptLanguageNoHeader.best_match
+    return request.accept_language.best_match(
+        [default_lang] + const.LANGUAGE_ENDONYMS.keys(),
+        default_match=default_lang)
+
 def get_repo_options(request, lang):
     """Returns a list of the names and titles of the launched repositories."""
     options = []


Test patch:
diff --git a/tests/scrape.py b/tests/scrape.py
--- a/tests/scrape.py
+++ b/tests/scrape.py
@@ -121,12 +121,18 @@ def setcookies(cookiejar, host, lines):
 RAW = object() # This sentinel value for 'charset' means "don't decode".
 
 def fetch(url, data='', agent=None, referrer=None, charset=None, verbose=0,
-          cookiejar={}, type=None, method=None):
-    """Make an HTTP or HTTPS request.  If 'data' is given, do a POST;
-    otherwise do a GET.  If 'agent' and/or 'referrer' are given, include
-    them as User-Agent and Referer headers in the request, respectively.
+          cookiejar={}, type=None, method=None, accept_language=None):
+    """Make an HTTP or HTTPS request.
+
+    If 'data' is given, do a POST; otherwise do a GET.
+
+    If 'agent', 'referrer' and/or 'accept_language' are given, include them as
+    User-Agent, Referer and Accept-Language headers in the request,
+    respectively.
+
     'cookiejar' should have the form {domain: {path: {name: value, ...}}};
     cookies will be sent from it and received cookies will be stored in it.
+
     Return the 5-element tuple (url, status, message, headers, content)
     where 'url' is the final URL retrieved, 'status' is the integer status
     code, 'message' is the reply status message, 'headers' is a dictionary of
@@ -134,9 +140,12 @@ def fetch(url, data='', agent=None, referrer=None, charset=None, verbose=0,
     For multiple occurrences of the same header, 'headers' will contain a
     single key-value pair where the values are joined together with newlines.
     If the Content-Type header specifies a 'charset' parameter, 'content'
-    will be a Unicode string, decoded using the given charset.  Giving the
-    'charset' argument overrides any received 'charset' parameter; a charset
-    of RAW ensures that the content is left undecoded in an 8-bit string."""
+    will be a Unicode string, decoded using the given charset.
+
+    Giving the 'charset' argument overrides any received 'charset' parameter; a
+    charset of RAW ensures that the content is left undecoded in an 8-bit
+    string.
+    """
     scheme, host, path, query, fragment = urlsplit(url)
     host = host.split('@')[-1]
 
@@ -173,6 +182,8 @@ def fetch(url, data='', agent=None, referrer=None, charset=None, verbose=0,
         headers['cookie'] = cookieheader
     if type:
         headers['content-type'] = type
+    if accept_language:
+        headers['accept-language'] = accept_language
 
     # Make the HTTP or HTTPS request using Python or cURL.
     if verbose:
@@ -280,20 +291,29 @@ def __init__(self, agent=None, verbose=0):
         self.history = []
 
     def go(self, url_or_doc, data='', redirects=10, referrer=True,
-           charset=None, type=None):
+           charset=None, type=None, accept_language=None):
         """Navigate to a given URL or a document.
 
-        If the URL is relative, it is resolved
-        with respect to the current URL.  If 'data' is provided, do a POST;
-        otherwise do a GET.  Follow redirections up to 'redirects' times.
+        If the URL is relative, it is resolved with respect to the current URL.
+
+        If 'data' is provided, do a POST; otherwise do a GET.
+
+        Follow redirections up to 'redirects' times.
+
         If 'referrer' is given, send it as the referrer; if 'referrer' is
         True (default), send the current URL as the referrer; if 'referrer'
-        is a false value, send no referrer.  If 'charset' is given, it
-        overrides any received 'charset' parameter; setting 'charset' to RAW
-        leaves the content undecoded in an 8-bit string.  If the document is
-        successfully fetched, return a Document spanning the entire document.
-        Any relevant previously stored cookies will be included in the
-        request, and any received cookies will be stored for future use.
+        is a false value, send no referrer.
+
+        If 'charset' is given, it overrides any received 'charset' parameter;
+        setting 'charset' to RAW leaves the content undecoded in an 8-bit
+        string.
+
+        If 'accept_language' is given, include it as Accept-Language headers in
+        the request.
+
+        If the document is successfully fetched, return a Document spanning the
+        entire document. Any relevant previously stored cookies will be included
+        in the request, and any received cookies will be stored for future use.
 
         If a scrape.Document instance is given, it just make it the current
         document (self.doc) in the session, without making any extra HTTP
@@ -314,8 +334,15 @@ def go(self, url_or_doc, data='', redirects=10, referrer=True,
             while 1:
                 (self.url, self.status, self.message, self.headers,
                  content_bytes) = fetch(
-                    url, data, self.agent, referrer, charset, self.verbose,
-                    self.cookiejar, type)
+                    url=url,
+                    data=data,
+                    agent=self.agent,
+                    referrer=referrer,
+                    charset=charset,
+                    verbose=self.verbose,
+                    cookiejar=self.cookiejar,
+                    type=type,
+                    accept_language=accept_language)
                 if redirects:
                     if self.status in [301, 302] and 'location' in self.headers:
                         url, data = urljoin(url, self.headers['location']), ''
@@ -453,6 +480,7 @@ def setcookie(self, cookieline):
         host = host.split('@')[-1]
         setcookies(self.cookiejar, host, [cookieline])
 
+
 urlquoted = dict((chr(i), '%%%02X' % i) for i in range(256))
 urlquoted.update(dict((c, c) for c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' +
                                       'abcdefghijklmnopqrstuvwxyz' +
diff --git a/tests/server_test_cases/read_only_tests.py b/tests/server_test_cases/read_only_tests.py
--- a/tests/server_test_cases/read_only_tests.py
+++ b/tests/server_test_cases/read_only_tests.py
@@ -63,6 +63,55 @@ def test_start_creole(self):
         doc = self.go('/haiti?lang=ht')
         assert u'Mwen ap ch\u00e8che yon moun' in doc.text
 
+    def test_language(self):
+        """Tests logic to choose the language of the page."""
+        # Defaults to the first language in the language menu for the repository
+        # if no other hint is available.
+        self.s = scrape.Session(verbose=1)
+        doc = self.go('/japan')
+        assert doc.xpath_one('/html').get('lang') == 'ja'
+
+        # Follows "lang" URL parameter when avaiable. Once you specify "lang"
+        # URL parameter, it remembers the choice in a cookie.
+        # "lang" URL parameter precedes the cookie.
+        self.s = scrape.Session(verbose=1)
+        doc = self.go('/haiti')
+        assert doc.xpath_one('/html').get('lang') == 'en'
+        doc = self.go('/haiti?lang=ja')
+        assert doc.xpath_one('/html').get('lang') == 'ja'
+        doc = self.go('/haiti')
+        assert doc.xpath_one('/html').get('lang') == 'ja'
+        doc = self.go('/haiti?lang=ko')
+        assert doc.xpath_one('/html').get('lang') == 'ko'
+
+        # Follows "Accept-Language" HTTP header when available.
+        self.s = scrape.Session(verbose=1)
+        doc = self.go('/haiti', accept_language='ko')
+        assert doc.xpath_one('/html').get('lang') == 'ko'
+
+        # Uses one with higher quality value (defaults to 1) in the header.
+        self.s = scrape.Session(verbose=1)
+        doc = self.go('/haiti', accept_language='ko,ja;q=0.9')
+        assert doc.xpath_one('/html').get('lang') == 'ko'
+
+        # Falls back to lower quality languages when the language is not
+        # supported.
+        self.s = scrape.Session(verbose=1)
+        doc = self.go('/haiti', accept_language='xx,ja;q=0.9,ko;q=0.8')
+        assert doc.xpath_one('/html').get('lang') == 'ja'
+
+        # "lang" URL parameter precedes "Accept-Language" HTTP header.
+        self.s = scrape.Session(verbose=1)
+        doc = self.go('/haiti?lang=ja', accept_language='ko')
+        assert doc.xpath_one('/html').get('lang') == 'ja'
+
+        # The cookie precedes "Accept-Language" HTTP header.
+        self.s = scrape.Session(verbose=1)
+        doc = self.go('/haiti?lang=ja')
+        assert doc.xpath_one('/html').get('lang') == 'ja'
+        doc = self.go('/haiti', accept_language='ko')
+        assert doc.xpath_one('/html').get('lang') == 'ja'
+
     def test_language_xss(self):
         """Regression test for an XSS vulnerability in the 'lang' parameter."""
         doc = self.go('/haiti?lang="<script>alert(1)</script>')