Repository: scrapy/w3lib

Issue:
basic_auth_header uses the wrong flavor of base64
I have reason to believe that `basic_auth_header` is wrong in using `urlsafe_b64encode` (which replaces `+/` with `-_`) instead of `b64encode`.

The first specification of HTTP basic auth according to [Wikipedia](https://en.wikipedia.org/wiki/Basic_access_authentication) is [HTTP 1.0](https://www.w3.org/Protocols/HTTP/1.0/spec.html#BasicAA), which does not mention any special flavor of base64, and points for a definition of base64 to RFC-1521, which [describes regular base64](https://datatracker.ietf.org/doc/html/rfc1521). The latest HTTP basic auth specification according to Wikipedia is [RFC-7617](https://datatracker.ietf.org/doc/html/rfc7617), which similarly does not specify any special flavor of base64, and points to [section 4 of RFC-4648](https://datatracker.ietf.org/doc/html/rfc4648#section-4), which also describes the regular base64.

I traced the origin of this bug, and it has been there [at least since the first Git commit of Scrapy](https://github.com/scrapy/scrapy/blame/b45d87d0fe313458d03cd533dee2d3d03e057a04/scrapy/trunk/scrapy/http/request.py#L82).

```python
>>> from w3lib.http import basic_auth_header
```

Actual:

```python
>>> basic_auth_header('aa~aa¿', '')
b'Basic YWF-YWG_Og=='
```

Expected:

```python
>>> basic_auth_header('aa~aa¿', '')
b'Basic YWF+YWG/Og=='
```

I believe this bug only affects ASCII credentials that include the `>`, `?` or `~` characters in certain positions.

For richer encodings like UTF-8, which is what `basic_auth_header` uses (~~and makes sense as a default, but it should be configurable~~ [rightly so](https://datatracker.ietf.org/doc/html/rfc7617#section-2.1)), many more characters can be affected (e.g. `¿` in the example above).

Comments:

Patch:
diff --git a/w3lib/encoding.py b/w3lib/encoding.py
index 35967a5..0879ead 100644
--- a/w3lib/encoding.py
+++ b/w3lib/encoding.py
@@ -227,8 +227,8 @@ def html_to_unicode(
 
     It will try in order:
 
-    * http content type header
     * BOM (byte-order mark)
+    * http content type header
     * meta or xml tag declarations
     * auto-detection, if the `auto_detect_fun` keyword argument is not ``None``
     * default encoding in keyword arg (which defaults to utf8)
@@ -281,27 +281,16 @@ def html_to_unicode(
     >>>
 
     '''
-
-    enc = http_content_type_encoding(content_type_header)
     bom_enc, bom = read_bom(html_body_str)
-    if enc is not None:
-        # remove BOM if it agrees with the encoding
-        if enc == bom_enc:
-            bom = cast(bytes, bom)
-            html_body_str = html_body_str[len(bom) :]
-        elif enc == "utf-16" or enc == "utf-32":
-            # read endianness from BOM, or default to big endian
-            # tools.ietf.org/html/rfc2781 section 4.3
-            if bom_enc is not None and bom_enc.startswith(enc):
-                enc = bom_enc
-                bom = cast(bytes, bom)
-                html_body_str = html_body_str[len(bom) :]
-            else:
-                enc += "-be"
-        return enc, to_unicode(html_body_str, enc)
     if bom_enc is not None:
         bom = cast(bytes, bom)
         return bom_enc, to_unicode(html_body_str[len(bom) :], bom_enc)
+
+    enc = http_content_type_encoding(content_type_header)
+    if enc is not None:
+        if enc == "utf-16" or enc == "utf-32":
+            enc += "-be"
+        return enc, to_unicode(html_body_str, enc)
     enc = html_body_declared_encoding(html_body_str)
     if enc is None and (auto_detect_fun is not None):
         enc = auto_detect_fun(html_body_str)
diff --git a/w3lib/http.py b/w3lib/http.py
index e14e434..10d1669 100644
--- a/w3lib/http.py
+++ b/w3lib/http.py
@@ -1,4 +1,4 @@
-from base64 import urlsafe_b64encode
+from base64 import b64encode
 from typing import Any, List, MutableMapping, Optional, AnyStr, Sequence, Union, Mapping
 from w3lib.util import to_bytes, to_unicode
 
@@ -101,4 +101,4 @@ def basic_auth_header(
     # XXX: RFC 2617 doesn't define encoding, but ISO-8859-1
     # seems to be the most widely used encoding here. See also:
     # http://greenbytes.de/tech/webdav/draft-ietf-httpauth-basicauth-enc-latest.html
-    return b"Basic " + urlsafe_b64encode(to_bytes(auth, encoding=encoding))
+    return b"Basic " + b64encode(to_bytes(auth, encoding=encoding))


Test patch:
diff --git a/tests/test_encoding.py b/tests/test_encoding.py
index 865cf72..be6d447 100644
--- a/tests/test_encoding.py
+++ b/tests/test_encoding.py
@@ -220,13 +220,12 @@ class HtmlConversionTests(unittest.TestCase):
     def test_BOM(self):
         # utf-16 cases already tested, as is the BOM detection function
 
-        # http header takes precedence, irrespective of BOM
+        # BOM takes precedence, ahead of the http header
         bom_be_str = codecs.BOM_UTF16_BE + "hi".encode("utf-16-be")
-        expected = "\ufffd\ufffd\x00h\x00i"
-        self._assert_encoding("utf-8", bom_be_str, "utf-8", expected)
+        expected = "hi"
+        self._assert_encoding("utf-8", bom_be_str, "utf-16-be", expected)
 
-        # BOM is stripped when it agrees with the encoding, or used to
-        # determine encoding
+        # BOM is stripped when present
         bom_utf8_str = codecs.BOM_UTF8 + b"hi"
         self._assert_encoding("utf-8", bom_utf8_str, "utf-8", "hi")
         self._assert_encoding(None, bom_utf8_str, "utf-8", "hi")
diff --git a/tests/test_http.py b/tests/test_http.py
index efabb0a..76a1ff1 100644
--- a/tests/test_http.py
+++ b/tests/test_http.py
@@ -17,7 +17,7 @@ class HttpTests(unittest.TestCase):
         )
         # Check url unsafe encoded header
         self.assertEqual(
-            b"Basic c29tZXVzZXI6QDx5dTk-Jm8_UQ==",
+            b"Basic c29tZXVzZXI6QDx5dTk+Jm8/UQ==",
             basic_auth_header("someuser", "@<yu9>&o?Q"),
         )
 
@@ -28,7 +28,7 @@ class HttpTests(unittest.TestCase):
         )
         # default encoding (ISO-8859-1)
         self.assertEqual(
-            b"Basic c29t5nVz6HI6c_htZXDkc3M=", basic_auth_header("somæusèr", "sømepäss")
+            b"Basic c29t5nVz6HI6c/htZXDkc3M=", basic_auth_header("somæusèr", "sømepäss")
         )
 
     def test_headers_raw_dict_none(self):