Repository: Pylons/webob

Issue:
validate media types consistently in acceptable_offers
```python
>>> create_accept_header('').acceptable_offers(['foo'])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/michael/work/oss/pyramid/env/lib/python3.6/site-packages/webob/acceptparse.py", line 804, in acceptable_offers
    raise ValueError(repr(offer) + ' is not a media type.')
ValueError: 'foo' is not a media type.
>>> create_accept_header('*/*').acceptable_offers(['foo'])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/michael/work/oss/pyramid/env/lib/python3.6/site-packages/webob/acceptparse.py", line 804, in acceptable_offers
    raise ValueError(repr(offer) + ' is not a media type.')
ValueError: 'foo' is not a media type.
>>> create_accept_header('invalid/x/y').acceptable_offers(['foo'])
[('foo', 1.0)]
>>> create_accept_header(None).acceptable_offers(['foo'])
[('foo', 1.0)]
```

It's not ok to raise here in some scenarios and not in others. The offer is server-side and webob should handle it consistently by validating those parameters the same always.

I went backward and found out that webob < 1.5 actually did this consistently but after that it stopped calling `_check_offer` in `MIMEAccept` but kept calling it in `MIMENilAccept`.

Comments:

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index fd34d21..7ccc765 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -33,3 +33,10 @@ Bugfix
   MIMEAccept to behave more like the old version. See
   https://github.com/Pylons/webob/pull/356
 
+- ``acceptparse.AcceptValidHeader``, ``acceptparse.AcceptInvalidHeader``, and
+  ``acceptparse.AcceptNoHeader`` will now always ignore offers that do not
+  match the required media type grammar when calling ``.acceptable_offers()``.
+  Previous versions raised a ``ValueError`` for invalid offers in
+  ``AcceptValidHeader`` and returned them as acceptable in the others.
+  See https://github.com/Pylons/webob/pull/372
+
diff --git a/src/webob/acceptparse.py b/src/webob/acceptparse.py
index d1d9d6f..0000667 100644
--- a/src/webob/acceptparse.py
+++ b/src/webob/acceptparse.py
@@ -408,6 +408,26 @@ class Accept(object):
                 )
         return generator(value=value)
 
+    def _parse_and_normalize_offers(self, offers):
+        """
+        Throw out any offers that do not match the media type ABNF.
+
+        :return: A list of offers split into the format ``[offer_index,
+                 offer_type_subtype, offer_media_type_params]``.
+
+        """
+        lowercased_offers_parsed = []
+        for index, offer in enumerate(offers):
+            match = self.media_type_compiled_re.match(offer.lower())
+            # we're willing to try to match any offer that matches the
+            # media type grammar can parse, but we'll throw out anything
+            # that doesn't fit the correct syntax - this is not saying that
+            # the media type is actually a real media type, just that it looks
+            # like one
+            if match:
+                lowercased_offers_parsed.append([index] + list(match.groups()))
+        return lowercased_offers_parsed
+
 
 class AcceptValidHeader(Accept):
     """
@@ -771,6 +791,8 @@ class AcceptValidHeader(Accept):
         This uses the matching rules described in :rfc:`RFC 7231, section 5.3.2
         <7231#section-5.3.2>`.
 
+        Any offers that do not match the media type grammar will be ignored.
+
         :param offers: ``iterable`` of ``str`` media types (media types can
                        include media type parameters)
         :return: A list of tuples of the form (media type, qvalue), in
@@ -793,21 +815,12 @@ class AcceptValidHeader(Accept):
             for media_range, qvalue, media_type_params, extension_params in
             parsed
         ]
-        lowercased_offers = [offer.lower() for offer in offers]
-
-        lowercased_offers_parsed = []
-        for offer in lowercased_offers:
-            match = self.media_type_compiled_re.match(offer)
-            # The regex here is only used for parsing, and not intended to
-            # validate the offer
-            if not match:
-                raise ValueError(repr(offer) + ' is not a media type.')
-            lowercased_offers_parsed.append(match.groups())
+        lowercased_offers_parsed = self._parse_and_normalize_offers(offers)
 
         acceptable_offers_n_quality_factors = {}
         for (
-            offer_index, (offer_type_subtype, offer_media_type_params)
-        ) in enumerate(lowercased_offers_parsed):
+            offer_index, offer_type_subtype, offer_media_type_params
+        ) in lowercased_offers_parsed:
             offer_media_type_params = self._parse_media_type_params(
                 media_type_params_segment=offer_media_type_params,
             )
@@ -1242,6 +1255,8 @@ class _AcceptInvalidOrNoHeader(Accept):
         """
         Return the offers that are acceptable according to the header.
 
+        Any offers that do not match the media type grammar will be ignored.
+
         :param offers: ``iterable`` of ``str`` media types (media types can
                        include media type parameters)
         :return: When the header is invalid, or there is no ``Accept`` header
@@ -1250,7 +1265,14 @@ class _AcceptInvalidOrNoHeader(Accept):
                  where each offer in `offers` is paired with the qvalue of 1.0,
                  in the same order as in `offers`.
         """
-        return [(offer, 1.0) for offer in offers]
+        return [
+            (offers[offer_index], 1.0)
+            for offer_index, _, _
+            # avoid returning any offers that don't match the grammar so
+            # that the return values here are consistent with what would be
+            # returned in AcceptValidHeader
+            in self._parse_and_normalize_offers(offers)
+        ]
 
     def best_match(self, offers, default_match=None):
         """


Test patch:
diff --git a/tests/test_acceptparse.py b/tests/test_acceptparse.py
index e9f3935..b8c0620 100644
--- a/tests/test_acceptparse.py
+++ b/tests/test_acceptparse.py
@@ -909,16 +909,30 @@ class TestAcceptValidHeader(object):
         instance = AcceptValidHeader(header_value=header_value)
         assert instance.accepts_html is returned
 
-    @pytest.mark.parametrize('offers', [
-        ['text/html;p=1;q=0.5'],
-        ['text/html;q=0.5'],
-        ['text/html;q=0.5;e=1'],
-        ['text/html', 'text/plain;p=1;q=0.5;e=1'],
+    @pytest.mark.parametrize('header, offers, expected_returned', [
+        (AcceptValidHeader('text/html'), ['text/html;p=1;q=0.5'], []),
+        (AcceptValidHeader('text/html'), ['text/html;q=0.5'], []),
+        (AcceptValidHeader('text/html'), ['text/html;q=0.5;e=1'], []),
+        (
+            AcceptValidHeader('text/html'),
+            ['text/html', 'text/plain;p=1;q=0.5;e=1', 'foo'],
+            [('text/html', 1.0)],
+        ),
+        (
+            AcceptInvalidHeader('foo'),
+            ['text/html', 'text/plain;p=1;q=0.5;e=1', 'foo'],
+            [('text/html', 1.0)],
+        ),
+        (
+            AcceptNoHeader(),
+            ['text/html', 'text/plain;p=1;q=0.5;e=1', 'foo'],
+            [('text/html', 1.0)],
+        ),
     ])
-    def test_acceptable_offers__invalid_offers(self, offers):
-        instance = AcceptValidHeader(header_value='text/html')
-        with pytest.raises(ValueError):
-            instance.acceptable_offers(offers=offers)
+    def test_acceptable_offers__invalid_offers(
+        self, header, offers, expected_returned,
+    ):
+        assert header.acceptable_offers(offers=offers) == expected_returned
 
     @pytest.mark.parametrize('header_value, offers, expected_returned', [
         # RFC 7231, section 5.3.2