Repository: scrapy/scrapy

Issue:
Add brotlicffi support
Currently, brotli compression is supported when using `brotli` or `brotlipy` (deprecated). We should also support it thorugh `brotlicffi`, the new name of `brotlipy`, which performs worse than `brotli` but works on PyPy.


Discussion:

Resolving patch:
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -29,7 +29,10 @@
 ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
 
 try:
-    import brotli  # noqa: F401
+    try:
+        import brotli  # noqa: F401
+    except ImportError:
+        import brotlicffi  # noqa: F401
 except ImportError:
     pass
 else:
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -5,7 +5,10 @@
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
-    import brotli
+    try:
+        import brotli
+    except ImportError:
+        import brotlicffi as brotli
 except ImportError:
     pass
 else:
@@ -17,9 +20,9 @@
                 "You have brotlipy installed, and Scrapy will use it, but "
                 "Scrapy support for brotlipy is deprecated and will stop "
                 "working in a future version of Scrapy. brotlipy itself is "
-                "deprecated, it has been superseded by brotlicffi (not "
-                "currently supported by Scrapy). Please, uninstall brotlipy "
-                "and install brotli instead. brotlipy has the same import "
+                "deprecated, it has been superseded by brotlicffi. "
+                "Please, uninstall brotlipy "
+                "and install brotli or brotlicffi instead. brotlipy has the same import "
                 "name as brotli, so keeping both installed is strongly "
                 "discouraged."
             ),


Test patch:
diff --git a/tests/requirements.txt b/tests/requirements.txt
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -11,8 +11,7 @@ uvloop; platform_system != "Windows"
 
 bpython  # optional for shell wrapper tests
 brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
-# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072
-brotli==1.0.9; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
+brotlicffi; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -130,7 +130,10 @@ def test_process_response_gzip(self):
 
     def test_process_response_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         response = self._getresponse("br")
@@ -448,7 +451,10 @@ def _test_compression_bomb_setting(self, compression_id):
 
     def test_compression_bomb_setting_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_setting("br")
@@ -486,7 +492,10 @@ class DownloadMaxSizeSpider(Spider):
 
     def test_compression_bomb_spider_attr_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_spider_attr("br")
@@ -522,7 +531,10 @@ def _test_compression_bomb_request_meta(self, compression_id):
 
     def test_compression_bomb_request_meta_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_request_meta("br")
@@ -568,7 +580,10 @@ def _test_download_warnsize_setting(self, compression_id):
 
     def test_download_warnsize_setting_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_setting("br")
@@ -616,7 +631,10 @@ class DownloadWarnSizeSpider(Spider):
 
     def test_download_warnsize_spider_attr_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_spider_attr("br")
@@ -662,7 +680,10 @@ def _test_download_warnsize_request_meta(self, compression_id):
 
     def test_download_warnsize_request_meta_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_request_meta("br")