Repository: scrapy/scrapy

Issue:
test_pipeline_images.py fails with "TypeError: Skipped expected string as 'msg' parameter, got 'bool' instead."
See e.g. https://github.com/scrapy/scrapy/pull/5019/checks?check_run_id=2012658916

This should be related to the skip attribute, though I'm not sure why did it start happening now.


Discussion:
OK, looks like it's because of the Twisted upgrade, because of https://github.com/twisted/twisted/commit/beafcd67fd3201c34a8fe0c6639ef0af92fbc7fc

Note that the docs have already said "None or a string".
Looks like changing `False` to `None` helps, but on the other hand the skipping code doesn't really work, as PIL is imported in `scrapy.pipelines.images` and so the test fails with an ImportError anyway.
My bad for getting this closed. Let’s keep it open until we figure out the right way forward.
So the easy way is just replacing False with None. A better one is either fixing the no-PIL code path and adding tests for it (I don't see a reason for this additional complexity TBH) or dropping it and saying "we require PIL for tests".
I still don’t see what’s wrong with our code. `skipIf(bool, reason)` should work, and instead is causing this issue about the reason being a boolean, as if somewhere in the middle the first parameter of `skipIf` got interpreted as the reason.
The class `skip` attribute previously was checked with `if`, now it is checked with `is None`: https://github.com/twisted/twisted/commit/beafcd67fd3201c34a8fe0c6639ef0af92fbc7fc  (respectively the old line 1076 and the new line 1040).
But is that a bug in Twisted that we should report, or does the fix need to happen in our code or code we use (e.g. `pytest-twisted`)?
It's a bug in our code as we set `skip` to `False` when the docs say `L{None} or a string explaining why this test is to be skipped. If defined, the test will not be run.`
I see. When I was looking at the code to check for this issue, I became convinced that our use of skipIf in https://github.com/scrapy/scrapy/blob/master/tests/test_pipeline_images.py#L187 was one of the cases where the issue was triggered. But it’s not, it’s the use of skip in other parts of the file as a string what’s triggering it.

OK, so it should be relatively easy to fix.

Resolving patch:
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -9,9 +9,8 @@
 from io import BytesIO
 
 from itemadapter import ItemAdapter
-from PIL import Image
 
-from scrapy.exceptions import DropItem
+from scrapy.exceptions import DropItem, NotConfigured
 from scrapy.http import Request
 from scrapy.pipelines.files import FileException, FilesPipeline
 # TODO: from scrapy.pipelines.media import MediaPipeline
@@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
+        try:
+            from PIL import Image
+            self._Image = Image
+        except ImportError:
+            raise NotConfigured(
+                'ImagesPipeline requires installing Pillow 4.0.0 or later'
+            )
+
         super().__init__(store_uri, settings=settings, download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
@@ -121,7 +128,7 @@ def image_downloaded(self, response, request, info, *, item=None):
 
     def get_images(self, response, request, info, *, item=None):
         path = self.file_path(request, response=response, info=info, item=item)
-        orig_image = Image.open(BytesIO(response.body))
+        orig_image = self._Image.open(BytesIO(response.body))
 
         width, height = orig_image.size
         if width < self.min_width or height < self.min_height:
@@ -139,12 +146,12 @@ def get_images(self, response, request, info, *, item=None):
 
     def convert_image(self, image, size=None):
         if image.format == 'PNG' and image.mode == 'RGBA':
-            background = Image.new('RGBA', image.size, (255, 255, 255))
+            background = self._Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert('RGB')
         elif image.mode == 'P':
             image = image.convert("RGBA")
-            background = Image.new('RGBA', image.size, (255, 255, 255))
+            background = self._Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert('RGB')
         elif image.mode != 'RGB':
@@ -152,7 +159,7 @@ def convert_image(self, image, size=None):
 
         if size:
             image = image.copy()
-            image.thumbnail(size, Image.ANTIALIAS)
+            image.thumbnail(size, self._Image.ANTIALIAS)
 
         buf = BytesIO()
         image.save(buf, 'JPEG')


Test patch:
diff --git a/tests/test_commands.py b/tests/test_commands.py
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -17,6 +17,8 @@
 from unittest import skipIf
 
 from pytest import mark
+from twisted import version as twisted_version
+from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
@@ -630,6 +632,7 @@ def test_asyncio_enabled_false(self):
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_asyncio_loop_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s',
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -8,7 +8,9 @@
 
 from pytest import raises, mark
 from testfixtures import LogCapture
+from twisted import version as twisted_version
 from twisted.internet import defer
+from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
@@ -358,6 +360,7 @@ def test_reactor_asyncio(self):
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_loop_asyncio(self):
         log = self.run_script("asyncio_custom_loop.py")
         self.assertIn("Spider closed (finished)", log)
@@ -366,6 +369,7 @@ def test_custom_loop_asyncio(self):
 
     @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
     @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
         self.assertIn("Spider closed (finished)", log)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -180,7 +180,18 @@ def test_download_media_redirected_allowed(self):
         self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
 
 
+try:
+    from PIL import Image  # noqa: imported just to check for the import error
+except ImportError:
+    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+    skip_pillow = None
+
+
 class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
+
+    skip = skip_pillow
+
     pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'
     store_setting_key = 'IMAGES_STORE'
     media_key = 'images'
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -23,15 +23,16 @@
     dataclass_field = None
 
 
-skip = False
 try:
     from PIL import Image
 except ImportError:
-    skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
     encoders = {'jpeg_encoder', 'jpeg_decoder'}
     if not encoders.issubset(set(Image.core.__dict__)):
-        skip = 'Missing JPEG encoders'
+        skip_pillow = 'Missing JPEG encoders'
+    else:
+        skip_pillow = None
 
 
 def _mocked_download_func(request, info):
@@ -41,7 +42,7 @@ def _mocked_download_func(request, info):
 
 class ImagesPipelineTestCase(unittest.TestCase):
 
-    skip = skip
+    skip = skip_pillow
 
     def setUp(self):
         self.tempdir = mkdtemp()
@@ -137,6 +138,8 @@ def thumb_key(self, url, thumb_id):
 
 class ImagesPipelineTestCaseFieldsMixin:
 
+    skip = skip_pillow
+
     def test_item_fields_default(self):
         url = 'http://www.example.com/images/1.jpg'
         item = self.item_class(name='item1', image_urls=[url])
@@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u
 
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
+
+    skip = skip_pillow
+
     img_cls_attribute_names = [
         # Pipeline attribute names with corresponding setting names.
         ("EXPIRES", "IMAGES_EXPIRES"),
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
@@ -17,6 +19,14 @@
 from scrapy import signals
 
 
+try:
+    from PIL import Image  # noqa: imported just to check for the import error
+except ImportError:
+    skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+    skip_pillow = None
+
+
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
     return response() if callable(response) else response
@@ -379,6 +389,7 @@ def image_downloaded(self, response, request, info):
 
 
 class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
+    skip = skip_pillow
 
     def setUp(self):
         self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)