Repository: scrapy/scrapy
Pull request #5801 (2023-01-20 10:21:09)

Support Path Objects Issue #5739
Located a couple of places where using Path object raises an exception. Relates to Issue https://github.com/scrapy/scrapy/issues/5739.

The issue can be reproduced by using a Path object in the "IMAGES_STORE" or "FILES_STORE" settings when configuring a file or image pipeline.

Code to reproduce:

```
from pathlib import Path
import scrapy

class MySpider(scrapy.Spider):
    name = 'myspider'
    custom_settings = {
        "IMAGES_URLS_FIELD": 'image_urls',
        "IMAGES_RESULT_FIELD": 'images',
        "IMAGES_STORE": Path.home() / 'Images',   # <-- here
        "ITEM_PIPELINES": {
            "scrapy.pipelines.images.ImagesPipeline": 1,
        }
    }

```

The unit tests I added will also trigger the error when tested without the changes.

And the exception that is raised:

```
2023-01-15 07:18:46 [twisted] CRITICAL:
Traceback (most recent call last):
  File "...\venv\lib\site-packages\twisted\internet\defer.py", line 1697, in _inlineCallbacks
    result = context.run(gen.send, result)

  ...
  File "...\venv\lib\site-packages\scrapy\pipelines\files.py", line 383, in _get_store
    return store_cls(uri)
  File "...\venv\lib\site-packages\scrapy\pipelines\files.py", line 43, in __init__
    if '://' in basedir:
TypeError: argument of type 'WindowsPath' is not iterable
```



Patch:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 0be28933e..6e9f661e5 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -13,8 +13,9 @@ from collections import defaultdict
 from contextlib import suppress
 from ftplib import FTP
 from io import BytesIO
+from os import PathLike
 from pathlib import Path
-from typing import DefaultDict, Optional, Set
+from typing import DefaultDict, Optional, Set, Union
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
@@ -36,24 +37,31 @@ from scrapy.utils.request import referer_str
 logger = logging.getLogger(__name__)
 
 
+def _to_string(path: Union[str, PathLike]) -> str:
+    return str(path)  # convert a Path object to string
+
+
 class FileException(Exception):
     """General media error exception"""
 
 
 class FSFilesStore:
-    def __init__(self, basedir: str):
+    def __init__(self, basedir: Union[str, PathLike]):
+        basedir = _to_string(basedir)
         if "://" in basedir:
             basedir = basedir.split("://", 1)[1]
         self.basedir = basedir
         self._mkdir(Path(self.basedir))
         self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
 
-    def persist_file(self, path: str, buf, info, meta=None, headers=None):
+    def persist_file(
+        self, path: Union[str, PathLike], buf, info, meta=None, headers=None
+    ):
         absolute_path = self._get_filesystem_path(path)
         self._mkdir(absolute_path.parent, info)
         absolute_path.write_bytes(buf.getvalue())
 
-    def stat_file(self, path: str, info):
+    def stat_file(self, path: Union[str, PathLike], info):
         absolute_path = self._get_filesystem_path(path)
         try:
             last_modified = absolute_path.stat().st_mtime
@@ -65,8 +73,8 @@ class FSFilesStore:
 
         return {"last_modified": last_modified, "checksum": checksum}
 
-    def _get_filesystem_path(self, path: str) -> Path:
-        path_comps = path.split("/")
+    def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path:
+        path_comps = _to_string(path).split("/")
         return Path(self.basedir, *path_comps)
 
     def _mkdir(self, dirname: Path, domain: Optional[str] = None):
@@ -332,12 +340,12 @@ class FilesPipeline(MediaPipeline):
     DEFAULT_FILES_RESULT_FIELD = "files"
 
     def __init__(self, store_uri, download_func=None, settings=None):
+        store_uri = _to_string(store_uri)
         if not store_uri:
             raise NotConfigured
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
-
         cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
         resolve = functools.partial(


Test patch:
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index c414e582d..9701e5d4e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -480,6 +480,22 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
             expected_value = settings.get(settings_attr)
             self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value)
 
+    def test_file_pipeline_using_pathlike_objects(self):
+        class CustomFilesPipelineWithPathLikeDir(FilesPipeline):
+            def file_path(self, request, response=None, info=None, *, item=None):
+                return Path("subdir") / Path(request.url).name
+
+        pipeline = CustomFilesPipelineWithPathLikeDir.from_settings(
+            Settings({"FILES_STORE": Path("./Temp")})
+        )
+        request = Request("http://example.com/image01.jpg")
+        self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg"))
+
+    def test_files_store_constructor_with_pathlike_object(self):
+        path = Path("./FileDir")
+        fs_store = FSFilesStore(path)
+        self.assertEqual(fs_store.basedir, str(path))
+
 
 class TestS3FilesStore(unittest.TestCase):
     @defer.inlineCallbacks