Repository: ejulio/spider-feeder

Issue:
Add a setting for input format
Currently the input format is inferred from the URI extension.
However, it may fail in cases of URLs or files without extension.

Add a new setting, `SPIDERFEEDER_INPUT_FORMAT`? that sets the file format.
If it is not set, the it falls back to the file extension

Discussion:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index c610c2f..ed13d9e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -11,6 +11,12 @@ Whenever possible, link the given PR with the feature/fix.
 
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Development
+
+### Added
+
+* New setting `SPIDERFEEDER_INPUT_FORMAT` to set the file format or fall back to the file extension. [PR#18](https://github.com/ejulio/spider-feeder/pull/18)
+
 
 ## 0.2.0 (2019-08-27)
 
diff --git a/README.md b/README.md
index 1505701..d06fa3a 100644
--- a/README.md
+++ b/README.md
@@ -97,6 +97,11 @@ There are two extensions to load input data to your spiders.
 
 `SPIDERFEEDER_INPUT_FILE_ENCODING` sets the file encoding. DEFAULT = `'utf-8'`.
 
+`SPIDERFEEDER_INPUT_FORMAT` sets the file format (`txt`, `csv`, `json`). DEFAULT = `None`.
+This setting is preferred over the file extension in `SPIDERFEEDER_INPUT_URI`.
+So, if `SPIDERFEEDER_INPUT_FORMAT` is set, this is the one to be used, otherwise
+it will fall back to the file extension in `SPIDERFEEDER_INPUT_URI`.
+
 `SPIDERFEEDER_INPUT_FIELD` sets the url field when parsing `json` or `csv` files.
 
 `SPIDERFEEDER_FILE_HANDLERS` is a set of functions to be matched with the given file scheme.
diff --git a/spider_feeder/store/file_store.py b/spider_feeder/store/file_store.py
index 8cd7ba1..b88a1e0 100644
--- a/spider_feeder/store/file_store.py
+++ b/spider_feeder/store/file_store.py
@@ -38,7 +38,8 @@ class FileStore(BaseStore):
         super().__init__(settings)
         self._input_file_uri = input_file_uri
         self._settings = settings
-        self._file_encoding = settings.get('SPIDERFEEDER_INPUT_FILE_ENCODING', 'utf-8')
+        self._input_file_encoding = settings.get('SPIDERFEEDER_INPUT_FILE_ENCODING', 'utf-8')
+        self._input_format = settings.get('SPIDERFEEDER_INPUT_FORMAT', None)
 
         handlers = settings.getdict('SPIDERFEEDER_FILE_HANDLERS', {})
         self._handlers = dict(self.FILE_HANDLERS, **handlers)
@@ -46,17 +47,24 @@ class FileStore(BaseStore):
         parsers = settings.getdict('SPIDERFEEDER_FILE_PARSERS', {})
         self._parsers = dict(self.FILE_PARSERS, **parsers)
 
+    @property
+    def _file_format(self):
+        if self._input_format:
+            return self._input_format
+
+        (_, file_extension) = path.splitext(self._input_file_uri)
+        return file_extension[1:]  # remove the "."
+
     def _open(self):
         parsed = urlparse(self._input_file_uri)
         logger.info(f'Opening file {self._input_file_uri} with scheme {parsed.scheme}.')
         open = load_object(self._handlers[parsed.scheme])
-        return open(self._input_file_uri, encoding=self._file_encoding)
+        return open(self._input_file_uri, encoding=self._input_file_encoding)
 
     def _parse(self, fd):
-        (_, file_extension) = path.splitext(self._input_file_uri)
-        file_extension = file_extension[1:]
-        logger.info(f'Parsing file {self._input_file_uri} with format {file_extension}.')
-        parser = load_object(self._parsers[file_extension])
+        file_format = self._file_format
+        logger.info(f'Parsing file {self._input_file_uri} with format {file_format}.')
+        parser = load_object(self._parsers[file_format])
         return parser(fd, self._settings)
 
     def read_input_items(self):


Test patch:
diff --git a/tests/store/test_file_store.py b/tests/store/test_file_store.py
index a255af3..913f215 100644
--- a/tests/store/test_file_store.py
+++ b/tests/store/test_file_store.py
@@ -98,6 +98,54 @@ def test_load_json_file(mocker, uri_scheme, file_opener):
     ]
 
 
+@pytest.mark.parametrize('uri_scheme, file_opener', [
+    ('file://', 'spider_feeder.store.file_handler.local.open'),
+    ('s3://', 'spider_feeder.store.file_handler.s3.open'),
+    ('', 'spider_feeder.store.file_handler.local.open'),
+])
+def test_get_file_format_from_setting(mocker, uri_scheme, file_opener):
+    file_content = StringIO('\n'.join(['http://url1.com', 'http://url2.com']))
+    mock = mocker.patch(file_opener, return_value=file_content, autospec=True)
+
+    store = FileStore(f'{uri_scheme}temp', Settings({
+        'SPIDERFEEDER_INPUT_FORMAT': 'txt'
+    }))
+
+    store_meta = []
+    store_urls = []
+    for (url, meta) in store:
+        store_urls.append(url)
+        store_meta.append(meta)
+
+    mock.assert_called_with(f'{uri_scheme}temp', encoding='utf-8')
+    assert store_meta == [{}, {}]
+    assert store_urls == ['http://url1.com', 'http://url2.com']
+
+
+@pytest.mark.parametrize('uri_scheme, file_opener', [
+    ('file://', 'spider_feeder.store.file_handler.local.open'),
+    ('s3://', 'spider_feeder.store.file_handler.s3.open'),
+    ('', 'spider_feeder.store.file_handler.local.open'),
+])
+def test_get_file_format_setting_is_preferred_over_file_extension(mocker, uri_scheme, file_opener):
+    file_content = StringIO('\n'.join(['http://url1.com', 'http://url2.com']))
+    mock = mocker.patch(file_opener, return_value=file_content, autospec=True)
+
+    store = FileStore(f'{uri_scheme}temp.csv', Settings({
+        'SPIDERFEEDER_INPUT_FORMAT': 'txt'
+    }))
+
+    store_meta = []
+    store_urls = []
+    for (url, meta) in store:
+        store_urls.append(url)
+        store_meta.append(meta)
+
+    mock.assert_called_with(f'{uri_scheme}temp.csv', encoding='utf-8')
+    assert store_meta == [{}, {}]
+    assert store_urls == ['http://url1.com', 'http://url2.com']
+
+
 def test_fail_if_input_field_and_not_dict_data(mocker):
     mocker.patch(
         'spider_feeder.store.file_handler.local.open',