Repository: ejulio/spider-feeder

Issue:
Allow http inputs
Add a new format do allow http/https input files.
Also, add a test case for google drive public files.

Say the URL https://drive.google.com/drive/folders/file-id?usp=sharing is given
Then we can download it from https://drive.google.com/uc?export=download&id=file-id

Comments:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 31b5399..2557fdd 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -18,6 +18,7 @@ Whenever possible, link the given PR with the feature/fix.
 * New setting `SPIDERFEEDER_INPUT_FORMAT` to set the file format or fall back to the file extension. [PR#18](https://github.com/ejulio/spider-feeder/pull/18)
 * Fall back to `scrapy` AWS settings if not provided. [PR#20](https://github.com/ejulio/spider-feeder/pull/20)
 * Fixed AWS settings set in Dash (Scrapy Cloud) UI. [PR#21](https://github.com/ejulio/spider-feeder/pull/21)
+* New _schemes_ `http` and `https`. [PR#22](https://github.com/ejulio/spider-feeder/pull/22)
 
 ## 0.2.0 (2019-08-27)
 
diff --git a/README.md b/README.md
index 747b426..9bc97d8 100644
--- a/README.md
+++ b/README.md
@@ -88,14 +88,17 @@ There are two extensions to load input data to your spiders.
 ## Settings
 
 `SPIDERFEEDER_INPUT_URI` is the URI to load URLs from.
-* If _scheme_ (`local`, `s3`, `collections`) is not provided, it'll use `local`
+* If _scheme_ (`file`, `s3`, `collections`) is not provided, it'll default to `file`
 * It can be formatted using spider attributes like `%(param)s` (similar to `FEED_URI` in scrapy)
-* Supported schemes are: `''` or `file` for local files and `s3` for AWS S3 (requires `botocore`)
-* When using `s3`, the URI must be formatted as `s3://key_id:secret_key@bucket/blob.txt`
-* If `key_id` and `secret_key` are not provided in the URI, they can be provided by the following settings: `SPIDERFEEDER_AWS_ACCESS_KEY_ID` and `SPIDERFEEDER_AWS_SECRET_ACCESS_KEY`.
-    * If they are not provided by these settings, they will fall back to `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
-    * If not set, they can be set as environment variables from `botocore`, but a warning will be logged by `spider-feeder`.
-* When using `collections`, it'll load URLs from [Scrapinghub Collections](https://doc.scrapinghub.com/api/collections.html)
+* Supported schemes are:
+    * `''` or `file` for local files
+    * `s3` for AWS S3 (requires `botocore`)
+        * The URI can be formatted as `s3://key_id:secret_key@bucket/blob.txt`
+        * If `key_id` and `secret_key` are not provided in the URI, they can be provided by the following settings: `SPIDERFEEDER_AWS_ACCESS_KEY_ID` and `SPIDERFEEDER_AWS_SECRET_ACCESS_KEY`.
+        * If they are not provided by these settings, they will fall back to `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
+        * If not set, they can be set as environment variables from `botocore`, but a warning will be logged by `spider-feeder`.
+    * `collections` for [Scrapinghub Collections](https://doc.scrapinghub.com/api/collections.html)
+    * `http` or `https` to load from any URI
 
 `SPIDERFEEDER_INPUT_FILE_ENCODING` sets the file encoding. DEFAULT = `'utf-8'`.
 
diff --git a/spider_feeder/loaders.py b/spider_feeder/loaders.py
index 2853dc6..5dfed4a 100644
--- a/spider_feeder/loaders.py
+++ b/spider_feeder/loaders.py
@@ -23,6 +23,8 @@ class BaseLoader:
         '': 'spider_feeder.store.file_store.FileStore',
         'file': 'spider_feeder.store.file_store.FileStore',
         's3': 'spider_feeder.store.file_store.FileStore',
+        'http': 'spider_feeder.store.file_store.FileStore',
+        'https': 'spider_feeder.store.file_store.FileStore',
         'collections': 'spider_feeder.store.scrapinghub_collection.ScrapinghubCollectionStore',
     }
 
diff --git a/spider_feeder/store/file_handler/http.py b/spider_feeder/store/file_handler/http.py
new file mode 100644
index 0000000..e64fb01
--- /dev/null
+++ b/spider_feeder/store/file_handler/http.py
@@ -0,0 +1,7 @@
+from io import StringIO
+from urllib.request import urlopen
+
+
+def open(url, encoding, settings):
+    response = urlopen(url)
+    return StringIO(response.read())
diff --git a/spider_feeder/store/file_store.py b/spider_feeder/store/file_store.py
index 870c0ce..70b7f23 100644
--- a/spider_feeder/store/file_store.py
+++ b/spider_feeder/store/file_store.py
@@ -26,6 +26,8 @@ class FileStore(BaseStore):
         '': 'spider_feeder.store.file_handler.local.open',
         'file': 'spider_feeder.store.file_handler.local.open',
         's3': 'spider_feeder.store.file_handler.s3.open',
+        'http': 'spider_feeder.store.file_handler.http.open',
+        'https': 'spider_feeder.store.file_handler.http.open',
     }
 
     FILE_PARSERS = {


Test patch:
diff --git a/tests/store/file_handler/test_http.py b/tests/store/file_handler/test_http.py
new file mode 100644
index 0000000..7e9cc6e
--- /dev/null
+++ b/tests/store/file_handler/test_http.py
@@ -0,0 +1,12 @@
+from spider_feeder.store.file_handler import http
+
+
+def test_open_http_file(mocker):
+    urlopen_mock = mocker.patch('spider_feeder.store.file_handler.http.urlopen')
+    urlopen_mock().read.return_value = 'FILE CONTENT'
+
+    url = 'https://someurl.com/index?qs=1'
+    fd = http.open(url, encoding='utf-8', settings=None)
+
+    urlopen_mock.assert_called_with(url)
+    assert fd.read() == 'FILE CONTENT'
diff --git a/tests/store/test_file_store.py b/tests/store/test_file_store.py
index b74e6e2..fca5e60 100644
--- a/tests/store/test_file_store.py
+++ b/tests/store/test_file_store.py
@@ -15,11 +15,16 @@ def custom_parser():
     pass
 
 
-@pytest.mark.parametrize('uri_scheme, file_opener', [
+SCHEMES_AND_OPENERS_TO_MOCK = [
     ('file://', 'spider_feeder.store.file_handler.local.open'),
     ('s3://', 'spider_feeder.store.file_handler.s3.open'),
     ('', 'spider_feeder.store.file_handler.local.open'),
-])
+    ('http://', 'spider_feeder.store.file_handler.http.open'),
+    ('https://', 'spider_feeder.store.file_handler.http.open'),
+]
+
+
+@pytest.mark.parametrize('uri_scheme, file_opener', SCHEMES_AND_OPENERS_TO_MOCK)
 def test_load_txt_file(mocker, uri_scheme, file_opener):
     file_content = StringIO('\n'.join(['http://url1.com', 'http://url2.com']))
     mock = mocker.patch(file_opener, return_value=file_content, autospec=True)
@@ -38,11 +43,7 @@ def test_load_txt_file(mocker, uri_scheme, file_opener):
     assert store_urls == ['http://url1.com', 'http://url2.com']
 
 
-@pytest.mark.parametrize('uri_scheme, file_opener', [
-    ('file://', 'spider_feeder.store.file_handler.local.open'),
-    ('s3://', 'spider_feeder.store.file_handler.s3.open'),
-    ('', 'spider_feeder.store.file_handler.local.open'),
-])
+@pytest.mark.parametrize('uri_scheme, file_opener', SCHEMES_AND_OPENERS_TO_MOCK)
 def test_load_csv_file(mocker, uri_scheme, file_opener):
     file_content = StringIO('\n'.join([
         '"url_id","url"',
@@ -70,11 +71,7 @@ def test_load_csv_file(mocker, uri_scheme, file_opener):
     ]
 
 
-@pytest.mark.parametrize('uri_scheme, file_opener', [
-    ('file://', 'spider_feeder.store.file_handler.local.open'),
-    ('s3://', 'spider_feeder.store.file_handler.s3.open'),
-    ('', 'spider_feeder.store.file_handler.local.open'),
-])
+@pytest.mark.parametrize('uri_scheme, file_opener', SCHEMES_AND_OPENERS_TO_MOCK)
 def test_load_json_file(mocker, uri_scheme, file_opener):
     file_content = StringIO(json.dumps([
         {'url_id': '1', 'url': 'http://url1.com'},
@@ -101,11 +98,7 @@ def test_load_json_file(mocker, uri_scheme, file_opener):
     ]
 
 
-@pytest.mark.parametrize('uri_scheme, file_opener', [
-    ('file://', 'spider_feeder.store.file_handler.local.open'),
-    ('s3://', 'spider_feeder.store.file_handler.s3.open'),
-    ('', 'spider_feeder.store.file_handler.local.open'),
-])
+@pytest.mark.parametrize('uri_scheme, file_opener', SCHEMES_AND_OPENERS_TO_MOCK)
 def test_get_file_format_from_setting(mocker, uri_scheme, file_opener):
     file_content = StringIO('\n'.join(['http://url1.com', 'http://url2.com']))
     mock = mocker.patch(file_opener, return_value=file_content, autospec=True)
@@ -126,11 +119,7 @@ def test_get_file_format_from_setting(mocker, uri_scheme, file_opener):
     assert store_urls == ['http://url1.com', 'http://url2.com']
 
 
-@pytest.mark.parametrize('uri_scheme, file_opener', [
-    ('file://', 'spider_feeder.store.file_handler.local.open'),
-    ('s3://', 'spider_feeder.store.file_handler.s3.open'),
-    ('', 'spider_feeder.store.file_handler.local.open'),
-])
+@pytest.mark.parametrize('uri_scheme, file_opener', SCHEMES_AND_OPENERS_TO_MOCK)
 def test_get_file_format_setting_is_preferred_over_file_extension(mocker, uri_scheme, file_opener):
     file_content = StringIO('\n'.join(['http://url1.com', 'http://url2.com']))
     mock = mocker.patch(file_opener, return_value=file_content, autospec=True)
diff --git a/tests/test_loaders.py b/tests/test_loaders.py
index 4a9c140..01427fd 100644
--- a/tests/test_loaders.py
+++ b/tests/test_loaders.py
@@ -32,6 +32,8 @@ def test_start_urls_loader_not_configured(get_crawler):
     ('s3://', 'spider_feeder.store.file_store.FileStore'),
     ('file://', 'spider_feeder.store.file_store.FileStore'),
     ('', 'spider_feeder.store.file_store.FileStore'),
+    ('http://', 'spider_feeder.store.file_store.FileStore'),
+    ('https://', 'spider_feeder.store.file_store.FileStore'),
     ('collections://', 'spider_feeder.store.scrapinghub_collection.ScrapinghubCollectionStore'),
 ])
 def test_start_urls_loader_open_store_given_scheme(get_crawler, mocker, scheme, store_cls):