Repository: ejulio/spider-feeder

Issue:
Send extra data from input files as request.meta
After https://github.com/ejulio/spider-feeder/pull/8 we'll be able to read any type of input file.
Usually we don't want only the URL field, but also the extra content for further processing/filtering.
So, it would be nice to store this data in the spider as well.

However, scrapy spiders only allow `start_urls` and not `start_data`.
So, I wouldn't like to `start_urls` with some kind of `dict` instead of plain strings.

My proposed approach is to `setattr(spider, "start_data", data)` where data is a `list` of dicts.
However, this is not nice in the sense that the user has to write `start_requests` method to handle this data.
Also, we are relying in some property that is not currently supported in scrapy spiders, so it is a bit magical.

My other approach would be to include a base spider in `spider-feeder` and this spider would have the `start_data` property.
Then it would override `start_requests` making requests using `SPIDERFEEDER_INPUT_FIELD` and setting all other fields as `request.meta`.
Also, it would be easier to assert in `spider_opened` signal if the given spider is an instance of this base spider or not.

Finally, I think it would be nice to have this behavior in `StartDataLoader` to make it explicit that we are loading `spider_data` and leave `StartUrlsLoader` to load `start_urls`.

Any thoughts on this?

Discussion:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 3d51515..4832876 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -18,6 +18,7 @@ Whenever possible, link the given PR with the feature/fix.
 
 * Support for `.csv` and `.json` input files [PR#8](https://github.com/ejulio/spider-feeder/pull/8)
 * Support for Scrapinghub Collections [PR#11](https://github.com/ejulio/spider-feeder/pull/11)
+* Support for loading meta data in requests through `StartRequestsLoader` [PR #12](https://github.com/ejulio/spider-feeder/pull/12)
 
 ### Changed
 
diff --git a/README.md b/README.md
index 582bb4e..614269d 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,7 @@ id,input_url
 Then, in `settings.py`
 ```
 EXTENSIONS = {
-    'spider_feeder.loaders.StartUrlsLoader': 0
+    'spider_feeder.loaders.StartRequestsLoader': 0
 }
 
 SPIDERFEEDER_INPUT_URI = './urls.csv'
@@ -63,6 +63,13 @@ SPIDERFEEDER_INPUT_FIELD = 'input_url'
 The same applies for `json`, just requiring to update the file extension to `.json` instead of `.csv`.
 This means that the input file format is inferred from the given file extension.
 
+## Extensions
+
+There are two extensions to load input data to your spiders.
+
+* `spider_feeder.loaders.StartUrlsLoader`: sets a list of urls to `spider.start_urls`
+* `spider_feeder.loaders.StartRequestsLoader`: overrides `spider.start_requests` by making a request to the given URL and setting `meta` to extra metadata parsed from `json`, `csv` or `collections`.
+
 ## Settings
 
 `SPIDERFEEDER_INPUT_URI` is the URI to load URLs from.
diff --git a/spider_feeder/loaders.py b/spider_feeder/loaders.py
index 6b3c3c6..ad493e1 100644
--- a/spider_feeder/loaders.py
+++ b/spider_feeder/loaders.py
@@ -1,16 +1,16 @@
 from urllib.parse import urlparse
 import logging
 
-from scrapy import signals
+from scrapy import Request, signals
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)
 
 
-class StartUrlsLoader:
-    '''Loads a set of input urls to `spider.start_urls`.
-    The URLs are loaded from `SPIDERFEEDER_INPUT_URI`.
+class BaseLoader:
+    '''Base loader for `start_urls` and `start_requests` loaders.
+    The data is loaded from `SPIDERFEEDER_INPUT_URI`.
     The given store is loaded according to the scheme in `SPIDERFEEDER_INPUT_URI`.
     Currently, the support is for local file system, Amazon AWS S3, and Scrapinghub Collections.
     The stores can be overriden or aggregated through `SPIDERFEEDER_STORES`.
@@ -29,7 +29,7 @@ class StartUrlsLoader:
     def from_crawler(cls, crawler):
         input_uri = crawler.settings.get('SPIDERFEEDER_INPUT_URI', None)
         if not input_uri:
-            raise NotConfigured('StartUrlsLoader requires SPIDERFEEDER_INPUT_URI setting.')
+            raise NotConfigured('Loader requires SPIDERFEEDER_INPUT_URI setting.')
 
         stores = cls.STORES
         stores = dict(stores, **crawler.settings.getdict('SPIDERFEEDER_STORES', {}))
@@ -46,10 +46,7 @@ class StartUrlsLoader:
     def spider_opened(self, spider):
         input_uri = self._get_formatted_input_uri(spider)
         store = self._get_store(input_uri)
-        spider.start_urls = [url for (url, _) in store]
-        n_urls = len(spider.start_urls)
-        logger.info(f'Loaded {n_urls} urls from {input_uri}.')
-        self._crawler.stats.set_value(f'spider_feeder/{spider.name}/url_count', n_urls)
+        self.set_spider_input_data(spider, store)
 
     def _get_formatted_input_uri(self, spider):
         params = {k: getattr(spider, k) for k in dir(spider)}
@@ -59,3 +56,50 @@ class StartUrlsLoader:
         parsed = urlparse(input_uri)
         store_cls = load_object(self._stores[parsed.scheme])
         return store_cls(input_uri, self._crawler.settings)
+
+    def set_spider_input_data(self, spider, store):
+        raise NotImplementedError()
+
+
+class StartUrlsLoader(BaseLoader):
+    '''Loader setting spider.start_urls. For more information, please refer to BaseLoader.'''
+
+    def set_spider_input_data(self, spider, store):
+        spider.start_urls = _StartDataIter(
+            self._crawler,
+            spider,
+            store,
+            generator_fn=lambda url, meta: url
+        )
+
+
+class StartRequestsLoader(BaseLoader):
+    '''Loader setting spider.start_requests. For more information, please refer to BaseLoader.'''
+
+    def set_spider_input_data(self, spider, store):
+        spider.start_requests = _StartDataIter(
+            self._crawler,
+            spider,
+            store,
+            generator_fn=lambda url, meta: Request(url, meta=meta)
+        )
+
+
+class _StartDataIter:
+
+    def __init__(self, crawler, spider, store, generator_fn):
+        self._crawler = crawler
+        self._spider = spider
+        self._store = store
+        self._generator_fn = generator_fn
+
+    def __call__(self):
+        yield from self
+
+    def __iter__(self):
+        n_urls = 0
+        for (url, meta) in self._store:
+            yield self._generator_fn(url, meta)
+            n_urls += 1
+
+        self._crawler.stats.set_value(f'spider_feeder/{self._spider.name}/url_count', n_urls)


Test patch:
diff --git a/tests/test_loaders.py b/tests/test_loaders.py
index f9d251b..33e18c2 100644
--- a/tests/test_loaders.py
+++ b/tests/test_loaders.py
@@ -14,11 +14,7 @@ CustomAbcStore = Mock()
 
 @pytest.fixture
 def get_crawler():
-    def _crawler(extended_settings={}):
-        settings = {
-            "EXTENSIONS": {"spider_feeder.loaders.StartUrlsLoader": 500},
-        }
-        settings.update(extended_settings)
+    def _crawler(settings={}):
         crawler = Crawler(Spider, settings=settings)
         crawler.spider = Spider("dummy")
         return crawler
@@ -42,38 +38,40 @@ def test_start_urls_loader_open_store_given_scheme(get_crawler, mocker, scheme,
     mock = mocker.patch(store_cls)
     mock().__iter__.return_value = iter([('https://url1.com', {}), ('https://url2.com', {})])
 
-    crawler = get_crawler({'SPIDERFEEDER_INPUT_URI': f'{scheme}input_file.txt'})
-    StartUrlsLoader.from_crawler(crawler)
+    crawler = get_crawler({
+        'EXTENSIONS': {'spider_feeder.loaders.StartUrlsLoader': 500},
+        'SPIDERFEEDER_INPUT_URI': f'{scheme}input_file.txt'}
+    )
 
     crawler.signals.send_catch_log(signals.spider_opened, spider=crawler.spider)
 
-    assert crawler.spider.start_urls == ['https://url1.com', 'https://url2.com']
+    assert list(crawler.spider.start_urls) == ['https://url1.com', 'https://url2.com']
     mock.assert_called_with(f'{scheme}input_file.txt', crawler.settings)
     assert crawler.stats.get_value(f'spider_feeder/{crawler.spider.name}/url_count') == 2
 
 
 def test_should_override_store(get_crawler, mocker):
     crawler = get_crawler({
+        'EXTENSIONS': {'spider_feeder.loaders.StartUrlsLoader': 500},
         'SPIDERFEEDER_INPUT_URI': 's3://input_file.txt',
         'SPIDERFEEDER_STORES': {
             's3': 'tests.test_loaders.CustomS3Store'
         }
     })
-    StartUrlsLoader.from_crawler(crawler)
 
     crawler.signals.send_catch_log(signals.spider_opened, spider=crawler.spider)
 
     CustomS3Store.assert_called_once_with('s3://input_file.txt', crawler.settings)
 
 
-def test_should_custom_store(get_crawler, mocker):
+def test_should_load_custom_store(get_crawler, mocker):
     crawler = get_crawler({
+        'EXTENSIONS': {'spider_feeder.loaders.StartUrlsLoader': 500},
         'SPIDERFEEDER_INPUT_URI': 'abc://input_file.txt',
         'SPIDERFEEDER_STORES': {
             'abc': 'tests.test_loaders.CustomAbcStore'
         }
     })
-    StartUrlsLoader.from_crawler(crawler)
 
     crawler.signals.send_catch_log(signals.spider_opened, spider=crawler.spider)
 
@@ -84,11 +82,34 @@ def test_uri_format_spider_attributes(get_crawler, mocker):
     mock = mocker.patch('spider_feeder.store.file_store.FileStore')
     mock().__iter__.return_value = iter([('https://url1.com', {}), ('https://url2.com', {})])
 
-    crawler = get_crawler({'SPIDERFEEDER_INPUT_URI': '%(dir)s/%(input_file)s.txt'})
+    crawler = get_crawler({
+        'EXTENSIONS': {'spider_feeder.loaders.StartUrlsLoader': 500},
+        'SPIDERFEEDER_INPUT_URI': '%(dir)s/%(input_file)s.txt'}
+    )
     crawler.spider.dir = '/tmp'
     crawler.spider.input_file = 'spider_input'
-    StartUrlsLoader.from_crawler(crawler)
 
     crawler.signals.send_catch_log(signals.spider_opened, spider=crawler.spider)
 
     mock.assert_called_with('/tmp/spider_input.txt', crawler.settings)
+
+
+def test_load_start_requests(get_crawler, mocker):
+    mock = mocker.patch('spider_feeder.store.file_store.FileStore')
+    store_data = [
+        ('https://url1.com', {'url_id': '1'}),
+        ('https://url2.com', {'url_id': '2'}),
+        ('https://url2.com', {'url_id': '2'}),
+    ]
+    mock().__iter__.return_value = iter(store_data)
+
+    crawler = get_crawler({
+        'EXTENSIONS': {'spider_feeder.loaders.StartRequestsLoader': 500},
+        'SPIDERFEEDER_INPUT_URI': 'input_file.csv',
+    })
+
+    crawler.signals.send_catch_log(signals.spider_opened, spider=crawler.spider)
+
+    request_data = [(r.url, r.meta) for r in crawler.spider.start_requests()]
+    assert request_data == store_data
+    assert crawler.stats.get_value(f'spider_feeder/{crawler.spider.name}/url_count') == 3