Repository: scrapy/scrapy

Issue:
Having an option to define an Items output file in the `scrapy parse` commnad line
I use `scrapy parse` command to check the Items output for given URL, and I use it a lot.

Sometimes, I need to latter explore the Items extracted by `scrapy parse`, e.g . to check that something I want to fix was fixed for that specific URL without having to touch the source code.

### Proposed solution:

Add a new parameter `-o` or `--output` in the parse command line. Allowing the client to set a desired output file for the items. 

I could also `|` the result of `scrapy parse`, filter out want I want and create a `json` file out of it, but would be much easier if I could simple write `scrapy parse {url} --callback=fixed_function -o items.json`


Discussion:
Shall I work on this issue?
@adityaa30 Please, go ahead, no need to ask for permission :slightly_smiling_face: 
Okay sure!
Hi, I m working on this issue and made some changes in scrapy/commands/parse.py but running "scrapy parse" in command line not reflecting changes. Any suggestions?
@Gallaecio how to run scrapy from cloned repository in my local computer? I think "scrapy parse" command running scrapy package installed through "pip install scrapy".
@AKSHAYSHARMAJS you should be able to install Scrapy in "development mode" by executing `pip install -e .`
@elacuesta Thanks!


Resolving patch:
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -108,7 +108,7 @@ def run(self, args, opts):
 
 class BaseRunSpiderCommand(ScrapyCommand):
     """
-    Common class used to share functionality between the crawl and runspider commands
+    Common class used to share functionality between the crawl, parse and runspider commands
     """
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -4,18 +4,16 @@
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import BaseRunSpiderCommand
 from scrapy.http import Request
 from scrapy.utils import display
-from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 
 logger = logging.getLogger(__name__)
 
 
-class Command(ScrapyCommand):
-
+class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
@@ -31,11 +29,9 @@ def short_desc(self):
         return "Parse URL (using its spider) and print the results"
 
     def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+        BaseRunSpiderCommand.add_options(self, parser)
         parser.add_option("--spider", dest="spider", default=None,
                           help="use this spider without looking for one")
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
         parser.add_option("--pipelines", action="store_true",
                           help="process items through pipelines")
         parser.add_option("--nolinks", dest="nolinks", action="store_true",
@@ -200,12 +196,15 @@ def callback(response, **cb_kwargs):
             self.add_items(depth, items)
             self.add_requests(depth, requests)
 
+            scraped_data = items if opts.output else []
             if depth < opts.depth:
                 for req in requests:
                     req.meta['_depth'] = depth + 1
                     req.meta['_callback'] = req.callback
                     req.callback = callback
-                return requests
+                scraped_data += requests
+
+            return scraped_data
 
         # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
@@ -221,18 +220,11 @@ def callback(response, **cb_kwargs):
         return request
 
     def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
+        BaseRunSpiderCommand.process_options(self, args, opts)
 
-        self.process_spider_arguments(opts)
         self.process_request_meta(opts)
         self.process_request_cb_kwargs(opts)
 
-    def process_spider_arguments(self, opts):
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-
     def process_request_meta(self, opts):
         if opts.meta:
             try:


Test patch:
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,5 +1,5 @@
 import os
-from os.path import join, abspath
+from os.path import join, abspath, isfile, exists
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
@@ -218,3 +218,24 @@ def test_crawlspider_no_matching_rule(self):
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
+
+    @defer.inlineCallbacks
+    def test_output_flag(self):
+        """Checks if a file was created successfully having
+        correct format containing correct data in it.
+        """
+        file_name = 'data.json'
+        file_path = join(self.proj_path, file_name)
+        yield self.execute([
+            '--spider', self.spider_name,
+            '-c', 'parse',
+            '-o', file_name,
+            self.url('/html')
+        ])
+
+        self.assertTrue(exists(file_path))
+        self.assertTrue(isfile(file_path))
+
+        content = '[\n{},\n{"foo": "bar"}\n]'
+        with open(file_path, 'r') as f:
+            self.assertEqual(f.read(), content)