Repository: scrapy/scrapy

Issue:
`scrapy genspider` should not overwrite existing file

## Summary

If the file mentioned in `scrapy genspider` already exists, then genspider should refuse to generate the the file.

## Motivation

As it stands, existing code can be blown away if this command runs twice.

## Describe alternatives you've considered

Prompting the user for overwriting existing spider.




Discussion:
Hey, is this Issue up for grabs??
As far as I am concerned, yes.

On Mon, May 11, 2020, 4:02 AM Jay Rajput <notifications@github.com> wrote:

> Hey, is this Issue up for grabs??
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/scrapy/scrapy/issues/4561#issuecomment-626540486>, or
> unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AAAFGLK7MXQJNHDX4I4AB2DRQ6WIHANCNFSM4M5FRRDQ>
> .
>

@Gallaecio @elacuesta thoughts??
I would not have thought this was an issue, as I'd expect spider code to be covered by source control. On the other hand, it should be only a matter of doing a file system check, so I think a clean implementation could be approved.
Alright! I will work on this one
Hi @metaperl @elacuesta,

I would like to understand this properly. I've tried to create a spider with the same name twice but it does not allow me to do so. 

```
(p3env) bash-3.2$ scrapy genspider toi 'https://timesofindia.indiatimes.com/'
Created spider 'toi' using template 'basic' in module:
  testscrapy.spiders.toi
(p3env) bash-3.2$
(p3env) bash-3.2$
(p3env) bash-3.2$ scrapy genspider toi 'https://timesofindia.indiatimes.com/'
Spider 'toi' already exists in module:
  testscrapy.spiders.toi
```

Could you please let me know what exactly is the issue so that I can try attempting to resolve or enhance?

Thanks & Regards.

Resolving patch:
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -66,16 +66,9 @@ def run(self, args, opts):
             print("Cannot create a spider with the same name as your project")
             return
 
-        try:
-            spidercls = self.crawler_process.spider_loader.load(name)
-        except KeyError:
-            pass
-        else:
-            # if spider already exists and not --force then halt
-            if not opts.force:
-                print("Spider %r already exists in module:" % name)
-                print("  %s" % spidercls.__module__)
-                return
+        if not opts.force and self._spider_exists(name):
+            return
+
         template_file = self._find_template(opts.template)
         if template_file:
             self._genspider(module, name, domain, opts.template, template_file)
@@ -119,6 +112,34 @@ def _list_templates(self):
             if filename.endswith('.tmpl'):
                 print("  %s" % splitext(filename)[0])
 
+    def _spider_exists(self, name):
+        if not self.settings.get('NEWSPIDER_MODULE'):
+            # if run as a standalone command and file with same filename already exists
+            if exists(name + ".py"):
+                print("%s already exists" % (abspath(name + ".py")))
+                return True
+            return False
+
+        try:
+            spidercls = self.crawler_process.spider_loader.load(name)
+        except KeyError:
+            pass
+        else:
+            # if spider with same name exists
+            print("Spider %r already exists in module:" % name)
+            print("  %s" % spidercls.__module__)
+            return True
+
+        # a file with the same name exists in the target directory
+        spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
+        spiders_dir = dirname(spiders_module.__file__)
+        spiders_dir_abs = abspath(spiders_dir)
+        if exists(join(spiders_dir_abs, name + ".py")):
+            print("%s already exists" % (join(spiders_dir_abs, (name + ".py"))))
+            return True
+
+        return False
+
     @property
     def templates_dir(self):
         return join(


Test patch:
diff --git a/tests/test_commands.py b/tests/test_commands.py
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -8,7 +8,7 @@
 import tempfile
 from contextlib import contextmanager
 from itertools import chain
-from os.path import exists, join, abspath
+from os.path import exists, join, abspath, getmtime
 from pathlib import Path
 from shutil import rmtree, copytree
 from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
@@ -337,8 +337,11 @@ def test_template(self, tplname='crawl'):
         p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
+        modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
         p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Spider %r already exists in module" % spname, out)
+        modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
+        self.assertEqual(modify_time_after, modify_time_before)
 
     def test_template_basic(self):
         self.test_template('basic')
@@ -360,6 +363,40 @@ def test_same_name_as_project(self):
         self.assertEqual(2, self.call('genspider', self.project_name))
         assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name))
 
+    def test_same_filename_as_existing_spider(self, force=False):
+        file_name = 'example'
+        file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name)
+        self.assertEqual(0, self.call('genspider', file_name, 'example.com'))
+        assert exists(file_path)
+
+        # change name of spider but not its file name
+        with open(file_path, 'r+') as spider_file:
+            file_data = spider_file.read()
+            file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
+            spider_file.seek(0)
+            spider_file.write(file_data)
+            spider_file.truncate()
+        modify_time_before = getmtime(file_path)
+        file_contents_before = file_data
+
+        if force:
+            p, out, err = self.proc('genspider', '--force', file_name, 'example.com')
+            self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out)
+            modify_time_after = getmtime(file_path)
+            self.assertNotEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertNotEqual(file_contents_after, file_contents_before)
+        else:
+            p, out, err = self.proc('genspider', file_name, 'example.com')
+            self.assertIn("%s already exists" % (file_path), out)
+            modify_time_after = getmtime(file_path)
+            self.assertEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertEqual(file_contents_after, file_contents_before)
+
+    def test_same_filename_as_existing_spider_force(self):
+        self.test_same_filename_as_existing_spider(force=True)
+
 
 class GenspiderStandaloneCommandTest(ProjectTest):
 
@@ -367,6 +404,34 @@ def test_generate_standalone_spider(self):
         self.call('genspider', 'example', 'example.com')
         assert exists(join(self.temp_path, 'example.py'))
 
+    def test_same_name_as_existing_file(self, force=False):
+        file_name = 'example'
+        file_path = join(self.temp_path, file_name + '.py')
+        p, out, err = self.proc('genspider', file_name, 'example.com')
+        self.assertIn("Created spider %r using template \'basic\' " % file_name, out)
+        assert exists(file_path)
+        modify_time_before = getmtime(file_path)
+        file_contents_before = open(file_path, 'r').read()
+
+        if force:
+            # use different template to ensure contents were changed
+            p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')
+            self.assertIn("Created spider %r using template \'crawl\' " % file_name, out)
+            modify_time_after = getmtime(file_path)
+            self.assertNotEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertNotEqual(file_contents_after, file_contents_before)
+        else:
+            p, out, err = self.proc('genspider', file_name, 'example.com')
+            self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out)
+            modify_time_after = getmtime(file_path)
+            self.assertEqual(modify_time_after, modify_time_before)
+            file_contents_after = open(file_path, 'r').read()
+            self.assertEqual(file_contents_after, file_contents_before)
+
+    def test_same_name_as_existing_file_force(self):
+        self.test_same_name_as_existing_file(force=True)
+
 
 class MiscCommandsTest(CommandTest):