Repository: scrapy/scrapy

Issue:
rename spider will load the old left .pyc instead of renamed spider
Hi all, I'm not sure if this is an issue.
If got `y.py` and `y.pyc` in the `spider` directory.
Then rename it with `mv y.py x.py` and update the new `x.py`.
After updates, run `scrapy crawl spider` will use the old version of spider instead of updates in the `x.py`.
Have to remove `y.pyc` first.
Thanks!



Discussion:
Isn't this just normal Python .pyc problems? Pretty sure it's not scrapy specific. If you're actively moving/editing source code, you should just clear all .pyc files to avoid any issues (they'll get regenerated automatically).

@candlejack297 You're right. 
However scrapy automatically load the spider code in the `spiders` directory. I thought scrapy should have handled this situation. 

That is just how Python works. (No?) Scrapy is written in Python. That means it is restricted by all constraints that applies to Python scripts/programs.

I think it is not a Scrapy bug.
As a feature request, I would say that it does not worth the needed code.

May not be worth the effort, but if it's to be investigated, I believe it'd be around
[SpiderLoader](https://github.com/scrapy/scrapy/blob/ebef6d7c6dd8922210db8a4a44f48fe27ee0cd16/scrapy/spiderloader.py#L12) and how it uses `importlib.import_module` in [`walk_modules`](https://github.com/scrapy/scrapy/blob/master/scrapy/utils/misc.py#L54)
Maybe look at [imp.load_source](https://docs.python.org/2/library/imp.html#imp.load_source) (hint from http://stackoverflow.com/a/8422549)

I think we shouldn't mess with pyc files and implement special handling of them; pyc files are standard Python feature and it'd be too opinionated to assume that .pyc modules shouldn't be loaded - e.g. someone may choose to deploy pyc files instead of .py files.

What I think we should do is to warn if there are several spiders with the same name. It is a more general problem; it is specific to scrapy, not to .pyc files handling.


Resolving patch:
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+from collections import defaultdict
 import traceback
 import warnings
 
@@ -19,10 +20,24 @@ class SpiderLoader(object):
     def __init__(self, settings):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self._spiders = {}
+        self._found = defaultdict(list)
         self._load_all_spiders()
 
+    def _check_name_duplicates(self):
+        dupes = ["\n".join("  {cls} named {name!r} (in {module})".format(
+                                module=mod, cls=cls, name=name)
+                           for (mod, cls) in locations)
+                 for name, locations in self._found.items()
+                 if len(locations)>1]
+        if dupes:
+            msg = ("There are several spiders with the same name:\n\n"
+                   "{}\n\n  This can cause unexpected behavior.".format(
+                        "\n\n".join(dupes)))
+            warnings.warn(msg, UserWarning)
+
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
+            self._found[spcls.name].append((module.__name__, spcls.__name__))
             self._spiders[spcls.name] = spcls
 
     def _load_all_spiders(self):
@@ -35,6 +50,7 @@ def _load_all_spiders(self):
                        "Check SPIDER_MODULES setting".format(
                             modname=name, tb=traceback.format_exc()))
                 warnings.warn(msg, RuntimeWarning)
+        self._check_name_duplicates()
 
     @classmethod
     def from_settings(cls, settings):


Test patch:
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -101,3 +101,55 @@ def test_bad_spider_modules_warning(self):
 
             spiders = spider_loader.list()
             self.assertEqual(spiders, [])
+
+
+class DuplicateSpiderNameLoaderTest(unittest.TestCase):
+
+    def setUp(self):
+        orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
+        self.tmpdir = self.mktemp()
+        os.mkdir(self.tmpdir)
+        self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
+        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        sys.path.append(self.tmpdir)
+        self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
+
+    def tearDown(self):
+        del sys.modules['test_spiders_xxx']
+        sys.path.remove(self.tmpdir)
+
+    def test_dupename_warning(self):
+        # copy 1 spider module so as to have duplicate spider name
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py'))
+
+        with warnings.catch_warnings(record=True) as w:
+            spider_loader = SpiderLoader.from_settings(self.settings)
+
+            self.assertEqual(len(w), 1)
+            msg = str(w[0].message)
+            self.assertIn("several spiders with the same name", msg)
+            self.assertIn("'spider3'", msg)
+
+            spiders = set(spider_loader.list())
+            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
+
+    def test_multiple_dupename_warning(self):
+        # copy 2 spider modules so as to have duplicate spider name
+        # This should issue 2 warning, 1 for each duplicate spider name
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py'))
+
+        with warnings.catch_warnings(record=True) as w:
+            spider_loader = SpiderLoader.from_settings(self.settings)
+
+            self.assertEqual(len(w), 1)
+            msg = str(w[0].message)
+            self.assertIn("several spiders with the same name", msg)
+            self.assertIn("'spider1'", msg)
+            self.assertIn("'spider2'", msg)
+
+            spiders = set(spider_loader.list())
+            self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))