Repository: scrapinghub/scrapy-autounit

Issue:
Allow "." in spider names
I created a dummy spider for `quotes.toscrape.com` and set `name = 'quotes.toscrape.com'`.
After running `scrapy crawl quotes.toscrape.com` it generated the tests as expected.
However, when I ran `python -m unittest` I got
```
E
======================================================================
ERROR: autounit.tests.quotes.toscrape (unittest.loader._FailedTest)
----------------------------------------------------------------------
ImportError: Failed to import test module: autounit.tests.quotes.toscrape
Traceback (most recent call last):
  File "/usr/lib/python3.6/unittest/loader.py", line 462, in _find_test_path
    package = self._get_module_from_name(name)
  File "/usr/lib/python3.6/unittest/loader.py", line 369, in _get_module_from_name
    __import__(name)
ModuleNotFoundError: No module named 'autounit.tests.quotes'
```

Also `ls autounit/tests/` returned `__init__.py  __pycache__  quotes.toscrape.com`.

My fix idea is to replace `.` with `_` as done when we run `scrapy genspider quotes.toscrape.com quotes.toscrape.com`


Discussion:

Patch:
diff --git a/scrapy_autounit/middleware.py b/scrapy_autounit/middleware.py
index 0568541..f333bc8 100644
--- a/scrapy_autounit/middleware.py
+++ b/scrapy_autounit/middleware.py
@@ -4,6 +4,7 @@ from pathlib import Path
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.commands.genspider import sanitize_module_name
 
 from .utils import (
     add_sample,
@@ -91,7 +92,7 @@ class AutounitMiddleware:
 
         test_dir, test_name = get_or_create_test_dir(
             self.base_path,
-            spider.name,
+            sanitize_module_name(spider.name),
             callback_name,
             settings.get('AUTOUNIT_EXTRA_PATH'),
         )
diff --git a/scrapy_autounit/utils.py b/scrapy_autounit/utils.py
index 81a3a30..50acd52 100644
--- a/scrapy_autounit/utils.py
+++ b/scrapy_autounit/utils.py
@@ -238,7 +238,7 @@ def binary_check(fx_obj, cb_obj, encoding):
     return fx_obj
 
 
-def test_generator(fixture_path, encoding):
+def test_generator(fixture_path, encoding='utf-8'):
     with open(str(fixture_path), 'rb') as f:
         data = f.read()
 
@@ -260,7 +260,7 @@ def test_generator(fixture_path, encoding):
 
     def test(self):
         fx_result = data['result']
-        fx_version = data['python_version']
+        fx_version = data.get('python_version')
 
         request = request_from_dict(data['request'], spider)
         response = HtmlResponse(request=request, **data['response'])
diff --git a/setup.py b/setup.py
index a0e652f..65fdd46 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name='scrapy-autounit',
-    version='0.0.16',
+    version='0.0.17',
     author='',
     author_email='',
     description='Automatic unit test generation for Scrapy.',


Test patch:
diff --git a/tests/test_record.py b/tests/test_record.py
index a4865ad..33ed48e 100644
--- a/tests/test_record.py
+++ b/tests/test_record.py
@@ -10,7 +10,7 @@ import scrapy
 
 
 class MySpider(scrapy.Spider):
-    name = 'myspider'
+    name = '{name}'
 
     custom_settings = dict(
         SPIDER_MIDDLEWARES={{
@@ -66,6 +66,7 @@ class CaseSpider(object):
             dest.write('SPIDER_MODULES = ["myproject"]\n')
         self._start_requests = None
         self._parse = None
+        self._spider_name = 'myspider'
 
     def __enter__(self):
         return self
@@ -73,17 +74,19 @@ class CaseSpider(object):
     def __exit__(self, exc_type, exc_value, traceback):
         shutil.rmtree(self.dir)
 
+    def name(self, string):
+        self._spider_name = string
+
     def start_requests(self, string):
         self._start_requests = string
-        self._write_spider()
 
     def parse(self, string):
         self._parse = string
-        self._write_spider()
 
     def _write_spider(self):
         with open(os.path.join(self.proj_dir, 'myspider.py'), 'w') as dest:
             dest.write(SPIDER_TEMPLATE.format(
+                name=self._spider_name,
                 start_requests=self._start_requests,
                 parse=self._parse
             ))
@@ -91,11 +94,12 @@ class CaseSpider(object):
     def record(self, args=None, settings=None):
         if self._start_requests is None or self._parse is None:
             raise AssertionError()
+        self._write_spider()
         env = os.environ.copy()
         env['PYTHONPATH'] = self.dir  # doesn't work if == cwd
         env['SCRAPY_SETTINGS_MODULE'] = 'myproject.settings'
         command_args = [
-            'scrapy', 'crawl', 'myspider',
+            'scrapy', 'crawl', self._spider_name,
             '-s', 'AUTOUNIT_ENABLED=1',
         ]
         for k, v in (args or {}).items():
@@ -160,3 +164,18 @@ class TestRecording(unittest.TestCase):
             ''')
             spider.record(settings=dict(AUTOUNIT_EXTRA_PATH='abc'))
             spider.test()
+
+    def test_empty(self):
+        with CaseSpider() as spider:
+            spider.start_requests("yield scrapy.Request('data:text/plain,')")
+            spider.parse('pass')
+            spider.record()
+            spider.test()
+
+    def test_dotted_name(self):
+        with CaseSpider() as spider:
+            spider.name('my.spider')
+            spider.start_requests("yield scrapy.Request('data:text/plain,')")
+            spider.parse('pass')
+            spider.record()
+            spider.test()