Repository: scrapy/scrapy

Issue:
Test Windows builds with Appveyor integration
Just like scrapy has Travis CI integration, scrapy can also be tested on Windows with http://www.appveyor.com/



Discussion:
See https://github.com/scrapy/scrapy/pull/1736


Resolving patch:
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -107,8 +107,8 @@ def run(self, args, opts):
                 string.Template(path).substitute(project_name=project_name))
             render_templatefile(tplfile, project_name=project_name,
                 ProjectName=string_camelcase(project_name))
-        print("New Scrapy project %r, using template directory %r, created in:" % \
-              (project_name, self.templates_dir))
+        print("New Scrapy project '%s', using template directory '%s', "
+              "created in:" % (project_name, self.templates_dir))
         print("    %s\n" % abspath(project_dir))
         print("You can start your first spider with:")
         print("    cd %s" % project_dir)


Test patch:
diff --git a/tests/mockserver.py b/tests/mockserver.py
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -209,7 +209,7 @@ def __exit__(self, exc_type, exc_value, traceback):
         time.sleep(0.2)
 
     def url(self, path, is_secure=False):
-        host = self.http_address
+        host = self.http_address.replace('0.0.0.0', '127.0.0.1')
         if is_secure:
             host = self.https_address
         return host + path
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -3,9 +3,10 @@ pytest-twisted
 pytest-cov==2.5.1
 testfixtures
 jmespath
-leveldb
+leveldb; sys_platform != "win32"
 botocore
 # optional for shell wrapper tests
 bpython
 ipython
 brotlipy
+pywin32; sys_platform == "win32"
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -52,7 +52,8 @@ def test_profiling(self):
             stats.print_stats()
             out.seek(0)
             stats = out.read()
-            self.assertIn('scrapy/commands/version.py', stats)
+            self.assertIn(os.path.join('scrapy', 'commands', 'version.py'),
+                          stats)
             self.assertIn('tottime', stats)
         finally:
             shutil.rmtree(path)
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,3 +1,4 @@
+import os
 from os.path import join, abspath
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -7,6 +8,11 @@
 from tests.test_commands import CommandTest
 
 
+def _textmode(bstr):
+    """Normalize input the same as writing to a file
+    and reading from it in text mode"""
+    return to_native_str(bstr).replace(os.linesep, '\n')
+
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
 
@@ -97,7 +103,7 @@ def test_spider_arguments(self):
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_with_meta(self):
@@ -106,13 +112,13 @@ def test_request_with_meta(self):
                                            '--meta', raw_json_string,
                                            '-c', 'parse_request_with_meta',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '-m', raw_json_string,
                                            '-c', 'parse_request_with_meta',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
 
     @defer.inlineCallbacks
@@ -120,7 +126,7 @@ def test_request_without_meta(self):
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                           '-c', 'parse_request_without_meta',
                                            self.url('/html')])
-        self.assertIn("DEBUG: It Works!", to_native_str(stderr))
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
 
     @defer.inlineCallbacks
@@ -129,29 +135,29 @@ def test_pipelines(self):
                                            '--pipelines',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assertIn("INFO: It Works!", to_native_str(stderr))
+        self.assertIn("INFO: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_parse_items(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-c', 'parse', self.url('/html')]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_parse_items_no_callback_passed(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, self.url('/html')]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_wrong_callback_passed(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-        self.assertIn("""Cannot find callback""", to_native_str(stderr))
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""Cannot find callback""", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_crawlspider_matching_rule_callback_set(self):
@@ -159,7 +165,7 @@ def test_crawlspider_matching_rule_callback_set(self):
         status, out, stderr = yield self.execute(
             ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_crawlspider_matching_rule_default_callback(self):
@@ -167,7 +173,7 @@ def test_crawlspider_matching_rule_default_callback(self):
         status, out, stderr = yield self.execute(
             ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
         )
-        self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))
+        self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out))
 
     @defer.inlineCallbacks
     def test_spider_with_no_rules_attribute(self):
@@ -175,15 +181,15 @@ def test_spider_with_no_rules_attribute(self):
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-        self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr))
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_crawlspider_missing_callback(self):
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
 
     @defer.inlineCallbacks
     def test_crawlspider_no_matching_rule(self):
@@ -191,5 +197,5 @@ def test_crawlspider_no_matching_rule(self):
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
         )
-        self.assertRegexpMatches(to_native_str(out), """# Scraped Items  -+\n\[\]""")
-        self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr))
+        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
diff --git a/tests/test_commands.py b/tests/test_commands.py
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -3,18 +3,17 @@
 import sys
 import subprocess
 import tempfile
-from time import sleep
 from os.path import exists, join, abspath
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from contextlib import contextmanager
+from threading import Timer
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
 import scrapy
 from scrapy.utils.python import to_native_str
-from scrapy.utils.python import retry_on_eintr
 from scrapy.utils.test import get_testenv
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
@@ -46,16 +45,18 @@ def proc(self, *new_args, **popen_kwargs):
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              **popen_kwargs)
 
-        waited = 0
-        interval = 0.2
-        while p.poll() is None:
-            sleep(interval)
-            waited += interval
-            if waited > 15:
-                p.kill()
-                assert False, 'Command took too much time to complete'
+        def kill_proc():
+            p.kill()
+            assert False, 'Command took too much time to complete'
 
-        return p
+        timer = Timer(15, kill_proc)
+        try:
+            timer.start()
+            stdout, stderr = p.communicate()
+        finally:
+            timer.cancel()
+
+        return p, to_native_str(stdout), to_native_str(stderr)
 
 
 class StartprojectTest(ProjectTest):
@@ -111,9 +112,9 @@ def test_startproject_template_override(self):
         assert exists(join(self.tmpl_proj, 'root_template'))
 
         args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
-        p = self.proc('startproject', self.project_name, *args)
-        out = to_native_str(retry_on_eintr(p.stdout.read))
-        self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
+        p, out, err = self.proc('startproject', self.project_name, *args)
+        self.assertIn("New Scrapy project '%s', using template directory"
+                      % self.project_name, out)
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
 
@@ -140,12 +141,10 @@ def test_arguments(self):
     def test_template(self, tplname='crawl'):
         args = ['--template=%s' % tplname] if tplname else []
         spname = 'test_spider'
-        p = self.proc('genspider', spname, 'test.com', *args)
-        out = to_native_str(retry_on_eintr(p.stdout.read))
+        p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
-        p = self.proc('genspider', spname, 'test.com', *args)
-        out = to_native_str(retry_on_eintr(p.stdout.read))
+        p, out, err = self.proc('genspider', spname, 'test.com', *args)
         self.assertIn("Spider %r already exists in module" % spname, out)
 
     def test_template_basic(self):
@@ -212,8 +211,8 @@ def runspider(self, code, name='myspider.py', args=()):
             return self.proc('runspider', fname, *args)
 
     def get_log(self, code, name='myspider.py', args=()):
-        p = self.runspider(code, name=name, args=args)
-        return to_native_str(p.stderr.read())
+        p, stdout, stderr = self.runspider(code, name=name, args=args)
+        return stderr
 
     def test_runspider(self):
         log = self.get_log(self.debug_log_spider)
@@ -223,12 +222,12 @@ def test_runspider(self):
         self.assertIn("INFO: Spider closed (finished)", log)
 
     def test_run_fail_spider(self):
-        proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
+        proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider))
         ret = proc.returncode
         self.assertNotEqual(ret, 0)
 
     def test_run_good_spider(self):
-        proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
+        proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider))
         ret = proc.returncode
         self.assertEqual(ret, 0)
 
@@ -279,8 +278,7 @@ def test_runspider_no_spider_found(self):
         self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
-        p = self.proc('runspider', 'some_non_existent_file')
-        log = to_native_str(p.stderr.read())
+        _, _, log = self.proc('runspider', 'some_non_existent_file')
         self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
@@ -304,8 +302,7 @@ def start_requests(self):
 class BenchCommandTest(CommandTest):
 
     def test_run(self):
-        p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
-                '-s', 'CLOSESPIDER_TIMEOUT=0.01')
-        log = to_native_str(p.stderr.read())
+        _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
+                           '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         self.assertIn('INFO: Crawled', log)
         self.assertNotIn('Unhandled Error', log)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,10 +1,9 @@
 import logging
 import tempfile
 import warnings
-import unittest
 
 from twisted.internet import defer
-import twisted.trial.unittest
+from twisted.trial import unittest
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
@@ -94,26 +93,29 @@ class MySpider(scrapy.Spider):
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
-        with tempfile.NamedTemporaryFile() as log_file:
-            class MySpider(scrapy.Spider):
-                name = 'spider'
-                custom_settings = {
-                    'LOG_LEVEL': 'INFO',
-                    'LOG_FILE': log_file.name,
-                    # disable telnet if not available to avoid an extra warning
-                    'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
-                }
-
-            configure_logging()
-            self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
-            crawler = Crawler(MySpider, {})
-            self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
-            info_count = crawler.stats.get_value('log_count/INFO')
-            logging.debug('debug message')
-            logging.info('info message')
-            logging.warning('warning message')
-            logging.error('error message')
-            logged = log_file.read().decode('utf8')
+        log_file = self.mktemp()
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+            custom_settings = {
+                'LOG_LEVEL': 'INFO',
+                'LOG_FILE': log_file,
+                # disable telnet if not available to avoid an extra warning
+                'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
+            }
+
+        configure_logging()
+        self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
+        crawler = Crawler(MySpider, {})
+        self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
+        info_count = crawler.stats.get_value('log_count/INFO')
+        logging.debug('debug message')
+        logging.info('info message')
+        logging.warning('warning message')
+        logging.error('error message')
+
+        with open(log_file, 'rb') as fo:
+            logged = fo.read().decode('utf8')
+
         self.assertNotIn('debug message', logged)
         self.assertIn('info message', logged)
         self.assertIn('warning message', logged)
@@ -141,9 +143,8 @@ def test_spider_manager_verify_interface(self):
         settings = Settings({
             'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
         })
-        with warnings.catch_warnings(record=True) as w, \
-                self.assertRaises(AttributeError):
-            CrawlerRunner(settings)
+        with warnings.catch_warnings(record=True) as w:
+            self.assertRaises(AttributeError, CrawlerRunner, settings)
             self.assertEqual(len(w), 1)
             self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
             self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
@@ -203,7 +204,7 @@ def start_requests(self):
         return []
 
 
-class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase):
+class CrawlerRunnerHasSpider(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,7 +1,8 @@
 import os
 import six
-import contextlib
 import shutil
+import tempfile
+import contextlib
 try:
     from unittest import mock
 except ImportError:
@@ -913,7 +914,9 @@ def _test(r):
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_local_filename(self):
-        local_fname = b"/tmp/file.txt"
+        f, local_fname = tempfile.mkstemp()
+        local_fname = to_bytes(local_fname)
+        os.close(f)
         meta = {"ftp_local_filename": local_fname}
         meta.update(self.req_meta)
         request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
@@ -922,7 +925,8 @@ def test_ftp_local_filename(self):
 
         def _test(r):
             self.assertEqual(r.body, local_fname)
-            self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']})
+            self.assertEqual(r.headers, {b'Local Filename': [local_fname],
+                                         b'Size': [b'17']})
             self.assertTrue(os.path.exists(local_fname))
             with open(local_fname, "rb") as f:
                 self.assertEqual(f.read(), b"I have the power!")
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2,11 +2,12 @@
 import os
 import csv
 import json
+import warnings
 from io import BytesIO
 import tempfile
 import shutil
-from six.moves.urllib.parse import urlparse
-import warnings
+from six.moves.urllib.parse import urljoin, urlparse
+from six.moves.urllib.request import pathname2url
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
@@ -226,9 +227,10 @@ class MyItem(scrapy.Item):
     def run_and_export(self, spider_cls, settings=None):
         """ Run spider with specified settings; return exported data. """
         tmpdir = tempfile.mkdtemp()
-        res_name = tmpdir + '/res'
+        res_path = os.path.join(tmpdir, 'res')
+        res_uri = urljoin('file:', pathname2url(res_path))
         defaults = {
-            'FEED_URI': 'file://' + res_name,
+            'FEED_URI': res_uri,
             'FEED_FORMAT': 'csv',
         }
         defaults.update(settings or {})
@@ -238,11 +240,13 @@ def run_and_export(self, spider_cls, settings=None):
                 spider_cls.start_urls = [s.url('/')]
                 yield runner.crawl(spider_cls)
 
-            with open(res_name, 'rb') as f:
-                defer.returnValue(f.read())
+            with open(res_path, 'rb') as f:
+                content = f.read()
 
         finally:
-            shutil.rmtree(tmpdir)
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     def exported_data(self, items, settings):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -7,7 +7,7 @@
 from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
-FOOBAR_NL = u"foo" + os.linesep + u"bar"
+FOOBAR_NL = u"foo\nbar"
 
 
 class XmliterTestCase(unittest.TestCase):
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -25,8 +25,12 @@ def inside_a_project():
 
 class ProjectUtilsTest(unittest.TestCase):
     def test_data_path_outside_project(self):
-        self.assertEqual('.scrapy/somepath', data_path('somepath'))
-        self.assertEqual('/absolute/path', data_path('/absolute/path'))
+        self.assertEqual(
+            os.path.join('.scrapy', 'somepath'),
+            data_path('somepath')
+        )
+        abspath = os.path.join(os.path.sep, 'absolute', 'path')
+        self.assertEqual(abspath, data_path(abspath))
 
     def test_data_path_inside_project(self):
         with inside_a_project() as proj_path:
@@ -35,4 +39,5 @@ def test_data_path_inside_project(self):
                 os.path.realpath(expected),
                 os.path.realpath(data_path('somepath'))
             )
-            self.assertEqual('/absolute/path', data_path('/absolute/path'))
+            abspath = os.path.join(os.path.sep, 'absolute', 'path')
+            self.assertEqual(abspath, data_path(abspath))