Repository: scrapy/scrapy

Issue:
Python 3.7 support
The goal is to add python 3.7 to travis and pass all tests, the first beta was already released at the end of January.


Discussion:
One issue was reported at https://stackoverflow.com/questions/48861287/why-am-i-getting-this-error-in-scrapy-python3-7-invalid-syntax

```$ scrapy shell http://quotes.toscrape.com/random
2018-02-26 19:11:02 [scrapy.utils.log] INFO: Scrapy 1.5.0 started (bot: scrapybot)
2018-02-26 19:11:02 [scrapy.utils.log] INFO: Versions: lxml 4.1.1.0, libxml2 2.9.3, cssselect 1.0.3, parsel 1.4.0, w3lib 1.19.0, Twisted 17.9.0, Python 3.7.0b1 (default, Feb 26 2018, 19:04:22) - [GCC 5.4.0 20160609], pyOpenSSL 17.5.0 (OpenSSL 1.0.2g  1 Mar 2016), cryptography 2.1.4, Platform Linux-4.4.0-116-generic-x86_64-with-debian-stretch-sid
2018-02-26 19:11:02 [scrapy.crawler] INFO: Overridden settings: {'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', 'EDITOR': 'vim', 'LOGSTATS_INTERVAL': 0, 'TELNETCONSOLE_ENABLED': '0'}
Traceback (most recent call last):
  File "/home/kostia/tmp/py37/bin/scrapy", line 11, in <module>
    sys.exit(execute())
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/cmdline.py", line 150, in execute
    _run_print_help(parser, _run_command, cmd, args, opts)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/cmdline.py", line 90, in _run_print_help
    func(*a, **kw)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/cmdline.py", line 157, in _run_command
    cmd.run(args, opts)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/commands/shell.py", line 65, in run
    crawler = self.crawler_process._create_crawler(spidercls)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/crawler.py", line 203, in _create_crawler
    return Crawler(spidercls, self.settings)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/crawler.py", line 55, in __init__
    self.extensions = ExtensionManager.from_crawler(self)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/middleware.py", line 58, in from_crawler
    return cls.from_settings(crawler.settings, crawler)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/middleware.py", line 34, in from_settings
    mwcls = load_object(clspath)
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/utils/misc.py", line 44, in load_object
    mod = import_module(module)
  File "/home/kostia/.pyenv/versions/3.7.0b1/lib/python3.7/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 994, in _gcd_import
  File "<frozen importlib._bootstrap>", line 971, in _find_and_load
  File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 723, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/scrapy/extensions/telnet.py", line 12, in <module>
    from twisted.conch import manhole, telnet
  File "/home/kostia/tmp/py37/lib/python3.7/site-packages/twisted/conch/manhole.py", line 154
    def write(self, data, async=False):
                              ^
SyntaxError: invalid syntax
```

This also happens if  ``-s TELNETCONSOLE_ENABLED=0`` is passed, because the module is still imported. 

Twisted issue: https://twistedmatrix.com/trac/ticket/9384#ticket and PR https://github.com/twisted/twisted/pull/966/
Is this an issue that needs someone to work on? I would love to give it a shot. :-) Looks like the PR to fix the above issue should be merged soon. 
@patiences yes, I'm not aware of anyone working on it, so it would be awesome if you give it a shot! And feel free to submit a work in progress PR. I think first step would be adding python 3.7 to tox and travis. There might be other issues besides this syntax error in twisted.

Resolving patch:
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -6,13 +6,15 @@
 
 import pprint
 import logging
+import traceback
 
 from twisted.internet import protocol
 try:
     from twisted.conch import manhole, telnet
     from twisted.conch.insults import insults
     TWISTED_CONCH_AVAILABLE = True
-except ImportError:
+except (ImportError, SyntaxError):
+    _TWISTED_CONCH_TRACEBACK = traceback.format_exc()
     TWISTED_CONCH_AVAILABLE = False
 
 from scrapy.exceptions import NotConfigured
@@ -40,7 +42,9 @@ def __init__(self, crawler):
         if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
             raise NotConfigured
         if not TWISTED_CONCH_AVAILABLE:
-            raise NotConfigured
+            raise NotConfigured(
+                'TELNETCONSOLE_ENABLED setting is True but required twisted '
+                'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK)
         self.crawler = crawler
         self.noisy = False
         self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     """
 
     encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
-    def _getrow(csv_r):
-        return [to_unicode(field, encoding) for field in next(csv_r)]
+
+    def row_to_unicode(row_):
+        return [to_unicode(field, encoding) for field in row_]
 
     # Python 3 csv reader input object needs to return strings
     if six.PY3:
@@ -113,10 +114,14 @@ def _getrow(csv_r):
     csv_r = csv.reader(lines, **kwargs)
 
     if not headers:
-        headers = _getrow(csv_r)
-
-    while True:
-        row = _getrow(csv_r)
+        try:
+            row = next(csv_r)
+        except StopIteration:
+            return
+        headers = row_to_unicode(row)
+
+    for row in csv_r:
+        row = row_to_unicode(row)
         if len(row) != len(headers):
             logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, "
                            "should be: %(csvheader)d)",


Test patch:
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,6 @@
-pytest==2.9.2
+pytest==3.6.3
 pytest-twisted
-pytest-cov==2.2.1
+pytest-cov==2.5.1
 testfixtures
 jmespath
 leveldb
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,5 +1,4 @@
 import logging
-import os
 import tempfile
 import warnings
 import unittest
@@ -14,8 +13,9 @@
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
-from scrapy.utils.test import get_crawler
 from scrapy.extensions.throttle import AutoThrottle
+from scrapy.extensions import telnet
+
 
 class BaseCrawlerTest(unittest.TestCase):
 
@@ -100,6 +100,8 @@ class MySpider(scrapy.Spider):
                 custom_settings = {
                     'LOG_LEVEL': 'INFO',
                     'LOG_FILE': log_file.name,
+                    # disable telnet if not available to avoid an extra warning
+                    'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
                 }
 
             configure_logging()
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -10,6 +10,7 @@
 from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter,
                               LogCounterHandler, StreamLogger)
 from scrapy.utils.test import get_crawler
+from scrapy.extensions import telnet
 
 
 class FailureToExcInfoTest(unittest.TestCase):
@@ -65,10 +66,14 @@ def test_different_name_logger(self):
 class LogCounterHandlerTest(unittest.TestCase):
 
     def setUp(self):
+        settings = {'LOG_LEVEL': 'WARNING'}
+        if not telnet.TWISTED_CONCH_AVAILABLE:
+            # disable it to avoid the extra warning
+            settings['TELNETCONSOLE_ENABLED'] = False
         self.logger = logging.getLogger('test')
         self.logger.setLevel(logging.NOTSET)
         self.logger.propagate = False
-        self.crawler = get_crawler(settings_dict={'LOG_LEVEL': 'WARNING'})
+        self.crawler = get_crawler(settings_dict=settings)
         self.handler = LogCounterHandler(self.crawler)
         self.logger.addHandler(self.handler)