Repository: scrapy/scrapy

Issue:
Set environment variable when running scrapy check
Sometimes it is nice to be able to enable/disable functionality, e.g. calculating things in settings.py when just checking spider contracts instead of running a crawl. I therefor propose setting an environment variable like `SCRAPY_CHECK` when using the check command.

Discussion:

Patch:
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 2fbe30a00..aac0f4496 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -167,9 +167,8 @@ Documentation policies
 
 For reference documentation of API members (classes, methods, etc.) use
 docstrings and make sure that the Sphinx documentation uses the autodoc_
-extension to pull the docstrings. API reference documentation should follow
-docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point,
-and it may provide short examples.
+extension to pull the docstrings. API reference documentation should be
+IDE-friendly: short, to the point, and it may provide short examples.
 
 Other types of documentation, such as tutorials or topics, should be covered in
 files within the ``docs/`` directory. This includes documentation that is
@@ -206,29 +205,6 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
     ``tox -- tests/test_loader.py``
 
-To run the tests on a specific tox_ environment, use ``-e <name>`` with an
-environment name from ``tox.ini``. For example, to run the tests with Python
-3.6 use::
-
-    tox -e py36
-
-You can also specify a comma-separated list of environmets, and use `tox’s
-parallel mode`_ to run the tests on multiple environments in parallel::
-
-    tox -e py27,py36 -p auto
-
-To pass command-line options to pytest_, add them after ``--`` in your call to
-tox_. Using ``--`` overrides the default positional arguments defined in
-``tox.ini``, so you must include those default positional arguments
-(``scrapy tests``) after ``--`` as well::
-
-    tox -- scrapy tests -x  # stop after first failure
-
-You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 tox_ environment using all your CPU cores::
-
-    tox -e py36 -- scrapy tests -n auto
-
 To see coverage report install `coverage`_ (``pip install coverage``) and run:
 
     ``coverage report``
@@ -261,9 +237,5 @@ And their unit-tests are in::
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
-.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
 .. _pull request: https://help.github.com/send-pull-requests/
-.. _pytest: https://docs.pytest.org/en/latest/usage.html
-.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
 .. _tox: https://pypi.python.org/pypi/tox
-.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 70f20d4ed..9337375bb 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -120,3 +120,23 @@ get the failures pretty printed::
             for header in self.args:
                 if header not in response.headers:
                     raise ContractFail('X-CustomHeader not present')
+
+
+Detecting check runs
+====================
+
+When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
+set to the ``true`` string. You can use `os.environ`_ to perform any change to
+your spiders or your settings when ``scrapy check`` is used::
+
+    import os
+    import scrapy
+
+    class ExampleSpider(scrapy.Spider):
+        name = 'example'
+
+        def __init__(self):
+            if os.environ.get('SCRAPY_CHECK'):
+                pass  # Do some scraper adjustments when a check is running
+
+.. _os.environ: https://docs.python.org/3/library/os.html#os.environ
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 371f21c72..4a5439bfc 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -897,16 +897,6 @@ Default: ``False``
 If ``True``, the logs will just contain the root path. If it is set to ``False``
 then it displays the component responsible for the log output
 
-.. setting:: LOGSTATS_INTERVAL
-
-LOGSTATS_INTERVAL
------------------
-
-Default: ``60.0``
-
-The interval (in seconds) between each logging printout of the stats 
-by :class:`~extensions.logstats.LogStats`.
-
 .. setting:: MEMDEBUG_ENABLED
 
 MEMDEBUG_ENABLED
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index b8a9ef989..ab73e85e7 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResult
 
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, set_environ
 from scrapy.utils.conf import build_component_list
 
 
@@ -68,16 +68,17 @@ class Command(ScrapyCommand):
 
         spider_loader = self.crawler_process.spider_loader
 
-        for spidername in args or spider_loader.list():
-            spidercls = spider_loader.load(spidername)
-            spidercls.start_requests = lambda s: conman.from_spider(s, result)
-
-            tested_methods = conman.tested_methods_from_spidercls(spidercls)
-            if opts.list:
-                for method in tested_methods:
-                    contract_reqs[spidercls.name].append(method)
-            elif tested_methods:
-                self.crawler_process.crawl(spidercls)
+        with set_environ(SCRAPY_CHECK='true'):
+            for spidername in args or spider_loader.list():
+                spidercls = spider_loader.load(spidername)
+                spidercls.start_requests = lambda s: conman.from_spider(s, result)
+
+                tested_methods = conman.tested_methods_from_spidercls(spidercls)
+                if opts.list:
+                    for method in tested_methods:
+                        contract_reqs[spidercls.name].append(method)
+                elif tested_methods:
+                    self.crawler_process.crawl(spidercls)
 
         # start checks
         if opts.list:
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index c6094643d..1b5e05b1b 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -31,7 +31,7 @@ class DummyPolicy(object):
     def should_cache_response(self, response, request):
         return response.status not in self.ignore_http_codes
 
-    def is_cached_response_fresh(self, cachedresponse, request):
+    def is_cached_response_fresh(self, response, request):
         return True
 
     def is_cached_response_valid(self, cachedresponse, response, request):
@@ -70,7 +70,7 @@ class RFC2616Policy(object):
         return True
 
     def should_cache_response(self, response, request):
-        # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+        # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
         # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
         # Status code 206 is not included because cache can not deal with partial contents
         cc = self._parse_cachecontrol(response)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index ddaa7f7bf..f638adb25 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,6 +1,8 @@
 """Helper functions which don't fit anywhere else"""
+import os
 import re
 import hashlib
+from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
 
@@ -142,3 +144,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
         return objcls.from_settings(settings, *args, **kwargs)
     else:
         return objcls(*args, **kwargs)
+
+
+@contextmanager
+def set_environ(**kwargs):
+    """Temporarily set environment variables inside the context manager and
+    fully restore previous environment afterwards
+    """
+
+    original_env = {k: os.environ.get(k) for k in kwargs}
+    os.environ.update(kwargs)
+    try:
+        yield
+    finally:
+        for k, v in original_env.items():
+            if v is None:
+                del os.environ[k]
+            else:
+                os.environ[k] = v


Test patch:
diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt
index be809b151..790f29d34 100644
--- a/tests/requirements-py2.txt
+++ b/tests/requirements-py2.txt
@@ -2,10 +2,9 @@
 mock
 mitmproxy==0.10.1
 netlib==0.10.1
-pytest
-pytest-cov
+pytest==2.9.2
 pytest-twisted
-pytest-xdist
+pytest-cov==2.2.1
 jmespath
 brotlipy
 testfixtures
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index ed7bf0be0..7c1aacd81 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,6 @@
-pytest
-pytest-cov
+pytest==3.6.3
 pytest-twisted
-pytest-xdist
+pytest-cov==2.5.1
 testfixtures
 jmespath
 leveldb; sys_platform != "win32"
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index fcb7772ab..e109d5343 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -3,12 +3,13 @@ import os
 import unittest
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules
+from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
 
 from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 
+
 class UtilsMiscTestCase(unittest.TestCase):
 
     def test_load_object(self):
@@ -130,5 +131,18 @@ class UtilsMiscTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             create_instance(m, None, None)
 
+    def test_set_environ(self):
+        assert os.environ.get('some_test_environ') is None
+        with set_environ(some_test_environ='test_value'):
+            assert os.environ.get('some_test_environ') == 'test_value'
+        assert os.environ.get('some_test_environ') is None
+
+        os.environ['some_test_environ'] = 'test'
+        assert os.environ.get('some_test_environ') == 'test'
+        with set_environ(some_test_environ='test_value'):
+            assert os.environ.get('some_test_environ') == 'test_value'
+        assert os.environ.get('some_test_environ') == 'test'
+
+
 if __name__ == "__main__":
     unittest.main()