Repository: scrapy/scrapy
Pull request #5727 (2022-11-22 17:21:58)

Better language surrounding logging
Fixes outstanding comments within #5722 

1. Changed comments referencing "Current" memory usage to "Peak".
2. Changed comments referencing "M" to "MiB" where applicable.



Patch:
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index bf2ee4e6d..4fdf86479 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -75,16 +75,16 @@ class MemoryUsage:
         self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
 
     def _check_limit(self):
-        current_mem_usage = self.get_virtual_size()
-        if current_mem_usage > self.limit:
+        peak_mem_usage = self.get_virtual_size()
+        if peak_mem_usage > self.limit:
             self.crawler.stats.set_value('memusage/limit_reached', 1)
             mem = self.limit / 1024 / 1024
-            logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
+            logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
                          {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
                 subj = (
                     f"{self.crawler.settings['BOT_NAME']} terminated: "
-                    f"memory usage exceeded {mem}M at {socket.gethostname()}"
+                    f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
                 )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/limit_notified', 1)
@@ -94,7 +94,7 @@ class MemoryUsage:
             else:
                 self.crawler.stop()
         else:
-            logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024})
+            logger.info("Peak memory usage is %(virtualsize)dMiB", {'virtualsize': peak_mem_usage / 1024 / 1024})
 
     def _check_warning(self):
         if self.warned:  # warn only once
@@ -102,12 +102,12 @@ class MemoryUsage:
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value('memusage/warning_reached', 1)
             mem = self.warning / 1024 / 1024
-            logger.warning("Memory usage reached %(memusage)dM",
+            logger.warning("Memory usage reached %(memusage)dMiB",
                            {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
                 subj = (
                     f"{self.crawler.settings['BOT_NAME']} warning: "
-                    f"memory usage reached {mem}M at {socket.gethostname()}"
+                    f"memory usage reached {mem}MiB at {socket.gethostname()}"
                 )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/warning_notified', 1)
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index fce198db4..c66af497e 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -6,7 +6,7 @@ from os.path import join, dirname, abspath, isabs, exists
 
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
@@ -67,16 +67,23 @@ def get_project_settings():
     if settings_module_path:
         settings.setmodule(settings_module_path, priority='project')
 
+    scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
+                      k.startswith('SCRAPY_')}
     valid_envvars = {
         'CHECK',
         'PROJECT',
         'PYTHON_SHELL',
         'SETTINGS_MODULE',
     }
-
-    scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
-                      k.startswith('SCRAPY_') and k.replace('SCRAPY_', '') in valid_envvars}
-
+    setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars}
+    if setting_envvars:
+        setting_envvar_list = ', '.join(sorted(setting_envvars))
+        warnings.warn(
+            'Use of environment variables prefixed with SCRAPY_ to override '
+            'settings is deprecated. The following environment variables are '
+            f'currently defined: {setting_envvar_list}',
+            ScrapyDeprecationWarning
+        )
     settings.setdict(scrapy_envvars, priority='project')
 
     return settings


Test patch:
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 802f5c198..8233e0101 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -31,6 +31,10 @@ class CmdlineTest(unittest.TestCase):
         self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s',
                                        'TEST1=override'), 'override')
 
+    def test_override_settings_using_envvar(self):
+        self.env['SCRAPY_TEST1'] = 'override'
+        self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
+
     def test_profiling(self):
         path = tempfile.mkdtemp()
         filename = os.path.join(path, 'res.prof')
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index e77ffa18b..46452415a 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -5,6 +5,9 @@ import shutil
 import contextlib
 import warnings
 
+from pytest import warns
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.project import data_path, get_project_settings
 
 
@@ -77,10 +80,10 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         envvars = {
             'SCRAPY_FOO': 'bar',
         }
-        with set_env(**envvars):
-            settings = get_project_settings()
-
-        assert settings.get("SCRAPY_FOO") is None
+        with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+            with set_env(**envvars):
+                get_project_settings()
+        assert len(record) == 1
 
     def test_valid_and_invalid_envvars(self):
         value = 'tests.test_cmdline.settings'
@@ -88,7 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase):
             'SCRAPY_FOO': 'bar',
             'SCRAPY_SETTINGS_MODULE': value,
         }
-        with set_env(**envvars):
-            settings = get_project_settings()
+        with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+            with set_env(**envvars):
+                settings = get_project_settings()
+        assert len(record) == 1
         assert settings.get('SETTINGS_MODULE') == value
-        assert settings.get('SCRAPY_FOO') is None