Repository: scrapy/scrapy

Issue:
Usage of HttpCompressionMiddleware needs to be reflected in Scrapy stats
## Summary
Usage of `HttpCompressionMiddleware` needs to be relfected in Scrapy stats.
## Motivation
In order to estimate scrapy memory usage efficiency and prevent.. memory leaks like [this](https://stackoverflow.com/q/63936759/10884791).
I will need to know:
1. number of request/response objects that can be active (can be achieved by using [`trackref`](https://docs.scrapy.org/en/latest/topics/leaks.html#debugging-memory-leaks-with-trackref) )
2. size of memory required to store that number of request/response objects. 

A lot of websites use compression to reduce traffic. In this case I would like to calculate average size of **decompressed** responses to estimate p.2.

Decompression process means that at some point application will require to allocate memory to store both compressed and decompressed response body and I will need to know this sizes to have more complete vision of scrapy memory usage.

Also size of  decompressed body will be several times more than size of compressed response and it will affect scrapy memory usage.

## Describe alternatives you've considered
The easiest one - is to change priority of `DownloaderStats` middleware and check difference in `downloader/response_bytes` stats parameter.
```
    custom_settings = {"DOWNLOAD_DELAY":1,
                       "DOWNLOADER_MIDDLEWARES":{
                           'scrapy.downloadermiddlewares.stats.DownloaderStats':50
                       }
```
Stats from quotes.toscrape.com spider (it uses `gzip` compression) with default settings:

```
{'downloader/request_bytes': 2642,
 'downloader/request_count': 10,
 'downloader/request_method_count/GET': 10,
 'downloader/response_bytes': 24534,
```
 
And with changed priority of `DownloaderStats` middleware:

```
{'downloader/request_bytes': 912, # size reduced as it didn't count size of request headers populated by downloader middlewares
 'downloader/request_count': 10,
 'downloader/request_method_count/GET': 10,
 'downloader/response_bytes': 110191,   # it counted size of decompressed data 
```

Average size of compressed response (by default) - 2453 bytes.
Average size of decompressed response  - 11019 bytes (~4.5 times more).

## Additional context
Potential solution is to add something like this:
` self.stats.inc_value('decompressed_bytes', spider=spider)`
into `process_response` method of `HttpCompressionMiddleware`


Discussion:

Resolving patch:
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,10 +1,12 @@
 import io
+import warnings
 import zlib
 
-from scrapy.utils.gz import gunzip
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.exceptions import NotConfigured
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.gz import gunzip
 
 
 ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
@@ -25,11 +27,25 @@
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
+    def __init__(self, stats=None):
+        self.stats = stats
+
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
             raise NotConfigured
-        return cls()
+        try:
+            return cls(stats=crawler.stats)
+        except TypeError:
+            warnings.warn(
+                "HttpCompressionMiddleware subclasses must either modify "
+                "their '__init__' method to support a 'stats' parameter or "
+                "reimplement the 'from_crawler' method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls()
+            result.stats = crawler.stats
+            return result
 
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding',
@@ -44,6 +60,9 @@ def process_response(self, request, response, spider):
             if content_encoding:
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
+                if self.stats:
+                    self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+                    self.stats.inc_value('httpcompression/response_count', spider=spider)
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )


Test patch:
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,13 +1,16 @@
+from gzip import GzipFile
 from io import BytesIO
-from unittest import TestCase, SkipTest
 from os.path import join
-from gzip import GzipFile
+from unittest import TestCase, SkipTest
+from warnings import catch_warnings
 
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
 from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
+from scrapy.utils.test import get_crawler
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
@@ -32,8 +35,10 @@
 class HttpCompressionTest(TestCase):
 
     def setUp(self):
-        self.spider = Spider('foo')
-        self.mw = HttpCompressionMiddleware()
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('scrapytest.org')
+        self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
+        self.crawler.stats.open_spider(self.spider)
 
     def _getresponse(self, coding):
         if coding not in FORMAT:
@@ -56,6 +61,34 @@ def _getresponse(self, coding):
         response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
         return response
 
+    def assertStatsEqual(self, key, value):
+        self.assertEqual(
+            self.crawler.stats.get_value(key, spider=self.spider),
+            value,
+            str(self.crawler.stats.get_stats(self.spider))
+        )
+
+    def test_setting_false_compression_enabled(self):
+        self.assertRaises(
+            NotConfigured,
+            HttpCompressionMiddleware.from_crawler,
+            get_crawler(settings_dict={'COMPRESSION_ENABLED': False})
+        )
+
+    def test_setting_default_compression_enabled(self):
+        self.assertIsInstance(
+            HttpCompressionMiddleware.from_crawler(get_crawler()),
+            HttpCompressionMiddleware
+        )
+
+    def test_setting_true_compression_enabled(self):
+        self.assertIsInstance(
+            HttpCompressionMiddleware.from_crawler(
+                get_crawler(settings_dict={'COMPRESSION_ENABLED': True})
+            ),
+            HttpCompressionMiddleware
+        )
+
     def test_process_request(self):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
@@ -72,6 +105,20 @@ def test_process_response_gzip(self):
         assert newresponse is not response
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+
+    def test_process_response_gzip_no_stats(self):
+        mw = HttpCompressionMiddleware()
+        response = self._getresponse('gzip')
+        request = response.request
+
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        newresponse = mw.process_response(request, response, self.spider)
+        self.assertEqual(mw.stats, None)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b'<!DOCTYPE')
+        assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_br(self):
         try:
@@ -85,6 +132,8 @@ def test_process_response_br(self):
         assert newresponse is not response
         assert newresponse.body.startswith(b"<!DOCTYPE")
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_zstd(self):
         try:
@@ -116,6 +165,8 @@ def test_process_response_rawdeflate(self):
         assert newresponse is not response
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74840)
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse('zlibdeflate')
@@ -126,6 +177,8 @@ def test_process_response_zlibdelate(self):
         assert newresponse is not response
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74840)
 
     def test_process_response_plain(self):
         response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
@@ -135,6 +188,8 @@ def test_process_response_plain(self):
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is response
         assert newresponse.body.startswith(b'<!DOCTYPE')
+        self.assertStatsEqual('httpcompression/response_count', None)
+        self.assertStatsEqual('httpcompression/response_bytes', None)
 
     def test_multipleencodings(self):
         response = self._getresponse('gzip')
@@ -162,6 +217,8 @@ def test_process_response_encoding_inside_body(self):
         assert isinstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 104)
 
     def test_process_response_force_recalculate_encoding(self):
         headers = {
@@ -181,6 +238,8 @@ def test_process_response_force_recalculate_encoding(self):
         assert isinstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 104)
 
     def test_process_response_no_content_type_header(self):
         headers = {
@@ -196,6 +255,8 @@ def test_process_response_no_content_type_header(self):
         assert isinstance(newresponse, respcls)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 104)
 
     def test_process_response_gzipped_contenttype(self):
         response = self._getresponse('gzip')
@@ -206,6 +267,8 @@ def test_process_response_gzipped_contenttype(self):
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse('gzip')
@@ -216,6 +279,8 @@ def test_process_response_gzip_app_octetstream_contenttype(self):
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse('x-gzip')
@@ -226,6 +291,8 @@ def test_process_response_gzip_binary_octetstream_contenttype(self):
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -268,6 +335,8 @@ def test_process_response_gzipped_gzip_file(self):
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertEqual(gunzip(newresponse.body), plainbody)
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 230)
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
@@ -278,3 +347,32 @@ def test_process_response_head_request_no_decode_required(self):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
         self.assertEqual(response.body, b'')
+        self.assertStatsEqual('httpcompression/response_count', None)
+        self.assertStatsEqual('httpcompression/response_bytes', None)
+
+
+class HttpCompressionSubclassTest(TestCase):
+
+    def test_init_missing_stats(self):
+        class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
+
+            def __init__(self):
+                super().__init__()
+
+        crawler = get_crawler(Spider)
+        with catch_warnings(record=True) as caught_warnings:
+            HttpCompressionMiddlewareSubclass.from_crawler(crawler)
+        messages = tuple(
+            str(warning.message) for warning in caught_warnings
+            if warning.category is ScrapyDeprecationWarning
+        )
+        self.assertEqual(
+            messages,
+            (
+                (
+                    "HttpCompressionMiddleware subclasses must either modify "
+                    "their '__init__' method to support a 'stats' parameter "
+                    "or reimplement the 'from_crawler' method."
+                ),
+            )
+        )