Repository: pypa/bandersnatch (python)

Issue:
Refactor verify.py to use less memory
Refactor verify.py to iterate through the file system JSON files better and use less memory.
- Potentially use a asyncio.Queue and have X (workers) consumers
-- Store only filepath in the queue and not coros

At the moment loading all the coros into a list and uses asyncio.gather is using ~1.3gb of RAM. This is not cool nor scalable.

Pull request description:
bugfix/46/verify-less-memory
Hello! Please, review my attempt to fix #46

Patch:
diff --git a/.gitignore b/.gitignore
index 43c098c..4aea12d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
+test.conf
 
 # Translations
 *.mo
diff --git a/src/bandersnatch/default.conf b/src/bandersnatch/default.conf
index 22bec3d..804ea7b 100644
--- a/src/bandersnatch/default.conf
+++ b/src/bandersnatch/default.conf
@@ -52,6 +52,9 @@ stop-on-error = false
 ; which requires serving packages from https://files.pythonhosted.org
 ; root_uri = https://example.com
 
+; Number of consumers which verify metadata
+verifiers = 3
+
 ; blacklist
 ; List of PyPI packages not to sync - Useful if malicious packages are mirrored
 [blacklist]
diff --git a/src/bandersnatch/verify.py b/src/bandersnatch/verify.py
index 4129570..472b473 100644
--- a/src/bandersnatch/verify.py
+++ b/src/bandersnatch/verify.py
@@ -4,13 +4,14 @@ import hashlib
 import json
 import logging
 import os
+from asyncio.queues import Queue
 from functools import partial
 from pathlib import Path
 from sys import stderr
 from urllib.parse import urlparse
-
 import aiohttp
 
+from bandersnatch.configuration import BandersnatchConfig
 from bandersnatch.utils import user_agent
 
 ASYNC_USER_AGENT = user_agent(f"aiohttp {aiohttp.__version__}")
@@ -137,15 +138,22 @@ async def url_fetch(url, file_path, executor, chunk_size=65536, timeout=60):
 async def async_verify(
     config, all_package_files, mirror_base, json_files, args, executor
 ) -> None:
-    coros = []
-    logger.debug("Loading JSON files to verify")
-    for json_file in json_files:
-        coros.append(
-            verify(config, json_file, mirror_base, all_package_files, args, executor)
-        )
-
-    logger.debug("Gathering all the verify threads")
-    await asyncio.gather(*coros)
+    queue = asyncio.Queue()  # type: Queue
+    for jf in json_files:
+        queue.put_nowait(jf)
+
+    async def consume(q: Queue):
+        while not q.empty():
+            json_file = q.get_nowait()
+            await verify(
+                config, json_file, mirror_base, all_package_files, args, executor
+            )
+
+    config = BandersnatchConfig().config
+    verifiers = config.getint("mirror", "verifiers", fallback=3)
+    consumers = [consume(queue)] * verifiers
+
+    await asyncio.gather(*consumers)
 
 
 async def metadata_verify(config, args):


Test patch:
diff --git a/src/bandersnatch/tests/test_configuration.py b/src/bandersnatch/tests/test_configuration.py
index 5d7d187..b226a45 100644
--- a/src/bandersnatch/tests/test_configuration.py
+++ b/src/bandersnatch/tests/test_configuration.py
@@ -1,4 +1,5 @@
 import os
+import unittest
 from tempfile import TemporaryDirectory
 from unittest import TestCase
 
@@ -48,6 +49,7 @@ class TestBandersnatchConf(TestCase):
                 "master",
                 "stop-on-error",
                 "timeout",
+                "verifiers",
                 "workers",
             ],
         )
@@ -104,3 +106,7 @@ class TestBandersnatchConf(TestCase):
 
         instance2 = BandersnatchConfig()
         self.assertEqual(instance2.config["mirror"]["master"], "https://foo.bar.baz")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test-requirements.txt b/test-requirements.txt
index ccedda9..aba760c 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -4,3 +4,4 @@ pytest-timeout
 pytest-cache
 setuptools
 tox
+black