Repository: AustinScola/pytest-motor (language: python)

Issue:
Use `aiohttp` instead of `requests`
Use `aiohttp` to fetch the mongod binary instead of `requests` because `requests` is not async.
Dangling `mongod`s
After running tests using the plugin, `mongod`s seem to accumulate as can be seen with `ps -C "mongod` or `ps -C "mongod | wc -l` to see how many such dangling `mongod`s there are.

Right now I have just been periodically killing them with `pkill "mongod"` as a temporary workaround but this really needs to be fixed.


Discussion / hints:

Fix patch:
diff --git a/mypy.ini b/mypy.ini
index 7c83d9d..db528a6 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -17,3 +17,6 @@ ignore_missing_imports = True
 
 [mypy-motor.*]
 ignore_missing_imports = True
+
+[mypy-pymongo.*]
+ignore_missing_imports = True
diff --git a/requirements/frozen/frozen_coverage_requirements.txt b/requirements/frozen/frozen_coverage_requirements.txt
index 43b8954..81fefc2 100644
--- a/requirements/frozen/frozen_coverage_requirements.txt
+++ b/requirements/frozen/frozen_coverage_requirements.txt
@@ -1,16 +1,20 @@
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
 attrs==21.2.0
+chardet==4.0.0
 coverage==5.5
-importlib-metadata==4.6.1
+idna==3.2
 iniconfig==1.1.1
 motor==2.4.0
+multidict==5.1.0
 packaging==21.0
 pluggy==0.13.1
 py==1.10.0
-pymongo==3.11.4
+pymongo==3.12.0
 pyparsing==2.4.7
 pytest==6.2.4
 pytest-asyncio==0.15.1
 pytest-cov==2.12.1
 toml==0.10.2
 typing-extensions==3.10.0.0
-zipp==3.5.0
+yarl==1.6.3
diff --git a/requirements/frozen/frozen_deployment_requirements.txt b/requirements/frozen/frozen_deployment_requirements.txt
index 5afe294..9dd9d39 100644
--- a/requirements/frozen/frozen_deployment_requirements.txt
+++ b/requirements/frozen/frozen_deployment_requirements.txt
@@ -1,11 +1,11 @@
-bleach==3.3.0
+bleach==3.3.1
 certifi==2021.5.30
 cffi==1.14.6
-chardet==4.0.0
+charset-normalizer==2.0.2
 colorama==0.4.4
 cryptography==3.4.7
 docutils==0.17.1
-idna==2.10
+idna==3.2
 importlib-metadata==4.6.1
 jeepney==0.7.0
 keyring==23.0.1
@@ -15,14 +15,13 @@ pycparser==2.20
 Pygments==2.9.0
 pyparsing==2.4.7
 readme-renderer==29.0
-requests==2.25.1
+requests==2.26.0
 requests-toolbelt==0.9.1
 rfc3986==1.5.0
 SecretStorage==3.3.1
 six==1.16.0
 tqdm==4.61.2
 twine==3.4.1
-typing-extensions==3.10.0.0
 urllib3==1.26.6
 webencodings==0.5.1
 zipp==3.5.0
diff --git a/requirements/frozen/frozen_lint_requirements.txt b/requirements/frozen/frozen_lint_requirements.txt
index 861aea9..4b630a8 100644
--- a/requirements/frozen/frozen_lint_requirements.txt
+++ b/requirements/frozen/frozen_lint_requirements.txt
@@ -1,21 +1,24 @@
+aiohttp==3.7.4.post0
 astroid==2.5.6
+async-timeout==3.0.1
 attrs==21.2.0
-importlib-metadata==4.6.1
+chardet==4.0.0
+idna==3.2
 iniconfig==1.1.1
 isort==5.9.2
 lazy-object-proxy==1.6.0
 mccabe==0.6.1
 motor==2.4.0
+multidict==5.1.0
 packaging==21.0
 pluggy==0.13.1
 py==1.10.0
 pylint==2.8.3
-pymongo==3.11.4
+pymongo==3.12.0
 pyparsing==2.4.7
 pytest==6.2.4
 pytest-asyncio==0.15.1
 toml==0.10.2
-typed-ast==1.4.3
 typing-extensions==3.10.0.0
 wrapt==1.12.1
-zipp==3.5.0
+yarl==1.6.3
diff --git a/requirements/frozen/frozen_type_check_requirements.txt b/requirements/frozen/frozen_type_check_requirements.txt
index 92a2474..9adba7d 100644
--- a/requirements/frozen/frozen_type_check_requirements.txt
+++ b/requirements/frozen/frozen_type_check_requirements.txt
@@ -1,17 +1,20 @@
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
 attrs==21.2.0
-importlib-metadata==4.6.1
+chardet==4.0.0
+idna==3.2
 iniconfig==1.1.1
 motor==2.4.0
+multidict==5.1.0
 mypy==0.902
 mypy-extensions==0.4.3
 packaging==21.0
 pluggy==0.13.1
 py==1.10.0
-pymongo==3.11.4
+pymongo==3.12.0
 pyparsing==2.4.7
 pytest==6.2.4
 pytest-asyncio==0.15.1
 toml==0.10.2
-typed-ast==1.4.3
 typing-extensions==3.10.0.0
-zipp==3.5.0
+yarl==1.6.3
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 7c5ed18..9ad97b3 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -1,3 +1,4 @@
 motor==2.4.0
 pytest==6.2.4
 pytest-asyncio==0.15.1 # Needed for async pytest fixtures
+aiohttp==3.7.4.post0
diff --git a/setup.py b/setup.py
index 69923d8..da860a6 100644
--- a/setup.py
+++ b/setup.py
@@ -36,5 +36,5 @@
         'Typing :: Typed',
     ],
     python_requires='>=3.6',
-    install_requires=['pytest', 'motor'],
+    install_requires=['pytest', 'motor', 'aiohttp'],
 )


Test patch:
diff --git a/pytest_motor/plugin.py b/pytest_motor/plugin.py
index 26e3aa2..1c59120 100644
--- a/pytest_motor/plugin.py
+++ b/pytest_motor/plugin.py
@@ -2,12 +2,13 @@
 import asyncio
 import secrets
 import shutil
+import socket
 import tarfile
-import urllib
-import urllib.request
+import tempfile
 from pathlib import Path
 from typing import AsyncIterator, Iterator, List
 
+import aiohttp
 import pytest
 from _pytest.config import Config as PytestConfig
 from motor.motor_asyncio import AsyncIOMotorClient
@@ -51,45 +52,38 @@ async def root_directory(pytestconfig: PytestConfig) -> Path:
 async def mongod_binary(root_directory: Path) -> Path:  # pylint: disable=redefined-outer-name
     """Return a path to a mongod binary."""
     destination: Path = root_directory.joinpath('.mongod')
-    mongod_binary_filename = destination.joinpath(
-        'mongodb-linux-x86_64-ubuntu1804-4.4.6/bin/mongod')
+    mongod_binary_relative_path = 'mongodb-linux-x86_64-ubuntu1804-4.4.6/bin/mongod'
+    mongod_binary_path = destination.joinpath(mongod_binary_relative_path)
 
-    if not mongod_binary_filename.exists():
+    if not mongod_binary_path.exists():
         download_url = 'https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.4.6.tgz'
-        (tar_filename, _) = urllib.request.urlretrieve(download_url)
-
-        with tarfile.open(tar_filename) as tar:
-            tar.extract(member='mongodb-linux-x86_64-ubuntu1804-4.4.6/bin/mongod', path=destination)
-
-        mongod_binary_filename = destination.joinpath(
-            'mongodb-linux-x86_64-ubuntu1804-4.4.6/bin/mongod')
-
-    return mongod_binary_filename
-
-
-@pytest.fixture(scope='function')
-# pylint: disable=redefined-outer-name
-async def sockets_directory(root_directory: Path) -> AsyncIterator[Path]:
-    """Yield a directory for MongodDB unix sockets."""
-    sockets_directory = root_directory.joinpath('.mongod_sockets')
-    sockets_directory.mkdir(exist_ok=True)
-
-    yield sockets_directory
+        async with aiohttp.ClientSession() as session:
+            async with session.get(download_url) as resp:
+                with tempfile.TemporaryFile(mode='w+b') as binary_file:
+                    # Read by chunks to avoid big RAM consumption
+                    while True:
+                        # read by 100 bytes
+                        chunk = await resp.content.read(100)
+                        if not chunk:
+                            break
+                        binary_file.write(chunk)
+                    binary_file.flush()
+                    binary_file.seek(0)
+                    # Extract tar
+                    with tarfile.open(fileobj=binary_file) as tar:
+                        tar.extract(member=mongod_binary_relative_path, path=destination)
+
+    return mongod_binary_path
 
 
 @pytest.fixture(scope='function')
-# pylint: disable=redefined-outer-name
-async def unix_socket(sockets_directory: Path) -> AsyncIterator[Path]:
-    """Yield a random unix socket in the given sockets directory."""
-    name: str = secrets.token_hex(12)
-    unix_socket = sockets_directory.joinpath(f'{name}.sock')
-
-    yield unix_socket
-
-    try:
-        unix_socket.unlink()
-    except FileNotFoundError:  # pragma: no cover
-        pass
+def new_port() -> int:
+    """Return an unused port for mongod to run on."""
+    port: int = 27017
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as opened_socket:
+        opened_socket.bind(("127.0.0.1", 0))  # system will automaticly assign port
+        port = opened_socket.getsockname()[1]
+    return port
 
 
 @pytest.fixture(scope='function')
@@ -117,31 +111,35 @@ async def database_path(databases_directory: Path) -> AsyncIterator[Path]:
 
 @pytest.fixture(scope='function')
 # pylint: disable=redefined-outer-name
-async def mongod_socket(unix_socket: Path, database_path: Path,
-                        mongod_binary: Path) -> AsyncIterator[Path]:
+async def mongod_socket(new_port: int, database_path: Path,
+                        mongod_binary: Path) -> AsyncIterator[str]:
     """Yield a mongod."""
+    # yapf: disable
     arguments: List[str] = [
-        str(mongod_binary), '--bind_ip',
-        str(unix_socket), '--storageEngine', 'ephemeralForTest', '--fork', '--logpath', '/dev/null',
-        '--dbpath',
-        str(database_path)
+        str(mongod_binary),
+        '--port', str(new_port),
+        '--storageEngine', 'ephemeralForTest',
+        '--logpath', '/dev/null',
+        '--dbpath', str(database_path)
     ]
+    # yapf: enable
 
     mongod = await asyncio.create_subprocess_exec(*arguments)
 
-    yield unix_socket
+    # mongodb binds to localhost by default
+    yield f'localhost:{new_port}'
 
     try:
         mongod.terminate()
-    except ProcessLookupError:
+    except ProcessLookupError:  # pragma: no cover
         pass
 
 
 @pytest.fixture(scope='function')
 # pylint: disable=redefined-outer-name
-async def motor_client(mongod_socket: Path) -> AsyncIterator[AsyncIOMotorClient]:
+async def motor_client(mongod_socket: str) -> AsyncIterator[AsyncIOMotorClient]:
     """Yield a Motor client."""
-    connection_string = 'mongodb://' + urllib.parse.quote(str(mongod_socket), safe='')
+    connection_string = f'mongodb://{mongod_socket}'
 
     motor_client_: AsyncIOMotorClient = \
         AsyncIOMotorClient(connection_string, serverSelectionTimeoutMS=3000)
diff --git a/requirements/frozen/frozen_test_requirements.txt b/requirements/frozen/frozen_test_requirements.txt
index 1867705..36f1e53 100644
--- a/requirements/frozen/frozen_test_requirements.txt
+++ b/requirements/frozen/frozen_test_requirements.txt
@@ -1,14 +1,18 @@
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
 attrs==21.2.0
-importlib-metadata==4.6.1
+chardet==4.0.0
+idna==3.2
 iniconfig==1.1.1
 motor==2.4.0
+multidict==5.1.0
 packaging==21.0
 pluggy==0.13.1
 py==1.10.0
-pymongo==3.11.4
+pymongo==3.12.0
 pyparsing==2.4.7
 pytest==6.2.4
 pytest-asyncio==0.15.1
 toml==0.10.2
 typing-extensions==3.10.0.0
-zipp==3.5.0
+yarl==1.6.3
diff --git a/test_data/files/binary_downloader_tests.py b/test_data/files/binary_downloader_tests.py
new file mode 100644
index 0000000..667c3e7
--- /dev/null
+++ b/test_data/files/binary_downloader_tests.py
@@ -0,0 +1,7 @@
+"""A test file with a multiple tests for mongodb binary downloader."""
+from pathlib import Path
+
+
+def test_binary_downloads(mongod_binary: Path) -> None:
+    """Tests if mongod_binary exists"""
+    assert mongod_binary.exists()
diff --git a/test_data/files/multiple_tests.py b/test_data/files/independence_tests.py
similarity index 100%
rename from test_data/files/multiple_tests.py
rename to test_data/files/independence_tests.py
diff --git a/test_data/files/port_tests.py b/test_data/files/port_tests.py
new file mode 100644
index 0000000..c69e36b
--- /dev/null
+++ b/test_data/files/port_tests.py
@@ -0,0 +1,30 @@
+"""A test file with a multiple tests for TCP/IP socket."""
+import asyncio
+import socket
+from pathlib import Path
+
+import pytest
+from pytest_motor.plugin import new_port
+# pylint: disable=redefined-outer-name
+
+
+def test_port_in_range(new_port: int) -> None:
+    """Test that port is user available"""
+    assert 1024 <= new_port <= 65535
+
+
+second_port = new_port
+
+
+def test_ports_not_equals(new_port: int, second_port: int) -> None:
+    """Test that fixture doesn't return same ports"""
+    assert new_port != second_port
+
+
+@pytest.mark.asyncio
+async def test_port_not_allocated(mongod_binary: Path, new_port: int) -> None:
+    """Tests that mongod leaves port free"""
+    mongod = await asyncio.create_subprocess_exec(*[str(mongod_binary), '--port', str(new_port)])
+    mongod.terminate()
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as opened_socket:
+        assert opened_socket.connect_ex(('127.0.0.1', new_port)) != 0
diff --git a/test_data/files/server_info_test.py b/test_data/files/server_info_test.py
index 5d74166..9541480 100644
--- a/test_data/files/server_info_test.py
+++ b/test_data/files/server_info_test.py
@@ -1,9 +1,13 @@
 """A test file with a single test that retrieves the Motor client server info."""
 import pytest
+from pymongo.errors import ServerSelectionTimeoutError
 from motor.motor_asyncio import AsyncIOMotorClient
 
 
 @pytest.mark.asyncio
 async def test_using_motor_client_server_info(motor_client: AsyncIOMotorClient) -> None:
     """Test retrieving the Motor client server info."""
-    await motor_client.server_info()
+    try:
+        await motor_client.server_info()
+    except ServerSelectionTimeoutError as exc:
+        assert False, f'MongoDB connection unsuccessful: {exc}'
diff --git a/tests/conftest.py b/tests/conftest.py
index bf4c71f..cd9d915 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,3 +1,3 @@
 """Pytest configuration."""
 
-pytest_plugins = 'pytester'  # pylint: disable=invalid-name
+pytest_plugins = ['pytester', 'pytest_asyncio', 'pytest_motor.plugin']
diff --git a/tests/test_plugin.py b/tests/test_plugin.py
index 4d37baa..d49648f 100644
--- a/tests/test_plugin.py
+++ b/tests/test_plugin.py
@@ -8,6 +8,24 @@
 
 from pytest_motor.plugin import _event_loop
 
+# pylint: disable=redefined-outer-name
+
+test_files_directory = Path(__file__).parent.parent / 'test_data' / 'files'
+# yapf: disable
+test_files = {
+    path_to_file.name: path_to_file
+    for path_to_file in test_files_directory.glob('*.py')
+    if path_to_file.name != '__init__.py'
+}
+# yapf: enable
+
+
+@pytest.fixture
+def read_conftest() -> str:
+    """Reads conftest.py"""
+    conftest_file = Path(__file__).parent / 'conftest.py'
+    return conftest_file.read_text()
+
 
 def test_event_loop() -> None:
     """Test pytest_motor.plugin._event_loop."""
@@ -26,19 +44,33 @@ def test_event_loop() -> None:
     mock_close.assert_called_once()
 
 
-def test_motor_client(testdir: Testdir) -> None:
-    """Test pytest_motor.plugin.motor_client."""
-    testdir.makeconftest("""
-    pytest_plugins=["pytest_asyncio", "pytest_motor.plugin"]
-    """)
+def test_new_port(testdir: Testdir, read_conftest: str) -> None:
+    """Test pytest_motor.plugin.new_port."""
+    assert 'port_tests.py' in test_files.keys()
+    testdir.makeconftest(read_conftest)
+    testdir.makepyfile(test_files['port_tests.py'].read_text())
+    testdir.runpytest().assert_outcomes(passed=3)
 
-    test_files_directory = Path(__file__).parent.parent / 'test_data' / 'files'
-    not_init_file = lambda path: path.name != '__init__.py'
-    test_files = filter(not_init_file, test_files_directory.glob('*.py'))
 
-    for test_file in test_files:
-        testdir.makepyfile(test_file.read_text())
+def test_mongod_binary_downloader(testdir: Testdir, read_conftest: str) -> None:
+    """Test pytest_motor.plugin.mongod_binary."""
+    assert 'binary_downloader_tests.py' in test_files.keys()
+    testdir.makeconftest(read_conftest)
+    testdir.makepyfile(test_files['binary_downloader_tests.py'].read_text())
+    testdir.runpytest().assert_outcomes(passed=1)
 
-    result = testdir.runpytest()
 
-    result.assert_outcomes(passed=2)
+def test_integration_motor_client(testdir: Testdir, read_conftest: str) -> None:
+    """Test pytest_motor.plugin.motor_client."""
+    assert 'server_info_test.py' in test_files.keys()
+    testdir.makeconftest(read_conftest)
+    testdir.makepyfile(test_files['server_info_test.py'].read_text())
+    testdir.runpytest().assert_outcomes(passed=1)
+
+
+def test_integration_clients_independence(testdir: Testdir, read_conftest: str) -> None:
+    """Test pytest_motor.plugin.motor_client."""
+    assert 'independence_tests.py' in test_files.keys()
+    testdir.makeconftest(read_conftest)
+    testdir.makepyfile(test_files['independence_tests.py'].read_text())
+    testdir.runpytest().assert_outcomes(passed=2)