Repository: certbot/certbot

Issue:
Replace psutils dependency
It's compiled from C.  It's causing problems (#1080 / #3101).  If there's something equally effective that we can do in pure Python, let's do it.



Discussion:
Part of #1301.

I don't know how much Linux-specificity bullet we're willing to bite here (the relevant functionality might not even be available in psutil itself on some other systems?) but I found that you can look in `/proc/net/fd` for a line that contains

`:3039 00000000:0000`

where :3039 is the hexadecimal value of the local port you're interested in, and then read the `inode` field from that line, and then look at the link target of every single file in `/proc/*/fd` to see if it is a link to that numbered inode. This is apparently how `lsof` does it (from `strace lsof`) and maybe also how psutil is doing it.

Here's a quick-and-dirty and not very robust proof-of-concept version. This would be completely Linux-specific.

```
#!/usr/bin/env python

import glob
import os
import re

def find_culprit(port):
    hex_port_search = ":%04X 00000000:0000 0A 00000000:00000000 00:00000000 00000000 + \d+ +\d+ +(\d+)" % port

    for line in open("/proc/net/tcp", "r"):
        x = re.search(hex_port_search, line)
        if x:
            break
    else:
        return None
    link_target = "socket:[%s]" % x.groups()[0]
    for pid in glob.glob("/proc/[0-9]*"):
        try:
            for fd in os.listdir(os.path.join(pid, "fd")):
                target = os.readlink(os.path.join(pid, "fd", fd))
                if target == link_target:
                    return open(os.path.join(pid, "stat")).read().split()[:2]
        except OSError: 
            continue
```

Yes, that's how it's done, more specifically here:

https://github.com/giampaolo/psutil/blob/master/psutil/_pslinux.py#L516

psutil sports BSD license, so it could be feasible to vendor the above linux code, and for other OSes just find out if port is in use (by for example trying to open it), and print error message accordingly.

However the effort needed to keep the functionality for all or even most OSes by vendoring and maintaining the code in the future seems to be a bit more than bargained for, so I'd like to open this issue up for discussion once more.

In my mind it boils down to either of:
1) Keep psutil dependency
2) Drop psutil dependency and implement simple socket test to be able to tell the user that the needed port(s) is/are in use, and perhaps guide them forward.

I'm personally in favor of option 2, however, I believe @schoen and @pde feel that this is a significant loss in UX.

Conclusion was: downgrade the dependency to a `Recommends:` in package managers that support it; try the import, use the data if `psutils` is available and be less specific otherwise.

(On some systems we could possibly shell out to netstat, too....)

`netstat -ntlp` on Linux, though it might not be compatible with other systems.


Resolving patch:
diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py
--- a/certbot/plugins/util.py
+++ b/certbot/plugins/util.py
@@ -3,15 +3,27 @@
 import os
 import socket
 
-import psutil
 import zope.component
 
 from certbot import interfaces
 from certbot import util
 
+try:
+    import psutil
+    USE_PSUTIL = True
+except ImportError:
+    USE_PSUTIL = False
 
 logger = logging.getLogger(__name__)
 
+RENEWER_EXTRA_MSG = (
+    " For automated renewal, you may want to use a script that stops"
+    " and starts your webserver. You can find an example at"
+    " https://certbot.eff.org/docs/using.html#renewal ."
+    " Alternatively you can use the webroot plugin to renew without"
+    " needing to stop and start your webserver.")
+
+
 def path_surgery(restart_cmd):
     """Attempt to perform PATH surgery to find restart_cmd
 
@@ -38,9 +50,11 @@ def path_surgery(restart_cmd):
         return True
     else:
         expanded = " expanded" if any(added) else ""
-        logger.warning("Failed to find %s in%s PATH: %s", restart_cmd, expanded, path)
+        logger.warning("Failed to find %s in%s PATH: %s", restart_cmd,
+                       expanded, path)
         return False
 
+
 def already_listening(port, renewer=False):
     """Check if a process is already listening on the port.
 
@@ -53,6 +67,50 @@ def already_listening(port, renewer=False):
     :param int port: The TCP port in question.
     :returns: True or False.
 
+    """
+
+    if USE_PSUTIL:
+        return already_listening_psutil(port, renewer=renewer)
+    else:
+        logger.debug("Psutil not found, using simple socket check.")
+        return already_listening_socket(port, renewer=renewer)
+
+
+def already_listening_socket(port, renewer=False):
+    """Simple socket based check to find out if port is already in use
+
+    :param int port: The TCP port in question.
+    :returns: True or False
+    """
+
+    try:
+        testsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+        try:
+            testsocket.bind(("", port))
+        except socket.error:
+            display = zope.component.getUtility(interfaces.IDisplay)
+            extra = ""
+            if renewer:
+                extra = RENEWER_EXTRA_MSG
+            display.notification(
+                "Port {0} is already in use by another process. This will "
+                "prevent us from binding to that port. Please stop the "
+                "process that is populating the port in question and try "
+                "again. {1}".format(port, extra), height=13)
+            return True
+        finally:
+            testsocket.close()
+    except socket.error:
+        pass
+    return False
+
+
+def already_listening_psutil(port, renewer=False):
+    """Psutil variant of the open port check
+
+    :param int port: The TCP port in question.
+    :returns: True or False.
+
     """
     try:
         net_connections = psutil.net_connections()
@@ -81,12 +139,7 @@ def already_listening(port, renewer=False):
             display = zope.component.getUtility(interfaces.IDisplay)
             extra = ""
             if renewer:
-                extra = (
-                    " For automated renewal, you may want to use a script that stops"
-                    " and starts your webserver. You can find an example at"
-                    " https://letsencrypt.org/howitworks/#writing-your-own-renewal-script"
-                    ". Alternatively you can use the webroot plugin to renew without"
-                    " needing to stop and start your webserver.")
+                extra = RENEWER_EXTRA_MSG
             display.notification(
                 "The program {0} (process ID {1}) is already listening "
                 "on TCP port {2}. This will prevent us from binding to "
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -40,7 +40,6 @@ def read_file(filename, encoding='utf8'):
     'configobj',
     'cryptography>=0.7',  # load_pem_x509_certificate
     'parsedatetime>=1.3',  # Calendar.parseDT
-    'psutil>=2.2.1',  # 2.1.0 for net_connections and 2.2.1 resolves #1080
     'PyOpenSSL',
     'pyrfc3339',
     'python2-pythondialog>=3.2.2rc1',  # Debian squeeze support, cf. #280
@@ -70,6 +69,7 @@ def read_file(filename, encoding='utf8'):
     'coverage',
     'nose',
     'pep8',
+    'psutil>=2.2.1',  # for tests, optional
     'pylint==1.4.2',  # upstream #248
     'tox',
     'twine',


Test patch:
diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py
--- a/certbot/plugins/util_test.py
+++ b/certbot/plugins/util_test.py
@@ -1,9 +1,17 @@
 """Tests for certbot.plugins.util."""
 import os
 import unittest
+import sys
 
 import mock
-import psutil
+
+try:
+    # Python 3.5+
+    from importlib import reload as refresh  # pylint: disable=no-name-in-module
+except ImportError:
+    # Python 2-3.4
+    from imp import reload as refresh
+
 
 class PathSurgeryTest(unittest.TestCase):
     """Tests for certbot.plugins.path_surgery."""
@@ -29,7 +37,51 @@ def test_path_surgery(self, mock_debug, mock_warn):
             self.assertTrue("/usr/local/bin" in os.environ["PATH"])
             self.assertTrue("/tmp" in os.environ["PATH"])
 
-class AlreadyListeningTest(unittest.TestCase):
+
+class AlreadyListeningTestNoPsutil(unittest.TestCase):
+    """Tests for certbot.plugins.already_listening when
+    psutil is not available"""
+    def setUp(self):
+        import certbot.plugins.util
+        # Ensure we get importerror
+        self.psutil = None
+        if "psutil" in sys.modules:
+            self.psutil = sys.modules['psutil']
+        sys.modules['psutil'] = None
+        # Reload hackery to ensure getting non-psutil version
+        # loaded to memory
+        refresh(certbot.plugins.util)
+
+    def tearDown(self):
+        # Need to reload the module to ensure
+        # getting back to normal
+        import certbot.plugins.util
+        sys.modules["psutil"] = self.psutil
+        refresh(certbot.plugins.util)
+
+    @mock.patch("certbot.plugins.util.zope.component.getUtility")
+    def test_ports_available(self, mock_getutil):
+        import certbot.plugins.util as plugins_util
+        # Ensure we don't get error
+        with mock.patch("socket._socketobject.bind"):
+            self.assertFalse(plugins_util.already_listening(80))
+            self.assertFalse(plugins_util.already_listening(80, True))
+            self.assertEqual(mock_getutil.call_count, 0)
+
+    @mock.patch("certbot.plugins.util.zope.component.getUtility")
+    def test_ports_blocked(self, mock_getutil):
+        sys.modules["psutil"] = None
+        import certbot.plugins.util as plugins_util
+        import socket
+        with mock.patch("socket._socketobject.bind", side_effect=socket.error):
+            self.assertTrue(plugins_util.already_listening(80))
+            self.assertTrue(plugins_util.already_listening(80, True))
+        with mock.patch("socket.socket", side_effect=socket.error):
+            self.assertFalse(plugins_util.already_listening(80))
+        self.assertEqual(mock_getutil.call_count, 2)
+
+
+class AlreadyListeningTestPsutil(unittest.TestCase):
     """Tests for certbot.plugins.already_listening."""
     def _call(self, *args, **kwargs):
         from certbot.plugins.util import already_listening
@@ -42,6 +94,7 @@ def test_race_condition(self, mock_get_utility, mock_process, mock_net):
         # This tests a race condition, or permission problem, or OS
         # incompatibility in which, for some reason, no process name can be
         # found to match the identified listening PID.
+        import psutil
         from psutil._common import sconn
         conns = [
             sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30),
@@ -94,7 +147,7 @@ def test_listening_ipv4(self, mock_get_utility, mock_process, mock_net):
                   raddr=(), status="LISTEN", pid=4416)]
         mock_net.return_value = conns
         mock_process.name.return_value = "inetd"
-        result = self._call(17)
+        result = self._call(17, True)
         self.assertTrue(result)
         self.assertEqual(mock_get_utility.call_count, 1)
         mock_process.assert_called_once_with(4416)
@@ -124,6 +177,7 @@ def test_listening_ipv6(self, mock_get_utility, mock_process, mock_net):
 
     @mock.patch("certbot.plugins.util.psutil.net_connections")
     def test_access_denied_exception(self, mock_net):
+        import psutil
         mock_net.side_effect = psutil.AccessDenied("")
         self.assertFalse(self._call(12345))