Repository: buildbot/buildbot

Issue:
PasswordManhole 'no host keys, failing'
I'm trying to go through [Quick Tour](http://docs.buildbot.net/current/tutorial/tour.html#debugging-with-manhole) tutorial, but it looks like `PasswordManhole` isn't implemented properly for new Twisted version:

```
	Traceback (most recent call last):
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
	    current.result = callback(current.result, *args, **kw)
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1357, in gotResult
	    _inlineCallbacks(r, g, deferred)
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1299, in _inlineCallbacks
	    result = result.throwExceptionIntoGenerator(g)
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/twisted/python/failure.py", line 393, in throwExceptionIntoGenerator
	    return g.throw(self.type, self.value, self.tb)
	--- <exception caught here> ---
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/buildbot/master.py", line 308, in startService
	    yield self.reconfigServiceWithBuildbotConfig(self.config)
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1299, in _inlineCallbacks
	    result = result.throwExceptionIntoGenerator(g)
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/twisted/python/failure.py", line 393, in throwExceptionIntoGenerator
	    return g.throw(self.type, self.value, self.tb)
	  File "/home/krna/dev/buildbot-master/.env/lib/python3.6/site-packages/buildbot/util/service.py", line 53, in reconfigServiceWithBuildbotConfig
	    yield svc.reconfigServiceWithBuildbotConfig(new_config)
	twisted.conch.error.ConchError: ('no host keys, failing', None)
```

I'm on:
```
asn1crypto==0.22.0
attrs==16.3.0
autobahn==0.18.2
Automat==0.5.0
buildbot==0.9.6
buildbot-console-view==0.9.6
buildbot-waterfall-view==0.9.6
buildbot-worker==0.9.6
buildbot-www==0.9.6
cffi==1.10.0
constantly==15.1.0
cryptography==1.8.1
decorator==4.0.11
future==0.16.0
idna==2.5
incremental==16.10.1
Jinja2==2.9.6
MarkupSafe==1.0
packaging==16.8
pbr==3.0.0
pyasn1==0.2.3
pycparser==2.17
PyJWT==1.5.0
pyparsing==2.2.0
python-dateutil==2.6.0
six==1.10.0
SQLAlchemy==1.1.9
sqlalchemy-migrate==0.11.0
sqlparse==0.2.3
Tempita==0.5.2
Twisted==17.1.0
txaio==2.7.0
zope.interface==4.4.0
```


Discussion:
Just hit the same issue
It is related to twisted update
* https://stackoverflow.com/questions/40235033/not-able-to-run-carbon-cache-with-manhole-enabled
* https://twistedmatrix.com/trac/ticket/8229

and looks like @rodrigc touched it a bit :) 
Installing Twisted < 16.0 fixed it for me.

Resolving patch:
diff --git a/master/buildbot/config.py b/master/buildbot/config.py
--- a/master/buildbot/config.py
+++ b/master/buildbot/config.py
@@ -521,7 +521,7 @@ def copy_str_or_callable_param(name, alt_key=None):
         if 'manhole' in config_dict:
             # we don't check that this is a manhole instance, since that
             # requires importing buildbot.manhole for every user, and currently
-            # that will fail if pycrypto isn't installed
+            # that will fail if cryptography isn't installed
             self.manhole = config_dict['manhole']
 
         if 'revlink' in config_dict:
diff --git a/master/buildbot/manhole.py b/master/buildbot/manhole.py
--- a/master/buildbot/manhole.py
+++ b/master/buildbot/manhole.py
@@ -16,12 +16,12 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from future.utils import string_types
 
 import base64
 import binascii
 import os
 import types
-from builtins import str
 
 from twisted.application import strports
 from twisted.conch import manhole
@@ -36,14 +36,17 @@
 from buildbot import config
 from buildbot.util import ComparableMixin
 from buildbot.util import service
+from buildbot.util import unicode2bytes
 
 try:
     from twisted.conch import checkers as conchc, manhole_ssh
-    _hush_pyflakes = [manhole_ssh, conchc]
+    from twisted.conch.openssh_compat.factory import OpenSSHFactory
+    _hush_pyflakes = [manhole_ssh, conchc, OpenSSHFactory]
     del _hush_pyflakes
 except ImportError:
     manhole_ssh = None
     conchc = None
+    OpenSSHFactory = None
 
 
 # makeTelnetProtocol and _TelnetRealm are for the TelnetManhole
@@ -108,7 +111,7 @@ def __init__(self, authorized_keys_file):
                 authorized_keys_file)
 
         def checkKey(self, credentials):
-            with open(self.authorized_keys_file) as f:
+            with open(self.authorized_keys_file, "rb") as f:
                 for l in f.readlines():
                     l2 = l.split()
                     if len(l2) < 2:
@@ -129,7 +132,7 @@ class _BaseManhole(service.AsyncMultiService):
     buildbot developers. Connect to this by running an ssh client.
     """
 
-    def __init__(self, port, checker, using_ssh=True):
+    def __init__(self, port, checker, ssh_hostkey_dir=None):
         """
         @type port: string or int
         @param port: what port should the Manhole listen on? This is a
@@ -148,9 +151,9 @@ def __init__(self, port, checker, using_ssh=True):
             c = credc.FilePasswordDB(passwd_filename) # file of name:passwd
             c = conchc.UNIXPasswordDatabase # getpwnam() (probably /etc/passwd)
 
-        @type using_ssh: bool
-        @param using_ssh: If True, accept SSH connections. If False, accept
-                          regular unencrypted telnet connections.
+        @type ssh_hostkey_dir: str
+        @param ssh_hostkey_dir: directory which contains ssh host keys for
+                                this server
         """
 
         # unfortunately, these don't work unless we're running as root
@@ -178,13 +181,22 @@ def makeProtocol():
             p = insults.ServerProtocol(manhole.ColoredManhole, namespace)
             return p
 
-        self.using_ssh = using_ssh
-        if using_ssh:
+        self.ssh_hostkey_dir = ssh_hostkey_dir
+        if self.ssh_hostkey_dir:
+            self.using_ssh = True
+            if not self.ssh_hostkey_dir:
+                raise ValueError("Most specify a value for ssh_hostkey_dir")
             r = manhole_ssh.TerminalRealm()
             r.chainedProtocolFactory = makeProtocol
             p = portal.Portal(r, [self.checker])
             f = manhole_ssh.ConchFactory(p)
+            openSSHFactory = OpenSSHFactory()
+            openSSHFactory.dataRoot = self.ssh_hostkey_dir
+            openSSHFactory.dataModuliRoot = self.ssh_hostkey_dir
+            f.publicKeys = openSSHFactory.getPublicKeys()
+            f.privateKeys = openSSHFactory.getPrivateKeys()
         else:
+            self.using_ssh = False
             r = _TelnetRealm(makeNamespace)
             p = portal.Portal(r, [self.checker])
             f = protocol.ServerFactory()
@@ -226,9 +238,9 @@ def __init__(self, port, username, password):
         self.password = password
 
         c = checkers.InMemoryUsernamePasswordDatabaseDontUse()
-        c.addUser(username, password)
+        c.addUser(unicode2bytes(username), unicode2bytes(password))
 
-        _BaseManhole.__init__(self, port, c, using_ssh=False)
+        _BaseManhole.__init__(self, port, c)
 
 
 class PasswordManhole(_BaseManhole, ComparableMixin):
@@ -237,9 +249,9 @@ class PasswordManhole(_BaseManhole, ComparableMixin):
     username and password to authorize access.
     """
 
-    compare_attrs = ("port", "username", "password")
+    compare_attrs = ("port", "username", "password", "ssh_hostkey_dir")
 
-    def __init__(self, port, username, password):
+    def __init__(self, port, username, password, ssh_hostkey_dir):
         """
         @type port: string or int
         @param port: what port should the Manhole listen on? This is a
@@ -250,17 +262,21 @@ def __init__(self, port, username, password):
         @param username:
         @param password: username= and password= form a pair of strings to
                          use when authenticating the remote user.
+        @type ssh_hostkey_dir: str
+        @param ssh_hostkey_dir: directory which contains ssh host keys for
+                                this server
         """
 
         if not manhole_ssh:
-            config.error("pycrypto required for ssh mahole.")
+            config.error("cryptography required for ssh mahole.")
         self.username = username
         self.password = password
+        self.ssh_hostkey_dir = ssh_hostkey_dir
 
         c = checkers.InMemoryUsernamePasswordDatabaseDontUse()
-        c.addUser(username, password)
+        c.addUser(unicode2bytes(username), unicode2bytes(password))
 
-        _BaseManhole.__init__(self, port, c)
+        _BaseManhole.__init__(self, port, c, ssh_hostkey_dir)
 
 
 class AuthorizedKeysManhole(_BaseManhole, ComparableMixin):
@@ -270,9 +286,9 @@ class AuthorizedKeysManhole(_BaseManhole, ComparableMixin):
     keys in our authorized_keys file. It is created with the name of a file
     that contains the public keys that we will accept."""
 
-    compare_attrs = ("port", "keyfile")
+    compare_attrs = ("port", "keyfile", "ssh_hostkey_dir")
 
-    def __init__(self, port, keyfile):
+    def __init__(self, port, keyfile, ssh_hostkey_dir):
         """
         @type port: string or int
         @param port: what port should the Manhole listen on? This is a
@@ -284,16 +300,19 @@ def __init__(self, port, keyfile):
                         basedir) that contains SSH public keys of authorized
                         users, one per line. This is the exact same format
                         as used by sshd in ~/.ssh/authorized_keys .
+        @type ssh_hostkey_dir: str
+        @param ssh_hostkey_dir: directory which contains ssh host keys for
+                                this server
         """
 
         if not manhole_ssh:
-            config.error("pycrypto required for ssh mahole.")
+            config.error("cryptography required for ssh mahole.")
 
         # TODO: expanduser this, and make it relative to the buildmaster's
         # basedir
         self.keyfile = keyfile
         c = AuthorizedKeysChecker(keyfile)
-        _BaseManhole.__init__(self, port, c)
+        _BaseManhole.__init__(self, port, c, ssh_hostkey_dir)
 
 
 class ArbitraryCheckerManhole(_BaseManhole, ComparableMixin):
@@ -316,7 +335,7 @@ def __init__(self, port, checker):
         """
 
         if not manhole_ssh:
-            config.error("pycrypto required for ssh mahole.")
+            config.error("cryptography required for ssh mahole.")
 
         _BaseManhole.__init__(self, port, checker)
 
@@ -330,19 +349,18 @@ def show(x):
     maxlen = max([0] + [len(n) for n in names])
     for k in names:
         v = getattr(x, k)
-        t = type(v)
-        if t == types.MethodType:
+        if isinstance(v, types.MethodType):
             continue
         if k[:2] == '__' and k[-2:] == '__':
             continue
-        if t is str:
+        if isinstance(v, string_types):
             if len(v) > 80 - maxlen - 5:
                 v = repr(v[:80 - maxlen - 5]) + "..."
-        elif t in (int, type(None)):
+        elif isinstance(v, (int, type(None))):
             v = str(v)
-        elif v in (list, tuple, dict):
+        elif isinstance(v, (list, tuple, dict)):
             v = "%s (%d elements)" % (v, len(v))
         else:
-            v = str(t)
+            v = str(type(v))
         print("%*s : %s" % (maxlen, k, v))
     return x


Test patch: