Repository: buildbot/buildbot

Issue:
sshHostKey does not support known_hosts file format
Hi all,

thanks a lot for giving us such an amazing project! I have been struggling with this for 2 days, and I decided to open an issue because I can't find my way out.

In order to avoid any trouble, I am using the official worker provided by Docker Hub: `buildbot/buildbot-worker:2.1.0`. Both master and worker are run using Docker Compose.

Secrets configuration:
```
c['secretsProviders'] = [
    secrets.SecretInAFile(dirname="/var/lib/buildbot/secrets", strip=True),
]
```

In the `secrets` folder I have two files, named `id_rsa` and `bitbucket_host_key`, each one with the right permissions.

I have done a manual test, copying both it in the worker container and running
* `ssh -i id_rsa -T git@bitbucket.org` (it works)
* `GIT_SSH_COMMAND=ssh -i "id_rsa" -o "UserKnownHostsFile=bitbucket_host_key" git clone --branch dev git@bitbucket.org:ubiquitypress/zipper.git test` (it works)

```
    steps.Git(
        repourl='git@bitbucket.org:ubiquitypress/zipper.git',
        mode='incremental',
        useProgress=True,
        sshPrivateKey=util.Secret('id_rsa'),
        sshHostKey=util.Secret('bitbucket_host_key'),
    ),
```

During the build I see the following command being triggered:

```
git clone --branch dev git@bitbucket.org:ubiquitypress/zipper.git .
 in dir /buildbot/zipper/build (timeout 1200 secs)
 watching logfiles {}
 argv: ['git', 'clone', '--branch', 'dev', 'git@bitbucket.org:ubiquitypress/zipper.git', '.']
 environment:
  BUILDMASTER=buildbot
  BUILDMASTER_PORT=9989
  GIT_SSH_COMMAND=ssh -i "/buildbot/.build.buildbot/ssh-key" -o "UserKnownHostsFile=/buildbot/.build.buildbot/ssh-known-hosts"
  HOME=/home/buildbot
  HOSTNAME=b388d4351b42
  PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
  PWD=/buildbot/zipper/build
  WORKERNAME=worker1
  WORKERPASS=pass
  WORKER_ENVIRONMENT_BLACKLIST="DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST"
  security_updates_as_of=2018-06-15
 using PTY: False
Cloning into '.'...
Permission denied (publickey).
fatal: Could not read from remote repository.
Please make sure you have the correct access rights
and the repository exists.
program finished with exit code 128
```

Any idea? @p12tic I am tagging you here since I went through the code in #4160 in order to understand how the SSH key works, before opening this :D


Discussion:
Hi all,
just to add that I have been able to reproduce the same issue on Gitlab as well. I am not sure if I am missing something, but it seems like the worker is not picking up the SSH key I have specified.

I don't believe the SSH agent is needed in the worker to get it to use the right key (at least, I haven't found this mentioned anywhere in the docs).
I have the same issue using private key.
Buildbot expect the secret file contains host key only, not in "known_hosts" file format.
https://github.com/buildbot/buildbot/blob/master/master/buildbot/util/git.py#L107-L109

The secret file should looks like

> ssh-rsa AAAA<...>WsHQ==

not 

> |1|xx2342k3= ssh-rsa AAAA.....


I need two ssh host keys in my setup, my workaround is add a new line to the top of known_hosts file.

@fradeve: Could you confirm that the workaround by @sinofool works? We indeed currently support only the bare key being passed, but ideally known_hosts file format could also be supported.

Resolving patch:
diff --git a/master/buildbot/changes/gitpoller.py b/master/buildbot/changes/gitpoller.py
--- a/master/buildbot/changes/gitpoller.py
+++ b/master/buildbot/changes/gitpoller.py
@@ -45,9 +45,10 @@ class GitPoller(base.PollingChangeSource, StateMixin, GitMixin):
     compare_attrs = ("repourl", "branches", "workdir",
                      "pollInterval", "gitbin", "usetimestamps",
                      "category", "project", "pollAtLaunch",
-                     "buildPushesWithNoCommits", "sshPrivateKey", "sshHostKey")
+                     "buildPushesWithNoCommits", "sshPrivateKey", "sshHostKey",
+                     "sshKnownHosts")
 
-    secrets = ("sshPrivateKey", "sshHostKey")
+    secrets = ("sshPrivateKey", "sshHostKey", "sshKnownHosts")
 
     def __init__(self, repourl, branches=None, branch=None,
                  workdir=None, pollInterval=10 * 60,
@@ -56,7 +57,7 @@ def __init__(self, repourl, branches=None, branch=None,
                  pollinterval=-2, fetch_refspec=None,
                  encoding='utf-8', name=None, pollAtLaunch=False,
                  buildPushesWithNoCommits=False, only_tags=False,
-                 sshPrivateKey=None, sshHostKey=None):
+                 sshPrivateKey=None, sshHostKey=None, sshKnownHosts=None):
 
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
@@ -65,16 +66,12 @@ def __init__(self, repourl, branches=None, branch=None,
         if name is None:
             name = repourl
 
-        if sshHostKey is not None and sshPrivateKey is None:
-            config.error('GitPoller: sshPrivateKey must be provided in order '
-                         'use sshHostKey')
-            sshPrivateKey = None
-
         super().__init__(name=name,
                          pollInterval=pollInterval,
                          pollAtLaunch=pollAtLaunch,
                          sshPrivateKey=sshPrivateKey,
-                         sshHostKey=sshHostKey)
+                         sshHostKey=sshHostKey,
+                         sshKnownHosts=sshKnownHosts)
 
         if project is None:
             project = ''
@@ -105,7 +102,8 @@ def __init__(self, repourl, branches=None, branch=None,
         self.lastRev = {}
         self.sshPrivateKey = sshPrivateKey
         self.sshHostKey = sshHostKey
-        self.setupGit()
+        self.sshKnownHosts = sshKnownHosts
+        self.setupGit(logname='GitPoller')
 
         if fetch_refspec is not None:
             config.error("GitPoller: fetch_refspec is no longer supported. "
@@ -384,7 +382,11 @@ def _downloadSshPrivateKey(self, keyPath):
         writeLocalFile(keyPath, self.sshPrivateKey, mode=stat.S_IRUSR)
 
     def _downloadSshKnownHosts(self, path):
-        writeLocalFile(path, getSshKnownHostsContents(self.sshHostKey))
+        if self.sshKnownHosts is not None:
+            contents = self.sshKnownHosts
+        else:
+            contents = getSshKnownHostsContents(self.sshHostKey)
+        writeLocalFile(path, contents)
 
     def _getSshPrivateKeyPath(self, ssh_data_path):
         return os.path.join(ssh_data_path, 'ssh-key')
@@ -412,7 +414,7 @@ def _dovccmdImpl(self, command, args, path, ssh_workdir):
             self._downloadSshPrivateKey(key_path)
 
             known_hosts_path = None
-            if self.sshHostKey is not None:
+            if self.sshHostKey is not None or self.sshKnownHosts is not None:
                 known_hosts_path = self._getSshKnownHostsPath(ssh_workdir)
                 self._downloadSshKnownHosts(known_hosts_path)
 
diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py
--- a/master/buildbot/steps/source/git.py
+++ b/master/buildbot/steps/source/git.py
@@ -74,7 +74,7 @@ class Git(Source, GitStepMixin):
     def __init__(self, repourl=None, branch='HEAD', mode='incremental', method=None,
                  reference=None, submodules=False, shallow=False, progress=False, retryFetch=False,
                  clobberOnFailure=False, getDescription=False, config=None,
-                 origin=None, sshPrivateKey=None, sshHostKey=None, **kwargs):
+                 origin=None, sshPrivateKey=None, sshHostKey=None, sshKnownHosts=None, **kwargs):
 
         if not getDescription and not isinstance(getDescription, dict):
             getDescription = False
@@ -92,6 +92,7 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', method=None,
         self.getDescription = getDescription
         self.sshPrivateKey = sshPrivateKey
         self.sshHostKey = sshHostKey
+        self.sshKnownHosts = sshKnownHosts
         self.config = config
         self.srcdir = 'source'
         self.origin = origin
@@ -536,7 +537,7 @@ class GitPush(buildstep.BuildStep, GitStepMixin, CompositeStepMixin):
 
     def __init__(self, workdir=None, repourl=None, branch=None, force=False,
                  env=None, timeout=20 * 60, logEnviron=True,
-                 sshPrivateKey=None, sshHostKey=None,
+                 sshPrivateKey=None, sshHostKey=None, sshKnownHosts=None,
                  config=None, **kwargs):
 
         self.workdir = workdir
@@ -548,6 +549,7 @@ def __init__(self, workdir=None, repourl=None, branch=None, force=False,
         self.logEnviron = logEnviron
         self.sshPrivateKey = sshPrivateKey
         self.sshHostKey = sshHostKey
+        self.sshKnownHosts = sshKnownHosts
         self.config = config
 
         super().__init__(**kwargs)
@@ -615,6 +617,7 @@ def __init__(self, workdir=None, tagName=None,
         self.repourl = " "
         self.sshHostKey = None
         self.sshPrivateKey = None
+        self.sshKnownHosts = None
 
         super().__init__(**kwargs)
 
@@ -688,6 +691,7 @@ def __init__(self, workdir=None, paths=None, messages=None, env=None,
         self.repourl = " "
         self.sshPrivateKey = None
         self.sshHostKey = None
+        self.sshKnownHosts = None
 
         super().__init__(**kwargs)
 
diff --git a/master/buildbot/util/git.py b/master/buildbot/util/git.py
--- a/master/buildbot/util/git.py
+++ b/master/buildbot/util/git.py
@@ -19,7 +19,7 @@
 from twisted.internet import defer
 from twisted.python import log
 
-from buildbot import config as bbconfig
+from buildbot import config
 from buildbot.process import buildstep
 from buildbot.process import remotecommand
 from buildbot.process.properties import Properties
@@ -50,7 +50,22 @@ def getSshCommand(keyPath, knownHostsPath):
 
 class GitMixin:
 
-    def setupGit(self):
+    def setupGit(self, logname=None):
+        if logname is None:
+            logname = 'GitMixin'
+
+        if self.sshHostKey is not None and self.sshPrivateKey is None:
+            config.error('{}: sshPrivateKey must be provided in order use sshHostKey'.format(
+                logname))
+
+        if self.sshKnownHosts is not None and self.sshPrivateKey is None:
+            config.error('{}: sshPrivateKey must be provided in order use sshKnownHosts'.format(
+                logname))
+
+        if self.sshHostKey is not None and self.sshKnownHosts is not None:
+            config.error('{}: only one of sshPrivateKey and sshHostKey can be provided'.format(
+                logname))
+
         self.gitInstalled = False
         self.supportsBranch = False
         self.supportsSubmoduleForce = False
@@ -113,15 +128,10 @@ class GitStepMixin(GitMixin):
 
     def setupGitStep(self):
         self.didDownloadSshPrivateKey = False
-        self.setupGit()
-
-        if self.sshHostKey is not None and self.sshPrivateKey is None:
-            bbconfig.error('Git: sshPrivateKey must be provided in order '
-                           'use sshHostKey')
-            self.sshPrivateKey = None
+        self.setupGit(logname='Git')
 
         if not self.repourl:
-            bbconfig.error("Git: must provide repourl.")
+            config.error("Git: must provide repourl.")
 
     def _isSshPrivateKeyNeededForGitCommand(self, command):
         if not command or self.sshPrivateKey is None:
@@ -172,7 +182,7 @@ def _adjustCommandParamsForSshPrivateKey(self, full_command, full_env):
         key_path = self._getSshPrivateKeyPath(ssh_data_path)
         ssh_wrapper_path = self._getSshWrapperScriptPath(ssh_data_path)
         host_key_path = None
-        if self.sshHostKey is not None:
+        if self.sshHostKey is not None or self.sshKnownHosts is not None:
             host_key_path = self._getSshHostKeyPath(ssh_data_path)
 
         self.adjustCommandParamsForSshPrivateKey(full_command, full_env,
@@ -254,6 +264,7 @@ def _downloadSshPrivateKeyIfNeeded(self):
         p.master = self.master
         private_key = yield p.render(self.sshPrivateKey)
         host_key = yield p.render(self.sshHostKey)
+        known_hosts_contents = yield p.render(self.sshKnownHosts)
 
         # not using self.workdir because it may be changed depending on step
         # options
@@ -276,9 +287,11 @@ def _downloadSshPrivateKeyIfNeeded(self):
                                                private_key,
                                                workdir=workdir, mode=0o400)
 
-        if self.sshHostKey is not None:
+        if self.sshHostKey is not None or self.sshKnownHosts is not None:
             known_hosts_path = self._getSshHostKeyPath(ssh_data_path)
-            known_hosts_contents = getSshKnownHostsContents(host_key)
+
+            if self.sshHostKey is not None:
+                known_hosts_contents = getSshKnownHostsContents(host_key)
             yield self.downloadFileContentToWorker(known_hosts_path,
                                                    known_hosts_contents,
                                                    workdir=workdir, mode=0o400)


Test patch:
diff --git a/master/buildbot/test/unit/test_changes_gitpoller.py b/master/buildbot/test/unit/test_changes_gitpoller.py
--- a/master/buildbot/test/unit/test_changes_gitpoller.py
+++ b/master/buildbot/test/unit/test_changes_gitpoller.py
@@ -1583,6 +1583,74 @@ def test_poll_initial_2_10(self, write_local_file_mock, temp_dir_mock):
                          write_local_file_mock.call_args_list)
 
 
+class TestGitPollerWithSshKnownHosts(TestGitPollerBase):
+
+    def createPoller(self):
+        return gitpoller.GitPoller(self.REPOURL, sshPrivateKey='ssh-key',
+                                   sshKnownHosts='ssh-known-hosts')
+
+    @mock.patch('buildbot.util.private_tempdir.PrivateTemporaryDirectory',
+                new_callable=MockPrivateTemporaryDirectory)
+    @mock.patch('buildbot.changes.gitpoller.writeLocalFile')
+    @defer.inlineCallbacks
+    def test_poll_initial_2_10(self, write_local_file_mock, temp_dir_mock):
+
+        key_path = os.path.join('gitpoller-work', '.buildbot-ssh@@@', 'ssh-key')
+        known_hosts_path = os.path.join('gitpoller-work', '.buildbot-ssh@@@',
+                                        'ssh-known-hosts')
+
+        self.expectCommands(
+            gpo.Expect('git', '--version')
+            .stdout(b'git version 2.10.0\n'),
+            gpo.Expect('git', 'init', '--bare', 'gitpoller-work'),
+            gpo.Expect('git',
+                       '-c',
+                       'core.sshCommand=ssh -i "{0}" '
+                       '-o "UserKnownHostsFile={1}"'.format(
+                               key_path, known_hosts_path),
+                       'ls-remote', '--refs', self.REPOURL),
+            gpo.Expect('git',
+                       '-c',
+                       'core.sshCommand=ssh -i "{0}" '
+                       '-o "UserKnownHostsFile={1}"'.format(
+                               key_path, known_hosts_path),
+                       'fetch', self.REPOURL,
+                       '+master:refs/buildbot/' + self.REPOURL_QUOTED + '/master')
+            .path('gitpoller-work'),
+            gpo.Expect('git', 'rev-parse',
+                       'refs/buildbot/' + self.REPOURL_QUOTED + '/master')
+            .path('gitpoller-work')
+            .stdout(b'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5\n'),
+        )
+
+        yield self.poller.poll()
+
+        self.assertAllCommandsRan()
+        self.assertEqual(self.poller.lastRev, {
+            'master': 'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5'
+        })
+        self.master.db.state.assertStateByClass(
+            name=bytes2unicode(self.REPOURL), class_name='GitPoller',
+            lastRev={
+                'master': 'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5'
+            })
+
+        temp_dir_path = os.path.join('gitpoller-work', '.buildbot-ssh@@@')
+        self.assertEqual(temp_dir_mock.dirs,
+                         [(temp_dir_path, 0o700),
+                          (temp_dir_path, 0o700)])
+
+        expected_file_writes = [
+            mock.call(key_path, 'ssh-key', mode=0o400),
+            mock.call(known_hosts_path, 'ssh-known-hosts'),
+            mock.call(key_path, 'ssh-key', mode=0o400),
+            mock.call(known_hosts_path, 'ssh-known-hosts'),
+        ]
+
+        self.assertEqual(expected_file_writes,
+                         write_local_file_mock.call_args_list)
+
+
 class TestGitPollerConstructor(unittest.TestCase, config.ConfigErrorsMixin):
 
     def test_deprecatedFetchRefspec(self):
diff --git a/master/buildbot/test/unit/test_reporter_zulip.py b/master/buildbot/test/unit/test_reporter_zulip.py
--- a/master/buildbot/test/unit/test_reporter_zulip.py
+++ b/master/buildbot/test/unit/test_reporter_zulip.py
@@ -26,8 +26,8 @@
 from buildbot.test.fake import httpclientservice as fakehttpclientservice
 from buildbot.test.util.config import ConfigErrorsMixin
 from buildbot.test.util.logging import LoggingMixin
-from buildbot.test.util.reporter import ReporterTestMixin
 from buildbot.test.util.misc import TestReactorMixin
+from buildbot.test.util.reporter import ReporterTestMixin
 
 
 class TestZulipStatusPush(unittest.TestCase, ReporterTestMixin, LoggingMixin, ConfigErrorsMixin, TestReactorMixin):
diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py
--- a/master/buildbot/test/unit/test_steps_source_git.py
+++ b/master/buildbot/test/unit/test_steps_source_git.py
@@ -13,6 +13,8 @@
 #
 # Copyright Buildbot Team Members
 
+from parameterized import parameterized
+
 from twisted.internet import defer
 from twisted.internet import error
 from twisted.trial import unittest
@@ -270,11 +272,14 @@ def test_mode_full_clean_ssh_key_1_7(self):
             'got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', self.sourceName)
         return self.runStep()
 
-    def test_mode_full_clean_ssh_host_key_2_10(self):
+    @parameterized.expand([
+        ('host_key', dict(sshHostKey='sshhostkey')),
+        ('known_hosts', dict(sshKnownHosts='known_hosts')),
+    ])
+    def test_mode_full_clean_ssh_host_key_2_10(self, name, class_params):
         self.setupStep(
             self.stepClass(repourl='http://github.com/buildbot/buildbot.git',
-                           mode='full', method='clean', sshPrivateKey='sshkey',
-                           sshHostKey='sshhostkey'))
+                           mode='full', method='clean', sshPrivateKey='sshkey', **class_params))
 
         ssh_workdir = '/wrk/.wkdir.buildbot'
         ssh_key_path = '/wrk/.wkdir.buildbot/ssh-key'
diff --git a/master/buildbot/test/unit/test_util_git.py b/master/buildbot/test/unit/test_util_git.py
--- a/master/buildbot/test/unit/test_util_git.py
+++ b/master/buildbot/test/unit/test_util_git.py
@@ -13,8 +13,11 @@
 #
 # Copyright Buildbot Team Members
 
+from parameterized import parameterized
+
 from twisted.trial import unittest
 
+from buildbot.test.util import config
 from buildbot.util.git import GitMixin
 from buildbot.util.git import escapeShellArgIfNeeded
 from buildbot.util.git import getSshKnownHostsContents
@@ -51,10 +54,40 @@ def test_no_escape(self):
         self.assert_does_not_escape('--opt')
 
 
+class TestSetUpGit(GitMixin, unittest.TestCase, config.ConfigErrorsMixin):
+
+    @parameterized.expand([
+        ('no_keys', None, None, None, None),
+        ('only_private_key', 'key', None, None, None),
+        ('private_key_host_key', 'key', 'host', None, None),
+        ('private_key_known_hosts', 'key', None, 'hosts', None),
+        ('no_private_key_host_key', None, 'host', None,
+         'sshPrivateKey must be provided in order use sshHostKey'),
+        ('no_private_key_known_hosts', None, None, 'hosts',
+         'sshPrivateKey must be provided in order use sshKnownHosts'),
+        ('both_host_key_known_hosts', 'key', 'host', 'hosts',
+         'only one of sshPrivateKey and sshHostKey can be provided'),
+    ])
+    def test_config(self, name, private_key, host_key, known_hosts, config_error):
+        self.sshPrivateKey = private_key
+        self.sshHostKey = host_key
+        self.sshKnownHosts = known_hosts
+        if config_error is None:
+            self.setupGit()
+        else:
+            with self.assertRaisesConfigError(config_error):
+                self.setupGit()
+
+
 class TestParseGitFeatures(GitMixin, unittest.TestCase):
 
-    def test_no_output(self):
+    def setUp(self):
+        self.sshPrivateKey = None
+        self.sshHostKey = None
+        self.sshKnownHosts = None
         self.setupGit()
+
+    def test_no_output(self):
         self.parseGitFeatures('')
         self.assertFalse(self.gitInstalled)
         self.assertFalse(self.supportsBranch)
@@ -64,7 +97,6 @@ def test_no_output(self):
         self.assertFalse(self.supportsSshPrivateKeyAsConfigOption)
 
     def test_git_noversion(self):
-        self.setupGit()
         self.parseGitFeatures('git')
         self.assertFalse(self.gitInstalled)
         self.assertFalse(self.supportsBranch)
@@ -74,7 +106,6 @@ def test_git_noversion(self):
         self.assertFalse(self.supportsSshPrivateKeyAsConfigOption)
 
     def test_git_zero_version(self):
-        self.setupGit()
         self.parseGitFeatures('git version 0.0.0')
         self.assertTrue(self.gitInstalled)
         self.assertFalse(self.supportsBranch)
@@ -84,7 +115,6 @@ def test_git_zero_version(self):
         self.assertFalse(self.supportsSshPrivateKeyAsConfigOption)
 
     def test_git_2_10_0(self):
-        self.setupGit()
         self.parseGitFeatures('git version 2.10.0')
         self.assertTrue(self.gitInstalled)
         self.assertTrue(self.supportsBranch)