Repository: buildbot/buildbot

Issue:
Secrets leaked by `util.Secret`
The	`util.Secret` API is designed to allow users to reference sensitive information without exposing it in log files. In Buildbot versoin 1.1.0, the data is incorrectly emitted to the log files if it has not previously been rendered via the `util.Interpolate` API.

I've included the contents of a `master.cfg` file which demonstrates this behavior and a screenshot of a build created using that file.

    $ buildbot --version
    Buildbot version: 1.1.0
    Twisted version: 17.9.0

(Installed via pip)

```python
import os
from buildbot.plugins import *

secrets_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'secrets')

try:
    os.mkdir(secrets_dir)
except OSError:
    pass

with open('%s/my_secret' % secrets_dir, 'w') as handle:
    handle.write('this is a secret')

os.chmod('%s/my_secret' % secrets_dir, 0600)

secret = util.Secret('my_secret')
interpolation = util.Interpolate('%(secret:my_secret)s')
demo_factory = util.BuildFactory([
    steps.ShellCommand(name='Debug ShellCommand / util.Secret #1',
                       command=['echo', secret]),
    steps.ShellCommand(name='Debug ShellCommand / util.Secret #2',
                       command=['echo', secret]),
    steps.ShellCommand(name='Debug ShellCommand / util.Interpolate',
                       command=['echo', interpolation]),
    steps.ShellCommand(name='Debug ShellCommand / util.Secret #3',
                       command=['echo', secret]),
])

c = BuildmasterConfig = {}
c['schedulers'] = [
  schedulers.ForceScheduler(name='demo', builderNames=['demoBuilder'])
]
c['builders'] = [
    util.BuilderConfig(name='demoBuilder',
                       workernames=['moe'],
                       factory=demo_factory)
]
c['secretsProviders'] = [secrets.SecretInAFile(dirname=secrets_dir)]
c['workers'] = [worker.LocalWorker('moe')]
c['protocols'] = {'pb': {'port': 9989}}
c['services'] = []
c['title'] = 'Demo'
c['titleURL'] = 'Demo'
c['buildbotURL'] = 'http://localhost/'
c['www'] = {'port': 80}
c['db'] = {'db_url': 'sqlite:///state.sqlite'}
```

![screenshot-2018-3-20 buildbot builder demobuilder build 8](https://user-images.githubusercontent.com/677252/37677243-e7f3fe8c-2c50-11e8-9c48-a9551ec15e69.png)


Discussion:
Hi good findings.

Would you like to make a patch for this issue?

It is about adding the line:

        properties.useSecret(secret_detail.value, self.secret_name)

from:
https://github.com/buildbot/buildbot/blob/master/master/buildbot/process/properties.py#L500

in line 524:
https://github.com/buildbot/buildbot/blob/master/master/buildbot/process/properties.py#L524

the fix the integration test to also test that Secret is not leaking values:

https://github.com/buildbot/buildbot/blob/master/master/buildbot/test/integration/interop/test_integration_secrets.py#L90


Hi, I just had to roll a password because of this. Please fix it, considering you seem to already know how to.

Resolving patch:
diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py
--- a/master/buildbot/process/properties.py
+++ b/master/buildbot/process/properties.py
@@ -478,24 +478,7 @@ def getRenderingFor(self, properties):
 class Secret(_SecretRenderer):
 
     def __repr__(self):
-        return "Secret({0})".format(self.secretKey)
-
-    def __init__(self, secretKey):
-        self.secretKey = secretKey
-
-    @defer.inlineCallbacks
-    def getRenderingFor(self, props):
-        secretsSrv = props.master.namedServices.get("secrets")
-        if not secretsSrv:
-            error_message = "secrets service not started, need to configure" \
-                            " SecretManager in c['services'] to use 'secrets'" \
-                            "in Interpolate"
-            raise KeyError(error_message)
-        credsservice = props.master.namedServices['secrets']
-        secret_detail = yield credsservice.get(self.secretKey)
-        if secret_detail is None:
-            raise KeyError("secret key %s is not found in any provider" % self.secretKey)
-        return secret_detail.value
+        return "Secret({0})".format(self.secret_name)
 
 
 class _SecretIndexer:


Test patch:
diff --git a/master/buildbot/test/integration/interop/test_integration_secrets.py b/master/buildbot/test/integration/interop/test_integration_secrets.py
--- a/master/buildbot/test/integration/interop/test_integration_secrets.py
+++ b/master/buildbot/test/integration/interop/test_integration_secrets.py
@@ -15,6 +15,8 @@
 
 import os
 
+from parameterized import parameterized
+
 from twisted.internet import defer
 
 from buildbot.process.properties import Interpolate
@@ -31,24 +33,40 @@ def send(self, build):
 
 class SecretsConfig(RunMasterBase):
 
+    @parameterized.expand([
+        ('with_interpolation', True),
+        ('plain_command', False),
+    ])
     @defer.inlineCallbacks
-    def test_secret(self):
-        c = masterConfig()
+    def test_secret(self, name, use_interpolation):
+        c = masterConfig(use_interpolation)
         yield self.setupConfig(c)
         build = yield self.doForceBuild(wantSteps=True, wantLogs=True)
         self.assertEqual(build['buildid'], 1)
+
+        # check the command line
         res = yield self.checkBuildStepLogExist(build, "echo <foo>")
-        if os.name == "posix":
+
+        # also check the secrets are replaced in argv
+        yield self.checkBuildStepLogExist(build, "argv:.*echo.*<foo>", regex=True)
+
+        # also check that the correct value goes to the command
+        if os.name == "posix" and use_interpolation:
             res &= yield self.checkBuildStepLogExist(build, "The password was there")
+
         self.assertTrue(res)
         # at this point, build contains all the log and steps info that is in the db
         # we check that our secret is not in there!
         self.assertNotIn("bar", repr(build))
         self.assertTrue(c['services'][0].reported)
 
+    @parameterized.expand([
+        ('with_interpolation', True),
+        ('plain_command', False),
+    ])
     @defer.inlineCallbacks
-    def test_secretReconfig(self):
-        c = masterConfig()
+    def test_secretReconfig(self, name, use_interpolation):
+        c = masterConfig(use_interpolation)
         yield self.setupConfig(c)
         c['secretsProviders'] = [FakeSecretStorage(
             secretdict={"foo": "different_value", "something": "more"})]
@@ -67,11 +85,11 @@ class SecretsConfigPB(SecretsConfig):
 
 
 # master configuration
-def masterConfig():
+def masterConfig(use_interpolation):
     c = {}
     from buildbot.config import BuilderConfig
     from buildbot.process.factory import BuildFactory
-    from buildbot.plugins import schedulers, steps
+    from buildbot.plugins import schedulers, steps, util
 
     c['services'] = [FakeSecretReporter('http://example.com/hook',
                                         auth=('user', Interpolate('%(secret:httppasswd)s')))]
@@ -83,12 +101,18 @@ def masterConfig():
     c['secretsProviders'] = [FakeSecretStorage(
         secretdict={"foo": "bar", "something": "more", 'httppasswd': 'myhttppasswd'})]
     f = BuildFactory()
-    if os.name == "posix":
-        f.addStep(steps.ShellCommand(command=Interpolate(
-            'echo %(secret:foo)s | sed "s/bar/The password was there/"')))
+
+    if use_interpolation:
+        if os.name == "posix":
+            # on posix we can also check whether the password was passed to the command
+            command = Interpolate('echo %(secret:foo)s | sed "s/bar/The password was there/"')
+        else:
+            command = Interpolate('echo %(secret:foo)s')
     else:
-        f.addStep(steps.ShellCommand(command=Interpolate(
-            'echo %(secret:foo)s')))
+        command = ['echo', util.Secret('foo')]
+
+    f.addStep(steps.ShellCommand(command=command))
+
     c['builders'] = [
         BuilderConfig(name="testy",
                       workernames=["local1"],
diff --git a/master/buildbot/test/util/integration.py b/master/buildbot/test/util/integration.py
--- a/master/buildbot/test/util/integration.py
+++ b/master/buildbot/test/util/integration.py
@@ -14,6 +14,7 @@
 # Copyright Buildbot Team Members
 
 import os
+import re
 import sys
 from io import StringIO
 
@@ -291,15 +292,19 @@ def printBuild(self, build, out=sys.stdout, withLogs=False):
                     self.printLog(log, out)
 
     @defer.inlineCallbacks
-    def checkBuildStepLogExist(self, build, expectedLog, onlyStdout=False):
+    def checkBuildStepLogExist(self, build, expectedLog, onlyStdout=False, regex=False):
         yield self.enrichBuild(build, wantSteps=True, wantProperties=True, wantLogs=True)
         for step in build['steps']:
             for log in step['logs']:
                 for line in log['contents']['content'].splitlines():
                     if onlyStdout and line[0] != 'o':
                         continue
-                    if expectedLog in line:
-                        return True
+                    if regex:
+                        if re.search(expectedLog, line):
+                            return True
+                    else:
+                        if expectedLog in line:
+                            return True
         return False
 
     def printLog(self, log, out):