Change:
As far as I can tell, container liveness probes of type exec flat out don't work against clusters built at head. I had the kubelet always log the output of the exec'ed command, and it's always empty no matter what I exec, causing the probe to fail since it requires the output to be equal to the string "ok". If you want to test it, try adding a liveness probe of this form to any container: I also tried these lovely forms, to more closely match the liveness example under examples/: On the plus side, this may be an opportunistic time to switch over to
The reason that the e2e test for exec liveness probes hasn't caught this is that it only tests that containers are restarted when they fail liveness probes. It doesn't test that containers aren't restarted when their liveness checks should be succeeding. The e2e test's exec'ed commands also only ever produced empty output, rather than the expected "fail" output.
Is it possible to include this in a new pre-release, since it fixes a bug in health checking? I see it merged in master and I'm wondering what the policy is of releasing new code in a separate (0.16.*) release.

Discussion:
package dockertools import ( <del> "bufio" </del> "bytes" "errors" "fmt"

return nil, fmt.Errorf("failed to run in container - Exec setup failed - %v", err) } var buf bytes.Buffer <del> wrBuf := bufio.NewWriter(&buf) </del> startOpts := docker.StartExecOptions{ Detach:       false, Tty:          false, <del> OutputStream: wrBuf, ErrorStream:  wrBuf, </del> <ins> OutputStream: &buf, ErrorStream:  &buf, </ins> RawTerminal:  false, } <del> errChan := make(chan error, 1) go func() { errChan <- dm.client.StartExec(execObj.ID, startOpts) }() wrBuf.Flush() return buf.Bytes(), <-errChan </del> <ins> err = dm.client.StartExec(execObj.ID, startOpts) return buf.Bytes(), err </ins> } // ExecInContainer uses nsenter to run the command inside the container identified by containerID.

if err != nil { return probe.Unknown, err } <del> if strings.ToLower(string(data)) != defaultHealthyOutput { </del> <ins> if !strings.HasPrefix(strings.ToLower(string(data)), defaultHealthyOutput) { </ins> return probe.Failure, nil } return probe.Success, nil

. "github.com/onsi/gomega" ) <del> func runLivenessTest(c *client.Client, podDescr *api.Pod) { </del> <ins> func runLivenessTest(c *client.Client, podDescr *api.Pod, expectRestart bool) { </ins> ns := "e2e-test-" + string(util.NewUUID()) By(fmt.Sprintf("Creating pod %s in namespace %s", podDescr.Name, ns))

By(fmt.Sprintf("Initial restart count of pod %s is %d", podDescr.Name, initialRestartCount)) // Wait for at most 48 * 5 = 240s = 4 minutes until restartCount is incremented <del> pass := false </del> <ins> restarts := false </ins> for i := 0; i < 48; i++ { // Wait until restartCount is incremented. time.Sleep(5 * time.Second)

By(fmt.Sprintf("Restart count of pod %s in namespace %s is now %d", podDescr.Name, ns, restartCount)) if restartCount > initialRestartCount { By(fmt.Sprintf("Restart count of pod %s in namespace %s increased from %d to %d during the test", podDescr.Name, ns, initialRestartCount, restartCount)) <del> pass = true </del> <ins> restarts = true </ins> break } } <del> if !pass { Fail(fmt.Sprintf("Did not see the restart count of pod %s in namespace %s increase from %d during the test", podDescr.Name, ns, initialRestartCount)) </del> <ins> if restarts != expectRestart { Fail(fmt.Sprintf("pod %s in namespace %s - expected restarts: %v, found restarts: %v", podDescr.Name, ns, expectRestart, restarts)) </ins> } }

}, }, }, <del> }) </del> <ins> }, true) }) It("should *not* be restarted with a docker exec "cat /tmp/health" liveness probe", func() { runLivenessTest(c, &api.Pod{ ObjectMeta: api.ObjectMeta{ Name:   "liveness-exec", Labels: map[string]string{"test": "liveness"}, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name:    "liveness", Image:   "gcr.io/google_containers/busybox", Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 600"}, LivenessProbe: &api.Probe{ Handler: api.Handler{ Exec: &api.ExecAction{ Command: []string{"cat", "/tmp/health"}, }, }, InitialDelaySeconds: 15, }, }, }, }, }, false) </ins> }) It("should be restarted with a /healthz http liveness probe", func() {

}, }, }, <del> }) </del> <ins> }, true) </ins> }) // The following tests for remote command execution and port forwarding are