Change:
We are running heapster, dns server, kube-proxy etc. in kube-system namespace. Some e2e tests try to make sure all those kube-system pods are at ready state before firing the real tests, others just simply ingore them. But both cases might cause e2e flakiness when some of those kube-system containers are in crashloop, for example, for heapster crashloop, for influxdb crashloop, etc. When such cases happen, the developer doesn't have much information on why since the nodes might be teared down. We should watch those kube-system pods, and onContainerFailure, we should collect the related logs. One simple way is using kubectl logs <Pod<Container--previous cc/
is this related enough to and that you can bundle it in?
The logs from the pods should be retrieved via the api server. This is different from the kubelet logs. I'll upload a PR for this.
Great job It will be very helpful for debugging.

Discussion:
import ( "fmt" <del> "strings" </del> "time" "k8s.io/kubernetes/pkg/api" <del> client "k8s.io/kubernetes/pkg/client/unversioned" </del> "k8s.io/kubernetes/pkg/util" . "github.com/onsi/ginkgo"

// How long to wait for a log pod to be displayed const podLogTimeout = 45 * time.Second <del> // utility function for gomega Eventually func getPodLogs(c *client.Client, namespace, podName, containerName string) (string, error) { logs, err := c.Get().Resource("pods").Namespace(namespace).Name(podName).SubResource("log").Param("container", containerName).Do().Raw() if err != nil { return "", err } if err == nil && strings.Contains(string(logs), "Internal Error") { return "", fmt.Errorf("Internal Error") } return string(logs), err } </del> var _ = Describe("Downward API volume", func() { f := NewFramework("downward-api") It("should provide podname only [Conformance]", func() {

} } <ins> func printStatusAndLogsForNotReadyPods(c *client.Client, oldPods, newPods []*api.Pod) { printFn := func(id, log string, err error, previous bool) { prefix := "Retrieving log for container" if previous { prefix = "Retrieving log for the last terminated container" } if err != nil { Logf("%s %s, err: %v:n%sn", prefix, id, log) } else { Logf("%s %s:n%sn", prefix, id, log) } } for _, oldPod := range oldPods { for _, p := range newPods { if p.Namespace != oldPod.Namespace || p.Name != oldPod.Name { continue } if ok, _ := podRunningReady(p); !ok { Logf("Status for not ready pod %s/%s: %+v", p.Namespace, p.Name, p.Status) // Print the log of the containers if pod is not running and ready. for _, container := range p.Status.ContainerStatuses { cIdentifer := fmt.Sprintf("%s/%s/%s", p.Namespace, p.Name, container.Name) log, err := getPodLogs(c, p.Namespace, p.Name, container.Name) printFn(cIdentifer, log, err, false) // Get log from the previous container. if container.RestartCount > 0 { printFn(cIdentifer, log, err, true) } } } break } } } </ins> // rebootNode takes node name on provider through the following steps using c: //  - ensures the node is ready //  - ensures all pods on the node are running and ready

// Ensure all of the pods that we found on this node before the reboot are // running / healthy. if !checkPodsRunningReady(c, ns, podNames, rebootPodReadyAgainTimeout) { <ins> newPods := ps.List() printStatusAndLogsForNotReadyPods(c, pods, newPods) </ins> return false }

By(fmt.Sprintf("Trying to get logs from node %s pod %s container %s: %v", podStatus.Spec.NodeName, podStatus.Name, containerName, err)) <del> var logs []byte </del> <ins> var logs string </ins> start := time.Now() // Sometimes the actual containers take a second to get started, try to get logs for 60s for time.Now().Sub(start) < (60 * time.Second) { err = nil <del> logs, err = c.Get(). Resource("pods"). Namespace(ns). Name(pod.Name). SubResource("log"). Param("container", containerName). Do(). Raw() if err == nil && strings.Contains(string(logs), "Internal Error") { err = fmt.Errorf("Fetched log contains "Internal Error": %q.", string(logs)) } </del> <ins> logs, err = getPodLogs(c, ns, pod.Name, containerName) </ins> if err != nil { By(fmt.Sprintf("Warning: Failed to get logs from node %q pod %q container %q. %v", podStatus.Spec.NodeName, podStatus.Name, containerName, err))

continue } <del> By(fmt.Sprintf("Successfully fetched pod logs:%vn", string(logs))) </del> <ins> By(fmt.Sprintf("Successfully fetched pod logs:%vn", logs)) </ins> break } for _, m := range expectedOutput { <del> Expect(string(logs)).To(matcher(m), "%q in container output", m) </del> <ins> Expect(logs).To(matcher(m), "%q in container output", m) </ins> } }

client, ns, labels.SelectorFromSet(labels.Set(rc.Spec.Selector))) } } <ins> func getPodLogs(c *client.Client, namespace, podName, containerName string) (string, error) { return getPodLogsInternal(c, namespace, podName, containerName, false) } func getPreviousPodLogs(c *client.Client, namespace, podName, containerName string) (string, error) { return getPodLogsInternal(c, namespace, podName, containerName, true) } // utility function for gomega Eventually func getPodLogsInternal(c *client.Client, namespace, podName, containerName string, previous bool) (string, error) { logs, err := c.Get(). Resource("pods"). Namespace(namespace). Name(podName).SubResource("log"). Param("container", containerName). Param("previous", strconv.FormatBool(previous)). Do(). Raw() if err != nil { return "", err } if err == nil && strings.Contains(string(logs), "Internal Error") { return "", fmt.Errorf("Fetched log contains "Internal Error": %q.", string(logs)) } return string(logs), err } </ins>