Change:
<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): Pod status started null, started, 01 Jan 0001 Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"6", GitVersion:"v1.6.3", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-05-10T15:48:59Z", GoVersion:"go1.7.5", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"6", GitVersion:"v1.6.3", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-05-10T15:38:08Z", GoVersion:"go1.7.5", Compiler:"gc", Platform:"linux/amd64"}` Environment: Cloud provider or hardware configuration: - OS (e.g. from /etc/os-release): CentOS 7 Kernel (e.g. ): Install tools: kubeadm 1.6 What happened: When a pod is created and starts running, shows the correct container start time in Containers -containername -State -Started. After the container is terminated, the start time changes to . In addition, shows that is . What you expected to happen: I'm expecting the start time to not change after the pod is terminated, so that afterwards I could check how long a time the container execution took. How to reproduce it (as minimally and precisely as possible): Have a running kubernetes cluster, and create a pod with the following command: kubectl create -f - <<EOF apiVersion: v1 kind: Pod metadata: name: sleep spec: containers: name: sleep image: ubuntu command: ["sleep"] args: ["30"] restartPolicy: Never EOF This created a container that sleeps for 30 seconds and quits. During those 30 seconds, check the pod status with : ... State: Running Started: Wed, 17 May 2017 10:25:22 +0300 ... After 30 seconds, check the status of the terminated container with again: ... State: Terminated Reason: Completed Exit Code: 0 Started: Mon, 01 Jan 0001 00:00:00 +0000 Finished: Wed, 17 May 2017 10:25:52 +0300 ... With we see that the container's start time is null: ... "state": { "terminated": { "containerID": "docker://ca0dbe0dab3bb91ad0b05ac167e44d5fbf805f736ef98f61b9e106ba4456667f", "exitCode": 0, "finishedAt": "2017-05-17T07:25:52Z", "reason": "Completed", "startedAt": null } } ... Anything else we need to know: I need to measure container run durations and this bug prevents me from doing so.
/cc PR to fix this:

Discussion:
glog.Errorf("ContainerStatus for %s error: %v", c.Id, err) return nil, err } <del> annotatedInfo := getContainerInfoFromAnnotations(c.Annotations) labeledInfo := getContainerInfoFromLabels(c.Labels) cStatus := &kubecontainer.ContainerStatus{ ID: kubecontainer.ContainerID{ Type: m.runtimeName, ID:   c.Id, }, Name:         labeledInfo.ContainerName, Image:        status.Image.Image, ImageID:      status.ImageRef, Hash:         annotatedInfo.Hash, RestartCount: annotatedInfo.RestartCount, State:        toKubeContainerState(c.State), CreatedAt:    time.Unix(0, status.CreatedAt), } if c.State == runtimeapi.ContainerState_CONTAINER_RUNNING { cStatus.StartedAt = time.Unix(0, status.StartedAt) } else { cStatus.Reason = status.Reason cStatus.Message = status.Message cStatus.ExitCode = int(status.ExitCode) cStatus.FinishedAt = time.Unix(0, status.FinishedAt) </del> <ins> cStatus := toKubeContainerStatus(status, m.runtimeName) if status.State == runtimeapi.ContainerState_CONTAINER_EXITED { // Populate the termination message if needed. annotatedInfo := getContainerInfoFromAnnotations(status.Annotations) labeledInfo := getContainerInfoFromLabels(status.Labels) </ins> fallbackToLogs := annotatedInfo.TerminationMessagePolicy == v1.TerminationMessageFallbackToLogsOnError && (cStatus.ExitCode != 0 || cStatus.Reason == "OOMKilled") tMessage, checkLogs := getTerminationMessage(status, annotatedInfo.TerminationMessagePath, fallbackToLogs) if checkLogs {

cStatus.Message = tMessage } } statuses[i] = cStatus }

return statuses, nil } <ins> func toKubeContainerStatus(status *runtimeapi.ContainerStatus, runtimeName string) *kubecontainer.ContainerStatus { annotatedInfo := getContainerInfoFromAnnotations(status.Annotations) labeledInfo := getContainerInfoFromLabels(status.Labels) cStatus := &kubecontainer.ContainerStatus{ ID: kubecontainer.ContainerID{ Type: runtimeName, ID:   status.Id, }, Name:         labeledInfo.ContainerName, Image:        status.Image.Image, ImageID:      status.ImageRef, Hash:         annotatedInfo.Hash, RestartCount: annotatedInfo.RestartCount, State:        toKubeContainerState(status.State), CreatedAt:    time.Unix(0, status.CreatedAt), } if status.State != runtimeapi.ContainerState_CONTAINER_CREATED { // If container is not in the created state, we have tried and // started the container. Set the StartedAt time. cStatus.StartedAt = time.Unix(0, status.StartedAt) } if status.State == runtimeapi.ContainerState_CONTAINER_EXITED { cStatus.Reason = status.Reason cStatus.Message = status.Message cStatus.ExitCode = int(status.ExitCode) cStatus.FinishedAt = time.Unix(0, status.FinishedAt) } return cStatus } </ins> // generateContainerEvent generates an event for the container. func (m *kubeGenericRuntimeManager) generateContainerEvent(containerID kubecontainer.ContainerID, eventType, reason, message string) { ref, ok := m.containerRefManager.GetRef(containerID)

import ( "path/filepath" "testing" <ins> "time" </ins> "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/api/v1" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1" <ins> kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" </ins> containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" )

assert.NoError(t, err) assert.Empty(t, containers) } <ins> // TestToKubeContainerStatus tests the converting the CRI container status to // the internal type (i.e., toKubeContainerStatus()) for containers in // different states. func TestToKubeContainerStatus(t *testing.T) { cid := &kubecontainer.ContainerID{Type: "testRuntime", ID: "dummyid"} meta := &runtimeapi.ContainerMetadata{Name: "cname", Attempt: 3} imageSpec := &runtimeapi.ImageSpec{Image: "fimage"} var ( createdAt  int64 = 327 startedAt  int64 = 999 finishedAt int64 = 1278 ) for desc, test := range map[string]struct { input    *runtimeapi.ContainerStatus expected *kubecontainer.ContainerStatus }{ "created container": { input: &runtimeapi.ContainerStatus{ Id:        cid.ID, Metadata:  meta, Image:     imageSpec, State:     runtimeapi.ContainerState_CONTAINER_CREATED, CreatedAt: createdAt, }, expected: &kubecontainer.ContainerStatus{ ID:        *cid, Image:     imageSpec.Image, State:     kubecontainer.ContainerStateCreated, CreatedAt: time.Unix(0, createdAt), }, }, "running container": { input: &runtimeapi.ContainerStatus{ Id:        cid.ID, Metadata:  meta, Image:     imageSpec, State:     runtimeapi.ContainerState_CONTAINER_RUNNING, CreatedAt: createdAt, StartedAt: startedAt, }, expected: &kubecontainer.ContainerStatus{ ID:        *cid, Image:     imageSpec.Image, State:     kubecontainer.ContainerStateRunning, CreatedAt: time.Unix(0, createdAt), StartedAt: time.Unix(0, startedAt), }, }, "exited container": { input: &runtimeapi.ContainerStatus{ Id:         cid.ID, Metadata:   meta, Image:      imageSpec, State:      runtimeapi.ContainerState_CONTAINER_EXITED, CreatedAt:  createdAt, StartedAt:  startedAt, FinishedAt: finishedAt, ExitCode:   int32(121), Reason:     "GotKilled", Message:    "The container was killed", }, expected: &kubecontainer.ContainerStatus{ ID:         *cid, Image:      imageSpec.Image, State:      kubecontainer.ContainerStateExited, CreatedAt:  time.Unix(0, createdAt), StartedAt:  time.Unix(0, startedAt), FinishedAt: time.Unix(0, finishedAt), ExitCode:   121, Reason:     "GotKilled", Message:    "The container was killed", }, }, "unknown container": { input: &runtimeapi.ContainerStatus{ Id:        cid.ID, Metadata:  meta, Image:     imageSpec, State:     runtimeapi.ContainerState_CONTAINER_UNKNOWN, CreatedAt: createdAt, StartedAt: startedAt, }, expected: &kubecontainer.ContainerStatus{ ID:        *cid, Image:     imageSpec.Image, State:     kubecontainer.ContainerStateUnknown, CreatedAt: time.Unix(0, createdAt), StartedAt: time.Unix(0, startedAt), }, }, } { actual := toKubeContainerStatus(test.input, cid.Type) assert.Equal(t, test.expected, actual, desc) } } </ins>