Change:
The following e2e Conformance Tests are showing a similar pattern in the way that they are flaking. - When the initial pod is scheduled for the test it's timing out and the test fails. Checking shows the first reference for the pod has a missing volume not mounted. Also, inspecting a shows that the pod never gets to the state. for this flake shows a similar pattern to the flakes above. Further logs/details are listed below. Due to the complexity and knowledge required to deal with these flakes it would be helpful get some extra eyes and a SIG to understand the root cause further and help create a suitable fix. Flake details where gathered from the following testgrids. namespace: deployment-2394 node: kind-worker ï¿½[1mSTEPï¿½[0m: creating a Deployment ï¿½[1mSTEPï¿½[0m: waiting for Deployment to be created ï¿½[1mSTEPï¿½[0m: waiting for all Replicas to be Ready Feb 28 22:51:10.748: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:51:10.748: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:51:10.754: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:51:10.754: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:51:10.769: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:51:10.769: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:51:10.782: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:51:10.782: INFO: observed Deployment test-deployment in namespace deployment-2394 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 28 22:52:10.748: FAIL: failed to see replicas of test-deployment in namespace deployment-2394 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc00023e240: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Feb 28 22:52:20 kind-worker kubelet[243]: E0228 22:52:20. 243 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--fqqvk.", GenerateName:"", Namespace:"deployment-2394", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-2394", Name:"test-deployment--fqqvk", UID:"-fa20-419e-8ab4-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-2t44s" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0072589242b8012, ext:, loc:(time.Location)(0x3e95d80)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0072589242b8012, ext:, loc:(time.Location)(0x3e95d80)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-2394" not found' (will not retry!) volume logs: Feb 28 22:51:11 kind-worker kubelet[243]: I0228 22:51:11. 243 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-2t44s" (UniqueName: "") pod "test-deployment--fqqvk" (UID: "-fa20-419e-8ab4-") Feb 28 22:52:19 kind-worker kubelet[243]: I0228 22:52:19. 243 ] operationExecutor.UnmountVolume started for volume "kube-api-access-2t44s" (UniqueName: "") pod "-fa20-419e-8ab4-" (UID: "-fa20-419e-8ab4-") Feb 28 22:52:19 kind-worker kubelet[243]: I0228 22:52:19. 243 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-2t44s") pod "-fa20-419e-8ab4-" (UID: "-fa20-419e-8ab4-"). InnerVolumeSpecName "kube-api-access-2t44s". PluginName "", VolumeGidValue "" Feb 28 22:52:19 kind-worker kubelet[243]: I0228 22:52:19. 243 ] Volume detached for volume "kube-api-access-2t44s" (UniqueName: "") on node "kind-worker" DevicePath "" Feb 28 22:52:20 kind-worker kubelet[243]: E0228 22:52:20. 243 ] Mount cannot be satisfied for container "test-deployment", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-2t44s ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Feb 28 22:52:20 kind-worker kubelet[243]: E0228 22:52:20. 243 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-2t44s,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--fqqvkdeployment-2394(-fa20-419e-8ab4-): CreateContainerConfigError: cannot find volume "kube-api-access-2t44s" to mount into container "test-deployment" Feb 28 22:52:20 kind-worker kubelet[243]: E0228 22:52:20. 243 ] Error syncing pod -fa20-419e-8ab4- ("test-deployment--fqqvkdeployment-2394(-fa20-419e-8ab4-)"), skipping: failed to "StartContainer" for "test-deployment" with CreateContainerConfigError: "cannot find volume "kube-api-access-2t44s" to mount into container "test-deployment"" Feb 28 22:52:20 kind-worker kubelet[243]: E0228 22:52:20. 243 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--fqqvk.", GenerateName:"", Namespace:"deployment-2394", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-2394", Name:"test-deployment--fqqvk", UID:"-fa20-419e-8ab4-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-2t44s" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0072589242b8012, ext:, loc:(time.Location)(0x3e95d80)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0072589242b8012, ext:, loc:(time.Location)(0x3e95d80)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-2394" not found' (will not retry!) namespace: deployment-8926 node: kind-worker2 ¿½[1mSTEPï¿½[0m: creating a Deployment ï¿½[1mSTEPï¿½[0m: waiting for Deployment to be created ï¿½[1mSTEPï¿½[0m: waiting for all Replicas to be Ready Feb 27 18:30:19.521: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:30:19.521: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:30:19.526: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:30:19.527: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:30:19.545: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:30:19.545: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:30:19.572: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:30:19.572: INFO: observed Deployment test-deployment in namespace deployment-8926 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 18:31:19.520: FAIL: failed to see replicas of test-deployment in namespace deployment-8926 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc00023e240: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Feb 27 18:31:25 kind-worker2 kubelet[243]: E0227 18:31:25. 243 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-zgtz4,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--5kn2ddeployment-8926(-b89e-41bc-a323-): CreateContainerConfigError: cannot find volume "kube-api-access-zgtz4" to mount into container "test-deployment" Feb 27 18:30:19 kind-worker2 kubelet[243]: I0227 18:30:19. 243 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-zgtz4" (UniqueName: "") pod "test-deployment--5kn2d" (UID: "-b89e-41bc-a323-") Feb 27 18:31:25 kind-worker2 kubelet[243]: I0227 18:31:25. 243 ] operationExecutor.UnmountVolume started for volume "kube-api-access-zgtz4" (UniqueName: "") pod "-b89e-41bc-a323-" (UID: "-b89e-41bc-a323-") Feb 27 18:31:25 kind-worker2 kubelet[243]: I0227 18:31:25. 243 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-zgtz4") pod "-b89e-41bc-a323-" (UID: "-b89e-41bc-a323-"). InnerVolumeSpecName "kube-api-access-zgtz4". PluginName "", VolumeGidValue "" Feb 27 18:31:25 kind-worker2 kubelet[243]: I0227 18:31:25. 243 ] Volume detached for volume "kube-api-access-zgtz4" (UniqueName: "") on node "kind-worker2" DevicePath "" Feb 27 18:31:25 kind-worker2 kubelet[243]: E0227 18:31:25. 243 ] Mount cannot be satisfied for container "test-deployment", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-zgtz4 ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Feb 27 18:31:25 kind-worker2 kubelet[243]: E0227 18:31:25. 243 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-zgtz4,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--5kn2ddeployment-8926(-b89e-41bc-a323-): CreateContainerConfigError: cannot find volume "kube-api-access-zgtz4" to mount into container "test-deployment" Feb 27 18:31:25 kind-worker2 kubelet[243]: E0227 18:31:25. 243 ] Error syncing pod -b89e-41bc-a323- ("test-deployment--5kn2ddeployment-8926(-b89e-41bc-a323-)"), skipping: failed to "StartContainer" for "test-deployment" with CreateContainerConfigError: "cannot find volume "kube-api-access-zgtz4" to mount into container "test-deployment"" Feb 27 18:31:25 kind-worker2 kubelet[243]: E0227 18:31:25. 243 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--5kn2d.", GenerateName:"", Namespace:"deployment-8926", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-8926", Name:"test-deployment--5kn2d", UID:"-b89e-41bc-a323-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-zgtz4" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc006c1df680f3ea0, ext:, loc:(time.Location)(0x3e95d80)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc006c1df680f3ea0, ext:, loc:(time.Location)(0x3e95d80)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-8926" not found' (will not retry!) namespace: deployment-5921 node: kind-worker ï¿½[1mSTEPï¿½[0m: creating a Deployment ï¿½[1mSTEPï¿½[0m: waiting for Deployment to be created ï¿½[1mSTEPï¿½[0m: waiting for all Replicas to be Ready Feb 27 07:08:24.219: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:08:24.219: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:08:24.244: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:08:24.245: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:08:24.304: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:08:24.305: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:08:24.386: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:08:24.386: INFO: observed Deployment test-deployment in namespace deployment-5921 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 07:09:24.218: FAIL: failed to see replicas of test-deployment in namespace deployment-5921 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc000238230: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Feb 27 07:09:33 kind-worker kubelet[246]: E0227 07:09:33. 246 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-s6mbj,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--fk8kldeployment-5921(-84f3-4d7a-911e-): CreateContainerConfigError: cannot find volume "kube-api-access-s6mbj" to mount into container "test-deployment" Feb 27 07:08:24 kind-worker kubelet[246]: I0227 07:08:24. 246 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-s6mbj" (UniqueName: "") pod "test-deployment--fk8kl" (UID: "-84f3-4d7a-911e-") Feb 27 07:09:32 kind-worker kubelet[246]: I0227 07:09:32. 246 ] operationExecutor.UnmountVolume started for volume "kube-api-access-s6mbj" (UniqueName: "") pod "-84f3-4d7a-911e-" (UID: "-84f3-4d7a-911e-") Feb 27 07:09:32 kind-worker kubelet[246]: I0227 07:09:32. 246 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-s6mbj") pod "-84f3-4d7a-911e-" (UID: "-84f3-4d7a-911e-"). InnerVolumeSpecName "kube-api-access-s6mbj". PluginName "", VolumeGidValue "" Feb 27 07:09:33 kind-worker kubelet[246]: E0227 07:09:33. 246 ] Mount cannot be satisfied for container "test-deployment", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-s6mbj ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Feb 27 07:09:33 kind-worker kubelet[246]: E0227 07:09:33. 246 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-s6mbj,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--fk8kldeployment-5921(-84f3-4d7a-911e-): CreateContainerConfigError: cannot find volume "kube-api-access-s6mbj" to mount into container "test-deployment" Feb 27 07:09:33 kind-worker kubelet[246]: E0227 07:09:33. 246 ] Error syncing pod -84f3-4d7a-911e- ("test-deployment--fk8kldeployment-5921(-84f3-4d7a-911e-)"), skipping: failed to "StartContainer" for "test-deployment" with CreateContainerConfigError: "cannot find volume "kube-api-access-s6mbj" to mount into container "test-deployment"" Feb 27 07:09:33 kind-worker kubelet[246]: I0227 07:09:33. 246 ] Volume detached for volume "kube-api-access-s6mbj" (UniqueName: "") on node "kind-worker" DevicePath "" Feb 27 07:09:33 kind-worker kubelet[246]: E0227 07:09:33. 246 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--fk8kl.", GenerateName:"", Namespace:"deployment-5921", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-5921", Name:"test-deployment--fk8kl", UID:"-84f3-4d7a-911e-", APIVersion:"v1", ResourceVersion:"4618", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-s6mbj" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc00699eb446473a2, ext:, loc:(time.Location)(0x3e95d80)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc00699eb446473a2, ext:, loc:(time.Location)(0x3e95d80)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-5921" not found' (will not retry!) namespace: deployment-4634 node: kind-worker2 ï¿½[1mSTEPï¿½[0m: creating a Deployment ï¿½[1mSTEPï¿½[0m: waiting for Deployment to be created ï¿½[1mSTEPï¿½[0m: waiting for all Replicas to be Ready Mar 4 21:45:17.150: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:45:17.151: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:45:17.160: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:45:17.160: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:45:17.204: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:45:17.204: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:45:17.218: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:45:17.218: INFO: observed Deployment test-deployment in namespace deployment-4634 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 4 21:46:17.148: FAIL: failed to see replicas of test-deployment in namespace deployment-4634 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc00024a250: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 04 21:46:26 kind-worker2 kubelet[246]: E0304 21:46:26. 246 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-djhbp,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--ld5nddeployment-4634(-18fb-4a19-9bef-): CreateContainerConfigError: cannot find volume "kube-api-access-djhbp" to mount into container "test-deployment" Mar 04 21:45:17 kind-worker2 kubelet[246]: I0304 21:45:17. 246 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-djhbp" (UniqueName: "") pod "test-deployment--ld5nd" (UID: "-18fb-4a19-9bef-") Mar 04 21:46:26 kind-worker2 kubelet[246]: I0304 21:46:26. 246 ] operationExecutor.UnmountVolume started for volume "kube-api-access-djhbp" (UniqueName: "") pod "-18fb-4a19-9bef-" (UID: "-18fb-4a19-9bef-") Mar 04 21:46:26 kind-worker2 kubelet[246]: I0304 21:46:26. 246 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-djhbp") pod "-18fb-4a19-9bef-" (UID: "-18fb-4a19-9bef-"). InnerVolumeSpecName "kube-api-access-djhbp". PluginName "", VolumeGidValue "" Mar 04 21:46:26 kind-worker2 kubelet[246]: I0304 21:46:26. 246 ] Volume detached for volume "kube-api-access-djhbp" (UniqueName: "") on node "kind-worker2" DevicePath "" Mar 04 21:46:26 kind-worker2 kubelet[246]: E0304 21:46:26. 246 ] Mount cannot be satisfied for container "test-deployment", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-djhbp ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 04 21:46:26 kind-worker2 kubelet[246]: E0304 21:46:26. 246 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-djhbp,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--ld5nddeployment-4634(-18fb-4a19-9bef-): CreateContainerConfigError: cannot find volume "kube-api-access-djhbp" to mount into container "test-deployment" Mar 04 21:46:26 kind-worker2 kubelet[246]: E0304 21:46:26. 246 ] Error syncing pod -18fb-4a19-9bef- ("test-deployment--ld5nddeployment-4634(-18fb-4a19-9bef-)"), skipping: failed to "StartContainer" for "test-deployment" with CreateContainerConfigError: "cannot find volume "kube-api-access-djhbp" to mount into container "test-deployment"" Mar 04 21:46:26 kind-worker2 kubelet[246]: E0304 21:46:26. 246 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--ld5nd.", GenerateName:"", Namespace:"deployment-4634", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-4634", Name:"test-deployment--ld5nd", UID:"-18fb-4a19-9bef-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-djhbp" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc008732ca5cbe343, ext:, loc:(time.Location)(0x70f9ea0)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc008732ca5cbe343, ext:, loc:(time.Location)(0x70f9ea0)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-4634" not found' (will not retry!) namespace: deployment-674 node: kind-worker2 ï¿½[1mSTEPï¿½[0m: creating a Deployment ï¿½[1mSTEPï¿½[0m: waiting for Deployment to be created ï¿½[1mSTEPï¿½[0m: waiting for all Replicas to be Ready Mar 1 17:21:21.044: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:21:21.044: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:21:21.047: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:21:21.048: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:21:21.058: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:21:21.058: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:21:21.085: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:21:21.085: INFO: observed Deployment test-deployment in namespace deployment-674 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 17:22:21.043: FAIL: failed to see replicas of test-deployment in namespace deployment-674 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc00023a230: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 01 17:22:30 kind-worker2 kubelet[242]: E0301 17:22:30. 242 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-qwrgt,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--c4bqpdeployment-674(-69ec-4e07-a068-): CreateContainerConfigError: cannot find volume "kube-api-access-qwrgt" to mount into container "test-deployment" Mar 01 17:21:21 kind-worker2 kubelet[242]: I0301 17:21:21. 242 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-qwrgt" (UniqueName: "") pod "test-deployment--c4bqp" (UID: "-69ec-4e07-a068-") Mar 01 17:22:30 kind-worker2 kubelet[242]: I0301 17:22:30. 242 ] operationExecutor.UnmountVolume started for volume "kube-api-access-qwrgt" (UniqueName: "") pod "-69ec-4e07-a068-" (UID: "-69ec-4e07-a068-") Mar 01 17:22:30 kind-worker2 kubelet[242]: I0301 17:22:30. 242 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-qwrgt") pod "-69ec-4e07-a068-" (UID: "-69ec-4e07-a068-"). InnerVolumeSpecName "kube-api-access-qwrgt". PluginName "", VolumeGidValue "" Mar 01 17:22:30 kind-worker2 kubelet[242]: I0301 17:22:30. 242 ] Volume detached for volume "kube-api-access-qwrgt" (UniqueName: "") on node "kind-worker2" DevicePath "" Mar 01 17:22:30 kind-worker2 kubelet[242]: E0301 17:22:30. 242 ] Mount cannot be satisfied for container "test-deployment", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-qwrgt ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 01 17:22:30 kind-worker2 kubelet[242]: E0301 17:22:30. 242 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-qwrgt,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--c4bqpdeployment-674(-69ec-4e07-a068-): CreateContainerConfigError: cannot find volume "kube-api-access-qwrgt" to mount into container "test-deployment" Mar 01 17:22:30 kind-worker2 kubelet[242]: E0301 17:22:30. 242 ] Error syncing pod -69ec-4e07-a068- ("test-deployment--c4bqpdeployment-674(-69ec-4e07-a068-)"), skipping: failed to "StartContainer" for "test-deployment" with CreateContainerConfigError: "cannot find volume "kube-api-access-qwrgt" to mount into container "test-deployment"" Mar 01 17:22:30 kind-worker2 kubelet[242]: E0301 17:22:30. 242 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--c4bqp.", GenerateName:"", Namespace:"deployment-674", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-674", Name:"test-deployment--c4bqp", UID:"-69ec-4e07-a068-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-qwrgt" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0076695955f3efa, ext:, loc:(time.Location)(0x3e95d80)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0076695955f3efa, ext:, loc:(time.Location)(0x3e95d80)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-674" not found' (will not retry!) namespace: deployment-9744 node: kind-worker ï¿½[1mSTEPï¿½[0m: creating a Deployment ï¿½[1mSTEPï¿½[0m: waiting for Deployment to be created ï¿½[1mSTEPï¿½[0m: waiting for all Replicas to be Ready Mar 1 16:17:31.831: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:17:31.831: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:17:31.846: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:17:31.846: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:17:31.865: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:17:31.865: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:17:31.903: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:17:31.903: INFO: observed Deployment test-deployment in namespace deployment-9744 with ReadyReplicas 0 and labels map[test-deployment-static:true] Mar 1 16:18:31.829: FAIL: failed to see replicas of test-deployment in namespace deployment-9744 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc00023e240: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 01 16:18:39 kind-worker kubelet[245]: E0301 16:18:39. 245 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-fptj9,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--dnc7wdeployment-9744(-dab8-4c38-b58d-): CreateContainerConfigError: cannot find volume "kube-api-access-fptj9" to mount into container "test-deployment" Mar 01 16:17:32 kind-worker kubelet[245]: I0301 16:17:32. 245 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-fptj9" (UniqueName: "") pod "test-deployment--dnc7w" (UID: "-dab8-4c38-b58d-") Mar 01 16:18:38 kind-worker kubelet[245]: I0301 16:18:38. 245 ] operationExecutor.UnmountVolume started for volume "kube-api-access-fptj9" (UniqueName: "") pod "-dab8-4c38-b58d-" (UID: "-dab8-4c38-b58d-") Mar 01 16:18:38 kind-worker kubelet[245]: I0301 16:18:38. 245 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-fptj9") pod "-dab8-4c38-b58d-" (UID: "-dab8-4c38-b58d-"). InnerVolumeSpecName "kube-api-access-fptj9". PluginName "", VolumeGidValue "" Mar 01 16:18:38 kind-worker kubelet[245]: I0301 16:18:38. 245 ] Volume detached for volume "kube-api-access-fptj9" (UniqueName: "") on node "kind-worker" DevicePath "" Mar 01 16:18:39 kind-worker kubelet[245]: E0301 16:18:39. 245 ] Mount cannot be satisfied for container "test-deployment", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-fptj9 ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 01 16:18:39 kind-worker kubelet[245]: E0301 16:18:39. 245 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-fptj9,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--dnc7wdeployment-9744(-dab8-4c38-b58d-): CreateContainerConfigError: cannot find volume "kube-api-access-fptj9" to mount into container "test-deployment" Mar 01 16:18:39 kind-worker kubelet[245]: E0301 16:18:39. 245 ] Error syncing pod -dab8-4c38-b58d- ("test-deployment--dnc7wdeployment-9744(-dab8-4c38-b58d-)"), skipping: failed to "StartContainer" for "test-deployment" with CreateContainerConfigError: "cannot find volume "kube-api-access-fptj9" to mount into container "test-deployment"" Mar 01 16:18:39 kind-worker kubelet[245]: E0301 16:18:39. 245 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--dnc7w.", GenerateName:"", Namespace:"deployment-9744", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-9744", Name:"test-deployment--dnc7w", UID:"-dab8-4c38-b58d-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-fptj9" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc00762d7e20c923b, ext:, loc:(time.Location)(0x3e95d80)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc00762d7e20c923b, ext:, loc:(time.Location)(0x3e95d80)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-9744" not found' (will not retry!) namespace: deployment-820 node: kind-worker ï¿½[1mSTEPï¿½[0m: creating a Deployment ï¿½[1mSTEPï¿½[0m: waiting for Deployment to be created ï¿½[1mSTEPï¿½[0m: waiting for all Replicas to be Ready Feb 27 23:25:45.899: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:25:45.899: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:25:45.907: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:25:45.907: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:25:45.989: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:25:45.989: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:25:46.022: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:25:46.023: INFO: observed Deployment test-deployment in namespace deployment-820 with ReadyReplicas 0 and labels map[test-deployment-static:true] Feb 27 23:26:45.897: FAIL: failed to see replicas of test-deployment in namespace deployment-820 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc00023e250: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Feb 27 23:26:58 kind-worker kubelet[247]: E0227 23:26:58. 247 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--4cftvdeployment-820(-a251-4902-b45f-): CreateContainerConfigError: cannot find volume "kube-api-access-" to mount into container "test-deployment" namespace: deployment-3233 node: kind-worker ï¿½[1mSTEPï¿½[0m: fetching the DeploymentStatus Feb 25 05:34:10.780: INFO: observed Deployment test-deployment in namespace deployment-3233 with ReadyReplicas 1 and labels map[test-deployment:updated test-deployment-static:true] Feb 25 05:34:10.814: INFO: observed Deployment test-deployment in namespace deployment-3233 with ReadyReplicas 1 and labels map[test-deployment:updated test-deployment-static:true] Feb 25 05:34:10.853: INFO: observed Deployment test-deployment in namespace deployment-3233 with ReadyReplicas 1 and labels map[test-deployment:updated test-deployment-static:true] Feb 25 05:34:10.908: INFO: observed Deployment test-deployment in namespace deployment-3233 with ReadyReplicas 1 and labels map[test-deployment:updated test-deployment-static:true] Feb 25 05:34:10.921: INFO: observed Deployment test-deployment in namespace deployment-3233 with ReadyReplicas 1 and labels map[test-deployment:updated test-deployment-static:true] Feb 25 05:34:10.935: INFO: observed Deployment test-deployment in namespace deployment-3233 with ReadyReplicas 1 and labels map[test-deployment:updated test-deployment-static:true] Feb 25 05:34:10.947: INFO: observed Deployment test-deployment in namespace deployment-3233 with ReadyReplicas 1 and labels map[test-deployment:updated test-deployment-static:true] Feb 25 05:35:10.775: FAIL: failed to see replicas of test-deployment in namespace deployment-3233 scale to requested amount of 2 Unexpected error: <errors.errorString | 0xc0002ae230: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Feb 25 05:35:19 kind-worker2 kubelet[246]: E0225 05:35:19. 246 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-xns8t,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--92cmjdeployment-3233(-537a-49d3-9f5b-): CreateContainerConfigError: cannot find volume "kube-api-access-xns8t" to mount into container "test-deployment" Feb 25 05:34:11 kind-worker2 kubelet[246]: I0225 05:34:11. 246 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-xns8t" (UniqueName: "") pod "test-deployment--92cmj" (UID: "-537a-49d3-9f5b-") Feb 25 05:35:18 kind-worker2 kubelet[246]: I0225 05:35:18. 246 ] operationExecutor.UnmountVolume started for volume "kube-api-access-xns8t" (UniqueName: "") pod "-537a-49d3-9f5b-" (UID: "-537a-49d3-9f5b-") Feb 25 05:35:18 kind-worker2 kubelet[246]: I0225 05:35:18. 246 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-xns8t") pod "-537a-49d3-9f5b-" (UID: "-537a-49d3-9f5b-"). InnerVolumeSpecName "kube-api-access-xns8t". PluginName "", VolumeGidValue "" Feb 25 05:35:18 kind-worker2 kubelet[246]: I0225 05:35:18. 246 ] Volume detached for volume "kube-api-access-xns8t" (UniqueName: "") on node "kind-worker2" DevicePath "" Feb 25 05:35:19 kind-worker2 kubelet[246]: E0225 05:35:19. 246 ] Mount cannot be satisfied for container "test-deployment", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-xns8t ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Feb 25 05:35:19 kind-worker2 kubelet[246]: E0225 05:35:19. 246 ] container &Container{Name:test-deployment,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-xns8t,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod test-deployment--92cmjdeployment-3233(-537a-49d3-9f5b-): CreateContainerConfigError: cannot find volume "kube-api-access-xns8t" to mount into container "test-deployment" Feb 25 05:35:19 kind-worker2 kubelet[246]: E0225 05:35:19. 246 ] Error syncing pod -537a-49d3-9f5b- ("test-deployment--92cmjdeployment-3233(-537a-49d3-9f5b-)"), skipping: failed to "StartContainer" for "test-deployment" with CreateContainerConfigError: "cannot find volume "kube-api-access-xns8t" to mount into container "test-deployment"" Feb 25 05:35:19 kind-worker2 kubelet[246]: E0225 05:35:19. 246 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-deployment--92cmj.", GenerateName:"", Namespace:"deployment-3233", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"deployment-3233", Name:"test-deployment--92cmj", UID:"-537a-49d3-9f5b-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{test-deployment}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-xns8t" to mount into container "test-deployment"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc005eba5da30fde7, ext:, loc:(time.Location)(0x3e93d40)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc005eba5da30fde7, ext:, loc:(time.Location)(0x3e93d40)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "deployment-3233" not found' (will not retry!) namespace: replication-controller-9476 node: kind-worker2 ï¿½[1mSTEPï¿½[0m: creating a ReplicationController ï¿½[1mSTEPï¿½[0m: waiting for RC to be ï¿½[1mSTEPï¿½[0m: waiting for available Replicas Mar 8 08:00:01.557: FAIL: Wait for condition with watch events should not return an error Unexpected error: <errors.errorString | 0xc00024a250: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 08 08:00:10 kind-worker2 kubelet[244]: E0308 08:00:10. 244 ] container &Container{Name:rc-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-xs4mx,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod rc-test-x7tb4replication-controller-9476(-640a-4101-b089-): CreateContainerConfigError: cannot find volume "kube-api-access-xs4mx" to mount into container "rc-test" Mar 08 07:58:01 kind-worker2 kubelet[244]: I0308 07:58:01. 244 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-xs4mx" (UniqueName: "") pod "rc-test-x7tb4" (UID: "-640a-4101-b089-") Mar 08 08:00:08 kind-worker2 kubelet[244]: I0308 08:00:08. 244 ] operationExecutor.UnmountVolume started for volume "kube-api-access-xs4mx" (UniqueName: "") pod "-640a-4101-b089-" (UID: "-640a-4101-b089-") Mar 08 08:00:08 kind-worker2 kubelet[244]: I0308 08:00:08. 244 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-xs4mx") pod "-640a-4101-b089-" (UID: "-640a-4101-b089-"). InnerVolumeSpecName "kube-api-access-xs4mx". PluginName "", VolumeGidValue "" Mar 08 08:00:08 kind-worker2 kubelet[244]: I0308 08:00:08. 244 ] Volume detached for volume "kube-api-access-xs4mx" (UniqueName: "") on node "kind-worker2" DevicePath "" Mar 08 08:00:10 kind-worker2 kubelet[244]: E0308 08:00:10. 244 ] Mount cannot be satisfied for container "rc-test", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-xs4mx ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 08 08:00:10 kind-worker2 kubelet[244]: E0308 08:00:10. 244 ] container &Container{Name:rc-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-xs4mx,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod rc-test-x7tb4replication-controller-9476(-640a-4101-b089-): CreateContainerConfigError: cannot find volume "kube-api-access-xs4mx" to mount into container "rc-test" Mar 08 08:00:10 kind-worker2 kubelet[244]: E0308 08:00:10. 244 ] Error syncing pod -640a-4101-b089- ("rc-test-x7tb4replication-controller-9476(-640a-4101-b089-)"), skipping: failed to "StartContainer" for "rc-test" with CreateContainerConfigError: "cannot find volume "kube-api-access-xs4mx" to mount into container "rc-test"" Mar 08 08:00:10 kind-worker2 kubelet[244]: E0308 08:00:10. 244 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"rc-test-x7tb4.", GenerateName:"", Namespace:"replication-controller-9476", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"replication-controller-9476", Name:"rc-test-x7tb4", UID:"-640a-4101-b089-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{rc-test}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-xs4mx" to mount into container "rc-test"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0099442824f9ee1, ext:, loc:(time.Location)(0x7418700)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0099442824f9ee1, ext:, loc:(time.Location)(0x7418700)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "rc-test-x7tb4." is forbidden: unable to create new content in namespace replication-controller-9476 because it is being terminated' (will not retry!) namespace: replication-controller-5147 node: kind-worker2 ï¿½[1mSTEPï¿½[0m: creating a ReplicationController ï¿½[1mSTEPï¿½[0m: waiting for RC to be ï¿½[1mSTEPï¿½[0m: waiting for available Replicas Mar 5 20:56:36.365: FAIL: Wait for condition with watch events should not return an error Unexpected error: <errors.errorString | 0xc00024a250: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 05 20:56:44 kind-worker2 kubelet[247]: E0305 20:56:44. 247 ] container &Container{Name:rc-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-z4fpx,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod rc-test-46wcnreplication-controller-5147(-c966-418a-82ff-): CreateContainerConfigError: cannot find volume "kube-api-access-z4fpx" to mount into container "rc-test" Mar 05 20:54:36 kind-worker2 kubelet[247]: I0305 20:54:36. 247 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-z4fpx" (UniqueName: "") pod "rc-test-46wcn" (UID: "-c966-418a-82ff-") Mar 05 20:56:43 kind-worker2 kubelet[247]: I0305 20:56:43. 247 ] operationExecutor.UnmountVolume started for volume "kube-api-access-z4fpx" (UniqueName: "") pod "-c966-418a-82ff-" (UID: "-c966-418a-82ff-") Mar 05 20:56:43 kind-worker2 kubelet[247]: I0305 20:56:43. 247 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-z4fpx") pod "-c966-418a-82ff-" (UID: "-c966-418a-82ff-"). InnerVolumeSpecName "kube-api-access-z4fpx". PluginName "", VolumeGidValue "" Mar 05 20:56:43 kind-worker2 kubelet[247]: I0305 20:56:43. 247 ] Volume detached for volume "kube-api-access-z4fpx" (UniqueName: "") on node "kind-worker2" DevicePath "" Mar 05 20:56:44 kind-worker2 kubelet[247]: E0305 20:56:44. 247 ] Mount cannot be satisfied for container "rc-test", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-z4fpx ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 05 20:56:44 kind-worker2 kubelet[247]: E0305 20:56:44. 247 ] container &Container{Name:rc-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-z4fpx,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod rc-test-46wcnreplication-controller-5147(-c966-418a-82ff-): CreateContainerConfigError: cannot find volume "kube-api-access-z4fpx" to mount into container "rc-test" Mar 05 20:56:44 kind-worker2 kubelet[247]: E0305 20:56:44. 247 ] Error syncing pod -c966-418a-82ff- ("rc-test-46wcnreplication-controller-5147(-c966-418a-82ff-)"), skipping: failed to "StartContainer" for "rc-test" with CreateContainerConfigError: "cannot find volume "kube-api-access-z4fpx" to mount into container "rc-test"" Mar 05 20:56:44 kind-worker2 kubelet[247]: E0305 20:56:44. 247 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"rc-test-46wcn.", GenerateName:"", Namespace:"replication-controller-5147", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"replication-controller-5147", Name:"rc-test-46wcn", UID:"-c966-418a-82ff-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{rc-test}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-z4fpx" to mount into container "rc-test"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc008c4a318e8b4d5, ext:, loc:(time.Location)(0x711c200)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc008c4a318e8b4d5, ext:, loc:(time.Location)(0x711c200)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "rc-test-46wcn." is forbidden: unable to create new content in namespace replication-controller-5147 because it is being terminated' (will not retry!) namespace: replication-controller-4026 node: kind-worker2 ï¿½[1mSTEPï¿½[0m: creating a ReplicationController ï¿½[1mSTEPï¿½[0m: waiting for RC to be ï¿½[1mSTEPï¿½[0m: waiting for available Replicas Mar 2 19:38:49.544: FAIL: Wait for condition with watch events should not return an error Unexpected error: <errors.errorString | 0xc0002b0230: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 02 19:38:57 kind-worker2 kubelet[245]: E0302 19:38:57. 245 ] container &Container{Name:rc-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-9b7vw,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod rc-test-t24fwreplication-controller-4026(-59af-4082-a39a-): CreateContainerConfigError: cannot find volume "kube-api-access-9b7vw" to mount into container "rc-test" Mar 02 19:36:49 kind-worker2 kubelet[245]: I0302 19:36:49. 245 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-9b7vw" (UniqueName: "") pod "rc-test-t24fw" (UID: "-59af-4082-a39a-") Mar 02 19:38:57 kind-worker2 kubelet[245]: I0302 19:38:57. 245 ] operationExecutor.UnmountVolume started for volume "kube-api-access-9b7vw" (UniqueName: "") pod "-59af-4082-a39a-" (UID: "-59af-4082-a39a-") Mar 02 19:38:57 kind-worker2 kubelet[245]: I0302 19:38:57. 245 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-9b7vw") pod "-59af-4082-a39a-" (UID: "-59af-4082-a39a-"). InnerVolumeSpecName "kube-api-access-9b7vw". PluginName "", VolumeGidValue "" Mar 02 19:38:57 kind-worker2 kubelet[245]: I0302 19:38:57. 245 ] Volume detached for volume "kube-api-access-9b7vw" (UniqueName: "") on node "kind-worker2" DevicePath "" Mar 02 19:38:57 kind-worker2 kubelet[245]: E0302 19:38:57. 245 ] Mount cannot be satisfied for container "rc-test", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-9b7vw ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 02 19:38:57 kind-worker2 kubelet[245]: E0302 19:38:57. 245 ] container &Container{Name:rc-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-9b7vw,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod rc-test-t24fwreplication-controller-4026(-59af-4082-a39a-): CreateContainerConfigError: cannot find volume "kube-api-access-9b7vw" to mount into container "rc-test" Mar 02 19:38:57 kind-worker2 kubelet[245]: E0302 19:38:57. 245 ] Error syncing pod -59af-4082-a39a- ("rc-test-t24fwreplication-controller-4026(-59af-4082-a39a-)"), skipping: failed to "StartContainer" for "rc-test" with CreateContainerConfigError: "cannot find volume "kube-api-access-9b7vw" to mount into container "rc-test"" Mar 02 19:38:57 kind-worker2 kubelet[245]: E0302 19:38:57. 245 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"rc-test-t24fw.", GenerateName:"", Namespace:"replication-controller-4026", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"replication-controller-4026", Name:"rc-test-t24fw", UID:"-59af-4082-a39a-", APIVersion:"v1", ResourceVersion:"3528", FieldPath:"spec.containers{rc-test}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-9b7vw" to mount into container "rc-test"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc007c2f4680a8a7e, ext:, loc:(time.Location)(0x3e310c0)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc007c2f4680a8a7e, ext:, loc:(time.Location)(0x3e310c0)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "rc-test-t24fw." is forbidden: unable to create new content in namespace replication-controller-4026 because it is being terminated' (will not retry!) Namespace: pods-9355 Node: kind-worker ï¿½[1mSTEPï¿½[0m: creating a Pod with a static label ï¿½[1mSTEPï¿½[0m: watching for Pod to be ready Mar 6 17:12:06.810: INFO: observed Pod pod-test in namespace pods-9355 in phase Pending with labels: map[test-pod-static:true] & conditions [] Mar 6 17:12:06.812: INFO: observed Pod pod-test in namespace pods-9355 in phase Pending with labels: map[test-pod-static:true] & conditions [{PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2021-03-06 17:12:06 +0000 UTC }] Mar 6 17:12:06.836: INFO: observed Pod pod-test in namespace pods-9355 in phase Pending with labels: map[test-pod-static:true] & conditions [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2021-03-06 17:12:06 +0000 UTC } {Ready False 0001-01-01 00:00:00 +0000 UTC 2021-03-06 17:12:06 +0000 UTC ContainersNotReady containers with unready status: [pod-test]} {ContainersReady False 0001-01-01 00:00:00 +0000 UTC 2021-03-06 17:12:06 +0000 UTC ContainersNotReady containers with unready status: [pod-test]} {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2021-03-06 17:12:06 +0000 UTC }] Mar 6 17:14:06.809: FAIL: failed to see Pod pod-test in namespace pods-9355 running Unexpected error: <errors.errorString | 0xc0001ca250: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 06 17:14:14 kind-worker kubelet[241]: E0306 17:14:14. 241 ] container &Container{Name:pod-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-bbm4f,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod pod-testpods-9355(-d32d-4426-a427-): CreateContainerConfigError: cannot find volume "kube-api-access-bbm4f" to mount into container "pod-test" Mar 06 17:12:06 kind-worker kubelet[241]: I0306 17:12:06. 241 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-bbm4f" (UniqueName: "") pod "pod-test" (UID: "-d32d-4426-a427-") Mar 06 17:14:13 kind-worker kubelet[241]: I0306 17:14:13. 241 ] operationExecutor.UnmountVolume started for volume "kube-api-access-bbm4f" (UniqueName: "") pod "-d32d-4426-a427-" (UID: "-d32d-4426-a427-") Mar 06 17:14:13 kind-worker kubelet[241]: I0306 17:14:13. 241 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-bbm4f") pod "-d32d-4426-a427-" (UID: "-d32d-4426-a427-"). InnerVolumeSpecName "kube-api-access-bbm4f". PluginName "", VolumeGidValue "" Mar 06 17:14:14 kind-worker kubelet[241]: I0306 17:14:14. 241 ] Volume detached for volume "kube-api-access-bbm4f" (UniqueName: "") on node "kind-worker" DevicePath "" Mar 06 17:14:14 kind-worker kubelet[241]: E0306 17:14:14. 241 ] Mount cannot be satisfied for container "pod-test", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-bbm4f ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 06 17:14:14 kind-worker kubelet[241]: E0306 17:14:14. 241 ] container &Container{Name:pod-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-bbm4f,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod pod-testpods-9355(-d32d-4426-a427-): CreateContainerConfigError: cannot find volume "kube-api-access-bbm4f" to mount into container "pod-test" Mar 06 17:14:14 kind-worker kubelet[241]: E0306 17:14:14. 241 ] Error syncing pod -d32d-4426-a427- ("pod-testpods-9355(-d32d-4426-a427-)"), skipping: failed to "StartContainer" for "pod-test" with CreateContainerConfigError: "cannot find volume "kube-api-access-bbm4f" to mount into container "pod-test"" Mar 06 17:14:14 kind-worker kubelet[241]: E0306 17:14:14. 241 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"pod-test.", GenerateName:"", Namespace:"pods-9355", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"pods-9355", Name:"pod-test", UID:"-d32d-4426-a427-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{pod-test}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-bbm4f" to mount into container "pod-test"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0090bf9a7d54d9f, ext:, loc:(time.Location)(0x711e200)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0090bf9a7d54d9f, ext:, loc:(time.Location)(0x711e200)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "pods-9355" not found' (will not retry!) namespace: pods-1784 Node: kind-worker ï¿½[1mSTEPï¿½[0m: creating a Pod with a static label ï¿½[1mSTEPï¿½[0m: watching for Pod to be ready Mar 7 20:44:07.131: INFO: observed Pod pod-test in namespace pods-1784 in phase Pending with labels: map[test-pod-static:true] & conditions [] Mar 7 20:44:07.135: INFO: observed Pod pod-test in namespace pods-1784 in phase Pending with labels: map[test-pod-static:true] & conditions [{PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2021-03-07 20:44:07 +0000 UTC }] Mar 7 20:44:14.388: INFO: observed Pod pod-test in namespace pods-1784 in phase Pending with labels: map[test-pod-static:true] & conditions [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2021-03-07 20:44:07 +0000 UTC } {Ready False 0001-01-01 00:00:00 +0000 UTC 2021-03-07 20:44:07 +0000 UTC ContainersNotReady containers with unready status: [pod-test]} {ContainersReady False 0001-01-01 00:00:00 +0000 UTC 2021-03-07 20:44:07 +0000 UTC ContainersNotReady containers with unready status: [pod-test]} {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2021-03-07 20:44:07 +0000 UTC }] Mar 7 20:46:07.132: FAIL: failed to see Pod pod-test in namespace pods-1784 running Unexpected error: <errors.errorString | 0xc00024a250: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Mar 07 20:46:14 kind-worker kubelet[244]: E0307 20:46:14. 244 ] container &Container{Name:pod-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-zd68k,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod pod-testpods-1784(-a9c5-4b5e-af60-): CreateContainerConfigError: cannot find volume "kube-api-access-zd68k" to mount into container "pod-test" Mar 07 20:44:07 kind-worker kubelet[244]: I0307 20:44:07. 244 ] operationExecutor.VerifyControllerAttachedVolume started for volume "kube-api-access-zd68k" (UniqueName: "") pod "pod-test" (UID: "-a9c5-4b5e-af60-") Mar 07 20:46:13 kind-worker kubelet[244]: I0307 20:46:13. 244 ] operationExecutor.UnmountVolume started for volume "kube-api-access-zd68k" (UniqueName: "") pod "-a9c5-4b5e-af60-" (UID: "-a9c5-4b5e-af60-") Mar 07 20:46:13 kind-worker kubelet[244]: I0307 20:46:13. 244 ] UnmountVolume.TearDown succeeded for volume "" (OuterVolumeSpecName: "kube-api-access-zd68k") pod "-a9c5-4b5e-af60-" (UID: "-a9c5-4b5e-af60-"). InnerVolumeSpecName "kube-api-access-zd68k". PluginName "", VolumeGidValue "" Mar 07 20:46:13 kind-worker kubelet[244]: I0307 20:46:13. 244 ] Volume detached for volume "kube-api-access-zd68k" (UniqueName: "") on node "kind-worker" DevicePath "" Mar 07 20:46:14 kind-worker kubelet[244]: E0307 20:46:14. 244 ] Mount cannot be satisfied for container "pod-test", because the volume is missing (ok=false) or the volume mounter (vol.Mounter) is nil (vol={Mounter:<nilBlockVolumeMapper:<nilSELinuxLabeled:false ReadOnly:false InnerVolumeSpecName:}): {Name:kube-api-access-zd68k ReadOnly:true SubPath: MountPropagation:<nilSubPathExpr:} Mar 07 20:46:14 kind-worker kubelet[244]: E0307 20:46:14. 244 ] container &Container{Name:pod-test,,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-zd68k,ReadOnly:true,,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod pod-testpods-1784(-a9c5-4b5e-af60-): CreateContainerConfigError: cannot find volume "kube-api-access-zd68k" to mount into container "pod-test" Mar 07 20:46:14 kind-worker kubelet[244]: E0307 20:46:14. 244 ] Error syncing pod -a9c5-4b5e-af60- ("pod-testpods-1784(-a9c5-4b5e-af60-)"), skipping: failed to "StartContainer" for "pod-test" with CreateContainerConfigError: "cannot find volume "kube-api-access-zd68k" to mount into container "pod-test"" Mar 07 20:46:15 kind-worker kubelet[244]: E0307 20:46:15. 244 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"pod-test.", GenerateName:"", Namespace:"pods-1784", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"pods-1784", Name:"pod-test", UID:"-a9c5-4b5e-af60-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{pod-test}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-zd68k" to mount into container "pod-test"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0096cc5b57bb890, ext:, loc:(time.Location)(0x7418700)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0096cc5b57bb890, ext:, loc:(time.Location)(0x7418700)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'namespaces "pods-1784" not found' (will not retry!)
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
During the conformance meeting earlier today there was a discussion this group of flakes. The feedback given by is that issue isn't with the tests but likely with kubelet or the cri. Thanks for the helpful feedback
/sig node
Do we know what sort of volumes are missing? Is this happening in any of the GCE jobs, or only kind?
Only seen this for kind jobs so far
will check if there are any GCE jobs with the same issue
Most of the volumes prefixes I've seen are for
So we're failing on service account volumes? ... Interesting.
Looks like the type of flake are only showing up on kind jobs
GCE is also using kube-up versus kind using kubeadm. Given these appear to be projected service accounts we should probably look into what's happening setting those up. That doesn't sound like a CRI issue to me.
/cc
/assign
the "cannot find volume" error is surprising: are there other kubelet messages about being unable to obtain a token for the service account?
/sig storage auth storage for projected volume, auth for service account token
bumping priority since projected volumes are in the critical path by default in 1.21
looked at this failure found i am not sure why reconciler didn't try to mount again. after 2.5 minutes, kubelet bailed out with
That means when the kubelet requested the token, the pod no longer existed in the API. Could that have been an error that occurred later, after the test namespace was cleaned up? Did anything in the kubelet log indicate the kubelet knew the pod no longer existed in the API as well? Did the kubelet try to start that pod previously? I'm unclear about the order of operations the kubelet does for volume creation/teardown. I wonder if any of the following issues are related:
how does the volume lifecycle interact with a pod that is already deleted in the API? for volumes like configmap/secret/projected volumes, does the kubelet try to fetch data in the process of tearing down the volume? Taking a random failure from above looks like UnmountVolume/TearDown succeeds: there's an error about the mount we just tore down not being able to be satisfied and a container not being able to be started: kubelet tries to record an event, but is rejected becuase the namespace is terminating: Mar 08 08:00:10 kind-worker2 kubelet[244]: E0308 08:00:10. 244 ] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"rc-test-x7tb4.", GenerateName:"", Namespace:"replication-controller-9476", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, DeletionTimestamp:(v1.Time)(nil), DeletionGracePeriodSeconds:(int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Pod", Namespace:"replication-controller-9476", Name:"rc-test-x7tb4", UID:"-640a-4101-b089-", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers{rc-test}"}, Reason:"Failed", Message:"Error: cannot find volume "kube-api-access-xs4mx" to mount into container "rc-test"", Source:v1.EventSource{Component:"kubelet", Host:"kind-worker2"}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0099442824f9ee1, ext:, loc:(time.Location)(0x7418700)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0099442824f9ee1, ext:, loc:(time.Location)(0x7418700)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(time.Location)(nil)}}, Series:(v1.EventSeries)(nil), Action:"", Related:(v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "rc-test-x7tb4." is forbidden: unable to create new content in namespace replication-controller-9476 because it is being terminated' (will not retry!)
/cc
Looking at the test failure from These events seem a little suspicious. There was a quick scale down and scale back up. But on the scale up, the pod got recreated with the same name? That, exasperated with the race outlined in , may have caused volume manager to get into some weird state. cc to help investigate why we didn't try mounting again with this new pod (with same name as previously deleted pod). EDIT: This was not the correct order of events. The pod was created and then immediately deleted.
Actually, all of those events happened at the same time, so the ordering is not clear. I couldn't find references to the pod in apiserver or controller-manager logs. EDIT: I misinterpreted the scheduler logs. "Delete event for unscheduled pod" doesn't necessarily mean the pod got deleted. In this case, it indicates the pod moved to the scheduled pods cache. So I think the next sequence of things to investigate are: EDIT: what is the actual sequence of events of pod creation/deletion
Sorry I completely misinterpreted the scheduler logs. I think we need to figure out the detailed sequence of pod creation/delete events going on first.
Take 2. Pod got created and scheduled: Then got deleted soon afterwards. This is logged during or when a , however there are no subsequent "Add event" logs, so it should be the former. So the main question is why is kubelet not tearing down the pod?
Here's what I think is happening: is waiting for volumes to finish mounting. before calling , it . However, if the pod is terminated after that check, then we are stuck in in until it times out, a little over . The call probably needs to be enhanced to return early if the pod is deleted. It doesn't seem like pod deletion is captured as a . In fact, we seem to if the pod is deleted.
While we investigate a fix in volume manager to return an error earlier, can help look into if the rapid pod create then delete is expected?
/assign to look at the volumemanager fix
GitHub didn't allow me to assign the following users: mauriciopoppe. Note that only , repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign to look at the volumemanager fix Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
This test involves a number of phases, summarize as below deployment with 2 replicas, with Agnhost image deployment with 1 replica, with Pause image and run command (/bin/sh sleep ) deployment with 2 replicas, with httpd image Running the test, we found out that there is issue in second step After patch deployment, it will result in 2 pods with following condition So patch update is actual not successfully. However the test is only checking readyReplica number and is not verifying the pod is actually changed to use the correct image. I think it is wrong to use Pause image and also try to run command. Should remove command from Pod spec. Although the test passes the check in the second step, it is wrong. Also note: The deployment is StrategyType: RollingUpdate MinReadySeconds: 0 RollingUpdateStrategy: 25% max unavailable, 25% max surge This can explain the behavior when sale down and up, in the log, you might see multiple pods were created around the same time, and got deleted immediately. Because the pod graceperiod is set to 0, the pod is immediately removed from API server. However, looks like pod manager might still hold the cache of this pod for a little while, so there are errors showing "tokens not found...", this because pod is already deleted from API server, so that it will fail to get token.
Thanks for digging. It sounds like there are a couple issues to fix: test issue running a command that won't work against the pause image () - are all three tests using the same flawed approach? kubelet hanging in WaitForAttachAndMount in a failure state for a long time when the pod is already deleted ()
From my quick san of the first two tests, they are not using Pause image with commands.
There are multiple known races when pods are created and deleted quickly. I suspect the one in , where the volume was umounted, but containers still tried to come up, is a manifestation of That race was partially improved by recently so that should help reduce some of the flakes, but the full fix is going to require some more effort.
Investigate more about the deployment test issue. I think there are some confusion from deployment controller status update. As mentioned in the case the deployment is patched with a different image and replica number changed from 2 to 1. But the following happens new pod comes up with the new image fails to start successfully, enter crash loop. old pod has to continue to run due to RollingUpdateStrategy 25% max unavailable, 25% max surge In this case, strictly speaking, the patch update failed, there is no pod running with the updated image. Is it reasonable to update number of ReadyReplica to 1? Or it should be 0 to reflect that no pod is ready with correct update? The following is the check from the test deployment.Spec.Template.Spec.Containers[0].Image was initially set to the original image (Anghost). After new pod is created, it changed to Pause image. And because ReadyReplicas is also changed to 1 finally, so the check pass for phase 2, and continue with phase 3, which will update the deployment again with third image and changed replicas from 1 to 2. It can succeed after deployment controller starts new pods and terminate old ones. I think because of this deployment status update, after test phase 2, the phase 3 is also checking the wrong thing. In phase 3, only one new pod is created using the third image http, but deployment counts ReadyReplicas to 2. The immediate fix could be remove the sleep command in Pod spec with Pause image. But I think the behavior of deployment status update should be further investigated by sig-app.
Created a PR for fix pod spec issue and created in issue about deployment status update
For ReplicationController and Pod status test failures, they are failed to start the initial replicationController pod sandbox in 2 mins. Typically the sandbox is created a little bit after 2 mins. Example log This context deadline exceeded error message appears a couple of times, and then pod will be deleted by test due to timeout (2mins). In successful runs, the sandbox creation should succeeds and returns, and create container in following steps. The error message for volumes seems reasonable. After about 2 mins times out, volume will be unmounted, however pod lifecycle might still try to start pod for a very short period of time, and see volume is not longer exist. This race is known, and should resolve by itself quickly. I don't think this relates to the test failure. this seems relates to kind cluster containerD runtime. I could not reproduce in my e2e test cluster.
Removing SIG Node as this appears to be a SIG Storage issue. /remove-sig node
/reopen only fixed some of the flaking tests.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
-- is someone working on de-flaking the remaining tests?
my is merged yesterday. However, the test Deployment should run the lifecycle of a Deployment still failed twice after that. They have different cauess. failed to start within timeout (1min). test result: it shows that after volumes are mounted succesfully, pod is trying to start sandbox, but after 1 min, pod sandbox still not created successfully. This is the same issue I mentioned in comment here I think this issue only happens in Kind cluster, not gce cluster. Could you or someone else help investigate? failed during deployment patch After deployment is patched and scaled down to 1, the logs shows new replicaset is created, and one of the old pod is deleted. The readyReplica is reached to 1. But in my fix, I an additional check UnavailableReplicas should be 0. I think somehow this number is not reached to 0 during failed test. Deployment/ReplicaSet has many status including The reason I UnavailableReplicas is to avoid the problem mentioned in issue Now I am not certain what is the exception of the test, whether the status should be achievable within timeout (1min). Need sig-app help to take a look. Note: test only flaky in Kind cluster.
since all tests flakes emitted logs regarding service account token, let's focus on Pod & PodStatus tests, which help us to get rid of intricacy by the deployment/replicationcontroller take as an example, this test is trying to create a pod and wait for its running but timed out after two minutes. the major events in the build logs are the following: not a single watch event received by the watch client between Mar 26 05:52:14.644 and Mar 26 05:54:05.961 by in the kubelet logs: the token was successfully mounted the token errors were happened after the pod deletion because the volume is already removed from desired state: as mentioned in at this point, i don't think these flakes are caused by bound token. there is no flake happened in gce tests: https://k8s- then what caused the pod stuck in PENDING after 2 minutes? it took more than 2 minutes to create the sandbox. when the sandbox was created, the test already timed out.
I also observed sandbox took more than 2 mins to create This only happens in Kind cluster.
It's possible this is related to disk throttling in CI. We're looking to switch to local SSD for CI potentially but and I are stretched thin. Anyone can gain access to check the CI cluster, see: But if it's not that it's probably a kubernetes or containerd bug. It would be helpful to know if this worked anywhere else as well, the "GCE" jobs have various deltas like using special templated manifests vs kubeadm, the containerd version or use of docker for node runtime, the amount of resources we've reserved for the cluster ...
I don't know if you're aware, but there is a new that sets mounts kubelet root and container fs on local SSDs. It's one flag to enable, I don't know how difficult it will be to switch and test prow to it.
The issue is that terraform doesn't support this as an option yet, but I'm aware and very interested to try it
Was looking at the failure in "should run the lifecycle of a Deployment" in I noticed that the test logs just hang for the 1 minute timeout instead of printing out the status, I ~would expect the output to continue to print while the deployment is not ready. ~ Edit: I now see that this is a watch so will only respond to events. I also saw the pod doesn't respond to anything until right after test times out. At that point it looks like it starts to reconcile things but its to late and the delete is signal by the test. Is it possible something is getting hung? I looked through containerd logs and such but not seeing anything
following the same test flake in in the containerd logs: this took ~2m to complete. however, it finished pretty quick for other pods during the same period: seems not related to disk io throttling. what made this pod special? :(
BTW, it looks like the deployment lifecycle test uses a , and the pod lifecycle test uses a , which is shorter than the we use for other tests. Since these flakes mainly seem to happen on kind clusters, and not other types of clusters, should we increase the timeout to unblock others, and investigate the kind slowness separately? So far, the issues discovered in this bug do not appear to be due to anything introduced in 1.21.
I noticed the same thing about the inconsistency in pod startup timeouts while working on a few of the other timeout flakes: Increasing and having consistent timeouts would likely help with a subset of the flakes
can you open a PR aligning these timeouts to that we can unblock the release?
sure, as pointed out I don't think it will fix all these flakes but hopefully help with some.
We'll unblock the release and will continue investigating this.
Sounds good. I was looking at this last night and noticed many of these in this issue use The comments combined with the output we are seeing leads me to wonder if that is the function that should be used here: In we see toggles back and forth: The seems to be the only one concerned with the exact sequence of events (has ) . The and seem to only really care if the pod got modified and are back to the expected state. Maybe the list watcher isn't the right thing in those cases? Haven't dug deep enough but thought it might be worth asking
regarding the timeouts, looks like there are at least two other places these are set as well beyond what was listed in ()
Yes, the status should be achievable after Deployment has scaled down all old replicas, scaled up all new replicas, and the new replicas all become available. From where the patch failed: We need to also check that as well. Otherwise we might be looking at an old generation of status.
Hi, Bug Triage lead here, is this issue still blocking the 1.21 release? I saw above that you said you'd unblock the release and continue investigating, but I still see the release-blocker label.
cc: for a signal on what our CI feels. :)
FWIW this would be my decision tree for release-blocker or not: have we narrowed this down to a specific set of tests? no, still occurring randomly across tests: has the flake rate reduced since time of filing, and is it acceptably low? no -still release-blocking yes -not release-blocking yes, specific set of tests: do these tests exist in the previous release? no, new to this release: has the flake rate reduced since time of filing, and is it acceptably low? no -still release-blocking yes -not release-blocking yes, exist in previous release: is the flake rate ~same or lower than the previous release? no -still release-blocking (we've regressed) yes -not release-blocking A few things I would use for data shows clusters of test failures over time (not necessarily flakes); it also excludes test failures that didn't cluster with other test failures (cluster is defined by edit distance between normalized failure text) aggregates test flakes per job over the last week (job failed and passed against the same commit), sorted by total number of flakes; consistency is % of commits over last week that did not flake Messages in led me to believe it has been narrowed down to a specific set of tests? But I'm not sure. If it has been, for those tests, then... leads me to believe failure rate hasn't changed meaningfully filtering out flakes for the last week leads me to believe is one of the top 1.20 flakes, so if it's a top 1.21 flake that's no change I don't enjoy that a Conformance test is flaking this much. I'm tempted to demote, but if it was already in v1.20 like this, then maybe not. So, maybe not a release blocker, if indeed we have narrowed down to a specific set of tests. Anyway, just my 2c as a former CI Signal lead.
Let's go with removing the release-blocker label. I was leaning towards saying this is not a blocker yesterday (just a very annoying flake), but I wanted to see if would help reduce some of the flakiness (but looking at the data in Triage it doesn't seem like it). That being said, I think Aaron's suggestion to look at how these tests perform on 1.20 was a really good idea, and it looks like the ones this issue covers - , , and - are about as flaky on 1.20 as 1.21, which feels like a more concrete indicator that we shouldn't treat this as a release blocker.
Moving it to the next milestone for now as it is not a 1.21 release blocker anymore. /milestone v1.22
After increasing pod timeout to 5mins , I still see pod sandbox fail to create within 5mins in kind cluster.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
clearing milestone, looks like we carried this over from 1.21 /milestone clear
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close [1]:
Closing this issue. <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
i am still facing this issue on k8s 1.21.2 , kubectl get events Error: cannot find volume "kube-api-access-4rdxn" to mount into container "busybox" as per my understanding, it's fix was provided for k8s 1.21.2 ()
An FYI that we started hitting this occasionally in our production 1.21 1.cluster with ~500 nodes at Woven Planet (i.e. Toyota) after we just recently updated customer jobs to use OIDC-enabled service accounts. The volume sometimes fails to mount correctly while the container is being set up. Here are a few messages from my :
we are seeing the same for IBM Cloud Kubernetes Service version 1.21.

Discussion:
}) // TODO: add tests that cover deployment.Spec.MinReadySeconds once we solved clock-skew issues // See https://github.com/kubernetes/kubernetes/issues/29229 <ins> // Add UnavailableReplicas check because ReadyReplicas or UpdatedReplicas might not represent // the actual number of pods running successfully if some pods failed to start after update or patch. // See issue ##100192 </ins> /* Release: v1.20

"spec": map[string]interface{}{ "TerminationGracePeriodSeconds": &zero, "containers": [1]map[string]interface{}{{ <del> "name":    testDeploymentName, "image":   testDeploymentPatchImage, "command": []string{"/bin/sleep", "100000"}, </del> <ins> "name":  testDeploymentName, "image": testDeploymentPatchImage, </ins> }}, }, },

found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.ObjectMeta.Labels["test-deployment-static"] == "true" && deployment.Status.ReadyReplicas == testDeploymentMinimumReplicas && <ins> deployment.Status.UpdatedReplicas == testDeploymentMinimumReplicas && deployment.Status.UnavailableReplicas == 0 && </ins> deployment.Spec.Template.Spec.Containers[0].Image == testDeploymentPatchImage if !found { framework.Logf("observed Deployment %v in namespace %v with ReadyReplicas %v", deployment.ObjectMeta.Name, deployment.ObjectMeta.Namespace, deployment.Status.ReadyReplicas)

if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.ObjectMeta.Labels["test-deployment-static"] == "true" && <del> deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas </del> <ins> deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas && deployment.Status.UpdatedReplicas == testDeploymentDefaultReplicas && deployment.Status.UnavailableReplicas == 0 </ins> if !found { framework.Logf("observed Deployment %v in namespace %v with ReadyReplicas %v and labels %v", deployment.ObjectMeta.Name, deployment.ObjectMeta.Namespace, deployment.Status.ReadyReplicas, deployment.ObjectMeta.Labels) }

found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.ObjectMeta.Labels["test-deployment-static"] == "true" && deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas && <ins> deployment.Status.UpdatedReplicas == testDeploymentDefaultReplicas && deployment.Status.UnavailableReplicas == 0 && </ins> deployment.Spec.Template.Spec.Containers[0].Image == testDeploymentUpdateImage if !found { framework.Logf("observed Deployment %v in namespace %v with ReadyReplicas %v", deployment.ObjectMeta.Name, deployment.ObjectMeta.Namespace, deployment.Status.ReadyReplicas)

appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" "k8s.io/apimachinery/pkg/types" <ins> "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" watchtools "k8s.io/client-go/tools/watch" </ins> v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors"

} rsName := "test-rs" <ins> label := "test-rs=patched" labelMap := map[string]string{"test-rs": "patched"} </ins> replicas := int32(1) rsPatchReplicas := int32(3) rsPatchImage := imageutils.GetE2EImage(imageutils.Pause) <ins> w := &cache.ListWatch{ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { options.LabelSelector = label return f.ClientSet.AppsV1().ReplicaSets(ns).Watch(context.TODO(), options) }, } rsList, err := f.ClientSet.AppsV1().ReplicaSets("").List(context.TODO(), metav1.ListOptions{LabelSelector: label}) framework.ExpectNoError(err, "failed to list rsList") </ins> // Create a ReplicaSet rs := newRS(rsName, replicas, rsPodLabels, WebserverImageName, WebserverImage, nil) <del> _, err := c.AppsV1().ReplicaSets(ns).Create(context.TODO(), rs, metav1.CreateOptions{}) </del> <ins> _, err = c.AppsV1().ReplicaSets(ns).Create(context.TODO(), rs, metav1.CreateOptions{}) </ins> framework.ExpectNoError(err) // Verify that the required pods have come up.

ginkgo.By("patching the ReplicaSet") rsPatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ <del> "labels": map[string]string{"test-rs": "patched"}, </del> <ins> "labels": labelMap, </ins> }, "spec": map[string]interface{}{ "replicas": rsPatchReplicas,

"spec": map[string]interface{}{ "TerminationGracePeriodSeconds": &zero, "containers": [1]map[string]interface{}{{ <del> "name":    rsName, "image":   rsPatchImage, "command": []string{"/bin/sleep", "100000"}, </del> <ins> "name":  rsName, "image": rsPatchImage, </ins> }}, }, },

_, err = f.ClientSet.AppsV1().ReplicaSets(ns).Patch(context.TODO(), rsName, types.StrategicMergePatchType, []byte(rsPatch), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch ReplicaSet") <del> rs, err = c.AppsV1().ReplicaSets(ns).Get(context.TODO(), rsName, metav1.GetOptions{}) framework.ExpectNoError(err, "Failed to get replicaset resource: %v", err) framework.ExpectEqual(*(rs.Spec.Replicas), rsPatchReplicas, "replicaset should have 3 replicas") framework.ExpectEqual(rs.Spec.Template.Spec.Containers[0].Image, rsPatchImage, "replicaset not using rsPatchImage. Is using %v", rs.Spec.Template.Spec.Containers[0].Image) </del> <ins> ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, rsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if rset, ok := event.Object.(*appsv1.ReplicaSet); ok { found := rset.ObjectMeta.Name == rsName && rset.ObjectMeta.Labels["test-rs"] == "patched" && rset.Status.ReadyReplicas == rsPatchReplicas && rset.Status.AvailableReplicas == rsPatchReplicas && rset.Spec.Template.Spec.Containers[0].Image == rsPatchImage if !found { framework.Logf("observed ReplicaSet %v in namespace %v with ReadyReplicas %v, AvailableReplicas %v", rset.ObjectMeta.Name, rset.ObjectMeta.Namespace, rset.Status.ReadyReplicas, rset.Status.AvailableReplicas) } else { framework.Logf("observed Replicaset %v in namespace %v with ReadyReplicas %v found %v", rset.ObjectMeta.Name, rset.ObjectMeta.Namespace, rset.Status.ReadyReplicas, found) } return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", rs.Name, ns, rsPatchReplicas) </ins> }