Code change:
} } <del> func TestSetDefaultReplicationControllerImagePullPolicy(t *testing.T) { containersWithoutPullPolicy, _ := json.Marshal([]map[string]interface{}{ { "name":  "install", "image": "busybox:latest", }, }) </del> <ins> type InitContainerValidator func(got, expected *v1.Container) error </ins> <del> containersWithPullPolicy, _ := json.Marshal([]map[string]interface{}{ { "name":            "install", "imagePullPolicy": "IfNotPresent", }, }) </del> <ins> func TestSetDefaultReplicationControllerInitContainers(t *testing.T) { assertEnvFieldRef := func(got, expected *v1.Container) error { if len(got.Env) != len(expected.Env) { return fmt.Errorf("different number of env: got <%v>, expected <%v>", len(got.Env), len(expected.Env)) } for j := range got.Env { ge := &got.Env[j] ee := &expected.Env[j] if ge.Name != ee.Name { return fmt.Errorf("different name of env: got <%v>, expected <%v>", ge.Name, ee.Name) } if ge.ValueFrom.FieldRef.APIVersion != ee.ValueFrom.FieldRef.APIVersion { return fmt.Errorf("different api version of FieldRef <%v>: got <%v>, expected <%v>", ge.Name, ge.ValueFrom.FieldRef.APIVersion, ee.ValueFrom.FieldRef.APIVersion) } } return nil } assertImagePullPolicy := func(got, expected *v1.Container) error { if got.ImagePullPolicy != expected.ImagePullPolicy { return fmt.Errorf("different image pull poicy: got <%v>, expected <%v>", got.ImagePullPolicy, expected.ImagePullPolicy) } return nil } assertContainerPort := func(got, expected *v1.Container) error { if len(got.Ports) != len(expected.Ports) { return fmt.Errorf("different number of ports: got <%v>, expected <%v>", len(got.Ports), len(expected.Ports)) } for i := range got.Ports { gp := &got.Ports[i] ep := &expected.Ports[i] if gp.Name != ep.Name { return fmt.Errorf("different name of port: got <%v>, expected <%v>", gp.Name, ep.Name) } if gp.Protocol != ep.Protocol { return fmt.Errorf("different port protocol <%v>: got <%v>, expected <%v>", gp.Name, gp.Protocol, ep.Protocol) } } return nil } assertResource := func(got, expected *v1.Container) error { if len(got.Resources.Limits) != len(expected.Resources.Limits) { return fmt.Errorf("different number of resources.Limits: got <%v>, expected <%v>", len(got.Resources.Limits), (expected.Resources.Limits)) } for k, v := range got.Resources.Limits { if ev, found := expected.Resources.Limits[v1.ResourceName(k)]; !found { return fmt.Errorf("failed to find resource <%v> in expected resources.Limits.", k) } else { if ev.Value() != v.Value() { return fmt.Errorf("different resource.Limits: got <%v>, expected <%v>.", v.Value(), ev.Value()) } } } if len(got.Resources.Requests) != len(expected.Resources.Requests) { return fmt.Errorf("different number of resources.Requests: got <%v>, expected <%v>", len(got.Resources.Requests), (expected.Resources.Requests)) } for k, v := range got.Resources.Requests { if ev, found := expected.Resources.Requests[v1.ResourceName(k)]; !found { return fmt.Errorf("failed to find resource <%v> in expected resources.Requests.", k) } else { if ev.Value() != v.Value() { return fmt.Errorf("different resource.Requests: got <%v>, expected <%v>.", v.Value(), ev.Value()) } } } return nil } assertProb := func(got, expected *v1.Container) error { // Assert LivenessProbe if got.LivenessProbe.Handler.HTTPGet.Path != expected.LivenessProbe.Handler.HTTPGet.Path || got.LivenessProbe.Handler.HTTPGet.Scheme != expected.LivenessProbe.Handler.HTTPGet.Scheme || got.LivenessProbe.FailureThreshold != expected.LivenessProbe.FailureThreshold || got.LivenessProbe.SuccessThreshold != expected.LivenessProbe.SuccessThreshold || got.LivenessProbe.PeriodSeconds != expected.LivenessProbe.PeriodSeconds || got.LivenessProbe.TimeoutSeconds != expected.LivenessProbe.TimeoutSeconds { return fmt.Errorf("different LivenessProbe: got <%v>, expected <%v>", got.LivenessProbe, expected.LivenessProbe) } // Assert ReadinessProbe if got.ReadinessProbe.Handler.HTTPGet.Path != expected.ReadinessProbe.Handler.HTTPGet.Path || got.ReadinessProbe.Handler.HTTPGet.Scheme != expected.ReadinessProbe.Handler.HTTPGet.Scheme || got.ReadinessProbe.FailureThreshold != expected.ReadinessProbe.FailureThreshold || got.ReadinessProbe.SuccessThreshold != expected.ReadinessProbe.SuccessThreshold || got.ReadinessProbe.PeriodSeconds != expected.ReadinessProbe.PeriodSeconds || got.ReadinessProbe.TimeoutSeconds != expected.ReadinessProbe.TimeoutSeconds { return fmt.Errorf("different ReadinessProbe: got <%v>, expected <%v>", got.ReadinessProbe, expected.ReadinessProbe) } return nil } assertLifeCycle := func(got, expected *v1.Container) error { if got.Lifecycle.PostStart.HTTPGet.Path != expected.Lifecycle.PostStart.HTTPGet.Path || got.Lifecycle.PostStart.HTTPGet.Scheme != expected.Lifecycle.PostStart.HTTPGet.Scheme { return fmt.Errorf("different LifeCycle: got <%v>, expected <%v>", got.Lifecycle, expected.Lifecycle) } return nil } cpu, _ := resource.ParseQuantity("100Gi") mem, _ := resource.ParseQuantity("100Mi") </ins> tests := []struct { <del> rc               v1.ReplicationController expectPullPolicy v1.PullPolicy </del> <ins> name       string rc         v1.ReplicationController expected   []v1.Container validators []InitContainerValidator </ins> }{ { <ins> name: "imagePullIPolicy", rc: v1.ReplicationController{ Spec: v1.ReplicationControllerSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "pod.beta.kubernetes.io/init-containers": ` [ { "name": "install", "image": "busybox" } ]`, }, }, }, }, }, expected: []v1.Container{ { ImagePullPolicy: v1.PullAlways, }, }, validators: []InitContainerValidator{assertImagePullPolicy}, }, { name: "FieldRef", rc: v1.ReplicationController{ Spec: v1.ReplicationControllerSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "pod.beta.kubernetes.io/init-containers": ` [ { "name": "fun", "image": "alpine", "env": [ { "name": "MY_POD_IP", "valueFrom": { "fieldRef": { "apiVersion": "", "fieldPath": "status.podIP" } } } ] } ]`, }, }, }, }, }, expected: []v1.Container{ { Env: []v1.EnvVar{ { Name: "MY_POD_IP", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "status.podIP", }, }, }, }, }, }, validators: []InitContainerValidator{assertEnvFieldRef}, }, { name: "ContainerPort", </ins> rc: v1.ReplicationController{ Spec: v1.ReplicationControllerSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ <del> "pod.beta.kubernetes.io/init-containers": string(containersWithoutPullPolicy), </del> <ins> "pod.beta.kubernetes.io/init-containers": ` [ { "name": "fun", "image": "alpine", "ports": [ { "name": "default" } ] } ]`, </ins> }, }, }, }, }, <del> expectPullPolicy: v1.PullAlways, </del> <ins> expected: []v1.Container{ { Ports: []v1.ContainerPort{ { Name:     "default", Protocol: v1.ProtocolTCP, }, }, }, }, validators: []InitContainerValidator{assertContainerPort}, </ins> }, { <ins> name: "Resources", </ins> rc: v1.ReplicationController{ Spec: v1.ReplicationControllerSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ <del> "pod.beta.kubernetes.io/init-containers": string(containersWithPullPolicy), </del> <ins> "pod.beta.kubernetes.io/init-containers": ` [ { "name": "fun", "image": "alpine", "resources": { "limits": { "cpu": "100Gi", "memory": "100Mi" }, "requests": { "cpu": "100Gi", "memory": "100Mi" } } } ]`, </ins> }, }, }, }, }, <del> expectPullPolicy: v1.PullIfNotPresent, </del> <ins> expected: []v1.Container{ { Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ v1.ResourceCPU:    cpu, v1.ResourceMemory: mem, }, Requests: v1.ResourceList{ v1.ResourceCPU:    cpu, v1.ResourceMemory: mem, }, }, }, }, validators: []InitContainerValidator{assertResource}, }, { name: "Prob", rc: v1.ReplicationController{ Spec: v1.ReplicationControllerSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "pod.beta.kubernetes.io/init-containers": ` [ { "name": "fun", "image": "alpine", "livenessProbe": { "httpGet": { "host": "localhost" } }, "readinessProbe": { "httpGet": { "host": "localhost" } } } ]`, }, }, }, }, }, expected: []v1.Container{ { LivenessProbe: &v1.Probe{ Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path:   "/", Scheme: v1.URISchemeHTTP, }, }, TimeoutSeconds:   1, PeriodSeconds:    10, SuccessThreshold: 1, FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path:   "/", Scheme: v1.URISchemeHTTP, }, }, TimeoutSeconds:   1, PeriodSeconds:    10, SuccessThreshold: 1, FailureThreshold: 3, }, }, }, validators: []InitContainerValidator{assertProb}, }, { name: "LifeCycle", rc: v1.ReplicationController{ Spec: v1.ReplicationControllerSpec{ Template: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "pod.beta.kubernetes.io/init-containers": ` [ { "name": "fun", "image": "alpine", "lifecycle": { "postStart": { "httpGet": { "host": "localhost" } }, "preStop": { "httpGet": { "host": "localhost" } } } } ]`, }, }, }, }, }, expected: []v1.Container{ { Lifecycle: &v1.Lifecycle{ PostStart: &v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path:   "/", Scheme: v1.URISchemeHTTP, }, }, PreStop: &v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path:   "/", Scheme: v1.URISchemeHTTP, }, }, }, }, }, validators: []InitContainerValidator{assertLifeCycle}, </ins> }, } <ins> assertInitContainers := func(got, expected []v1.Container, validators []InitContainerValidator) error { if len(got) != len(expected) { return fmt.Errorf("different number of init container: got <%d>, expected <%d>", len(got), len(expected)) } for i := range got { g := &got[i] e := &expected[i] for _, validator := range validators { if err := validator(g, e); err != nil { return err } } } return nil } </ins> for _, test := range tests { rc := &test.rc obj2 := roundTrip(t, runtime.Object(rc))

Related discussion:
text: What happened: kubelet doesn't seem to be able to manage containers started with CRI-O any more I'm testing kubelet + CRI-O on an Ubuntu 18.04 server installation, inside a qemu/kvm VM managed by Vagrant (see ), in order to be able to test k8s + KubeVirt + KataContainers, which are all compatible with CRI-O kubelet will obsess on starting new containers, which looks like and endless process, and the cluster never gets ready. For instance here are some syslogs : What you expected to happen: It used to work (see asciinema recording at ) a few days ago, but I fear some recent changes broke it. How to reproduce it (as minimally and precisely as possible): Testing with vagrant up --provider=libvirt on Linux with a clone of that commit: vagrant ssh into the VM and ps -edf f or tail -f /var/log/syslog would do... Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: qemu/kvm VM +OS (e.g: ): Ubuntu 18.04.2 LTS Kernel (e.g. ): Linux kubernetes-vagrant-01 4.15.0-48-generic -Ubuntu SMP Wed Apr 3 08:28:49 UTC 2019 x8664 x8664 x8664 GNU/Linux Install tools: kubeadm as in Network plugin and version (if this is a network-related bug): Others:
/sig node
I can confirm this. If I start cri-o from a systemd service it fails with the above mentioned error message. Starting crio with without systemd works just fine. systemd file:
Thanks for your detailed report :) Just want to do some quick version confirming - you've been using k8s 1.14.1 both when it did and did not work? Any other version changes?
Not sure for before, when it used to work, but according to the asciinema recording mentioned above (), it seems the same. However, not sure for CRI-O, underlying OS stuff (systemd, kernel, etc.). Sorry.
I was looking for some information for days. I have exactly the same issue when I use podman + systemd on 18.04.2.