Machine configuration OpenShift Container Platform 4.18 Managing and applying configuration and updates of the base operating system and container runtimes in OpenShift Container Platform Red Hat OpenShift Documentation Team

Commands:
oc get mcp worker
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-404caf3180818d8ac1f50c32f14b57c3 False True True 2 1 1 1 5h51m
oc describe mcp worker
Last Transition Time: 2021-12-20T18:54:00Z Message: Node ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4 is reporting: "content mismatch for file \"/etc/mco-test-file\"" 1 Reason: 1 nodes are reporting degraded status on sync Status: True Type: NodeDegraded 2
oc describe node/ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4
Annotations: cloud.network.openshift.io/egress-ipconfig: [{"interface":"nic0","ifaddr":{"ipv4":"10.0.128.0/17"},"capacity":{"ip":10}}] csi.volume.kubernetes.io/nodeid: {"pd.csi.storage.gke.io":"projects/openshift-gce-devel-ci/zones/us-central1-a/instances/ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4"} machine.openshift.io/machine: openshift-machine-api/ci-ln-j4h8nkb-72292-pxqxz-worker-a-fjks4 machineconfiguration.openshift.io/controlPlaneTopology: HighlyAvailable machineconfiguration.openshift.io/currentConfig: rendered-worker-67bd55d0b02b0f659aef33680693a9f9 machineconfiguration.openshift.io/desiredConfig: rendered-worker-67bd55d0b02b0f659aef33680693a9f9 machineconfiguration.openshift.io/reason: content mismatch for file "/etc/mco-test-file" 1 machineconfiguration.openshift.io/state: Degraded 2
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-06c9c4... True False False 3 3 3 0 4h42m worker rendered-worker-f4b64... False True False 3 2 2 0 4h42m
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-06c9c4... True False False 3 3 3 0 4h42m worker rendered-worker-c1b41a... False True False 3 2 3 0 4h42m
oc describe mcp worker
Degraded Machine Count: 0 Machine Count: 3 Observed Generation: 2 Ready Machine Count: 3 Unavailable Machine Count: 0 Updated Machine Count: 3 Events: <none>
Degraded Machine Count: 0 Machine Count: 3 Observed Generation: 2 Ready Machine Count: 2 Unavailable Machine Count: 1 Updated Machine Count: 3
oc get machineconfigs
NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m 00-worker 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m 01-master-container-runtime 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m 01-master-kubelet 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m rendered-master-dde... 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m rendered-worker-fde... 2c9371fbb673b97a6fe8b1c52... 3.2.0 5h18m
oc describe machineconfigs 01-master-kubelet
Name: 01-master-kubelet Spec: Config: Ignition: Version: 3.2.0 Storage: Files: Contents: Source: data:, Mode: 420 Overwrite: true Path: /etc/kubernetes/cloud.conf Contents: Source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fkubelet-ca.crt%0A%20%20anonymous Mode: 420 Overwrite: true Path: /etc/kubernetes/kubelet.conf Systemd: Units: Contents: [Unit] Description=Kubernetes Kubelet Wants=rpc-statd.service network-online.target crio.service After=network-online.target crio.service ExecStart=/usr/bin/hyperkube kubelet --config=/etc/kubernetes/kubelet.conf \
oc delete -f ./myconfig.yaml
oc get machineconfig
rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 c00e2c941bc6e236b50e0bf3988e6c790cf2bbb2 3.4.0 6d15h rendered-master-a386c2d1550b927d274054124f58be68 c00e2c941bc6e236b50e0bf3988e6c790cf2bbb2 3.4.0 7m26s rendered-worker-01f27f752eb84eba917450e43636b210 c00e2c941bc6e236b50e0bf3988e6c790cf2bbb2 3.4.0 6d15h 1 rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 c00e2c941bc6e236b50e0bf3988e6c790cf2bbb2 3.4.0 7m26s 2
oc get machineconfignodes
NAME POOLNAME DESIREDCONFIG CURRENTCONFIG UPDATED ci-ln-ds73n5t-72292-9xsm9-master-0 master rendered-master-a386c2d1550b927d274054124f58be68 rendered-master-a386c2d1550b927d274054124f58be68 True ci-ln-ds73n5t-72292-9xsm9-master-1 master rendered-master-a386c2d1550b927d274054124f58be68 rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 False ci-ln-ds73n5t-72292-9xsm9-master-2 master rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 True ci-ln-ds73n5t-72292-9xsm9-worker-a-2d8tz worker rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 True 1 ci-ln-ds73n5t-72292-9xsm9-worker-b-gw5sd worker rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 rendered-worker-01f27f752eb84eba917450e43636b210 False 2 ci-ln-ds73n5t-72292-9xsm9-worker-c-t227w worker rendered-worker-01f27f752eb84eba917450e43636b210 rendered-worker-01f27f752eb84eba917450e43636b210 True 3
oc get machineconfignodes -o wide
oc get machineconfignode -o wide NAME POOLNAME DESIREDCONFIG CURRENTCONFIG UPDATED UPDATEPREPARED UPDATEEXECUTED UPDATEPOSTACTIONCOMPLETE UPDATECOMPLETE RESUMED UPDATECOMPATIBLE UPDATEDFILESANDOS CORDONEDNODE DRAINEDNODE REBOOTEDNODE RELOADEDCRIO UNCORDONEDNODE ci-ln-ds73n5t-72292-9xsm9-master-0 master rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 True False False False False False False False False False False False False ci-ln-ds73n5t-72292-9xsm9-master-1 master rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 True False False False False False False False False False False False False ci-ln-ds73n5t-72292-9xsm9-master-2 master rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 rendered-master-23cf200e4ee97daa6e39fdce24c9fb67 True False False False False False False False False False False False False ci-ln-ds73n5t-72292-9xsm9-worker-a-2d8tz worker rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 True False False False False False False False False False False False False ci-ln-ds73n5t-72292-9xsm9-worker-b-gw5sd worker rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 rendered-worker-01f27f752eb84eba917450e43636b210 False True True Unknown False False True True True True Unknown False False ci-ln-ds73n5t-72292-9xsm9-worker-c-t227w worker rendered-worker-01f27f752eb84eba917450e43636b210 rendered-worker-01f27f752eb84eba917450e43636b210 True False False False False False False False False False False False False
oc describe machineconfignode/<machine_config_node_name> -o yaml
Name: <machine_config_node_name> 1 Namespace: Labels: <none> Annotations: <none> API Version: machineconfiguration.openshift.io/v1alpha1 Kind: MachineConfigNode Metadata: Creation Timestamp: 2023-10-17T13:08:58Z Generation: 1 Resource Version: 49443 UID: 4bd758ab-2187-413c-ac42-882e61761b1d Spec: Node Ref: Name: <node_name> Pool: Name: worker ConfigVersion: Desired: rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 2 Status: Conditions: Last Transition Time: 2025-01-14T17:01:16Z Message: Node ci-ln-ds73n5t-72292-9xsm9-worker-b-gw5sd needs an update Reason: Updated Status: False Type: Updated Last Transition Time: 2025-01-14T17:01:18Z Message: Update is Compatible. Reason: UpdateCompatible Status: True Type: UpdatePrepared Last Transition Time: 2025-01-14T17:04:08Z Message: Updated the Files and OS on disk as a part of the in progress phase Reason: AppliedFilesAndOS Status: True Type: UpdateExecuted Last Transition Time: 2025-01-14T17:04:08Z Message: Node will reboot into config rendered-worker-db01b33f959e5645a721da50a6db1fbb Reason: RebootedNode Status: Unknown Type: UpdatePostActionComplete Last Transition Time: 2025-01-14T16:04:27Z Message: Action during update to rendered-worker-f351f6947f15cd0380514f4b1c89f8f2: UnCordoned Node as part of completing upgrade phase Reason: Uncordoned Status: False Type: UpdateComplete Last Transition Time: 2025-01-14T16:04:27Z Message: Action during update to rendered-worker-f351f6947f15cd0380514f4b1c89f8f2: In desired config rendered-worker-01f27f752eb84eba917450e43636b210. Resumed normal operations. Reason: Resumed Status: False Type: Resumed Last Transition Time: 2025-01-14T17:01:18Z Message: Update Compatible. Post Cfg Actions []: Drain Required: true Reason: UpdatePreparedUpdateCompatible Status: True Type: UpdateCompatible Last Transition Time: 2025-01-14T17:03:57Z Message: Drained node. The drain is complete as the desired drainer matches current drainer: drain-rendered-worker-db01b33f959e5645a721da50a6db1fbb Reason: UpdateExecutedDrained Status: True Type: Drained Last Transition Time: 2025-01-14T17:04:08Z Message: Applied files and new OS config to node. OS did not need an update. SSH Keys did not need an update Reason: UpdateExecutedAppliedFilesAndOS Status: True Type: AppliedFilesAndOS Last Transition Time: 2025-01-14T17:01:23Z Message: Cordoned node. The node is reporting Unschedulable = true Reason: UpdateExecutedCordoned Status: True Type: Cordoned Last Transition Time: 2025-01-14T17:04:08Z Message: Upgrade requires a reboot. Currently doing this as the post update action. Reason: UpdatePostActionCompleteRebootedNode Status: Unknown Type: RebootedNode Last Transition Time: 2025-01-14T15:30:57Z Message: This node has not yet entered the ReloadedCRIO phase Reason: NotYetOccured Status: False Type: ReloadedCRIO Last Transition Time: 2025-01-14T16:04:27Z Message: Action during update to rendered-worker-f351f6947f15cd0380514f4b1c89f8f2: UnCordoned node. The node is reporting Unschedulable = false Reason: UpdateCompleteUncordoned Status: False Type: Uncordoned Last Transition Time: 2025-01-14T16:04:07Z Message: All is good Reason: AsExpected Status: False Type: PinnedImageSetsDegraded Last Transition Time: 2025-01-14T16:04:07Z Message: All pinned image sets complete Reason: AsExpected Status: False Type: PinnedImageSetsProgressing Config Version: Current: rendered-worker-01f27f752eb84eba917450e43636b210 3 Desired: rendered-worker-f351f6947f15cd0380514f4b1c89f8f2 Observed Generation: 6
oc edit featuregate cluster
apiVersion: config.openshift.io/v1 kind: FeatureGate metadata: name: cluster spec: featureSet: TechPreviewNoUpgrade
oc get machineconfignodes
NAME POOLNAME DESIREDCONFIG CURRENTCONFIG UPDATED ci-ln-mdb23yt-72292-kzdsg-master-0 master rendered-master-f21b093d20f68a7c06f922ed3ea5fbc8 rendered-master-1abc053eec29e6c945670f39d6dc8afa False ci-ln-mdb23yt-72292-kzdsg-master-1 master rendered-master-1abc053eec29e6c945670f39d6dc8afa rendered-master-1abc053eec29e6c945670f39d6dc8afa True ci-ln-mdb23yt-72292-kzdsg-master-2 master rendered-master-1abc053eec29e6c945670f39d6dc8afa rendered-master-1abc053eec29e6c945670f39d6dc8afa True ci-ln-mdb23yt-72292-kzdsg-worker-a-gfqjr worker rendered-worker-d0130cd74e9e576d7ba78ce166272bfb rendered-worker-8f61bf839898a4487c3b5263a430e94a False ci-ln-mdb23yt-72292-kzdsg-worker-b-gknq4 worker rendered-worker-8f61bf839898a4487c3b5263a430e94a rendered-worker-8f61bf839898a4487c3b5263a430e94a True ci-ln-mdb23yt-72292-kzdsg-worker-c-mffrx worker rendered-worker-8f61bf839898a4487c3b5263a430e94a rendered-worker-8f61bf839898a4487c3b5263a430e94a True
oc get machineconfignodes -o wide
NAME POOLNAME DESIREDCONFIG CURRENTCONFIG UPDATED UPDATEPREPARED UPDATEEXECUTED UPDATEPOSTACTIONCOMPLETE UPDATECOMPLETE RESUMED UPDATECOMPATIBLE UPDATEDFILESANDOS CORDONEDNODE DRAINEDNODE REBOOTEDNODE RELOADEDCRIO UNCORDONEDNODE ci-ln-g6dr34b-72292-g9btv-master-0 master rendered-master-d4e122320b351cdbe1df59ddb63ddcfc rendered-master-6f2064fcb36d2a914de5b0c660dc49ff False True Unknown False False False True Unknown False False False False False ci-ln-g6dr34b-72292-g9btv-master-1 master rendered-master-6f2064fcb36d2a914de5b0c660dc49ff rendered-master-6f2064fcb36d2a914de5b0c660dc49ff True False False False False False False False False False False False False ci-ln-g6dr34b-72292-g9btv-master-2 master rendered-master-6f2064fcb36d2a914de5b0c660dc49ff rendered-master-6f2064fcb36d2a914de5b0c660dc49ff True False False False False False False False False False False False False ci-ln-g6dr34b-72292-g9btv-worker-a-sjh5r worker rendered-worker-671b88c8c569fa3f60dc1a27cf9c91f2 rendered-worker-d5534cb730e5e108905fc285c2a42b6c False True Unknown False False False True Unknown False False False False False ci-ln-g6dr34b-72292-g9btv-worker-b-xthbz worker rendered-worker-d5534cb730e5e108905fc285c2a42b6c rendered-worker-d5534cb730e5e108905fc285c2a42b6c True False False False False False False False False False False False False ci-ln-g6dr34b-72292-g9btv-worker-c-gnpd6 worker rendered-worker-d5534cb730e5e108905fc285c2a42b6c rendered-worker-d5534cb730e5e108905fc285c2a42b6c True False False False False False False False False False False False False
oc get machineconfignodes USD(oc get machineconfignodes -o json | jq -r '.items[]|select(.spec.pool.name=="<pool_name>")|.metadata.name') 1
NAME POOLNAME DESIREDCONFIG CURRENTCONFIG UPDATED ci-ln-g6dr34b-72292-g9btv-worker-a-sjh5r worker rendered-worker-d5534cb730e5e108905fc285c2a42b6c rendered-worker-d5534cb730e5e108905fc285c2a42b6c True ci-ln-g6dr34b-72292-g9btv-worker-b-xthbz worker rendered-worker-d5534cb730e5e108905fc285c2a42b6c rendered-worker-faf6b50218a8bbce21f1370866283de5 False ci-ln-g6dr34b-72292-g9btv-worker-c-gnpd6 worker rendered-worker-faf6b50218a8bbce21f1370866283de5 rendered-worker-faf6b50218a8bbce21f1370866283de5 True
oc describe machineconfignode/<node_name>
Name: <node_name> Namespace: Labels: <none> Annotations: <none> API Version: machineconfiguration.openshift.io/v1alpha1 Kind: MachineConfigNode Metadata: Creation Timestamp: 2023-10-17T13:08:58Z Generation: 1 Resource Version: 49443 UID: 4bd758ab-2187-413c-ac42-882e61761b1d Spec: Node Ref: Name: <node_name> Pool: Name: master ConfigVersion: Desired: rendered-worker-823ff8dc2b33bf444709ed7cd2b9855b Status: Message: Drained node. The drain is complete as the desired drainer matches current drainer: drain-rendered-worker-01f27f752eb84eba917450e43636b210 Reason: UpdateExecutedDrained Status: True Type: Drained Last Transition Time: 2025-01-14T15:45:55Z Config Version: Current: rendered-master-8110974a5cea69dff5b263237b58abd8 Desired: rendered-worker-823ff8dc2b33bf444709ed7cd2b9855b Observed Generation: 6
oc get controllerconfig/machine-config-controller -o yaml | yq -y '.status.controllerCertificates'
- bundleFile: KubeAPIServerServingCAData notAfter: '2034-10-23T13:13:02Z' notBefore: '2024-10-25T13:13:02Z' signer: CN=admin-kubeconfig-signer,OU=openshift subject: CN=admin-kubeconfig-signer,OU=openshift - bundleFile: KubeAPIServerServingCAData notAfter: '2024-10-26T13:13:05Z' notBefore: '2024-10-25T13:27:14Z' signer: CN=kubelet-signer,OU=openshift subject: CN=kube-csr-signer_@1729862835 - bundleFile: KubeAPIServerServingCAData notAfter: '2024-10-26T13:13:05Z' notBefore: '2024-10-25T13:13:05Z' signer: CN=kubelet-signer,OU=openshift subject: CN=kubelet-signer,OU=openshift
oc get mcp master -o yaml | yq -y '.status.certExpirys'
- bundle: KubeAPIServerServingCAData expiry: '2034-10-23T13:13:02Z' subject: CN=admin-kubeconfig-signer,OU=openshift - bundle: KubeAPIServerServingCAData expiry: '2024-10-26T13:13:05Z' subject: CN=kube-csr-signer_@1729862835 - bundle: KubeAPIServerServingCAData expiry: '2024-10-26T13:13:05Z' subject: CN=kubelet-signer,OU=openshift - bundle: KubeAPIServerServingCAData expiry: '2025-10-25T13:13:05Z' subject: CN=kube-apiserver-to-kubelet-signer,OU=openshift
oc debug node/<node_name>
sh-5.1# chroot /host
sh-5.1# ls /etc/docker/certs.d
image-registry.openshift-image-registry.svc.cluster.local:5000 image-registry.openshift-image-registry.svc:5000
variant: openshift version: 4.18.0 metadata: name: 99-worker-chrony 1 labels: machineconfiguration.openshift.io/role: worker 2 storage: files: - path: /etc/chrony.conf mode: 0644 3 overwrite: true contents: inline: | pool 0.rhel.pool.ntp.org iburst 4 driftfile /var/lib/chrony/drift makestep 1.0 3 rtcsync logdir /var/log/chrony
butane 99-worker-chrony.bu -o 99-worker-chrony.yaml
oc apply -f ./99-worker-chrony.yaml
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: <node_role> 1 name: disable-chronyd spec: config: ignition: version: 3.2.0 systemd: units: - contents: | [Unit] Description=NTP client/server Documentation=man:chronyd(8) man:chrony.conf(5) After=ntpdate.service sntp.service ntpd.service Conflicts=ntpd.service systemd-timesyncd.service ConditionCapability=CAP_SYS_TIME [Service] Type=forking PIDFile=/run/chrony/chronyd.pid EnvironmentFile=-/etc/sysconfig/chronyd ExecStart=/usr/sbin/chronyd USDOPTIONS ExecStartPost=/usr/libexec/chrony-helper update-daemon PrivateTmp=yes ProtectHome=yes ProtectSystem=full [Install] WantedBy=multi-user.target enabled: false name: "chronyd.service" - name: "kubelet-dependencies.target" contents: | [Unit] Description=Dependencies necessary to run kubelet Documentation=https://github.com/openshift/machine-config-operator/ Requires=basic.target network-online.target Wants=NetworkManager-wait-online.service crio-wipe.service Wants=rpc-statd.service
oc create -f disable-chronyd.yaml
oc get MachineConfig
NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 00-worker 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-ssh 3.2.0 40m 99-worker-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-worker-ssh 3.2.0 40m rendered-master-23e785de7587df95a4b517e0647e5ab7 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m rendered-worker-5d596d9293ca3ea80c896a1191735bb1 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker 1 name: 05-worker-kernelarg-selinuxpermissive 2 spec: kernelArguments: - enforcing=0 3
oc create -f 05-worker-kernelarg-selinuxpermissive.yaml
oc get MachineConfig
NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 00-worker 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 05-worker-kernelarg-selinuxpermissive 3.2.0 105s 99-master-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-ssh 3.2.0 40m 99-worker-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-worker-ssh 3.2.0 40m rendered-master-23e785de7587df95a4b517e0647e5ab7 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m rendered-worker-5d596d9293ca3ea80c896a1191735bb1 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-136-161.ec2.internal Ready worker 28m v1.31.3 ip-10-0-136-243.ec2.internal Ready master 34m v1.31.3 ip-10-0-141-105.ec2.internal Ready,SchedulingDisabled worker 28m v1.31.3 ip-10-0-142-249.ec2.internal Ready master 34m v1.31.3 ip-10-0-153-11.ec2.internal Ready worker 28m v1.31.3 ip-10-0-153-150.ec2.internal Ready master 34m v1.31.3
oc debug node/ip-10-0-141-105.ec2.internal
Starting pod/ip-10-0-141-105ec2internal-debug To use host binaries, run `chroot /host` sh-4.2# cat /host/proc/cmdline BOOT_IMAGE=/ostree/rhcos-... console=tty0 console=ttyS0,115200n8 rootflags=defaults,prjquota rw root=UUID=fd0... ostree=/ostree/boot.0/rhcos/16 coreos.oem.id=qemu coreos.oem.id=ec2 ignition.platform.id=ec2 enforcing=0 sh-4.2# exit
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: "master" name: 99-master-kargs-mpath spec: kernelArguments: - 'rd.multipath=default' - 'root=/dev/disk/by-label/dm-mpath-root'
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: "worker" name: 99-worker-kargs-mpath spec: kernelArguments: - 'rd.multipath=default' - 'root=/dev/disk/by-label/dm-mpath-root'
oc create -f ./99-worker-kargs-mpath.yaml
oc get MachineConfig
NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 00-worker 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-master-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-ssh 3.2.0 40m 99-worker-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-worker-kargs-mpath 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 105s 99-worker-ssh 3.2.0 40m rendered-master-23e785de7587df95a4b517e0647e5ab7 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m rendered-worker-5d596d9293ca3ea80c896a1191735bb1 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-136-161.ec2.internal Ready worker 28m v1.31.3 ip-10-0-136-243.ec2.internal Ready master 34m v1.31.3 ip-10-0-141-105.ec2.internal Ready,SchedulingDisabled worker 28m v1.31.3 ip-10-0-142-249.ec2.internal Ready master 34m v1.31.3 ip-10-0-153-11.ec2.internal Ready worker 28m v1.31.3 ip-10-0-153-150.ec2.internal Ready master 34m v1.31.3
oc debug node/ip-10-0-141-105.ec2.internal
Starting pod/ip-10-0-141-105ec2internal-debug To use host binaries, run `chroot /host` sh-4.2# cat /host/proc/cmdline rd.multipath=default root=/dev/disk/by-label/dm-mpath-root sh-4.2# exit
cat << EOF > 99-worker-realtime.yaml apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: "worker" name: 99-worker-realtime spec: kernelType: realtime EOF
oc create -f 99-worker-realtime.yaml
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-143-147.us-east-2.compute.internal Ready worker 103m v1.31.3 ip-10-0-146-92.us-east-2.compute.internal Ready worker 101m v1.31.3 ip-10-0-169-2.us-east-2.compute.internal Ready worker 102m v1.31.3
oc debug node/ip-10-0-143-147.us-east-2.compute.internal
Starting pod/ip-10-0-143-147us-east-2computeinternal-debug To use host binaries, run `chroot /host` sh-4.4# uname -a Linux <worker_node> 4.18.0-147.3.1.rt24.96.el8_1.x86_64 #1 SMP PREEMPT RT Wed Nov 27 18:29:55 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
oc delete -f 99-worker-realtime.yaml
variant: openshift version: 4.18.0 metadata: name: 40-worker-custom-journald labels: machineconfiguration.openshift.io/role: worker storage: files: - path: /etc/systemd/journald.conf mode: 0644 overwrite: true contents: inline: | # Disable rate limiting RateLimitInterval=1s RateLimitBurst=10000 Storage=volatile Compress=no MaxRetentionSec=30s
butane 40-worker-custom-journald.bu -o 40-worker-custom-journald.yaml
oc apply -f 40-worker-custom-journald.yaml
oc get machineconfigpool NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-35 True False False 3 3 3 0 34m worker rendered-worker-d8 False True False 3 1 1 0 34m
oc get node | grep worker ip-10-0-0-1.us-east-2.compute.internal Ready worker 39m v0.0.0-master+USDFormat:%hUSD oc debug node/ip-10-0-0-1.us-east-2.compute.internal Starting pod/ip-10-0-141-142us-east-2computeinternal-debug sh-4.2# chroot /host sh-4.4# cat /etc/systemd/journald.conf Disable rate limiting RateLimitInterval=1s RateLimitBurst=10000 Storage=volatile Compress=no MaxRetentionSec=30s sh-4.4# exit
cat << EOF > 80-extensions.yaml apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: 80-worker-extensions spec: config: ignition: version: 3.2.0 extensions: - usbguard EOF
oc create -f 80-extensions.yaml
oc get machineconfig 80-worker-extensions
NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 80-worker-extensions 3.2.0 57s
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-35 True False False 3 3 3 0 34m worker rendered-worker-d8 False True False 3 1 1 0 34m
oc get node | grep worker
NAME STATUS ROLES AGE VERSION ip-10-0-169-2.us-east-2.compute.internal Ready worker 102m v1.31.3
oc debug node/ip-10-0-169-2.us-east-2.compute.internal
To use host binaries, run `chroot /host` sh-4.4# chroot /host sh-4.4# rpm -q usbguard usbguard-0.7.4-4.el8.x86_64.rpm
variant: openshift version: 4.18.0 metadata: labels: machineconfiguration.openshift.io/role: worker name: 98-worker-firmware-blob storage: files: - path: /var/lib/firmware/<package_name> 1 contents: local: <package_name> 2 mode: 0644 3 openshift: kernel_arguments: - 'firmware_class.path=/var/lib/firmware' 4
butane 98-worker-firmware-blob.bu -o 98-worker-firmware-blob.yaml --files-dir <directory_including_package_name>
oc apply -f 98-worker-firmware-blob.yaml
mkpasswd -m SHA-512 testpass
USD6USDCBZwA6s6AVFOtiZeUSDaUKDWpthhJEyR3nnhM02NM1sKCpHn9XN.NPrJNQ3HYewioaorpwL3mKGLxvW0AOb4pJxqoqP4nFX77y0p00.8.
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: set-core-user-password spec: config: ignition: version: 3.2.0 passwd: users: - name: core 1 passwordHash: <password> 2
oc create -f <file-name>.yaml
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-d686a3ffc8fdec47280afec446fce8dd True False False 3 3 3 0 64m worker rendered-worker-4605605a5b1f9de1d061e9d350f251e5 False True False 3 0 0 0 64m
oc debug node/<node_name>
sh-4.4# chroot /host
core:USD6USD2sE/010goDuRSxxvUSDo18K52wor.wIwZp:19418:0:99999:7:::
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: logLevel: Normal managementState: Managed operatorLogLevel: Normal status: nodeDisruptionPolicyStatus: clusterPolicies: files: - actions: - type: None path: /etc/mco/internal-registry-pull-secret.json - actions: - type: None path: /var/lib/kubelet/config.json - actions: - reload: serviceName: crio.service type: Reload path: /etc/machine-config-daemon/no-reboot/containers-gpg.pub - actions: - reload: serviceName: crio.service type: Reload path: /etc/containers/policy.json - actions: - type: Special path: /etc/containers/registries.conf - actions: - reload: serviceName: crio.service type: Reload path: /etc/containers/registries.d - actions: - type: None path: /etc/nmstate/openshift - actions: - restart: serviceName: coreos-update-ca-trust.service type: Restart - restart: serviceName: crio.service type: Restart path: /etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt sshkey: actions: - type: None observedGeneration: 9
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: sshkey: actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: files: - actions: - restart: serviceName: chronyd.service type: Restart path: /etc/chrony.conf - actions: - type: None path: /var/run
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: units: - name: auditd.service actions: - type: Drain - type: Reload reload: serviceName: crio.service - type: DaemonReload - type: Restart restart: serviceName: crio.service
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: files: - actions: - type: None path: /etc/containers/registries.conf
oc edit MachineConfiguration cluster -n openshift-machine-config-operator
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster spec: nodeDisruptionPolicy: 1 files: 2 - actions: 3 - restart: 4 serviceName: chronyd.service 5 type: Restart path: /etc/chrony.conf 6 sshkey: 7 actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart units: 8 - actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart name: test.service
oc get MachineConfiguration/cluster -o yaml
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: labels: machineconfiguration.openshift.io/role: worker name: cluster status: nodeDisruptionPolicyStatus: 1 clusterPolicies: files: - actions: - restart: serviceName: chronyd.service type: Restart path: /etc/chrony.conf sshkey: actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart units: - actions: - type: Drain - reload: serviceName: crio.service type: Reload - type: DaemonReload - restart: serviceName: crio.service type: Restart name: test.se
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: name: infra spec: machineConfigSelector: matchExpressions: - {key: machineconfiguration.openshift.io/role, operator: In, values: [worker,infra]}
oc get kubeletconfig
NAME AGE set-kubelet-config 15m
oc get mc | grep kubelet
99-worker-generated-kubelet-1 b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 26m
oc describe machineconfigpool <name>
oc describe machineconfigpool worker
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: creationTimestamp: 2019-02-08T14:52:39Z generation: 1 labels: custom-kubelet: set-kubelet-config 1
oc label machineconfigpool worker custom-kubelet=set-kubelet-config
oc get machineconfig
oc describe node <node_name>
oc describe node ci-ln-5grqprb-f76d1-ncnqq-worker-a-mdv94
Allocatable: attachable-volumes-aws-ebs: 25 cpu: 3500m hugepages-1Gi: 0 hugepages-2Mi: 0 memory: 15341844Ki pods: 250
apiVersion: machineconfiguration.openshift.io/v1 kind: KubeletConfig metadata: name: set-kubelet-config spec: machineConfigPoolSelector: matchLabels: custom-kubelet: set-kubelet-config 1 kubeletConfig: 2 podPidsLimit: 8192 containerLogMaxSize: 50Mi maxPods: 500
apiVersion: machineconfiguration.openshift.io/v1 kind: KubeletConfig metadata: name: set-kubelet-config spec: machineConfigPoolSelector: matchLabels: custom-kubelet: set-kubelet-config kubeletConfig: maxPods: <pod_count> kubeAPIBurst: <burst_rate> kubeAPIQPS: <QPS>
oc label machineconfigpool worker custom-kubelet=set-kubelet-config
oc create -f change-maxPods-cr.yaml
oc get kubeletconfig
NAME AGE set-kubelet-config 15m
oc describe node <node_name>
Allocatable: attachable-volumes-gce-pd: 127 cpu: 3500m ephemeral-storage: 123201474766 hugepages-1Gi: 0 hugepages-2Mi: 0 memory: 14225400Ki pods: 500 1
oc get kubeletconfigs set-kubelet-config -o yaml
spec: kubeletConfig: containerLogMaxSize: 50Mi maxPods: 500 podPidsLimit: 8192 machineConfigPoolSelector: matchLabels: custom-kubelet: set-kubelet-config status: conditions: - lastTransitionTime: "2021-06-30T17:04:07Z" message: Success status: "True" type: Success
oc get ctrcfg
NAME AGE ctr-overlay 15m ctr-level 5m45s
oc get mc | grep container
01-master-container-runtime b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 57m 01-worker-container-runtime b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 57m 99-worker-generated-containerruntime b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 26m 99-worker-generated-containerruntime-1 b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 17m 99-worker-generated-containerruntime-2 b5c5119de007945b6fe6fb215db3b8e2ceb12511 3.2.0 7m26s
apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: overlay-size spec: machineConfigPoolSelector: matchLabels: pools.operator.machineconfiguration.openshift.io/worker: '' 1 containerRuntimeConfig: logLevel: debug 2 overlaySize: 8G 3 defaultRuntime: "runc" 4
apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: overlay-size spec: machineConfigPoolSelector: matchLabels: pools.operator.machineconfiguration.openshift.io/worker: '' 1 containerRuntimeConfig: 2 logLevel: debug overlaySize: 8G defaultRuntime: "runc"
oc create -f <file_name>.yaml
oc get ContainerRuntimeConfig
NAME AGE overlay-size 3m19s
oc get machineconfigs | grep containerrun
99-worker-generated-containerruntime 2c9371fbb673b97a6fe8b1c52691999ed3a1bfc2 3.2.0 31s
oc get mcp worker
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-169 False True False 3 1 1 0 9h
oc debug node/<node_name>
sh-4.4# chroot /host
sh-4.4# crio config | grep 'log_level'
log_level = "debug"
sh-4.4# head -n 7 /etc/containers/storage.conf
[storage] driver = "overlay" runroot = "/var/run/containers/storage" graphroot = "/var/lib/containers/storage" [storage.options] additionalimagestores = [] size = "8G"
sh-5.1# cat /etc/crio/crio.conf.d/01-ctrcfg-defaultRuntime
[crio] [crio.runtime] default_runtime = "runc"
apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: overlay-size spec: machineConfigPoolSelector: matchLabels: custom-crio: overlay-size containerRuntimeConfig: logLevel: debug overlaySize: 8G
oc apply -f overlaysize.yml
oc edit machineconfigpool worker
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: creationTimestamp: "2020-07-09T15:46:34Z" generation: 3 labels: custom-crio: overlay-size machineconfiguration.openshift.io/mco-built-in: ""
oc get machineconfigs
99-worker-generated-containerruntime 4173030d89fbf4a7a0976d1665491a4d9a6e54f1 3.2.0 7m42s rendered-worker-xyz 4173030d89fbf4a7a0976d1665491a4d9a6e54f1 3.2.0 7m36s
oc get mcp worker
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-xyz False True False 3 2 2 0 20h
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE worker rendered-worker-xyz True False False 3 3 3 0 20h
head -n 7 /etc/containers/storage.conf [storage] driver = "overlay" runroot = "/var/run/containers/storage" graphroot = "/var/lib/containers/storage" [storage.options] additionalimagestores = [] size = "8G"
~ USD df -h Filesystem Size Used Available Use% Mounted on overlay 8.0G 8.0K 8.0G 0% /
oc get ctrcfg
NAME AGE ctr-overlay 15m ctr-level 5m45s
cat /proc/1/status | grep Cap
capsh --decode=<decode_CapBnd_value> 1
apiVersion: machine.openshift.io/v1beta1 kind: MachineSet metadata: name: ci-ln-hmy310k-72292-5f87z-worker-a namespace: openshift-machine-api spec: template: spec: providerSpec: value: disks: - autoDelete: true boot: true image: projects/rhcos-cloud/global/images/rhcos-412-85-202203181601-0-gcp-x86-64 1
oc edit MachineConfiguration cluster
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster namespace: openshift-machine-config-operator spec: managedBootImages: 1 machineManagers: - resource: machinesets apiGroup: machine.openshift.io selection: mode: All 2
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster namespace: openshift-machine-config-operator spec: managedBootImages: 1 machineManagers: - resource: machinesets apiGroup: machine.openshift.io selection: mode: Partial partial: machineResourceSelector: matchLabels: update-boot-image: "true" 2
oc label machineset.machine ci-ln-hmy310k-72292-5f87z-worker-a update-boot-image=true -n openshift-machine-api
oc get machineconfiguration cluster -n openshift-machine-api -o yaml
kind: MachineConfiguration metadata: name: cluster status: conditions: - lastTransitionTime: "2024-09-09T13:51:37Z" 1 message: Reconciled 1 of 2 MAPI MachineSets | Reconciled 0 of 0 CAPI MachineSets | Reconciled 0 of 0 CAPI MachineDeployments reason: BootImageUpdateConfigurationAdded status: "True" type: BootImageUpdateProgressing - lastTransitionTime: "2024-09-09T13:51:37Z" 2 message: 0 Degraded MAPI MachineSets | 0 Degraded CAPI MachineSets | 0 CAPI MachineDeployments reason: BootImageUpdateConfigurationAdded status: "False" type: BootImageUpdateDegraded
oc get machinesets <machineset_name> -n openshift-machine-api -o yaml
apiVersion: machine.openshift.io/v1beta1 kind: MachineSet metadata: labels: machine.openshift.io/cluster-api-cluster: ci-ln-77hmkpt-72292-d4pxp update-boot-image: "true" name: ci-ln-77hmkpt-72292-d4pxp-worker-a namespace: openshift-machine-api spec: template: spec: providerSpec: value: disks: - autoDelete: true boot: true image: projects/rhcos-cloud/global/images/rhcos-416-92-202402201450-0-gcp-x86-64 1
oc edit MachineConfiguration cluster
apiVersion: operator.openshift.io/v1 kind: MachineConfiguration metadata: name: cluster namespace: openshift-machine-config-operator spec: managedBootImages: 1 machineManagers: - resource: machinesets apiGroup: machine.openshift.io selection: mode: All
oc adm prune renderedmachineconfigs list --in-use=false --pool-name=worker
worker rendered-worker-f38bf61ced3c920cf5a29a200ed43243 -- 2025-01-21 13:45:01 +0000 UTC (Currently in use: false) rendered-worker-fc94397dc7c43808c7014683c208956e-- 2025-01-30 17:20:53 +0000 UTC (Currently in use: false) rendered-worker-708c652868f7597eaa1e2622edc366ef -- 2025-01-31 18:01:16 +0000 UTC (Currently in use: true)
oc adm prune renderedmachineconfigs --pool-name=worker
Dry run enabled - no modifications will be made. Add --confirm to remove rendered machine configs. dry-run deleting rendered MachineConfig rendered-worker-f38bf61ced3c920cf5a29a200ed43243 dry-run deleting MachineConfig rendered-worker-fc94397dc7c43808c7014683c208956e Skip dry-run deleting rendered MachineConfig rendered-worker-708c652868f7597eaa1e2622edc366ef as it's currently in use
oc adm prune renderedmachineconfigs --pool-name=worker
Dry run enabled - no modifications will be made. Add --confirm to remove rendered machine configs. dry-run deleting rendered MachineConfig rendered-worker-f38bf61ced3c920cf5a29a200ed43243 dry-run deleting MachineConfig rendered-worker-fc94397dc7c43808c7014683c208956e Skip dry-run deleting rendered MachineConfig rendered-worker-708c652868f7597eaa1e2622edc366ef as it's currently in use
oc adm prune renderedmachineconfigs --pool-name=worker --count=2 --confirm
deleting rendered MachineConfig rendered-worker-f38bf61ced3c920cf5a29a200ed43243 deleting rendered MachineConfig rendered-worker-fc94397dc7c43808c7014683c208956e Skip deleting rendered MachineConfig rendered-worker-708c652868f7597eaa1e2622edc366ef as it's currently in use
Using a 4.17.0 image containerfileArch: noarch content: |- FROM configs AS final #Install hotfix rpm RUN dnf install -y https://example.com/myrepo/haproxy-1.0.16-5.el8.src.rpm && dnf clean all && ostree container commit
Using a 4.17.0 image FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 #Install hotfix rpm RUN dnf install -y https://example.com/myrepo/haproxy-1.0.16-5.el8.src.rpm && dnf clean all && ostree container commit
Get RHCOS base image of target cluster `oc adm release info --image-for rhel-coreos` hadolint ignore=DL3006 FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 Install our config file COPY my-host-to-host.conf /etc/ipsec.d/ RHEL entitled host is needed here to access RHEL packages Install libreswan as extra RHEL package RUN dnf install -y libreswan && dnf clean all && systemctl enable ipsec && ostree container commit
FROM configs AS final #Enable EPEL (more info at https://docs.fedoraproject.org/en-US/epel/ ) and install htop RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && dnf install -y htop && dnf clean all && ostree container commit
Get RHCOS base image of target cluster `oc adm release info --image-for rhel-coreos` hadolint ignore=DL3006 FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 #Enable EPEL (more info at https://docs.fedoraproject.org/en-US/epel/ ) and install htop RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && dnf install -y htop && dnf clean all && ostree container commit
FROM configs AS final RHEL entitled host is needed here to access RHEL packages Install fish as third party package from EPEL RUN dnf install -y https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/Packages/f/fish-3.3.1-3.el9.x86_64.rpm && dnf clean all && ostree container commit
Get RHCOS base image of target cluster `oc adm release info --image-for rhel-coreos` hadolint ignore=DL3006 FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256 RHEL entitled host is needed here to access RHEL packages Install fish as third party package from EPEL RUN dnf install -y https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/Packages/f/fish-3.3.1-3.el9.x86_64.rpm && dnf clean all && ostree container commit
NAME PREPARED BUILDING SUCCEEDED INTERRUPTED FAILED layered-c8765e26ebc87e1e17a7d6e0a78e8bae False False True False False
NAME READY STATUS RESTARTS AGE build-layered-c8765e26ebc87e1e17a7d6e0a78e8bae 2/2 Running 0 11m
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-a0b404d061a6183cc36d302363422aba True False False 3 3 3 0 4h14m worker rendered-worker-221507009cbcdec0eec8ab3ccd789d18 False False False 2 2 2 0 4h14m 1
NAME PREPARED BUILDING SUCCEEDED INTERRUPTED FAILED layered-c8765e26ebc87e1e17a7d6e0a78e8bae False False False False True
apiVersion: machineconfiguration.openshift.io/v1aplha1 1 kind: MachineOSConfig metadata: name: layered 2 spec: machineConfigPool: name: worker 3 buildInputs: containerFile: 4 - containerfileArch: noarch content: |- FROM configs AS final RUN rpm-ostree install tree && ostree container commit imageBuilder: 5 imageBuilderType: PodImageBuilder baseImagePullSecret: 6 name: global-pull-secret-copy renderedImagePushspec: image-registry.openshift-image-registry.svc:5000/openshift/os-image:latest 7 renderedImagePushSecret: 8 name: builder-dockercfg-7lzwl buildOutputs: 9 currentImagePullSecret: name: builder-dockercfg-7lzwl
oc create -f <file_name>.yaml
oc get machineosbuild
NAME PREPARED BUILDING SUCCEEDED INTERRUPTED FAILED layered-ad5a3cad36303c363cf458ab0524e7c0-builder False False True False False
oc label node <node_name> 'node-role.kubernetes.io/<mcp_name>='
oc get pods -n openshift-machine-config-operator
NAME READY STATUS RESTARTS AGE build-layered-ad5a3cad36303c363cf458ab0524e7c0-hxrws 2/2 Running 0 2m40s 1 machine-os-builder-6fb66cfb99-zcpvq 1/1 Running 0 2m42s 2
oc get machineosbuilds
NAME PREPARED BUILDING SUCCEEDED INTERRUPTED FAILED layered-ad5a3cad36303c363cf458ab0524e7c0 False True False False False 1
oc describe machineosbuild <object_name>
Name: layered-ad5a3cad36303c363cf458ab0524e7c0 API Version: machineconfiguration.openshift.io/v1alpha1 Kind: MachineOSBuild Spec: Config Generation: 1 Desired Config: Name: rendered-layered-ad5a3cad36303c363cf458ab0524e7c0 Machine OS Config: Name: layered Rendered Image Pushspec: image-registry.openshift-image-registry.svc:5000/openshift-machine-config-operator/os-images:layered-ad5a3cad36303c363cf458ab0524e7c0 Last Transition Time: 2025-02-12T19:21:28Z Message: Build Ready Reason: Ready Status: True Type: Succeeded Final Image Pullspec: image-registry.openshift-image-registry.svc:5000/openshift-machine-config-operator/os-images@sha256:312e48825e074b01a913deedd6de68abd44894ede50b2d14f99d722f13cda04b 1
oc debug node/<node_name>
sh-4.4# chroot /host
sh-5.1# rpm-ostree status
Deployments: * ostree-unverified-registry:image-registry.openshift-image-registry.svc:5000/openshift-machine-config-operator/os-images@sha256:312e48825e074b01a913deedd6de68abd44894ede50b2d14f99d722f13cda04b Digest: sha256:312e48825e074b01a913deedd6de68abd44894ede50b2d14f99d722f13cda04b 1 Version: 418.94.202502100215-0 (2025-02-12T19:20:44Z)
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-a0b404d061a6183cc36d302363422aba True False False 3 3 3 0 4h14m worker rendered-worker-221507009cbcdec0eec8ab3ccd789d18 False False False 2 2 2 0 4h14m 1
apiVersion: machineconfiguration.openshift.io/v1alpha1 kind: MachineOSConfig metadata: name: layered spec: machineConfigPool: name: worker buildInputs: containerFile: - containerfileArch: noarch content: |- 1 FROM configs AS final RUN rpm-ostree install rng-tools && systemctl enable rngd && rpm-ostree cleanup -m && ostree container commit RUN rpm-ostree install tree && ostree container commit imageBuilder: imageBuilderType: PodImageBuilder baseImagePullSecret: name: global-pull-secret-copy 2 renderedImagePushspec: image-registry.openshift-image-registry.svc:5000/openshift-machine-config-operator/os-images:latest 3 renderedImagePushSecret: 4 name: new-secret-name buildOutputs: currentImagePullSecret: name: new-secret-name 5
oc get machineosbuild
NAME PREPARED BUILDING SUCCEEDED INTERRUPTED FAILED layered-a5457b883f5239cdcb71b57e1a30b6ef False False True False False layered-f91f0f5593dd337d89bf4d38c877590b False True False False False 1
oc get machineconfigpools
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-a0b404d061a6183cc36d302363422aba True False False 3 3 3 0 3h38m worker rendered-worker-221507009cbcdec0eec8ab3ccd789d18 False True False 2 2 2 0 3h38m 1
oc debug node/<node_name>
sh-5.1# chroot /host
sh-5.1# rpm -qa |grep rng-tools
rng-tools-6.17-3.fc41.x86_64
sh-5.1# rngd -v
rngd 6.16
oc label node/<node_name> node-role.kubernetes.io/<mcp_name>-
oc get mcp
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-8332482204e0b76002f15ecad15b6c2d True False False 3 3 3 0 5h26m worker rendered-worker-bde4e4206442c0a48b1a1fb35ba56e85 False True False 3 2 2 0 5h26m 1
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-148-79.us-west-1.compute.internal Ready worker 32m v1.31.3 ip-10-0-155-125.us-west-1.compute.internal Ready,SchedulingDisabled worker 35m v1.31.3 ip-10-0-170-47.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-174-77.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-211-49.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-218-151.us-west-1.compute.internal Ready worker 31m v1.31.3
oc debug node/<node_name>
sh-5.1# chroot /host
sh-5.1# rpm-ostree status
State: idle Deployments: * ostree-unverified-image:containers-storage:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:76721c875a2b79688be46b1dca654c2c6619a6be28b29a2822cd86c3f9d8e3c1 Digest: sha256:76721c875a2b79688be46b1dca654c2c6619a6be28b29a2822cd86c3f9d8e3c1 Version: 418.94.202501300706-0 (2025-01-30T07:10:58Z)
Using a 4.18.0 image FROM quay.io/openshift-release/ocp-release@sha256... 1 #Install hotfix rpm RUN rpm-ostree override replace http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/kernel-{,core-,modules-,modules-core-,modules-extra-}5.14.0-295.el9.x86_64.rpm && \ 2 rpm-ostree cleanup -m && ostree container commit
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker 1 name: os-layer-custom spec: osImageURL: quay.io/my-registry/custom-image@sha256... 2
oc create -f <file_name>.yaml
oc get mc
NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE 00-master 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 00-worker 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-master-container-runtime 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-master-kubelet 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-worker-container-runtime 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 01-worker-kubelet 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 99-master-generated-registries 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 99-master-ssh 3.2.0 98m 99-worker-generated-registries 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m 99-worker-ssh 3.2.0 98m os-layer-custom 10s 1 rendered-master-15961f1da260f7be141006404d17d39b 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m rendered-worker-5aff604cb1381a4fe07feaf1595a797e 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 95m rendered-worker-5de4837625b1cbc237de6b22bc0bc873 5bdb57489b720096ef912f738b46330a8f577803 3.2.0 4s 2
oc describe mc rendered-worker-5de4837625b1cbc237de6b22bc0bc873
Name: rendered-worker-5de4837625b1cbc237de6b22bc0bc873 Namespace: Labels: <none> Annotations: machineconfiguration.openshift.io/generated-by-controller-version: 5bdb57489b720096ef912f738b46330a8f577803 machineconfiguration.openshift.io/release-image-version: 4.18.0-ec.3 API Version: machineconfiguration.openshift.io/v1 Kind: MachineConfig Os Image URL: quay.io/my-registry/custom-image@sha256
oc get mcp
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-15961f1da260f7be141006404d17d39b True False False 3 3 3 0 39m worker rendered-worker-5de4837625b1cbc237de6b22bc0bc873 True False False 3 0 0 0 39m 1
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-148-79.us-west-1.compute.internal Ready worker 32m v1.31.3 ip-10-0-155-125.us-west-1.compute.internal Ready,SchedulingDisabled worker 35m v1.31.3 ip-10-0-170-47.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-174-77.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-211-49.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-218-151.us-west-1.compute.internal Ready worker 31m v1.31.3
oc debug node/ip-10-0-155-125.us-west-1.compute.internal
sh-4.4# chroot /host
sh-4.4# sudo rpm-ostree status
State: idle Deployments: * ostree-unverified-registry:quay.io/my-registry/ Digest: sha256:
oc delete mc os-layer-custom
oc get mcp
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-6faecdfa1b25c114a58cf178fbaa45e2 True False False 3 3 3 0 39m worker rendered-worker-6b000dbc31aaee63c6a2d56d04cd4c1b False True False 3 0 0 0 39m 1
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-148-79.us-west-1.compute.internal Ready worker 32m v1.31.3 ip-10-0-155-125.us-west-1.compute.internal Ready,SchedulingDisabled worker 35m v1.31.3 ip-10-0-170-47.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-174-77.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-211-49.us-west-1.compute.internal Ready control-plane,master 42m v1.31.3 ip-10-0-218-151.us-west-1.compute.internal Ready worker 31m v1.31.3
oc debug node/<node_name>
sh-5.1# chroot /host
sh-5.1# sudo rpm-ostree status
State: idle Deployments: * ostree-unverified-registry:podman pull quay.io/openshift-release-dev/ocp-release@sha256:e2044c3cfebe0ff3a99fc207ac5efe6e07878ad59fd4ad5e41f88cb016dacd73 Digest: sha256:e2044c3cfebe0ff3a99fc207ac5efe6e07878ad59fd4ad5e41f88cb016dacd73