Updating clusters OpenShift Container Platform 4.12 Updating OpenShift Container Platform clusters Red Hat OpenShift Documentation Team

Commands:
oc adm upgrade --include-not-recommended
Cluster version is 4.10.22 Upstream is unset, so the cluster will use an appropriate default. Channel: fast-4.11 (available channels: candidate-4.10, candidate-4.11, eus-4.10, fast-4.10, fast-4.11, stable-4.10) Recommended updates: VERSION IMAGE 4.10.26 quay.io/openshift-release-dev/ocp-release@sha256:e1fa1f513068082d97d78be643c369398b0e6820afab708d26acda2262940954 4.10.25 quay.io/openshift-release-dev/ocp-release@sha256:ed84fb3fbe026b3bbb4a2637ddd874452ac49c6ead1e15675f257e28664879cc 4.10.24 quay.io/openshift-release-dev/ocp-release@sha256:aab51636460b5a9757b736a29bc92ada6e6e6282e46b06e6fd483063d590d62a 4.10.23 quay.io/openshift-release-dev/ocp-release@sha256:e40e49d722cb36a95fa1c03002942b967ccbd7d68de10e003f0baa69abad457b Supported but not recommended updates: Version: 4.11.0 Image: quay.io/openshift-release-dev/ocp-release@sha256:300bce8246cf880e792e106607925de0a404484637627edf5f517375517d54a4 Recommended: False Reason: RPMOSTreeTimeout Message: Nodes with substantial numbers of containers and CPU contention may not reconcile machine configuration https://bugzilla.redhat.com/show_bug.cgi?id=2111817#c22
oc get clusterversion version -o json | jq '.status.availableUpdates'
[ { "channels": [ "candidate-4.11", "candidate-4.12", "fast-4.11", "fast-4.12" ], "image": "quay.io/openshift-release-dev/ocp-release@sha256:400267c7f4e61c6bfa0a59571467e8bd85c9188e442cbd820cc8263809be3775", "url": "https://access.redhat.com/errata/RHBA-2023:3213", "version": "4.11.41" }, ]
oc get clusterversion version -o json | jq '.status.conditionalUpdates'
[ { "conditions": [ { "lastTransitionTime": "2023-05-30T16:28:59Z", "message": "The 4.11.36 release only resolves an installation issue https://issues.redhat.com//browse/OCPBUGS-11663 , which does not affect already running clusters. 4.11.36 does not include fixes delivered in recent 4.11.z releases and therefore upgrading from these versions would cause fixed bugs to reappear. Red Hat does not recommend upgrading clusters to 4.11.36 version for this reason. https://access.redhat.com/solutions/7007136", "reason": "PatchesOlderRelease", "status": "False", "type": "Recommended" } ], "release": { "channels": [...], "image": "quay.io/openshift-release-dev/ocp-release@sha256:8c04176b771a62abd801fcda3e952633566c8b5ff177b93592e8e8d2d1f8471d", "url": "https://access.redhat.com/errata/RHBA-2023:1733", "version": "4.11.36" }, "risks": [...] }, ]
oc adm release extract <release image>
oc adm release extract quay.io/openshift-release-dev/ocp-release:4.12.6-x86_64 Extracted release payload from digest sha256:800d1e39d145664975a3bb7cbc6e674fbf78e3c45b5dde9ff2c5a11a8690c87b created at 2023-03-01T12:46:29Z ls 0000_03_authorization-openshift_01_rolebindingrestriction.crd.yaml 0000_03_config-operator_01_proxy.crd.yaml 0000_03_marketplace-operator_01_operatorhub.crd.yaml 0000_03_marketplace-operator_02_operatorhub.cr.yaml 0000_03_quota-openshift_01_clusterresourcequota.crd.yaml 1 0000_90_service-ca-operator_02_prometheusrolebinding.yaml 2 0000_90_service-ca-operator_03_servicemonitor.yaml 0000_99_machine-api-operator_00_tombstones.yaml image-references 3 release-metadata
0000_<runlevel>_<component>_<manifest-name>.yaml
0000_03_config-operator_01_proxy.crd.yaml
oc get mcp
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-acd1358917e9f98cbdb599aea622d78b True False False 3 3 3 0 22h worker rendered-worker-1d871ac76e1951d32b2fe92369879826 False True False 2 1 1 0 22h
oc adm upgrade channel <channel>
oc get node
NAME STATUS ROLES AGE VERSION ip-10-0-137-31.us-east-2.compute.internal Ready,SchedulingDisabled worker 12d v1.23.5+3afdacb ip-10-0-151-208.us-east-2.compute.internal Ready master 12d v1.23.5+3afdacb ip-10-0-176-138.us-east-2.compute.internal Ready master 12d v1.23.5+3afdacb ip-10-0-183-194.us-east-2.compute.internal Ready worker 12d v1.23.5+3afdacb ip-10-0-204-102.us-east-2.compute.internal Ready master 12d v1.23.5+3afdacb ip-10-0-207-224.us-east-2.compute.internal Ready worker 12d v1.23.5+3afdacb
Cluster update time = CVO target update payload deployment time + (# node update iterations x MCO node update time)
Cluster update time = 60 + (6 x 5) = 90 minutes
Cluster update time = 60 + (3 x 5) = 75 minutes
oc get apirequestcounts
NAME REMOVEDINRELEASE REQUESTSINCURRENTHOUR REQUESTSINLAST24H poddisruptionbudgets.v1.policy 391 8114 poddisruptionbudgets.v1beta1.policy 1.25 2 23 podmonitors.v1.monitoring.coreos.com 3 70 podnetworkconnectivitychecks.v1alpha1.controlplane.operator.openshift.io 612 11748 pods.v1 1531 38634 podsecuritypolicies.v1beta1.policy 1.25 3 39 podtemplates.v1 2 79 preprovisioningimages.v1alpha1.metal3.io 2 39 priorityclasses.v1.scheduling.k8s.io 12 248 prioritylevelconfigurations.v1beta1.flowcontrol.apiserver.k8s.io 1.26 3 86
oc get apirequestcounts -o jsonpath='{range .items[?(@.status.removedInRelease!="")]}{.status.removedInRelease}{"\t"}{.metadata.name}{"\n"}{end}'
1.26 flowschemas.v1beta1.flowcontrol.apiserver.k8s.io 1.26 horizontalpodautoscalers.v2beta2.autoscaling 1.25 poddisruptionbudgets.v1beta1.policy 1.25 podsecuritypolicies.v1beta1.policy 1.26 prioritylevelconfigurations.v1beta1.flowcontrol.apiserver.k8s.io
oc get apirequestcounts <resource>.<version>.<group> -o yaml
oc get apirequestcounts poddisruptionbudgets.v1beta1.policy -o yaml
oc get apirequestcounts poddisruptionbudgets.v1beta1.policy -o jsonpath='{range .status.currentHour..byUser[*]}{..byVerb[*].verb}{","}{.username}{","}{.userAgent}{"\n"}{end}' | sort -k 2 -t, -u | column -t -s, -NVERBS,USERNAME,USERAGENT
VERBS USERNAME USERAGENT watch system:serviceaccount:openshift-operators:3scale-operator manager/v0.0.0 watch system:serviceaccount:openshift-operators:datadog-operator-controller-manager manager/v0.0.0
oc -n openshift-config patch cm admin-acks --patch '{"data":{"ack-4.11-kube-1.25-api-removals-in-4.12":"true"}}' --type=merge
oc get mcp
NAME CONFIG UPDATED UPDATING master rendered-master-ecbb9582781c1091e1c9f19d50cf836c True False worker rendered-worker-00a3f0c68ae94e747193156b491553d5 True False
oc adm upgrade channel eus-<4.y+2>
oc patch mcp/worker --type merge --patch '{"spec":{"paused":true}}'
oc adm upgrade --to-latest
Updating to latest version <4.y+1.z>
oc adm upgrade
Cluster version is <4.y+1.z>
oc adm upgrade --to-latest
oc adm upgrade
Cluster version is <4.y+2.z>
oc patch mcp/worker --type merge --patch '{"spec":{"paused":false}}'
oc get mcp
NAME CONFIG UPDATED UPDATING master rendered-master-52da4d2760807cb2b96a3402179a9a4c True False worker rendered-worker-4756f60eccae96fb9dcb4c392c69d497 True False
oc get cloudcredentials cluster -o=jsonpath={.spec.credentialsMode}
oc get secret <secret_name> -n=kube-system
oc get authentication cluster -o jsonpath --template='{ .spec.serviceAccountIssuer }'
RELEASE_IMAGE=USD(./openshift-install version | awk '/release image/ {print USD3}')
CCO_IMAGE=USD(oc adm release info --image-for='cloud-credential-operator' USDRELEASE_IMAGE -a ~/.pull-secret)
oc image extract USDCCO_IMAGE --file="/usr/bin/ccoctl" -a ~/.pull-secret
chmod 775 ccoctl
ccoctl --help
OpenShift credentials provisioning tool Usage: ccoctl [command] Available Commands: alibabacloud Manage credentials objects for alibaba cloud aws Manage credentials objects for AWS cloud gcp Manage credentials objects for Google cloud help Help about any command ibmcloud Manage credentials objects for IBM Cloud nutanix Manage credentials objects for Nutanix Flags: -h, --help help for ccoctl Use "ccoctl [command] --help" for more information about a command.
oc adm release extract --credentials-requests --cloud=<provider_type> --to=<path_to_directory_with_list_of_credentials_requests>/credrequests quay.io/<path_to>/ocp-release:<version>
apiVersion: cloudcredential.openshift.io/v1 kind: CredentialsRequest metadata: name: cloud-credential-operator-iam-ro namespace: openshift-cloud-credential-operator spec: providerSpec: apiVersion: cloudcredential.openshift.io/v1 kind: AWSProviderSpec statementEntries: - effect: Allow action: - iam:GetUser - iam:GetUserPolicy - iam:ListAccessKeys resource: "*" secretRef: name: cloud-credential-operator-iam-ro-creds namespace: openshift-cloud-credential-operator 1
oc create namespace <component_namespace>
ls <path_to_ccoctl_output_dir>/manifests/*-credentials.yaml | xargs -I{} oc apply -f {}
0000_30_machine-api-operator_00_credentials-request.yaml 1 0000_50_cloud-credential-operator_05-iam-ro-credentialsrequest.yaml 2 0000_50_cluster-image-registry-operator_01-registry-credentials-request.yaml 3 0000_50_cluster-ingress-operator_00-ingress-credentials-request.yaml 4 0000_50_cluster-network-operator_02-cncc-credentials.yaml 5 0000_50_cluster-storage-operator_03_credentials_request_aws.yaml 6
0000_26_cloud-controller-manager-operator_16_credentialsrequest-gcp.yaml 1 0000_30_machine-api-operator_00_credentials-request.yaml 2 0000_50_cloud-credential-operator_05-gcp-ro-credentialsrequest.yaml 3 0000_50_cluster-image-registry-operator_01-registry-credentials-request-gcs.yaml 4 0000_50_cluster-ingress-operator_00-ingress-credentials-request.yaml 5 0000_50_cluster-network-operator_02-cncc-credentials.yaml 6 0000_50_cluster-storage-operator_03_credentials_request_gcp.yaml 7
oc edit cloudcredential cluster
metadata: annotations: cloudcredential.openshift.io/upgradeable-to: <version_number>
0000_30_machine-api-operator_00_credentials-request.yaml 1 0000_50_cloud-credential-operator_05-iam-ro-credentialsrequest.yaml 2 0000_50_cluster-image-registry-operator_01-registry-credentials-request.yaml 3 0000_50_cluster-ingress-operator_00-ingress-credentials-request.yaml 4 0000_50_cluster-network-operator_02-cncc-credentials.yaml 5 0000_50_cluster-storage-operator_03_credentials_request_aws.yaml 6
0000_26_cloud-controller-manager-operator_16_credentialsrequest-gcp.yaml 1 0000_30_machine-api-operator_00_credentials-request.yaml 2 0000_50_cloud-credential-operator_05-gcp-ro-credentialsrequest.yaml 3 0000_50_cluster-image-registry-operator_01-registry-credentials-request-gcs.yaml 4 0000_50_cluster-ingress-operator_00-ingress-credentials-request.yaml 5 0000_50_cluster-network-operator_02-cncc-credentials.yaml 6 0000_50_cluster-storage-operator_03_credentials_request_gcp.yaml 7
spec: clusterID: db93436d-7b05-42cc-b856-43e11ad2d31a upstream: '<update-server-url>' 1
oc get machinehealthcheck -n openshift-machine-api
oc -n openshift-machine-api annotate mhc <mhc-name> cluster.x-k8s.io/paused=""
apiVersion: machine.openshift.io/v1beta1 kind: MachineHealthCheck metadata: name: example namespace: openshift-machine-api annotations: cluster.x-k8s.io/paused: "" spec: selector: matchLabels: role: worker unhealthyConditions: - type: "Ready" status: "Unknown" timeout: "300s" - type: "Ready" status: "False" timeout: "300s" maxUnhealthy: "40%" status: currentHealthy: 5 expectedMachines: 5
oc -n openshift-machine-api annotate mhc <mhc-name> cluster.x-k8s.io/paused-
oc adm upgrade
Cluster version is 4.9.23 Upstream is unset, so the cluster will use an appropriate default. Channel: stable-4.9 (available channels: candidate-4.10, candidate-4.9, fast-4.10, fast-4.9, stable-4.10, stable-4.9, eus-4.10) Recommended updates: VERSION IMAGE 4.9.24 quay.io/openshift-release-dev/ocp-release@sha256:6a899c54dda6b844bb12a247e324a0f6cde367e880b73ba110c056df6d018032 4.9.25 quay.io/openshift-release-dev/ocp-release@sha256:2eafde815e543b92f70839972f585cc52aa7c37aa72d5f3c8bc886b0fd45707a 4.9.26 quay.io/openshift-release-dev/ocp-release@sha256:3ccd09dd08c303f27a543351f787d09b83979cd31cf0b4c6ff56cd68814ef6c8 4.9.27 quay.io/openshift-release-dev/ocp-release@sha256:1c7db78eec0cf05df2cead44f69c0e4b2c3234d5635c88a41e1b922c3bedae16 4.9.28 quay.io/openshift-release-dev/ocp-release@sha256:4084d94969b186e20189649b5affba7da59f7d1943e4e5bc7ef78b981eafb7a8 4.9.29 quay.io/openshift-release-dev/ocp-release@sha256:b04ca01d116f0134a102a57f86c67e5b1a3b5da1c4a580af91d521b8fa0aa6ec 4.9.31 quay.io/openshift-release-dev/ocp-release@sha256:2a28b8ebb53d67dd80594421c39e36d9896b1e65cb54af81fbb86ea9ac3bf2d7 4.9.32 quay.io/openshift-release-dev/ocp-release@sha256:ecdb6d0df547b857eaf0edb5574ddd64ca6d9aff1fa61fd1ac6fb641203bedfa
oc adm upgrade channel <channel>
oc adm upgrade channel stable-4.12
oc adm upgrade --to-latest=true 1
oc adm upgrade --to=<version> 1
oc adm upgrade
oc get clusterversion
Cluster version is <version> Upstream is unset, so the cluster will use an appropriate default. Channel: stable-4.10 (available channels: candidate-4.10, candidate-4.11, eus-4.10, fast-4.10, fast-4.11, stable-4.10) No updates available. You may force an upgrade to a specific release image, but doing so might not be supported and might result in downtime or data loss.
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-168-251.ec2.internal Ready master 82m v1.25.0 ip-10-0-170-223.ec2.internal Ready master 82m v1.25.0 ip-10-0-179-95.ec2.internal Ready worker 70m v1.25.0 ip-10-0-182-134.ec2.internal Ready worker 70m v1.25.0 ip-10-0-211-16.ec2.internal Ready master 82m v1.25.0 ip-10-0-250-100.ec2.internal Ready worker 69m v1.25.0
oc adm upgrade --include-not-recommended
oc adm upgrade --allow-not-recommended --to <version> <.>
oc patch clusterversion/version --patch '{"spec":{"upstream":"<update-server-url>"}}' --type=merge
clusterversion.config.openshift.io/version patched
oc get -l 'node-role.kubernetes.io/master!=' -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' nodes
ci-ln-pwnll6b-f76d1-s8t9n-worker-a-s75z4 ci-ln-pwnll6b-f76d1-s8t9n-worker-b-dglj2 ci-ln-pwnll6b-f76d1-s8t9n-worker-c-lldbm
oc label node <node_name> node-role.kubernetes.io/<custom_label>=
oc label node ci-ln-0qv1yp2-f76d1-kl2tq-worker-a-j2ssz node-role.kubernetes.io/workerpool-canary=
node/ci-ln-gtrwm8t-f76d1-spbl7-worker-a-xk76k labeled
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: name: workerpool-canary 1 spec: machineConfigSelector: matchExpressions: - { key: machineconfiguration.openshift.io/role, operator: In, values: [worker,workerpool-canary] 2 } nodeSelector: matchLabels: node-role.kubernetes.io/workerpool-canary: "" 3
oc create -f <file_name>
machineconfigpool.machineconfiguration.openshift.io/workerpool-canary created
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-b0bb90c4921860f2a5d8a2f8137c1867 True False False 3 3 3 0 97m workerpool-canary rendered-workerpool-canary-87ba3dec1ad78cb6aecebf7fbb476a36 True False False 1 1 1 0 2m42s worker rendered-worker-87ba3dec1ad78cb6aecebf7fbb476a36 True False False 2 2 2 0 97m
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: name: worker-perf spec: machineConfigSelector: matchExpressions: - { key: machineconfiguration.openshift.io/role, operator: In, values: [worker,worker-perf] } nodeSelector: matchLabels: node-role.kubernetes.io/worker-perf: ""
oc create -f machineConfigPool.yaml
machineconfigpool.machineconfiguration.openshift.io/worker-perf created
oc label node worker-a node-role.kubernetes.io/worker-perf=''
oc label node worker-b node-role.kubernetes.io/worker-perf=''
oc label node worker-c node-role.kubernetes.io/worker-perf=''
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker-perf name: 06-kdump-enable-worker-perf spec: config: ignition: version: 3.2.0 systemd: units: - enabled: true name: kdump.service kernelArguments: - crashkernel=512M
oc create -f new-machineconfig.yaml
oc label node worker-a node-role.kubernetes.io/worker-perf-canary=''
oc label node worker-a node-role.kubernetes.io/worker-perf-
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: name: worker-perf-canary spec: machineConfigSelector: matchExpressions: - { key: machineconfiguration.openshift.io/role, operator: In, values: [worker,worker-perf,worker-perf-canary] 1 } nodeSelector: matchLabels: node-role.kubernetes.io/worker-perf-canary: ""
oc create -f machineConfigPool-Canary.yaml
machineconfigpool.machineconfiguration.openshift.io/worker-perf-canary created
oc get mcp
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-2bf1379b39e22bae858ea1a3ff54b2ac True False False 3 3 3 0 5d16h worker rendered-worker-b9576d51e030413cfab12eb5b9841f34 True False False 0 0 0 0 5d16h worker-perf rendered-worker-perf-b98a1f62485fa702c4329d17d9364f6a True False False 2 2 2 0 56m worker-perf-canary rendered-worker-perf-canary-b98a1f62485fa702c4329d17d9364f6a True False False 1 1 1 0 44m
oc get nodes
NAME STATUS ROLES AGE VERSION worker-a Ready worker,worker-perf-canary 5d15h v1.27.13+e709aa5 worker-b Ready worker,worker-perf 5d15h v1.27.13+e709aa5 worker-c Ready worker,worker-perf 5d15h v1.27.13+e709aa5
systemctl status kdump.service
NAME STATUS ROLES AGE VERSION kdump.service - Crash recovery kernel arming Loaded: loaded (/usr/lib/systemd/system/kdump.service; enabled; preset: disabled) Active: active (exited) since Tue 2024-09-03 12:44:43 UTC; 10s ago Process: 4151139 ExecStart=/usr/bin/kdumpctl start (code=exited, status=0/SUCCESS) Main PID: 4151139 (code=exited, status=0/SUCCESS)
cat /proc/cmdline
crashkernel=512M
oc label node worker-a node-role.kubernetes.io/worker-perf=''
oc label node worker-a node-role.kubernetes.io/worker-perf-canary-
oc patch mcp/<mcp_name> --patch '{"spec":{"paused":true}}' --type=merge
oc patch mcp/workerpool-canary --patch '{"spec":{"paused":true}}' --type=merge
machineconfigpool.machineconfiguration.openshift.io/workerpool-canary patched
oc patch mcp/<mcp_name> --patch '{"spec":{"paused":false}}' --type=merge
oc patch mcp/workerpool-canary --patch '{"spec":{"paused":false}}' --type=merge
machineconfigpool.machineconfiguration.openshift.io/workerpool-canary patched
oc get machineconfigpools
oc label node <node_name> node-role.kubernetes.io/<custom_label>-
oc label node ci-ln-0qv1yp2-f76d1-kl2tq-worker-a-j2ssz node-role.kubernetes.io/workerpool-canary-
node/ci-ln-0qv1yp2-f76d1-kl2tq-worker-a-j2ssz labeled
oc get mcp
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-1203f157d053fd987c7cbd91e3fbc0ed True False False 3 3 3 0 61m workerpool-canary rendered-mcp-noupdate-5ad4791166c468f3a35cd16e734c9028 True False False 0 0 0 0 21m worker rendered-worker-5ad4791166c468f3a35cd16e734c9028 True False False 3 3 3 0 61m
oc delete mcp <mcp_name>
# bootupctl status
Component EFI Installed: grub2-efi-x64-1:2.04-31.el8_4.1.x86_64,shim-x64-15-8.el8_1.x86_64 Update: At latest version
Component EFI Installed: grub2-efi-aa64-1:2.02-99.el8_4.1.aarch64,shim-aa64-15.4-2.el8_1.aarch64 Update: At latest version
# bootupctl adopt-and-update
Updated: grub2-efi-x64-1:2.04-31.el8_4.1.x86_64,shim-x64-15-8.el8_1.x86_64
# bootupctl update
Updated: grub2-efi-x64-1:2.04-31.el8_4.1.x86_64,shim-x64-15-8.el8_1.x86_64
variant: openshift version: 4.12.0 metadata: name: 99-worker-chrony 1 labels: machineconfiguration.openshift.io/role: worker 2 systemd: units: - name: bootupctl-update.service enabled: true contents: | [Unit] Description=Bootupd automatic update [Service] ExecStart=/usr/bin/bootupctl update RemainAfterExit=yes [Install] WantedBy=multi-user.target
butane 99-worker-bootupctl-update.bu -o 99-worker-bootupctl-update.yaml
oc apply -f ./99-worker-bootupctl-update.yaml
--- Trivial example forcing an operator to acknowledge the start of an upgrade file=/home/user/openshift-ansible/hooks/pre_compute.yml - name: note the start of a compute machine update debug: msg: "Compute machine upgrade of {{ inventory_hostname }} is about to start" - name: require the user agree to start an upgrade pause: prompt: "Press Enter to start the compute machine update"
[all:vars] openshift_node_pre_upgrade_hook=/home/user/openshift-ansible/hooks/pre_node.yml openshift_node_post_upgrade_hook=/home/user/openshift-ansible/hooks/post_node.yml
systemctl disable --now firewalld.service
subscription-manager repos --disable=rhocp-4.11-for-rhel-8-x86_64-rpms --disable=ansible-2.9-for-rhel-8-x86_64-rpms --enable=rhocp-4.12-for-rhel-8-x86_64-rpms
yum swap ansible ansible-core
yum update openshift-ansible openshift-clients
subscription-manager repos --disable=rhocp-4.11-for-rhel-8-x86_64-rpms --enable=rhocp-4.12-for-rhel-8-x86_64-rpms
[all:vars] ansible_user=root #ansible_become=True openshift_kubeconfig_path="~/.kube/config" [workers] mycluster-rhel8-0.example.com mycluster-rhel8-1.example.com mycluster-rhel8-2.example.com mycluster-rhel8-3.example.com
cd /usr/share/ansible/openshift-ansible
ansible-playbook -i /<path>/inventory/hosts playbooks/upgrade.yml 1
oc get node
NAME STATUS ROLES AGE VERSION mycluster-control-plane-0 Ready master 145m v1.25.0 mycluster-control-plane-1 Ready master 145m v1.25.0 mycluster-control-plane-2 Ready master 145m v1.25.0 mycluster-rhel8-0 Ready worker 98m v1.25.0 mycluster-rhel8-1 Ready worker 98m v1.25.0 mycluster-rhel8-2 Ready worker 98m v1.25.0 mycluster-rhel8-3 Ready worker 98m v1.25.0
yum update
tar xvf <file>
echo USDPATH
oc <command>
C:\> path
C:\> oc <command>
echo USDPATH
oc <command>
cat ./pull-secret | jq . > <path>/<pull_secret_file_in_json> 1
{ "auths": { "cloud.openshift.com": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "quay.io": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "registry.connect.redhat.com": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" }, "registry.redhat.io": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" } } }
mkdir -p <directory_name>
cp <path>/<pull_secret_file_in_json> <directory_name>/<auth_file>
echo -n '<user_name>:<password>' | base64 -w0 1 BGVtbYk3ZHAtqXs=
"auths": { "<mirror_registry>": { 1 "auth": "<credentials>", 2 "email": "you@example.com" } },
{ "auths": { "registry.example.com": { "auth": "BGVtbYk3ZHAtqXs=", "email": "you@example.com" }, "cloud.openshift.com": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "quay.io": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "registry.connect.redhat.com": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" }, "registry.redhat.io": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" } } }
tar xvzf oc-mirror.tar.gz
chmod +x oc-mirror
sudo mv oc-mirror /usr/local/bin/.
oc mirror help
oc mirror init --registry example.com/mirror/oc-mirror-metadata > imageset-config.yaml 1
kind: ImageSetConfiguration apiVersion: mirror.openshift.io/v1alpha2 archiveSize: 4 1 storageConfig: 2 registry: imageURL: example.com/mirror/oc-mirror-metadata 3 skipTLS: false mirror: platform: channels: - name: stable-4.12 4 type: ocp graph: true 5 operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 6 packages: - name: serverless-operator 7 channels: - name: stable 8 additionalImages: - name: registry.redhat.io/ubi8/ubi:latest 9 helm: {}
oc mirror --config=./imageset-config.yaml \ 1 docker://registry.example:5000 2
oc mirror --config=./imageset-config.yaml \ 1 file://<path_to_output_directory> 2
cd <path_to_output_directory>
ls
mirror_seq1_000000.tar
oc mirror --from=./mirror_seq1_000000.tar \ 1 docker://registry.example:5000 2
oc apply -f ./oc-mirror-workspace/results-1639608409/
oc apply -f ./oc-mirror-workspace/results-1639608409/release-signatures/
oc get imagecontentsourcepolicy
oc get catalogsource -n openshift-marketplace
oc mirror --config=./imageset-config.yaml \ 1 docker://registry.example:5000 \ 2 --dry-run 3
Checking push permissions for registry.example:5000 Creating directory: oc-mirror-workspace/src/publish Creating directory: oc-mirror-workspace/src/v2 Creating directory: oc-mirror-workspace/src/charts Creating directory: oc-mirror-workspace/src/release-signatures No metadata detected, creating new workspace wrote mirroring manifests to oc-mirror-workspace/operators.1658342351/manifests-redhat-operator-index info: Planning completed in 31.48s info: Dry run complete Writing image mapping to oc-mirror-workspace/mapping.txt
cd oc-mirror-workspace/
kind: ImageSetConfiguration apiVersion: mirror.openshift.io/v1alpha2 mirror: operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 packages: - name: aws-load-balancer-operator
oc mirror --config=./imageset-config.yaml \ 1 --use-oci-feature \ 2 --oci-feature-action=copy \ 3 oci://my-oci-catalog 4
[[registry]] location = "registry.redhat.io:5000" insecure = false blocked = false mirror-by-digest-only = true prefix = "" [[registry.mirror]] location = "preprod-registry.example.com" insecure = false
ls -l
my-oci-catalog 1 oc-mirror-workspace 2 olm_artifacts 3
kind: ImageSetConfiguration apiVersion: mirror.openshift.io/v1alpha2 mirror: operators: - catalog: oci:///home/user/oc-mirror/my-oci-catalog/redhat-operator-index 1 packages: - name: aws-load-balancer-operator
oc mirror --config=./imageset-config.yaml \ 1 --use-oci-feature \ 2 --oci-feature-action=mirror \ 3 docker://registry.example:5000 4
additionalImages: - name: registry.redhat.io/ubi8/ubi:latest
local: - name: podinfo path: /test/podinfo-5.0.0.tar.gz
repositories: - name: podinfo url: https://example.github.io/podinfo charts: - name: podinfo version: 5.0.0
operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 packages: - name: elasticsearch-operator minVersion: '2.4.0'
operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 packages: - name: elasticsearch-operator minVersion: '5.2.3-31'
architectures: - amd64 - arm64
channels: - name: stable-4.10 - name: stable-4.12
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: local: path: /home/user/metadata mirror: platform: channels: - name: stable-4.12 minVersion: 4.11.37 maxVersion: 4.12.15 shortestPath: true
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: registry: imageURL: example.com/mirror/oc-mirror-metadata skipTLS: false mirror: platform: channels: - name: stable-4.10 minVersion: 4.10.10
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: local: path: /home/user/metadata mirror: operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 packages: - name: rhacs-operator channels: - name: stable minVersion: 4.0.1
kind: ImageSetConfiguration apiVersion: mirror.openshift.io/v1alpha2 storageConfig: registry: imageURL: mylocalregistry/ocp-mirror/openshift4 skipTLS: false mirror: platform: channels: - name: stable-4.12 type: ocp graph: true operators: - catalog: registry.redhat.io/redhat/certified-operator-index:v4.12 packages: - name: nutanixcsioperator channels: - name: stable additionalImages: - name: registry.redhat.io/ubi9/ubi:latest
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: registry: imageURL: example.com/mirror/oc-mirror-metadata skipTLS: false mirror: operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 packages: - name: elasticsearch-operator channels: - name: stable-5.7 - name: stable
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: registry: imageURL: example.com/mirror/oc-mirror-metadata skipTLS: false mirror: operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 full: true
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: registry: imageURL: example.com/mirror/oc-mirror-metadata skipTLS: false mirror: operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration archiveSize: 4 storageConfig: registry: imageURL: example.com/mirror/oc-mirror-metadata skipTLS: false mirror: platform: architectures: - "s390x" channels: - name: stable-4.12 operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.12 helm: repositories: - name: redhat-helm-charts url: https://raw.githubusercontent.com/redhat-developer/redhat-helm-charts/master charts: - name: ibm-mongodb-enterprise-helm version: 0.2.0 additionalImages: - name: registry.redhat.io/ubi9/ubi:latest
export OCP_RELEASE=<release_version>
LOCAL_REGISTRY='<local_registry_host_name>:<local_registry_host_port>'
LOCAL_REPOSITORY='<local_repository_name>'
LOCAL_RELEASE_IMAGES_REPOSITORY='<local_release_images_repository_name>'
PRODUCT_REPO='openshift-release-dev'
LOCAL_SECRET_JSON='<path_to_pull_secret>'
RELEASE_NAME="ocp-release"
ARCHITECTURE=<server_architecture>
REMOVABLE_MEDIA_PATH=<path> 1
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --to-dir=USD{REMOVABLE_MEDIA_PATH}/mirror quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE} --dry-run
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --to-dir=USD{REMOVABLE_MEDIA_PATH}/mirror quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE}
oc image mirror -a USD{LOCAL_SECRET_JSON} --from-dir=USD{REMOVABLE_MEDIA_PATH}/mirror "file://openshift/release:USD{OCP_RELEASE}*" USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY} 1
oc apply -f USD{REMOVABLE_MEDIA_PATH}/mirror/config/<image_signature_file> 1
oc image mirror -a USD{LOCAL_SECRET_JSON} USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE} USD{LOCAL_REGISTRY}/USD{LOCAL_RELEASE_IMAGES_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --from=quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE} --to=USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY} --apply-release-image-signature
oc image mirror -a USD{LOCAL_SECRET_JSON} USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE} USD{LOCAL_REGISTRY}/USD{LOCAL_RELEASE_IMAGES_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}
apiVersion: v1 kind: ConfigMap metadata: name: my-registry-ca data: updateservice-registry: | 1 -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registry-with-port.example.com..5000: | 2 -----BEGIN CERTIFICATE----- -----END CERTIFICATE-----
oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' ><pull_secret_location> 1
oc registry login --registry="<registry>" \ 1 --auth-basic="<username>:<password>" \ 2 --to=<pull_secret_location> 3
oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=<pull_secret_location> 1
apiVersion: v1 kind: Namespace metadata: name: openshift-update-service annotations: openshift.io/node-selector: "" labels: openshift.io/cluster-monitoring: "true" 1
oc create -f <filename>.yaml
oc create -f update-service-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: update-service-operator-group spec: targetNamespaces: - openshift-update-service
oc -n openshift-update-service create -f <filename>.yaml
oc -n openshift-update-service create -f update-service-operator-group.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: update-service-subscription spec: channel: v1 installPlanApproval: "Automatic" source: "redhat-operators" 1 sourceNamespace: "openshift-marketplace" name: "cincinnati-operator"
oc create -f <filename>.yaml
oc -n openshift-update-service create -f update-service-subscription.yaml
oc -n openshift-update-service get clusterserviceversions
NAME DISPLAY VERSION REPLACES PHASE update-service-operator.v4.6.0 OpenShift Update Service 4.6.0 Succeeded
FROM registry.access.redhat.com/ubi8/ubi:8.1 RUN curl -L -o cincinnati-graph-data.tar.gz https://api.openshift.com/api/upgrades_info/graph-data RUN mkdir -p /var/lib/cincinnati-graph-data && tar xvzf cincinnati-graph-data.tar.gz -C /var/lib/cincinnati-graph-data/ --no-overwrite-dir --no-same-owner CMD ["/bin/bash", "-c" ,"exec cp -rp /var/lib/cincinnati-graph-data/* /var/lib/cincinnati/graph-data"]
podman build -f ./Dockerfile -t registry.example.com/openshift/graph-data:latest
podman push registry.example.com/openshift/graph-data:latest
NAMESPACE=openshift-update-service
NAME=service
RELEASE_IMAGES=registry.example.com/ocp4/openshift4-release-images
GRAPH_DATA_IMAGE=registry.example.com/openshift/graph-data:latest
oc -n "USD{NAMESPACE}" create -f - <<EOF apiVersion: updateservice.operator.openshift.io/v1 kind: UpdateService metadata: name: USD{NAME} spec: replicas: 2 releases: USD{RELEASE_IMAGES} graphDataImage: USD{GRAPH_DATA_IMAGE} EOF
while sleep 1; do POLICY_ENGINE_GRAPH_URI="USD(oc -n "USD{NAMESPACE}" get -o jsonpath='{.status.policyEngineURI}/api/upgrades_info/v1/graph{"\n"}' updateservice "USD{NAME}")"; SCHEME="USD{POLICY_ENGINE_GRAPH_URI%%:*}"; if test "USD{SCHEME}" = http -o "USD{SCHEME}" = https; then break; fi; done
while sleep 10; do HTTP_CODE="USD(curl --header Accept:application/json --output /dev/stderr --write-out "%{http_code}" "USD{POLICY_ENGINE_GRAPH_URI}?channel=stable-4.6")"; if test "USD{HTTP_CODE}" -eq 200; then break; fi; echo "USD{HTTP_CODE}"; done
NAMESPACE=openshift-update-service
NAME=service
POLICY_ENGINE_GRAPH_URI="USD(oc -n "USD{NAMESPACE}" get -o jsonpath='{.status.policyEngineURI}/api/upgrades_info/v1/graph{"\n"}' updateservice "USD{NAME}")"
PATCH="{\"spec\":{\"upstream\":\"USD{POLICY_ENGINE_GRAPH_URI}\"}}"
oc patch clusterversion version -p USDPATCH --type merge
oc get machinehealthcheck -n openshift-machine-api
oc -n openshift-machine-api annotate mhc <mhc-name> cluster.x-k8s.io/paused=""
apiVersion: machine.openshift.io/v1beta1 kind: MachineHealthCheck metadata: name: example namespace: openshift-machine-api annotations: cluster.x-k8s.io/paused: "" spec: selector: matchLabels: role: worker unhealthyConditions: - type: "Ready" status: "Unknown" timeout: "300s" - type: "Ready" status: "False" timeout: "300s" maxUnhealthy: "40%" status: currentHealthy: 5 expectedMachines: 5
oc -n openshift-machine-api annotate mhc <mhc-name> cluster.x-k8s.io/paused-
oc adm release info -o 'jsonpath={.digest}{"\n"}' quay.io/openshift-release-dev/ocp-release:USD{OCP_RELEASE_VERSION}-USD{ARCHITECTURE}
sha256:a8bfba3b6dddd1a2fbbead7dac65fe4fb8335089e4e7cae327f3bad334add31d
oc adm upgrade --allow-explicit-upgrade --to-image <defined_registry>/<defined_repository>@<digest>
skopeo copy docker://registry.access.redhat.com/ubi8/ubi-minimal@sha256:5cfbaf45ca96806917830c183e9f37df2e913b187adb32e89fd83fa455ebaa6 docker://example.io/example/ubi-minimal
apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: ubi8repo spec: repositoryDigestMirrors: - mirrors: - example.io/example/ubi-minimal 1 - example.com/example/ubi-minimal 2 source: registry.access.redhat.com/ubi8/ubi-minimal 3 - mirrors: - mirror.example.com/redhat source: registry.redhat.io/openshift4 4 - mirrors: - mirror.example.com source: registry.redhat.io 5 - mirrors: - mirror.example.net/image source: registry.example.com/example/myimage 6 - mirrors: - mirror.example.net source: registry.example.com/example 7 - mirrors: - mirror.example.net/registry-example-com source: registry.example.com 8
oc create -f registryrepomirror.yaml
oc get node
NAME STATUS ROLES AGE VERSION ip-10-0-137-44.ec2.internal Ready worker 7m v1.25.0 ip-10-0-138-148.ec2.internal Ready master 11m v1.25.0 ip-10-0-139-122.ec2.internal Ready master 11m v1.25.0 ip-10-0-147-35.ec2.internal Ready worker 7m v1.25.0 ip-10-0-153-12.ec2.internal Ready worker 7m v1.25.0 ip-10-0-154-10.ec2.internal Ready master 11m v1.25.0
oc debug node/ip-10-0-147-35.ec2.internal
Starting pod/ip-10-0-147-35ec2internal-debug To use host binaries, run `chroot /host`
sh-4.2# chroot /host
sh-4.2# cat /etc/containers/registries.conf
unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] short-name-mode = "" [[registry]] prefix = "" location = "registry.access.redhat.com/ubi8/ubi-minimal" mirror-by-digest-only = true [[registry.mirror]] location = "example.io/example/ubi-minimal" [[registry.mirror]] location = "example.com/example/ubi-minimal" [[registry]] prefix = "" location = "registry.example.com" mirror-by-digest-only = true [[registry.mirror]] location = "mirror.example.net/registry-example-com" [[registry]] prefix = "" location = "registry.example.com/example" mirror-by-digest-only = true [[registry.mirror]] location = "mirror.example.net" [[registry]] prefix = "" location = "registry.example.com/example/myimage" mirror-by-digest-only = true [[registry.mirror]] location = "mirror.example.net/image" [[registry]] prefix = "" location = "registry.redhat.io" mirror-by-digest-only = true [[registry.mirror]] location = "mirror.example.com" [[registry]] prefix = "" location = "registry.redhat.io/openshift4" mirror-by-digest-only = true [[registry.mirror]] location = "mirror.example.com/redhat"
sh-4.2# podman pull --log-level=debug registry.access.redhat.com/ubi8/ubi-minimal@sha256:5cfbaf45ca96806917830c183e9f37df2e913b187adb32e89fd83fa455ebaa6
oc adm catalog mirror <local_registry>/<pull_spec> <local_registry> -a <pull_secret_file> --icsp-scope=registry
oc apply -f imageContentSourcePolicy.yaml
oc get ImageContentSourcePolicy -o yaml
apiVersion: v1 items: - apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"operator.openshift.io/v1alpha1","kind":"ImageContentSourcePolicy","metadata":{"annotations":{},"name":"redhat-operator-index"},"spec":{"repositoryDigestMirrors":[{"mirrors":["local.registry:5000"],"source":"registry.redhat.io"}]}}
oc get updateservice -n openshift-update-service
NAME AGE service 6s
oc delete updateservice service -n openshift-update-service
updateservice.updateservice.operator.openshift.io "service" deleted
oc project openshift-update-service
Now using project "openshift-update-service" on server "https://example.com:6443".
oc get operatorgroup
NAME AGE openshift-update-service-fprx2 4m41s
oc delete operatorgroup openshift-update-service-fprx2
operatorgroup.operators.coreos.com "openshift-update-service-fprx2" deleted
oc get subscription
NAME PACKAGE SOURCE CHANNEL update-service-operator update-service-operator updateservice-index-catalog v1
oc get subscription update-service-operator -o yaml | grep " currentCSV"
currentCSV: update-service-operator.v0.0.1
oc delete subscription update-service-operator
subscription.operators.coreos.com "update-service-operator" deleted
oc delete clusterserviceversion update-service-operator.v0.0.1
clusterserviceversion.operators.coreos.com "update-service-operator.v0.0.1" deleted
oc get nodes -l node-role.kubernetes.io/master
NAME STATUS ROLES AGE VERSION control-plane-node-0 Ready master 75m v1.25.0 control-plane-node-1 Ready master 75m v1.25.0 control-plane-node-2 Ready master 75m v1.25.0
oc adm cordon <control_plane_node>
oc wait --for=condition=Ready node/<control_plane_node>
oc adm uncordon <control_plane_node>
oc get nodes -l node-role.kubernetes.io/worker
NAME STATUS ROLES AGE VERSION compute-node-0 Ready worker 30m v1.25.0 compute-node-1 Ready worker 30m v1.25.0 compute-node-2 Ready worker 30m v1.25.0
oc adm cordon <compute_node>
oc adm drain <compute_node> [--pod-selector=<pod_selector>]
oc wait --for=condition=Ready node/<compute_node>
oc adm uncordon <compute_node>
type PreflightValidationOCPSpec struct { // releaseImage describes the OCP release image that all Modules need to be checked against. // +kubebuilder:validation:Required ReleaseImage string `json:"releaseImage"` 1 // Boolean flag that determines whether images build during preflight must also // be pushed to a defined repository // +optional PushBuiltImage bool `json:"pushBuiltImage"` 2 }
type CRStatus struct { // Status of Module CR verification: true (verified), false (verification failed), // error (error during verification process), unknown (verification has not started yet) // +required // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=True;False VerificationStatus string `json:"verificationStatus"` 1 // StatusReason contains a string describing the status source. // +optional StatusReason string `json:"statusReason,omitempty"` 2 // Current stage of the verification process: // image (image existence verification), build(build process verification) // +required // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=Image;Build;Sign;Requeued;Done VerificationStage string `json:"verificationStage"` 3 // LastTransitionTime is the last time the CR status transitioned from one status to another. // This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. // +required // +kubebuilder:validation:Required // +kubebuilder:validation:Type=string // +kubebuilder:validation:Format=date-time LastTransitionTime metav1.Time `json:"lastTransitionTime" protobuf:"bytes,4,opt,name=lastTransitionTime"` 4 }
RUN depmod -b /opt USD{KERNEL_VERSION}
quay.io/openshift-release-dev/ocp-release@sha256:22e149142517dfccb47be828f012659b1ccf71d26620e6f62468c264a7ce7863
apiVersion: kmm.sigs.x-k8s.io/v1beta1 kind: PreflightValidationOCP metadata: name: preflight spec: releaseImage: quay.io/openshift-release-dev/ocp-release@sha256:22e149142517dfccb47be828f012659b1ccf71d26620e6f62468c264a7ce7863 pushBuiltImage: true