Updating clusters OpenShift Container Platform 4.7 Updating OpenShift Container Platform clusters Red Hat OpenShift Documentation Team

Commands:
Disabling ownership via cluster version overrides prevents upgrades. Please remove overrides before continuing.
apiVersion: v1 kind: ConfigMap metadata: name: my-registry-ca data: updateservice-registry: | 1 -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registry-with-port.example.com..5000: | 2 -----BEGIN CERTIFICATE----- -----END CERTIFICATE-----
oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' ><pull_secret_location> 1
oc registry login --registry="<registry>" \ 1 --auth-basic="<username>:<password>" \ 2 --to=<pull_secret_location> 3
oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=<pull_secret_location> 1
apiVersion: v1 kind: Namespace metadata: name: openshift-update-service annotations: openshift.io/node-selector: "" labels: openshift.io/cluster-monitoring: "true" 1
oc create -f <filename>.yaml
oc create -f update-service-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: update-service-operator-group spec: targetNamespaces: - openshift-update-service
oc -n openshift-update-service create -f <filename>.yaml
oc -n openshift-update-service create -f update-service-operator-group.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: update-service-subscription spec: channel: v1 installPlanApproval: "Automatic" source: "redhat-operators" 1 sourceNamespace: "openshift-marketplace" name: "cincinnati-operator"
oc create -f <filename>.yaml
oc -n openshift-update-service create -f update-service-subscription.yaml
oc -n openshift-update-service get clusterserviceversions
NAME DISPLAY VERSION REPLACES PHASE update-service-operator.v4.6.0 OpenShift Update Service 4.6.0 Succeeded
FROM registry.access.redhat.com/ubi8/ubi:8.1 RUN curl -L -o cincinnati-graph-data.tar.gz https://github.com/openshift/cincinnati-graph-data/archive/master.tar.gz CMD exec /bin/bash -c "tar xvzf cincinnati-graph-data.tar.gz -C /var/lib/cincinnati/graph-data/ --strip-components=1"
podman build -f ./Dockerfile -t registry.example.com/openshift/graph-data:latest
podman push registry.example.com/openshift/graph-data:latest
x509: certificate relies on legacy Common Name field, use SANs or temporarily enable Common Name matching with GODEBUG=x509ignoreCN=0
OCP_RELEASE=<release_version>
LOCAL_REGISTRY='<local_registry_host_name>:<local_registry_host_port>'
LOCAL_REPOSITORY='<local_repository_name>'
LOCAL_RELEASE_IMAGES_REPOSITORY='<local_release_images_repository_name>'
PRODUCT_REPO='openshift-release-dev'
LOCAL_SECRET_JSON='<path_to_pull_secret>'
RELEASE_NAME="ocp-release"
ARCHITECTURE=<server_architecture>
REMOVABLE_MEDIA_PATH=<path> 1
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --from=quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE} --to=USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY} --to-release-image=USD{LOCAL_REGISTRY}/USD{LOCAL_RELEASE_IMAGES_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE} --dry-run
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --to-dir=USD{REMOVABLE_MEDIA_PATH}/mirror quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE}
oc image mirror -a USD{LOCAL_SECRET_JSON} --from-dir=USD{REMOVABLE_MEDIA_PATH}/mirror "file://openshift/release:USD{OCP_RELEASE}*" USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY} 1
oc image mirror -a USD{LOCAL_SECRET_JSON} USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE} USD{LOCAL_REGISTRY}/USD{LOCAL_RELEASE_IMAGES_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --from=quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE} --to=USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY} --to-release-image=USD{LOCAL_REGISTRY}/USD{LOCAL_RELEASE_IMAGES_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}
NAMESPACE=openshift-update-service
NAME=service
RELEASE_IMAGES=registry.example.com/ocp4/openshift4-release-images
GRAPH_DATA_IMAGE=registry.example.com/openshift/graph-data:latest
oc -n "USD{NAMESPACE}" create -f - <<EOF apiVersion: updateservice.operator.openshift.io/v1 kind: UpdateService metadata: name: USD{NAME} spec: replicas: 2 releases: USD{RELEASE_IMAGES} graphDataImage: USD{GRAPH_DATA_IMAGE} EOF
while sleep 1; do POLICY_ENGINE_GRAPH_URI="USD(oc -n "USD{NAMESPACE}" get -o jsonpath='{.status.policyEngineURI}/api/upgrades_info/v1/graph{"\n"}' updateservice "USD{NAME}")"; SCHEME="USD{POLICY_ENGINE_GRAPH_URI%%:*}"; if test "USD{SCHEME}" = http -o "USD{SCHEME}" = https; then break; fi; done
while sleep 10; do HTTP_CODE="USD(curl --header Accept:application/json --output /dev/stderr --write-out "%{http_code}" "USD{POLICY_ENGINE_GRAPH_URI}?channel=stable-4.6")"; if test "USD{HTTP_CODE}" -eq 200; then break; fi; echo "USD{HTTP_CODE}"; done
NAMESPACE=openshift-update-service
NAME=service
POLICY_ENGINE_GRAPH_URI="USD(oc -n "USD{NAMESPACE}" get -o jsonpath='{.status.policyEngineURI}/api/upgrades_info/v1/graph{"\n"}' updateservice "USD{NAME}")"
PATCH="{\"spec\":{\"upstream\":\"USD{POLICY_ENGINE_GRAPH_URI}\"}}"
oc patch clusterversion version -p USDPATCH --type merge
oc get updateservice -n openshift-update-service
NAME AGE service 6s
oc delete updateservice service -n openshift-update-service
updateservice.updateservice.operator.openshift.io "service" deleted
oc project openshift-update-service
Now using project "openshift-update-service" on server "https://example.com:6443".
oc get operatorgroup
NAME AGE openshift-update-service-fprx2 4m41s
oc delete operatorgroup openshift-update-service-fprx2
operatorgroup.operators.coreos.com "openshift-update-service-fprx2" deleted
oc get subscription
NAME PACKAGE SOURCE CHANNEL update-service-operator update-service-operator updateservice-index-catalog v1
oc get subscription update-service-operator -o yaml | grep " currentCSV"
currentCSV: update-service-operator.v0.0.1
oc delete subscription update-service-operator
subscription.operators.coreos.com "update-service-operator" deleted
oc delete clusterserviceversion update-service-operator.v0.0.1
clusterserviceversion.operators.coreos.com "update-service-operator.v0.0.1" deleted
oc patch clusterversion version --type json -p '[{"op": "add", "path": "/spec/channel", "value": "<channel>"}]'
spec: clusterID: db93436d-7b05-42cc-b856-43e11ad2d31a upstream: '<update-server-url>' 1
oc get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.6.9 True False 158m Cluster version is 4.6.9
oc get clusterversion -o json|jq ".items[0].spec"
channel: stable-4.7
clusterID: 990f7ab8-109b-4c95-8480-2bd1deec55ff
oc adm upgrade
Cluster version is 4.1.0 Updates: VERSION IMAGE 4.1.2 quay.io/openshift-release-dev/ocp-release@sha256:9c5f0df8b192a0d7b46cd5f6a4da2289c155fd5302dec7954f8f06c878160b8b
oc adm upgrade --to-latest=true 1
oc adm upgrade --to=<version> 1
oc get clusterversion -o json|jq ".items[0].spec"
{ "channel": "stable-4.7", "clusterID": "990f7ab8-109b-4c95-8480-2bd1deec55ff", "desiredUpdate": { "force": false, "image": "quay.io/openshift-release-dev/ocp-release@sha256:9c5f0df8b192a0d7b46cd5f6a4da2289c155fd5302dec7954f8f06c878160b8b", "version": "4.7.0" 1 } }
oc get clusterversion -o json|jq ".items[0].status.history"
image: quay.io/openshift-release-dev/ocp-release@sha256:b8fa13e09d869089fc5957c32b02b7d3792a0b6f36693432acc0409615ab23b7
startedTime: 2021-01-28T20:30:50Z
state: Partial
verified: yes
version: 4.7.0
completionTime: 2021-01-28T20:30:50Z
image: quay.io/openshift-release-dev/ocp-release@sha256:b8fa13e09d869089fc5957c32b02b7d3792a0b6f36693432acc0409615ab23b7
startedTime: 2021-01-28T17:38:10Z
state: Completed
verified: no
version: 4.7.0
oc get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.7.0 True False 2m Cluster version is 4.7.0
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-168-251.ec2.internal Ready master 82m v1.20.0 ip-10-0-170-223.ec2.internal Ready master 82m v1.20.0 ip-10-0-179-95.ec2.internal Ready worker 70m v1.20.0 ip-10-0-182-134.ec2.internal Ready worker 70m v1.20.0 ip-10-0-211-16.ec2.internal Ready master 82m v1.20.0 ip-10-0-250-100.ec2.internal Ready worker 69m v1.20.0
oc patch clusterversion/version --patch '{"spec":{"upstream":"<update-server-url>"}}' --type=merge
clusterversion.config.openshift.io/version patched
oc get -l 'node-role.kubernetes.io/master!=' -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' nodes
ci-ln-pwnll6b-f76d1-s8t9n-worker-a-s75z4 ci-ln-pwnll6b-f76d1-s8t9n-worker-b-dglj2 ci-ln-pwnll6b-f76d1-s8t9n-worker-c-lldbm
oc label node <node name> node-role.kubernetes.io/<custom-label>=
oc label node ci-ln-0qv1yp2-f76d1-kl2tq-worker-a-j2ssz node-role.kubernetes.io/workerpool-canary=
node/ci-ln-gtrwm8t-f76d1-spbl7-worker-a-xk76k labeled
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool metadata: name: workerpool-canary 1 spec: machineConfigSelector: matchExpressions: 2 - { key: machineconfiguration.openshift.io/role, operator: In, values: [worker,workerpool-canary] } nodeSelector: matchLabels: node-role.kubernetes.io/workerpool-canary: "" 3
oc create -f <file_name>
machineconfigpool.machineconfiguration.openshift.io/workerpool-canary created
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-b0bb90c4921860f2a5d8a2f8137c1867 True False False 3 3 3 0 97m workerpool-canary rendered-workerpool-canary-87ba3dec1ad78cb6aecebf7fbb476a36 True False False 1 1 1 0 2m42s worker rendered-worker-87ba3dec1ad78cb6aecebf7fbb476a36 True False False 2 2 2 0 97m
oc patch mcp/<mcp_name> --patch '{"spec":{"paused":true}}' --type=merge
oc patch mcp/workerpool-canary --patch '{"spec":{"paused":true}}' --type=merge
machineconfigpool.machineconfiguration.openshift.io/workerpool-canary patched
oc patch mcp/<mcp_name> --patch '{"spec":{"paused":false}}' --type=merge
oc patch mcp/workerpool-canary --patch '{"spec":{"paused":false}}' --type=merge
machineconfigpool.machineconfiguration.openshift.io/workerpool-canary patched
oc label node <node_name> node-role.kubernetes.io/<custom-label>-
oc label node ci-ln-0qv1yp2-f76d1-kl2tq-worker-a-j2ssz node-role.kubernetes.io/workerpool-canary-
node/ci-ln-0qv1yp2-f76d1-kl2tq-worker-a-j2ssz labeled
USDoc get mcp
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE master rendered-master-1203f157d053fd987c7cbd91e3fbc0ed True False False 3 3 3 0 61m workerpool-canary rendered-mcp-noupdate-5ad4791166c468f3a35cd16e734c9028 True False False 0 0 0 0 21m worker rendered-worker-5ad4791166c468f3a35cd16e734c9028 True False False 3 3 3 0 61m
oc delete mcp <mcp_name>
--- Trivial example forcing an operator to acknowledge the start of an upgrade file=/home/user/openshift-ansible/hooks/pre_compute.yml - name: note the start of a compute machine update debug: msg: "Compute machine upgrade of {{ inventory_hostname }} is about to start" - name: require the user agree to start an upgrade pause: prompt: "Press Enter to start the compute machine update"
[all:vars] openshift_node_pre_upgrade_hook=/home/user/openshift-ansible/hooks/pre_node.yml openshift_node_post_upgrade_hook=/home/user/openshift-ansible/hooks/post_node.yml
systemctl disable --now firewalld.service
subscription-manager repos --disable=rhel-7-server-ose-4.6-rpms --enable=rhel-7-server-ansible-2.9-rpms --enable=rhel-7-server-ose-4.7-rpms
yum update openshift-ansible openshift-clients
subscription-manager repos --disable=rhel-7-server-ose-4.6-rpms --enable=rhel-7-server-ose-4.7-rpms --enable=rhel-7-fast-datapath-rpms --enable=rhel-7-server-optional-rpms
oc get node
NAME STATUS ROLES AGE VERSION mycluster-control-plane-0 Ready master 145m v1.20.0 mycluster-control-plane-1 Ready master 145m v1.20.0 mycluster-control-plane-2 Ready master 145m v1.20.0 mycluster-rhel7-0 NotReady,SchedulingDisabled worker 98m v1.14.6+97c81d00e mycluster-rhel7-1 Ready worker 98m v1.14.6+97c81d00e mycluster-rhel7-2 Ready worker 98m v1.14.6+97c81d00e mycluster-rhel7-3 Ready worker 98m v1.14.6+97c81d00e
[all:vars] ansible_user=root #ansible_become=True openshift_kubeconfig_path="~/.kube/config" [workers] mycluster-rhel7-0.example.com
cd /usr/share/ansible/openshift-ansible
ansible-playbook -i /<path>/inventory/hosts playbooks/upgrade.yml 1
oc get node
NAME STATUS ROLES AGE VERSION mycluster-control-plane-0 Ready master 145m v1.20.0 mycluster-control-plane-1 Ready master 145m v1.20.0 mycluster-control-plane-2 Ready master 145m v1.20.0 mycluster-rhel7-0 NotReady,SchedulingDisabled worker 98m v1.20.0 mycluster-rhel7-1 Ready worker 98m v1.20.0 mycluster-rhel7-2 Ready worker 98m v1.20.0 mycluster-rhel7-3 Ready worker 98m v1.20.0
yum update
tar xvzf <file>
echo USDPATH
oc <command>
C:\> path
C:\> oc <command>
echo USDPATH
oc <command>
echo -n '<user_name>:<password>' | base64 -w0 1 BGVtbYk3ZHAtqXs=
cat ./pull-secret.text | jq . > <path>/<pull_secret_file_in_json> 1
{ "auths": { "cloud.openshift.com": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "quay.io": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "registry.connect.redhat.com": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" }, "registry.redhat.io": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" } } }
"auths": { "<mirror_registry>": { 1 "auth": "<credentials>", 2 "email": "you@example.com" },
{ "auths": { "registry.example.com": { "auth": "BGVtbYk3ZHAtqXs=", "email": "you@example.com" }, "cloud.openshift.com": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "quay.io": { "auth": "b3BlbnNo...", "email": "you@example.com" }, "registry.connect.redhat.com": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" }, "registry.redhat.io": { "auth": "NTE3Njg5Nj...", "email": "you@example.com" } } }
export OCP_RELEASE=<release_version>
LOCAL_REGISTRY='<local_registry_host_name>:<local_registry_host_port>'
LOCAL_REPOSITORY='<local_repository_name>'
PRODUCT_REPO='openshift-release-dev'
LOCAL_SECRET_JSON='<path_to_pull_secret>'
RELEASE_NAME="ocp-release"
ARCHITECTURE=<server_architecture>
REMOVABLE_MEDIA_PATH=<path> 1
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --to-dir=USD{REMOVABLE_MEDIA_PATH}/mirror quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE} --dry-run
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --to-dir=USD{REMOVABLE_MEDIA_PATH}/mirror quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE}
oc image mirror -a USD{LOCAL_SECRET_JSON} --from-dir=USD{REMOVABLE_MEDIA_PATH}/mirror "file://openshift/release:USD{OCP_RELEASE}*" USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY} 1
oc apply -f USD{REMOVABLE_MEDIA_PATH}/mirror/config/<image_signature_file> 1
oc adm release mirror -a USD{LOCAL_SECRET_JSON} --from=quay.io/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE}-USD{ARCHITECTURE} --to=USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY} --apply-release-image-signature
OCP_RELEASE_NUMBER=<release_version> 1
ARCHITECTURE=<server_architecture> 1
DIGEST="USD(oc adm release info quay.io/openshift-release-dev/ocp-release:USD{OCP_RELEASE_NUMBER}-USD{ARCHITECTURE} | sed -n 's/Pull From: .*@//p')"
DIGEST_ALGO="USD{DIGEST%%:*}"
DIGEST_ENCODED="USD{DIGEST#*:}"
SIGNATURE_BASE64=USD(curl -s "https://mirror.openshift.com/pub/openshift-v4/signatures/openshift/release/USD{DIGEST_ALGO}=USD{DIGEST_ENCODED}/signature-1" | base64 -w0 && echo)
cat >checksum-USD{OCP_RELEASE_NUMBER}.yaml <<EOF apiVersion: v1 kind: ConfigMap metadata: name: release-image-USD{OCP_RELEASE_NUMBER} namespace: openshift-config-managed labels: release.openshift.io/verification-signatures: "" binaryData: USD{DIGEST_ALGO}-USD{DIGEST_ENCODED}: USD{SIGNATURE_BASE64} EOF
oc apply -f checksum-USD{OCP_RELEASE_NUMBER}.yaml
oc adm upgrade --allow-explicit-upgrade --to-image USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}<sha256_sum_value> 1
skopeo copy docker://registry.access.redhat.com/ubi8/ubi-minimal@sha256:5cfbaf45ca96806917830c183e9f37df2e913b187adb32e89fd83fa455ebaa6 docker://example.io/example/ubi-minimal
apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: ubi8repo spec: repositoryDigestMirrors: - mirrors: - example.io/example/ubi-minimal 1 source: registry.access.redhat.com/ubi8/ubi-minimal 2 - mirrors: - example.com/example/ubi-minimal source: registry.access.redhat.com/ubi8/ubi-minimal - mirrors: - mirror.example.com/redhat source: registry.redhat.io/openshift4 3
oc create -f registryrepomirror.yaml
oc get node
NAME STATUS ROLES AGE VERSION ip-10-0-137-44.ec2.internal Ready worker 7m v1.20.0 ip-10-0-138-148.ec2.internal Ready master 11m v1.20.0 ip-10-0-139-122.ec2.internal Ready master 11m v1.20.0 ip-10-0-147-35.ec2.internal Ready,SchedulingDisabled worker 7m v1.20.0 ip-10-0-153-12.ec2.internal Ready worker 7m v1.20.0 ip-10-0-154-10.ec2.internal Ready master 11m v1.20.0
oc debug node/ip-10-0-147-35.ec2.internal
Starting pod/ip-10-0-147-35ec2internal-debug To use host binaries, run `chroot /host`
sh-4.2# chroot /host
sh-4.2# cat /etc/containers/registries.conf
unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] location = "registry.access.redhat.com/ubi8/" insecure = false blocked = false mirror-by-digest-only = true prefix = "" [[registry.mirror]] location = "example.io/example/ubi8-minimal" insecure = false [[registry.mirror]] location = "example.com/example/ubi8-minimal" insecure = false
sh-4.2# podman pull --log-level=debug registry.access.redhat.com/ubi8/ubi-minimal@sha256:5cfbaf45ca96806917830c183e9f37df2e913b187adb32e89fd83fa455ebaa6
oc adm catalog mirror <local_registry>/<pull_spec> <local_registry> -a <pull_secret_file> --icsp-scope=registry
oc apply -f imageContentSourcePolicy.yaml
oc get ImageContentSourcePolicy -o yaml
apiVersion: v1 items: - apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"operator.openshift.io/v1alpha1","kind":"ImageContentSourcePolicy","metadata":{"annotations":{},"name":"redhat-operator-index"},"spec":{"repositoryDigestMirrors":[{"mirrors":["local.registry:5000"],"source":"registry.redhat.io"}]}}