Validation and troubleshooting OpenShift Container Platform 4.15 Validating and troubleshooting an OpenShift Container Platform installation Red Hat OpenShift Documentation Team

Commands:
cat <install_dir>/.openshift_install.log
time="2020-12-03T09:50:47Z" level=info msg="Install complete!" time="2020-12-03T09:50:47Z" level=info msg="To access the cluster as the system:admin user when using 'oc', run 'export KUBECONFIG=/home/myuser/install_dir/auth/kubeconfig'" time="2020-12-03T09:50:47Z" level=info msg="Access the OpenShift web-console here: https://console-openshift-console.apps.mycluster.example.com" time="2020-12-03T09:50:47Z" level=info msg="Login to the console with user: \"kubeadmin\", and password: \"password\"" time="2020-12-03T09:50:47Z" level=debug msg="Time elapsed per stage:" time="2020-12-03T09:50:47Z" level=debug msg=" Infrastructure: 6m45s" time="2020-12-03T09:50:47Z" level=debug msg="Bootstrap Complete: 11m30s" time="2020-12-03T09:50:47Z" level=debug msg=" Bootstrap Destroy: 1m5s" time="2020-12-03T09:50:47Z" level=debug msg=" Cluster Operators: 17m31s" time="2020-12-03T09:50:47Z" level=info msg="Time elapsed: 37m26s"
oc adm node-logs <node_name> -u crio
Mar 17 02:52:50 ip-10-0-138-140.ec2.internal crio[1366]: time="2021-08-05 10:33:21.594930907Z" level=info msg="Pulling image: quay.io/openshift-release-dev/ocp-release:4.10.0-ppc64le" id=abcd713b-d0e1-4844-ac1c-474c5b60c07c name=/runtime.v1alpha2.ImageService/PullImage Mar 17 02:52:50 ip-10-0-138-140.ec2.internal crio[1484]: time="2021-03-17 02:52:50.194341109Z" level=info msg="Trying to access \"li0317gcp1.mirror-registry.qe.gcp.devcluster.openshift.com:5000/ocp/release@sha256:1926eae7cacb9c00f142ec98b00628970e974284b6ddaf9a6a086cb9af7a6c31\"" Mar 17 02:52:50 ip-10-0-138-140.ec2.internal crio[1484]: time="2021-03-17 02:52:50.226788351Z" level=info msg="Trying to access \"li0317gcp1.mirror-registry.qe.gcp.devcluster.openshift.com:5000/ocp/release@sha256:1926eae7cacb9c00f142ec98b00628970e974284b6ddaf9a6a086cb9af7a6c31\""
Trying to access \"li0317gcp1.mirror-registry.qe.gcp.devcluster.openshift.com:5000/ocp/release@sha256:1926eae7cacb9c00f142ec98b00628970e974284b6ddaf9a6a086cb9af7a6c31\" Trying to access \"li0317gcp2.mirror-registry.qe.gcp.devcluster.openshift.com:5000/ocp/release@sha256:1926eae7cacb9c00f142ec98b00628970e974284b6ddaf9a6a086cb9af7a6c31\"
oc get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.6.4 True False 6m25s Cluster version is 4.6.4
oc get clusteroperators.config.openshift.io
oc describe clusterversion
oc get clusterversion -o jsonpath='{.items[0].spec}{"\n"}'
channel: stable-4.6
clusterID: 245539c1-72a3-41aa-9cec-72ed8cf25c5c
oc adm upgrade
Cluster version is 4.6.4 Updates: VERSION IMAGE 4.6.6 quay.io/openshift-release-dev/ocp-release@sha256:c7e8f18e8116356701bd23ae3a23fb9892dd5ea66c8300662ef30563d7104f39
oc get cloudcredentials cluster -o=jsonpath={.spec.credentialsMode}
Manual
oc get secrets -n kube-system <secret_name>
Error from server (NotFound): secrets "aws-creds" not found
oc get authentication cluster -o jsonpath --template='{ .spec.serviceAccountIssuer }'
oc get secrets -n openshift-image-registry installer-cloud-credentials -o jsonpath='{.data}'
oc get pods -n openshift-cloud-credential-operator
NAME READY STATUS RESTARTS AGE cloud-credential-operator-59cf744f78-r8pbq 2/2 Running 2 71m pod-identity-webhook-548f977b4c-859lz 1/1 Running 1 70m
oc get nodes
NAME STATUS ROLES AGE VERSION compute-1.example.com Ready worker 33m v1.28.5 control-plane-1.example.com Ready master 41m v1.28.5 control-plane-2.example.com Ready master 45m v1.28.5 compute-2.example.com Ready worker 38m v1.28.5 compute-3.example.com Ready worker 33m v1.28.5 control-plane-3.example.com Ready master 41m v1.28.5
oc adm top nodes
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% compute-1.example.com 128m 8% 1132Mi 16% control-plane-1.example.com 801m 22% 3471Mi 23% control-plane-2.example.com 1718m 49% 6085Mi 40% compute-2.example.com 935m 62% 5178Mi 75% compute-3.example.com 111m 7% 1131Mi 16% control-plane-3.example.com 942m 26% 4100Mi 27%
./openshift-install gather bootstrap --dir <installation_directory> 1
./openshift-install gather bootstrap --dir <installation_directory> \ 1 --bootstrap <bootstrap_address> \ 2 --master <master_1_address> \ 3 --master <master_2_address> \ 4 --master <master_3_address> 5
INFO Pulling debug logs from the bootstrap machine INFO Bootstrap gather logs captured here "<installation_directory>/log-bundle-<timestamp>.tar.gz"
journalctl -b -f -u bootkube.service
for pod in USD(sudo podman ps -a -q); do sudo podman logs USDpod; done
tail -f /var/lib/containers/storage/overlay-containers/*/userdata/ctr.log
journalctl -b -f -u kubelet.service -u crio.service
sudo tail -f /var/log/containers/*
oc adm node-logs --role=master -u kubelet
oc adm node-logs --role=master --path=openshift-apiserver
cat ~/<installation_directory>/.openshift_install.log 1
./openshift-install create cluster --dir <installation_directory> --log-level debug 1
./openshift-install destroy cluster --dir <installation_directory> 1
rm -rf <installation_directory>