Support OpenShift Container Platform 4.10 Getting support for OpenShift Container Platform Red Hat OpenShift Documentation Team

Commands:
oc api-resources -o name | grep config.openshift.io
oc explain <resource_name>.config.openshift.io
oc get <resource_name>.config -o yaml
oc edit <resource_name>.config -o yaml
oc get clusterversion -o jsonpath='{.items[].spec.clusterID}{"\n"}'
oc get route prometheus-k8s -n openshift-monitoring -o jsonpath="{.spec.host}"
{__name__=~"cluster:usage:.*|count:up0|count:up1|cluster_version|cluster_version_available_updates|cluster_operator_up|cluster_operator_conditions|cluster_version_payload|cluster_installer|cluster_infrastructure_provider|cluster_feature_set|instance:etcd_object_counts:sum|ALERTS|code:apiserver_request_total:rate:sum|cluster:capacity_cpu_cores:sum|cluster:capacity_memory_bytes:sum|cluster:cpu_usage_cores:sum|cluster:memory_usage_bytes:sum|openshift:cpu_usage_cores:sum|openshift:memory_usage_bytes:sum|workload:cpu_usage_cores:sum|workload:memory_usage_bytes:sum|cluster:virt_platform_nodes:sum|cluster:node_instance_type_count:sum|cnv:vmi_status_running:count|cluster:vmi_request_cpu_cores:sum|node_role_os_version_machine:cpu_capacity_cores:sum|node_role_os_version_machine:cpu_capacity_sockets:sum|subscription_sync_total|olm_resolution_duration_seconds|csv_succeeded|csv_abnormal|cluster:kube_persistentvolumeclaim_resource_requests_storage_bytes:provisioner:sum|cluster:kubelet_volume_stats_used_bytes:provisioner:sum|ceph_cluster_total_bytes|ceph_cluster_total_used_raw_bytes|ceph_health_status|job:ceph_osd_metadata:count|job:kube_pv:count|job:ceph_pools_iops:total|job:ceph_pools_iops_bytes:total|job:ceph_versions_running:count|job:noobaa_total_unhealthy_buckets:sum|job:noobaa_bucket_count:sum|job:noobaa_total_object_count:sum|noobaa_accounts_num|noobaa_total_usage|console_url|cluster:network_attachment_definition_instances:max|cluster:network_attachment_definition_enabled_instance_up:max|cluster:ingress_controller_aws_nlb_active:sum|insightsclient_request_send_total|cam_app_workload_migrations|cluster:apiserver_current_inflight_requests:sum:max_over_time:2m|cluster:alertmanager_integrations:max|cluster:telemetry_selected_series:count|openshift:prometheus_tsdb_head_series:sum|openshift:prometheus_tsdb_head_samples_appended_total:sum|monitoring:container_memory_working_set_bytes:sum|namespace_job:scrape_series_added:topk3_sum1h|namespace_job:scrape_samples_post_metric_relabeling:topk3|monitoring:haproxy_server_http_responses_total:sum|rhmi_status|cluster_legacy_scheduler_policy|cluster_master_schedulable|che_workspace_status|che_workspace_started_total|che_workspace_failure_total|che_workspace_start_time_seconds_sum|che_workspace_start_time_seconds_count|cco_credentials_mode|cluster:kube_persistentvolume_plugin_type_counts:sum|visual_web_terminal_sessions_total|acm_managed_cluster_info|cluster:vsphere_vcenter_info:sum|cluster:vsphere_esxi_version_total:sum|cluster:vsphere_node_hw_version_total:sum|openshift:build_by_strategy:sum|rhods_aggregate_availability|rhods_total_users|instance:etcd_disk_wal_fsync_duration_seconds:histogram_quantile|instance:etcd_mvcc_db_total_size_in_bytes:sum|instance:etcd_network_peer_round_trip_time_seconds:histogram_quantile|instance:etcd_mvcc_db_total_size_in_use_in_bytes:sum|instance:etcd_disk_backend_commit_duration_seconds:histogram_quantile|jaeger_operator_instances_storage_types|jaeger_operator_instances_strategies|jaeger_operator_instances_agent_strategies|appsvcs:cores_by_product:sum|nto_custom_profiles:count|openshift_csi_share_configmap|openshift_csi_share_secret|openshift_csi_share_mount_failures_total|openshift_csi_share_mount_requests_total",alertstate=~"firing|"}
INSIGHTS_OPERATOR_POD=USD(oc get pods --namespace=openshift-insights -o custom-columns=:metadata.name --no-headers --field-selector=status.phase=Running)
oc cp openshift-insights/USDINSIGHTS_OPERATOR_POD:/var/lib/insights-operator ./insights-data
oc extract secret/pull-secret -n openshift-config --to=.
"cloud.openshift.com":{"auth":"<hash>","email":"<email_address>"}
oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' ><pull_secret_location> 1
oc registry login --registry="<registry>" \ 1 --auth-basic="<username>:<password>" \ 2 --to=<pull_secret_location> 3
oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=<pull_secret_location> 1
{ "auths": { "cloud.openshift.com": { "auth": " <your_token> ", "email": " <email_address> " } }
oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' > pull-secret
cp pull-secret pull-secret-backup
set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=pull-secret
oc get pods --namespace=openshift-insights -o custom-columns=:metadata.name --no-headers --field-selector=status.phase=Running
oc cp openshift-insights/<insights_operator_pod_name>:/var/lib/insights-operator ./insights-data 1
{ "name": "clusterconfig/authentication", "duration_in_ms": 730, 1 "records_count": 1, "errors": null, "panic": null }
apiVersion: batch/v1 kind: Job metadata: name: insights-operator-job annotations: config.openshift.io/inject-proxy: insights-operator spec: backoffLimit: 6 ttlSecondsAfterFinished: 600 template: spec: restartPolicy: OnFailure serviceAccountName: operator nodeSelector: beta.kubernetes.io/os: linux node-role.kubernetes.io/master: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Exists - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 900 - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 900 volumes: - name: snapshots emptyDir: {} - name: service-ca-bundle configMap: name: service-ca-bundle optional: true initContainers: - name: insights-operator image: quay.io/openshift/origin-insights-operator:latest terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: snapshots mountPath: /var/lib/insights-operator - name: service-ca-bundle mountPath: /var/run/configmaps/service-ca-bundle readOnly: true ports: - containerPort: 8443 name: https resources: requests: cpu: 10m memory: 70Mi args: - gather - -v=4 - --config=/etc/insights-operator/server.yaml containers: - name: sleepy image: quay.io/openshift/origin-base:latest args: - /bin/sh - -c - sleep 10m volumeMounts: [{name: snapshots, mountPath: /var/lib/insights-operator}]
oc get -n openshift-insights deployment insights-operator -o yaml
initContainers: - name: insights-operator image: <your_insights_operator_image_version> terminationMessagePolicy: FallbackToLogsOnError volumeMounts:
oc apply -n openshift-insights -f gather-job.yaml
oc describe -n openshift-insights job/insights-operator-job
Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal SuccessfulCreate 7m18s job-controller Created pod: insights-operator-job- <your_job>
oc logs -n openshift-insights insights-operator-job- <your_job> insights-operator
I0407 11:55:38.192084 1 diskrecorder.go:34] Wrote 108 records to disk in 33ms
oc cp openshift-insights/insights-operator-job- <your_job> :/var/lib/insights-operator ./insights-data
oc delete -n openshift-insights job insights-operator-job
oc extract secret/pull-secret -n openshift-config --to=.
{ "auths": { "cloud.openshift.com": { "auth": " <your_token> ", "email": "asd@redhat.com" } }
curl -v -H "User-Agent: insights-operator/one10time200gather184a34f6a168926d93c330 cluster/ <cluster_id> " -H "Authorization: Bearer <your_token> " -F "upload=@ <path_to_archive> ; type=application/vnd.redhat.openshift.periodic+tar" https://console.redhat.com/api/ingress/v1/upload
* Connection #0 to host console.redhat.com left intact {"request_id":"393a7cf1093e434ea8dd4ab3eb28884c","upload":{"account_number":"6274079"}}%
oc adm must-gather --image=registry.redhat.io/container-native-virtualization/cnv-must-gather-rhel8:v4.10.0
oc adm must-gather -- /usr/bin/gather_audit_logs
NAMESPACE NAME READY STATUS RESTARTS AGE openshift-must-gather-5drcj must-gather-bklx4 2/2 Running 0 72s openshift-must-gather-5drcj must-gather-s8sdh 2/2 Running 0 72s
oc import-image is/must-gather -n openshift
oc adm must-gather
tar cvaf must-gather.tar.gz must-gather.local.5421342344627712289/ 1
oc adm must-gather --image-stream=openshift/must-gather \ 1 --image=registry.redhat.io/container-native-virtualization/cnv-must-gather-rhel8:v4.10.10 2
oc adm must-gather --image=USD(oc -n openshift-logging get deployment.apps/cluster-logging-operator -o jsonpath='{.spec.template.spec.containers[?(@.name == "cluster-logging-operator")].image}')
├── cluster-logging │ ├── clo │ │ ├── cluster-logging-operator-74dd5994f-6ttgt │ │ ├── clusterlogforwarder_cr │ │ ├── cr │ │ ├── csv │ │ ├── deployment │ │ └── logforwarding_cr │ ├── collector │ │ ├── fluentd-2tr64 │ ├── eo │ │ ├── csv │ │ ├── deployment │ │ └── elasticsearch-operator-7dc7d97b9d-jb4r4 │ ├── es │ │ ├── cluster-elasticsearch │ │ │ ├── aliases │ │ │ ├── health │ │ │ ├── indices │ │ │ ├── latest_documents.json │ │ │ ├── nodes │ │ │ ├── nodes_stats.json │ │ │ └── thread_pool │ │ ├── cr │ │ ├── elasticsearch-cdm-lp8l38m0-1-794d6dd989-4jxms │ │ └── logs │ │ ├── elasticsearch-cdm-lp8l38m0-1-794d6dd989-4jxms │ ├── install │ │ ├── co_logs │ │ ├── install_plan │ │ ├── olmo_logs │ │ └── subscription │ └── kibana │ ├── cr │ ├── kibana-9d69668d4-2rkvz ├── cluster-scoped-resources │ └── core │ ├── nodes │ │ ├── ip-10-0-146-180.eu-west-1.compute.internal.yaml │ └── persistentvolumes │ ├── pvc-0a8d65d9-54aa-4c44-9ecc-33d9381e41c1.yaml ├── event-filter.html ├── gather-debug.log └── namespaces ├── openshift-logging │ ├── apps │ │ ├── daemonsets.yaml │ │ ├── deployments.yaml │ │ ├── replicasets.yaml │ │ └── statefulsets.yaml │ ├── batch │ │ ├── cronjobs.yaml │ │ └── jobs.yaml │ ├── core │ │ ├── configmaps.yaml │ │ ├── endpoints.yaml │ │ ├── events │ │ │ ├── elasticsearch-im-app-1596020400-gm6nl.1626341a296c16a1.yaml │ │ │ ├── elasticsearch-im-audit-1596020400-9l9n4.1626341a2af81bbd.yaml │ │ │ ├── elasticsearch-im-infra-1596020400-v98tk.1626341a2d821069.yaml │ │ │ ├── elasticsearch-im-app-1596020400-cc5vc.1626341a3019b238.yaml │ │ │ ├── elasticsearch-im-audit-1596020400-s8d5s.1626341a31f7b315.yaml │ │ │ ├── elasticsearch-im-infra-1596020400-7mgv8.1626341a35ea59ed.yaml │ │ ├── events.yaml │ │ ├── persistentvolumeclaims.yaml │ │ ├── pods.yaml │ │ ├── replicationcontrollers.yaml │ │ ├── secrets.yaml │ │ └── services.yaml │ ├── openshift-logging.yaml │ ├── pods │ │ ├── cluster-logging-operator-74dd5994f-6ttgt │ │ │ ├── cluster-logging-operator │ │ │ │ └── cluster-logging-operator │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ └── cluster-logging-operator-74dd5994f-6ttgt.yaml │ │ ├── cluster-logging-operator-registry-6df49d7d4-mxxff │ │ │ ├── cluster-logging-operator-registry │ │ │ │ └── cluster-logging-operator-registry │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ ├── cluster-logging-operator-registry-6df49d7d4-mxxff.yaml │ │ │ └── mutate-csv-and-generate-sqlite-db │ │ │ └── mutate-csv-and-generate-sqlite-db │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ │ ├── elasticsearch-cdm-lp8l38m0-1-794d6dd989-4jxms │ │ ├── elasticsearch-im-app-1596030300-bpgcx │ │ │ ├── elasticsearch-im-app-1596030300-bpgcx.yaml │ │ │ └── indexmanagement │ │ │ └── indexmanagement │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ │ ├── fluentd-2tr64 │ │ │ ├── fluentd │ │ │ │ └── fluentd │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ ├── fluentd-2tr64.yaml │ │ │ └── fluentd-init │ │ │ └── fluentd-init │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ │ ├── kibana-9d69668d4-2rkvz │ │ │ ├── kibana │ │ │ │ └── kibana │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ ├── kibana-9d69668d4-2rkvz.yaml │ │ │ └── kibana-proxy │ │ │ └── kibana-proxy │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ └── route.openshift.io │ └── routes.yaml └── openshift-operators-redhat ├──
oc adm must-gather --image-stream=openshift/must-gather \ 1 --image=quay.io/kubevirt/must-gather 2
tar cvaf must-gather.tar.gz must-gather.local.5421342344627712289/ 1
oc adm must-gather -- /usr/bin/gather_audit_logs
tar cvaf must-gather.tar.gz must-gather.local.472290403699006248 1
oc get clusterversion -o jsonpath='{.items[].spec.clusterID}{"\n"}'
oc get nodes
oc debug node/my-cluster-node
oc new-project dummy
oc patch namespace dummy --type=merge -p '{"metadata": {"annotations": { "scheduler.alpha.kubernetes.io/defaultTolerations": "[{\"operator\": \"Exists\"}]"}}}'
oc debug node/my-cluster-node
chroot /host
toolbox
sosreport -k crio.all=on -k crio.logs=on 1
Your sosreport has been generated and saved in: /host/var/tmp/sosreport-my-cluster-node-01234567-2020-05-28-eyjknxt.tar.xz 1 The checksum is: 382ffc167510fd71b4f12a4f40b97a4e
redhat-support-tool addattachment -c 01234567 /host/var/tmp/my-sosreport.tar.xz 1
oc debug node/my-cluster-node -- bash -c 'cat /host/var/tmp/sosreport-my-cluster-node-01234567-2020-05-28-eyjknxt.tar.xz' > /tmp/sosreport-my-cluster-node-01234567-2020-05-28-eyjknxt.tar.xz 1
ssh core@<bootstrap_fqdn> journalctl -b -f -u bootkube.service
ssh core@<bootstrap_fqdn> 'for pod in USD(sudo podman ps -a -q); do sudo podman logs USDpod; done'
oc adm node-logs --role=master -u kubelet 1
oc adm node-logs --role=master --path=openshift-apiserver
oc adm node-logs --role=master --path=openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/openshift-apiserver/audit.log
oc adm must-gather --dest-dir /tmp/captures \ <.> --source-dir '/tmp/tcpdump/' \ <.> --image registry.redhat.io/openshift4/network-tools-rhel8:latest \ <.> --node-selector 'node-role.kubernetes.io/worker' \ <.> --host-network=true \ <.> --timeout 30s \ <.> -- tcpdump -i any \ <.> -w /tmp/tcpdump/%Y-%m-%dT%H:%M:%S.pcap -W 1 -G 300
tmp/captures ├── event-filter.html ├── ip-10-0-192-217-ec2-internal 1 │ └── registry-redhat-io-openshift4-network-tools-rhel8-sha256-bca │ └── 2022-01-13T19:31:31.pcap ├── ip-10-0-201-178-ec2-internal 2 │ └── registry-redhat-io-openshift4-network-tools-rhel8-sha256-bca │ └── 2022-01-13T19:31:30.pcap ├── ip- └── timestamp
oc get nodes
oc debug node/my-cluster-node
chroot /host
ip ad
toolbox
tcpdump -nn -s 0 -i ens5 -w /host/var/tmp/my-cluster-node_USD(date +%d_%m_%Y-%H_%M_%S-%Z).pcap 1
chroot /host crictl ps
chroot /host crictl inspect --output yaml a7fe32346b120 | grep 'pid' | awk '{print USD2}'
nsenter -n -t 49628 -- tcpdump -nn -i ens5 -w /host/var/tmp/my-cluster-node-my-container_USD(date +%d_%m_%Y-%H_%M_%S-%Z).pcap.pcap 1
redhat-support-tool addattachment -c 01234567 /host/var/tmp/my-tcpdump-capture-file.pcap 1
oc debug node/my-cluster-node -- bash -c 'cat /host/var/tmp/my-tcpdump-capture-file.pcap' > /tmp/my-tcpdump-capture-file.pcap 1
oc debug node/my-cluster-node -- bash -c 'cat /host/var/tmp/my-diagnostic-data.tar.gz' > /var/tmp/my-diagnostic-data.tar.gz 1
oc get nodes
oc debug node/my-cluster-node
chroot /host
toolbox
redhat-support-tool addattachment -c 01234567 /host/var/tmp/my-diagnostic-data.tar.gz 1
chroot /host
toolbox
dnf install -y <package_name>
chroot /host
vi ~/.toolboxrc
REGISTRY=quay.io 1 IMAGE=fedora/fedora:33-x86_64 2 TOOLBOX_NAME=toolbox-fedora-33 3
toolbox
oc get clusterversion
oc describe clusterversion
ssh <user_name>@<load_balancer> systemctl status haproxy
ssh <user_name>@<load_balancer> netstat -nltupe | grep -E ':80|:443|:6443|:22623'
ssh <user_name>@<load_balancer> ss -nltupe | grep -E ':80|:443|:6443|:22623'
dig <wildcard_fqdn> @<dns_server>
./openshift-install --dir <installation_directory> wait-for bootstrap-complete --log-level debug 1
./openshift-install create ignition-configs --dir=./install_dir
tail -f ~/<installation_directory>/.openshift_install.log
ssh core@<bootstrap_fqdn> journalctl -b -f -u bootkube.service
oc adm node-logs --role=master -u kubelet
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=master -u crio
ssh core@master-N.cluster_name.sub_domain.domain journalctl -b -f -u crio.service
curl -I http://<http_server_fqdn>:<port>/bootstrap.ign 1
grep -is 'bootstrap.ign' /var/log/httpd/access_log
ssh core@<bootstrap_fqdn> journalctl -b -f -u bootkube.service
ssh core@<bootstrap_fqdn> 'for pod in USD(sudo podman ps -a -q); do sudo podman logs USDpod; done'
curl -I http://<http_server_fqdn>:<port>/master.ign 1
grep -is 'master.ign' /var/log/httpd/access_log
oc get nodes
oc describe node <master_node>
oc get daemonsets -n openshift-sdn
oc get pods -n openshift-sdn
oc logs <sdn_pod> -n openshift-sdn
oc get network.config.openshift.io cluster -o yaml
./openshift-install create manifests
oc get pods -n openshift-network-operator
oc logs pod/<network_operator_pod_name> -n openshift-network-operator
oc adm node-logs --role=master -u kubelet
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=master -u crio
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u crio.service
oc adm node-logs --role=master --path=openshift-apiserver
oc adm node-logs --role=master --path=openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl ps -a
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
curl https://api-int.<cluster_name>:22623/config/master
dig api-int.<cluster_name> @<dns_server>
dig -x <load_balancer_mco_ip_address> @<dns_server>
ssh core@<bootstrap_fqdn> curl https://api-int.<cluster_name>:22623/config/master
ssh core@<node>.<cluster_name>.<base_domain> chronyc tracking
openssl s_client -connect api-int.<cluster_name>:22623 | openssl x509 -noout -text
oc get pods -n openshift-etcd
oc get pods -n openshift-etcd-operator
oc describe pod/<pod_name> -n <namespace>
oc logs pod/<pod_name> -n <namespace>
oc logs pod/<pod_name> -c <container_name> -n <namespace>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl pods --name=etcd-
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspectp <pod_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl ps | grep '<pod_id>'
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspect <container_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
oc adm node-logs --role=master -u kubelet
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=master -u kubelet | grep -is 'x509: certificate has expired'
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service | grep -is 'x509: certificate has expired'
curl -I http://<http_server_fqdn>:<port>/worker.ign 1
grep -is 'worker.ign' /var/log/httpd/access_log
oc get nodes
oc describe node <worker_node>
oc get pods -n openshift-machine-api
oc describe pod/<machine_api_operator_pod_name> -n openshift-machine-api
oc logs pod/<machine_api_operator_pod_name> -n openshift-machine-api -c machine-api-operator
oc logs pod/<machine_api_operator_pod_name> -n openshift-machine-api -c kube-rbac-proxy
oc adm node-logs --role=worker -u kubelet
ssh core@<worker-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=worker -u crio
ssh core@<worker-node>.<cluster_name>.<base_domain> journalctl -b -f -u crio.service
oc adm node-logs --role=worker --path=sssd
oc adm node-logs --role=worker --path=sssd/sssd.log
ssh core@<worker-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/sssd/sssd.log
ssh core@<worker-node>.<cluster_name>.<base_domain> sudo crictl ps -a
ssh core@<worker-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
curl https://api-int.<cluster_name>:22623/config/worker
dig api-int.<cluster_name> @<dns_server>
dig -x <load_balancer_mco_ip_address> @<dns_server>
ssh core@<bootstrap_fqdn> curl https://api-int.<cluster_name>:22623/config/worker
ssh core@<node>.<cluster_name>.<base_domain> chronyc tracking
openssl s_client -connect api-int.<cluster_name>:22623 | openssl x509 -noout -text
oc get clusteroperators
oc get csr
NAME AGE REQUESTOR CONDITION csr-8b2br 15m system:serviceaccount:openshift-machine-config-operator:node-bootstrapper Pending 1 csr-8vnps 15m system:serviceaccount:openshift-machine-config-operator:node-bootstrapper Pending csr-bfd72 5m26s system:node:ip-10-0-50-126.us-east-2.compute.internal Pending 2 csr-c57lv 5m26s system:node:ip-10-0-95-157.us-east-2.compute.internal Pending
oc adm certificate approve <csr_name> 1
oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve
oc describe clusteroperator <operator_name>
oc get pods -n <operator_namespace>
oc describe pod/<operator_pod_name> -n <operator_namespace>
oc logs pod/<operator_pod_name> -n <operator_namespace>
oc get pod -o "jsonpath={range .status.containerStatuses[*]}{.name}{'\t'}{.state}{'\t'}{.image}{'\n'}{end}" <operator_pod_name> -n <operator_namespace>
oc adm release info <image_path>:<tag> --commits
./openshift-install gather bootstrap --dir <installation_directory> 1
./openshift-install gather bootstrap --dir <installation_directory> \ 1 --bootstrap <bootstrap_address> \ 2 --master <master_1_address> \ 3 --master <master_2_address> \ 4 --master <master_3_address>" 5
INFO Pulling debug logs from the bootstrap machine INFO Bootstrap gather logs captured here "<installation_directory>/log-bundle-<timestamp>.tar.gz"
oc get nodes
oc adm top nodes
oc adm top node my-node
oc debug node/my-node
chroot /host
systemctl is-active kubelet
systemctl status kubelet
oc adm node-logs --role=master -u kubelet 1
oc adm node-logs --role=master --path=openshift-apiserver
oc adm node-logs --role=master --path=openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/openshift-apiserver/audit.log
oc debug node/my-node
chroot /host
systemctl is-active crio
systemctl status crio.service
oc adm node-logs --role=master -u crio
oc adm node-logs <node_name> -u crio
ssh core@<node>.<cluster_name>.<base_domain> journalctl -b -f -u crio.service
Failed to create pod sandbox: rpc error: code = Unknown desc = failed to mount container XXX: error recreating the missing symlinks: error reading name of symlink for XXX: open /var/lib/containers/storage/overlay/XXX/link: no such file or directory
can't stat lower layer ... because it does not exist. Going through storage to recreate the missing symlinks.
oc adm cordon <nodename>
oc adm drain <nodename> --ignore-daemonsets --delete-emptydir-data
ssh core@node1.example.com sudo -i
systemctl stop kubelet
.. for pod in USD(crictl pods -q); do if [[ "USD(crictl inspectp USDpod | jq -r .status.linux.namespaces.options.network)" != "NODE" ]]; then crictl rmp -f USDpod; fi; done
crictl rmp -fa
systemctl stop crio
crio wipe -f
systemctl start crio systemctl start kubelet
oc get nodes
NAME STATUS ROLES AGE VERSION ci-ln-tkbxyft-f76d1-nvwhr-master-1 Ready, SchedulingDisabled master 133m v1.23.0
oc adm uncordon <nodename>
NAME STATUS ROLES AGE VERSION ci-ln-tkbxyft-f76d1-nvwhr-master-1 Ready master 133m v1.23.0
rpm-ostree kargs --append='crashkernel=256M'
systemctl enable kdump.service
systemctl reboot
variant: openshift version: 4.10.0 metadata: name: 99-worker-kdump 1 labels: machineconfiguration.openshift.io/role: worker 2 openshift: kernel_arguments: 3 - crashkernel=256M storage: files: - path: /etc/kdump.conf 4 mode: 0644 overwrite: true contents: inline: | path /var/crash core_collector makedumpfile -l --message-level 7 -d 31 - path: /etc/sysconfig/kdump 5 mode: 0644 overwrite: true contents: inline: | KDUMP_COMMANDLINE_REMOVE="hugepages hugepagesz slub_debug quiet log_buf_len swiotlb" KDUMP_COMMANDLINE_APPEND="irqpoll nr_cpus=1 reset_devices cgroup_disable=memory mce=off numa=off udev.children-max=2 panic=10 rootflags=nofail acpi_no_memhotplug transparent_hugepage=never nokaslr novmcoredd hest_disable" KEXEC_ARGS="-s" KDUMP_IMG="vmlinuz" systemd: units: - name: kdump.service enabled: true
butane 99-worker-kdump.bu -o 99-worker-kdump.yaml
oc create -f ./99-worker-kdump.yaml
systemctl --failed
journalctl -u <unit>.service
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: 98-nodenet-override spec: config: ignition: version: 3.2.0 storage: files: - contents: source: data:text/plain;charset=utf-8;base64,<encoded_script> mode: 0755 overwrite: true path: /usr/local/bin/override-node-ip.sh systemd: units: - contents: | [Unit] Description=Override node IP detection Wants=network-online.target Before=kubelet.service After=network-online.target [Service] Type=oneshot ExecStart=/usr/local/bin/override-node-ip.sh ExecStart=systemctl daemon-reload [Install] WantedBy=multi-user.target enabled: true name: nodenet-override.service
E0514 12:47:17.998892 2281 daemon.go:1350] content mismatch for file /etc/systemd/system/ovs-vswitchd.service: [Unit]
oc debug node/<node_name>
chroot /host
ovs-appctl vlog/list
console syslog file ------- ------ ------ backtrace OFF INFO INFO bfd OFF INFO INFO bond OFF INFO INFO bridge OFF INFO INFO bundle OFF INFO INFO bundles OFF INFO INFO cfm OFF INFO INFO collectors OFF INFO INFO command_line OFF INFO INFO connmgr OFF INFO INFO conntrack OFF INFO INFO conntrack_tp OFF INFO INFO coverage OFF INFO INFO ct_dpif OFF INFO INFO daemon OFF INFO INFO daemon_unix OFF INFO INFO dns_resolve OFF INFO INFO dpdk OFF INFO INFO
Restart=always ExecStartPre=-/bin/sh -c '/usr/bin/chown -R :USDUSD{OVS_USER_ID##*:} /var/lib/openvswitch' ExecStartPre=-/bin/sh -c '/usr/bin/chown -R :USDUSD{OVS_USER_ID##*:} /etc/openvswitch' ExecStartPre=-/bin/sh -c '/usr/bin/chown -R :USDUSD{OVS_USER_ID##*:} /run/openvswitch' ExecStartPost=-/usr/bin/ovs-appctl vlog/set syslog:dbg ExecReload=-/usr/bin/ovs-appctl vlog/set syslog:dbg
systemctl daemon-reload
systemctl restart ovs-vswitchd
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: master 1 name: 99-change-ovs-loglevel spec: config: ignition: version: 3.2.0 systemd: units: - dropins: - contents: | [Service] ExecStartPost=-/usr/bin/ovs-appctl vlog/set syslog:dbg 2 ExecReload=-/usr/bin/ovs-appctl vlog/set syslog:dbg name: 20-ovs-vswitchd-restart.conf name: ovs-vswitchd.service
oc apply -f 99-change-ovs-loglevel.yaml
oc adm node-logs <node_name> -u ovs-vswitchd
journalctl -b -f -u ovs-vswitchd.service
oc get subs -n <operator_namespace>
oc describe sub <subscription_name> -n <operator_namespace>
Conditions: Last Transition Time: 2019-07-29T13:42:57Z Message: all available catalogsources are healthy Reason: AllCatalogSourcesHealthy Status: False Type: CatalogSourcesUnhealthy
oc get catalogsources -n openshift-marketplace
NAME DISPLAY TYPE PUBLISHER AGE certified-operators Certified Operators grpc Red Hat 55m community-operators Community Operators grpc Red Hat 55m example-catalog Example Catalog grpc Example Org 2m25s redhat-marketplace Red Hat Marketplace grpc Red Hat 55m redhat-operators Red Hat Operators grpc Red Hat 55m
oc describe catalogsource example-catalog -n openshift-marketplace
Name: example-catalog Namespace: openshift-marketplace Status: Connection State: Address: example-catalog.openshift-marketplace.svc:50051 Last Connect: 2021-09-09T17:07:35Z Last Observed State: TRANSIENT_FAILURE Registry Service: Created At: 2021-09-09T17:05:45Z Port: 50051 Protocol: grpc Service Name: example-catalog Service Namespace: openshift-marketplace
oc get pods -n openshift-marketplace
NAME READY STATUS RESTARTS AGE certified-operators-cv9nn 1/1 Running 0 36m community-operators-6v8lp 1/1 Running 0 36m marketplace-operator-86bfc75f9b-jkgbc 1/1 Running 0 42m example-catalog-bwt8z 0/1 ImagePullBackOff 0 3m55s redhat-marketplace-57p8c 1/1 Running 0 36m redhat-operators-smxx8 1/1 Running 0 36m
oc describe pod example-catalog-bwt8z -n openshift-marketplace
Name: example-catalog-bwt8z Namespace: openshift-marketplace Priority: 0 Node: ci-ln-jyryyg2-f76d1-ggdbq-worker-b-vsxjd/10.0.128.2 Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 48s default-scheduler Successfully assigned openshift-marketplace/example-catalog-bwt8z to ci-ln-jyryyf2-f76d1-fgdbq-worker-b-vsxjd Normal AddedInterface 47s multus Add eth0 [10.131.0.40/23] from openshift-sdn Normal BackOff 20s (x2 over 46s) kubelet Back-off pulling image "quay.io/example-org/example-catalog:v1" Warning Failed 20s (x2 over 46s) kubelet Error: ImagePullBackOff Normal Pulling 8s (x3 over 47s) kubelet Pulling image "quay.io/example-org/example-catalog:v1" Warning Failed 8s (x3 over 47s) kubelet Failed to pull image "quay.io/example-org/example-catalog:v1": rpc error: code = Unknown desc = reading manifest v1 in quay.io/example-org/example-catalog: unauthorized: access to the requested resource is not authorized Warning Failed 8s (x3 over 47s) kubelet Error: ErrImagePull
oc get clusteroperators
oc get pod -n <operator_namespace>
oc describe pod <operator_pod_name> -n <operator_namespace>
oc debug node/my-node
chroot /host
crictl ps
crictl ps --name network-operator
oc get pods -n <operator_namespace>
oc logs pod/<pod_name> -n <operator_namespace>
oc logs pod/<operator_pod_name> -c <container_name> -n <operator_namespace>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl pods
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspectp <operator_pod_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl ps --pod=<operator_pod_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspect <container_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool spec: paused: true 1
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool spec: paused: false 1
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/master
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/worker
oc get machineconfigpool/master --template='{{.spec.paused}}'
oc get machineconfigpool/worker --template='{{.spec.paused}}'
true
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING master rendered-master-33cf0a1254318755d7b48002c597bf91 True False worker rendered-worker-e405a5bdb0db1295acea08bcca33fa60 False False
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/master
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/worker
oc get machineconfigpool/master --template='{{.spec.paused}}'
oc get machineconfigpool/worker --template='{{.spec.paused}}'
false
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING master rendered-master-546383f80705bd5aeaba93 True False worker rendered-worker-b4c51bb33ccaae6fc4a6a5 False True
ImagePullBackOff for Back-off pulling image "example.com/openshift4/ose-elasticsearch-operator-bundle@sha256:6d2587129c846ec28d384540322b40b05833e7e00b25cca584e004af9a1d292e"
rpc error: code = Unknown desc = error pinging docker registry example.com: Get "https://example.com/v2/": dial tcp: lookup example.com on 10.0.0.1:53: no such host
oc get sub,csv -n <namespace>
NAME PACKAGE SOURCE CHANNEL subscription.operators.coreos.com/elasticsearch-operator elasticsearch-operator redhat-operators 5.0 NAME DISPLAY VERSION REPLACES PHASE clusterserviceversion.operators.coreos.com/elasticsearch-operator.5.0.0-65 OpenShift Elasticsearch Operator 5.0.0-65 Succeeded
oc delete subscription <subscription_name> -n <namespace>
oc delete csv <csv_name> -n <namespace>
oc get job,configmap -n openshift-marketplace
NAME COMPLETIONS DURATION AGE job.batch/1de9443b6324e629ddf31fed0a853a121275806170e34c926d69e53a7fcbccb 1/1 26s 9m30s NAME DATA AGE configmap/1de9443b6324e629ddf31fed0a853a121275806170e34c926d69e53a7fcbccb 3 9m30s
oc delete job <job_name> -n openshift-marketplace
oc delete configmap <configmap_name> -n openshift-marketplace
oc get sub,csv,installplan -n <namespace>
message: 'Failed to delete all resource types, 1 remaining: Internal error occurred: error resolving resource'
oc get namespaces
operator-ns-1 Terminating
oc get crds
oc delete crd <crd_name>
oc get EtcdCluster -n <namespace_name>
oc get EtcdCluster --all-namespaces
oc delete <cr_name> <cr_instance_name> -n <namespace_name>
oc get namespace <namespace_name>
oc get sub,csv,installplan -n <namespace>
oc project <project_name>
oc get pods
oc status
skopeo inspect docker://<image_reference>
oc edit deployment/my-deployment
oc get pods -w
oc get events
oc logs <pod_name>
oc logs <pod_name> -c <container_name>
oc exec <pod_name> ls -alh /var/log
oc exec <pod_name> cat /var/log/<path_to_log>
oc exec <pod_name> -c <container_name> ls /var/log
oc exec <pod_name> -c <container_name> cat /var/log/<path_to_log>
oc project <namespace>
oc rsh <pod_name> 1
oc rsh -c <container_name> pod/<pod_name>
oc port-forward <pod_name> <host_port>:<pod_port> 1
oc get deployment -n <project_name>
oc debug deployment/my-deployment --as-root -n <project_name>
oc get deploymentconfigs -n <project_name>
oc debug deploymentconfig/my-deployment-configuration --as-root -n <project_name>
oc cp <local_path> <pod_name>:/<path> -c <container_name> 1
oc cp <pod_name>:/<path> -c <container_name><local_path> 1
oc get pods -w 1
oc logs -f pod/<application_name>-<build_number>-build
oc logs -f pod/<application_name>-<build_number>-deploy
oc logs -f pod/<application_name>-<build_number>-<random_string>
oc describe pod/my-app-1-akdlg
oc logs -f pod/my-app-1-akdlg
oc exec my-app-1-akdlg -- cat /var/log/my-application.log
oc debug dc/my-deployment-configuration --as-root -- cat /var/log/my-application.log
oc exec -it my-app-1-akdlg /bin/bash
oc debug node/my-cluster-node
chroot /host
crictl ps
crictl inspect a7fe32346b120 --output yaml | grep 'pid:' | awk '{print USD2}'
nsenter -n -t 31150 -- ip ad
Unable to attach or mount volumes: unmounted volumes=[sso-mysql-pvol], unattached volumes=[sso-mysql-pvol default-token-x4rzc]: timed out waiting for the condition Multi-Attach error for volume "pvc-8837384d-69d7-40b2-b2e6-5df86943eef9" Volume is already used by pod(s) sso-mysql-1-ns6b4
oc delete pod <old_pod> --force=true --grace-period=0
oc logs -f deployment/windows-machine-config-operator -n openshift-windows-machine-config-operator
ssh -t -o StrictHostKeyChecking=no -o ProxyCommand='ssh -A -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -W %h:%p core@USD(oc get service --all-namespaces -l run=ssh-bastion -o go-template="{{ with (index (index .items 0).status.loadBalancer.ingress 0) }}{{ or .hostname .ip }}{{end}}")' <username>@<windows_node_internal_ip> 1 2
oc get nodes <node_name> -o jsonpath={.status.addresses[?\(@.type==\"InternalIP\"\)].address}
ssh -L 2020:<windows_node_internal_ip>:3389 \ 1 core@USD(oc get service --all-namespaces -l run=ssh-bastion -o go-template="{{ with (index (index .items 0).status.loadBalancer.ingress 0) }}{{ or .hostname .ip }}{{end}}")
oc get nodes <node_name> -o jsonpath={.status.addresses[?\(@.type==\"InternalIP\"\)].address}
C:\> net user <username> * 1
oc adm node-logs -l kubernetes.io/os=windows --path= /ip-10-0-138-252.us-east-2.compute.internal containers /ip-10-0-138-252.us-east-2.compute.internal hybrid-overlay /ip-10-0-138-252.us-east-2.compute.internal kube-proxy /ip-10-0-138-252.us-east-2.compute.internal kubelet /ip-10-0-138-252.us-east-2.compute.internal pods
oc adm node-logs -l kubernetes.io/os=windows --path=/kubelet/kubelet.log
oc adm node-logs -l kubernetes.io/os=windows --path=journal
oc adm node-logs -l kubernetes.io/os=windows --path=journal -u docker
C:\> powershell
C:\> Get-EventLog -LogName Application -Source Docker
oc -n ns1 get service prometheus-example-app -o yaml
labels: app: prometheus-example-app
oc -n ns1 get servicemonitor prometheus-example-monitor -o yaml
spec: endpoints: - interval: 30s port: web scheme: http selector: matchLabels: app: prometheus-example-app
oc -n openshift-user-workload-monitoring get pods
NAME READY STATUS RESTARTS AGE prometheus-operator-776fcbbd56-2nbfm 2/2 Running 0 132m prometheus-user-workload-0 5/5 Running 1 132m prometheus-user-workload-1 5/5 Running 1 132m thanos-ruler-user-workload-0 3/3 Running 0 132m thanos-ruler-user-workload-1 3/3 Running 0 132m
oc -n openshift-user-workload-monitoring logs prometheus-operator-776fcbbd56-2nbfm -c prometheus-operator
level=warn ts=2020-08-10T11:48:20.906739623Z caller=operator.go:1829 component=prometheusoperator msg="skipping servicemonitor" error="it accesses file system via bearer token file which Prometheus specification prohibits" servicemonitor=eagle/eagle namespace=openshift-user-workload-monitoring prometheus=user-workload
oc port-forward -n openshift-user-workload-monitoring pod/prometheus-user-workload-0 9090
oc -n openshift-user-workload-monitoring edit configmap user-workload-monitoring-config
apiVersion: v1 kind: ConfigMap metadata: name: user-workload-monitoring-config namespace: openshift-user-workload-monitoring data: config.yaml: | prometheusOperator: logLevel: debug
oc -n openshift-user-workload-monitoring get deploy prometheus-operator -o yaml | grep "log-level"
- --log-level=debug
oc -n openshift-user-workload-monitoring get pods
topk(10,count by (job)({__name__=~".+"}))
oc <options> --loglevel <log_level>
oc whoami -t