Support OpenShift Container Platform 4.14 Getting support for OpenShift Container Platform Red Hat OpenShift Documentation Team

Commands:
oc api-resources -o name | grep config.openshift.io
oc explain <resource_name>.config.openshift.io
oc get <resource_name>.config -o yaml
oc edit <resource_name>.config -o yaml
oc get clusterversion -o jsonpath='{.items[].spec.clusterID}{"\n"}'
curl -G -k -H "Authorization: Bearer USD(oc whoami -t)" https://USD(oc get route prometheus-k8s-federate -n openshift-monitoring -o jsonpath="{.spec.host}")/federate --data-urlencode 'match[]={__name__=~"cluster:usage:.*"}' --data-urlencode 'match[]={__name__="count:up0"}' --data-urlencode 'match[]={__name__="count:up1"}' --data-urlencode 'match[]={__name__="cluster_version"}' --data-urlencode 'match[]={__name__="cluster_version_available_updates"}' --data-urlencode 'match[]={__name__="cluster_version_capability"}' --data-urlencode 'match[]={__name__="cluster_operator_up"}' --data-urlencode 'match[]={__name__="cluster_operator_conditions"}' --data-urlencode 'match[]={__name__="cluster_version_payload"}' --data-urlencode 'match[]={__name__="cluster_installer"}' --data-urlencode 'match[]={__name__="cluster_infrastructure_provider"}' --data-urlencode 'match[]={__name__="cluster_feature_set"}' --data-urlencode 'match[]={__name__="instance:etcd_object_counts:sum"}' --data-urlencode 'match[]={__name__="ALERTS",alertstate="firing"}' --data-urlencode 'match[]={__name__="code:apiserver_request_total:rate:sum"}' --data-urlencode 'match[]={__name__="cluster:capacity_cpu_cores:sum"}' --data-urlencode 'match[]={__name__="cluster:capacity_memory_bytes:sum"}' --data-urlencode 'match[]={__name__="cluster:cpu_usage_cores:sum"}' --data-urlencode 'match[]={__name__="cluster:memory_usage_bytes:sum"}' --data-urlencode 'match[]={__name__="openshift:cpu_usage_cores:sum"}' --data-urlencode 'match[]={__name__="openshift:memory_usage_bytes:sum"}' --data-urlencode 'match[]={__name__="workload:cpu_usage_cores:sum"}' --data-urlencode 'match[]={__name__="workload:memory_usage_bytes:sum"}' --data-urlencode 'match[]={__name__="cluster:virt_platform_nodes:sum"}' --data-urlencode 'match[]={__name__="cluster:node_instance_type_count:sum"}' --data-urlencode 'match[]={__name__="cnv:vmi_status_running:count"}' --data-urlencode 'match[]={__name__="cluster:vmi_request_cpu_cores:sum"}' --data-urlencode 'match[]={__name__="node_role_os_version_machine:cpu_capacity_cores:sum"}' --data-urlencode 'match[]={__name__="node_role_os_version_machine:cpu_capacity_sockets:sum"}' --data-urlencode 'match[]={__name__="subscription_sync_total"}' --data-urlencode 'match[]={__name__="olm_resolution_duration_seconds"}' --data-urlencode 'match[]={__name__="csv_succeeded"}' --data-urlencode 'match[]={__name__="csv_abnormal"}' --data-urlencode 'match[]={__name__="cluster:kube_persistentvolumeclaim_resource_requests_storage_bytes:provisioner:sum"}' --data-urlencode 'match[]={__name__="cluster:kubelet_volume_stats_used_bytes:provisioner:sum"}' --data-urlencode 'match[]={__name__="ceph_cluster_total_bytes"}' --data-urlencode 'match[]={__name__="ceph_cluster_total_used_raw_bytes"}' --data-urlencode 'match[]={__name__="ceph_health_status"}' --data-urlencode 'match[]={__name__="odf_system_raw_capacity_total_bytes"}' --data-urlencode 'match[]={__name__="odf_system_raw_capacity_used_bytes"}' --data-urlencode 'match[]={__name__="odf_system_health_status"}' --data-urlencode 'match[]={__name__="job:ceph_osd_metadata:count"}' --data-urlencode 'match[]={__name__="job:kube_pv:count"}' --data-urlencode 'match[]={__name__="job:odf_system_pvs:count"}' --data-urlencode 'match[]={__name__="job:ceph_pools_iops:total"}' --data-urlencode 'match[]={__name__="job:ceph_pools_iops_bytes:total"}' --data-urlencode 'match[]={__name__="job:ceph_versions_running:count"}' --data-urlencode 'match[]={__name__="job:noobaa_total_unhealthy_buckets:sum"}' --data-urlencode 'match[]={__name__="job:noobaa_bucket_count:sum"}' --data-urlencode 'match[]={__name__="job:noobaa_total_object_count:sum"}' --data-urlencode 'match[]={__name__="odf_system_bucket_count", system_type="OCS", system_vendor="Red Hat"}' --data-urlencode 'match[]={__name__="odf_system_objects_total", system_type="OCS", system_vendor="Red Hat"}' --data-urlencode 'match[]={__name__="noobaa_accounts_num"}' --data-urlencode 'match[]={__name__="noobaa_total_usage"}' --data-urlencode 'match[]={__name__="console_url"}' --data-urlencode 'match[]={__name__="cluster:ovnkube_master_egress_routing_via_host:max"}' --data-urlencode 'match[]={__name__="cluster:network_attachment_definition_instances:max"}' --data-urlencode 'match[]={__name__="cluster:network_attachment_definition_enabled_instance_up:max"}' --data-urlencode 'match[]={__name__="cluster:ingress_controller_aws_nlb_active:sum"}' --data-urlencode 'match[]={__name__="cluster:route_metrics_controller_routes_per_shard:min"}' --data-urlencode 'match[]={__name__="cluster:route_metrics_controller_routes_per_shard:max"}' --data-urlencode 'match[]={__name__="cluster:route_metrics_controller_routes_per_shard:avg"}' --data-urlencode 'match[]={__name__="cluster:route_metrics_controller_routes_per_shard:median"}' --data-urlencode 'match[]={__name__="cluster:openshift_route_info:tls_termination:sum"}' --data-urlencode 'match[]={__name__="insightsclient_request_send_total"}' --data-urlencode 'match[]={__name__="cam_app_workload_migrations"}' --data-urlencode 'match[]={__name__="cluster:apiserver_current_inflight_requests:sum:max_over_time:2m"}' --data-urlencode 'match[]={__name__="cluster:alertmanager_integrations:max"}' --data-urlencode 'match[]={__name__="cluster:telemetry_selected_series:count"}' --data-urlencode 'match[]={__name__="openshift:prometheus_tsdb_head_series:sum"}' --data-urlencode 'match[]={__name__="openshift:prometheus_tsdb_head_samples_appended_total:sum"}' --data-urlencode 'match[]={__name__="monitoring:container_memory_working_set_bytes:sum"}' --data-urlencode 'match[]={__name__="namespace_job:scrape_series_added:topk3_sum1h"}' --data-urlencode 'match[]={__name__="namespace_job:scrape_samples_post_metric_relabeling:topk3"}' --data-urlencode 'match[]={__name__="monitoring:haproxy_server_http_responses_total:sum"}' --data-urlencode 'match[]={__name__="rhmi_status"}' --data-urlencode 'match[]={__name__="status:upgrading:version:rhoam_state:max"}' --data-urlencode 'match[]={__name__="state:rhoam_critical_alerts:max"}' --data-urlencode 'match[]={__name__="state:rhoam_warning_alerts:max"}' --data-urlencode 'match[]={__name__="rhoam_7d_slo_percentile:max"}' --data-urlencode 'match[]={__name__="rhoam_7d_slo_remaining_error_budget:max"}' --data-urlencode 'match[]={__name__="cluster_legacy_scheduler_policy"}' --data-urlencode 'match[]={__name__="cluster_master_schedulable"}' --data-urlencode 'match[]={__name__="che_workspace_status"}' --data-urlencode 'match[]={__name__="che_workspace_started_total"}' --data-urlencode 'match[]={__name__="che_workspace_failure_total"}' --data-urlencode 'match[]={__name__="che_workspace_start_time_seconds_sum"}' --data-urlencode 'match[]={__name__="che_workspace_start_time_seconds_count"}' --data-urlencode 'match[]={__name__="cco_credentials_mode"}' --data-urlencode 'match[]={__name__="cluster:kube_persistentvolume_plugin_type_counts:sum"}' --data-urlencode 'match[]={__name__="visual_web_terminal_sessions_total"}' --data-urlencode 'match[]={__name__="acm_managed_cluster_info"}' --data-urlencode 'match[]={__name__="cluster:vsphere_vcenter_info:sum"}' --data-urlencode 'match[]={__name__="cluster:vsphere_esxi_version_total:sum"}' --data-urlencode 'match[]={__name__="cluster:vsphere_node_hw_version_total:sum"}' --data-urlencode 'match[]={__name__="openshift:build_by_strategy:sum"}' --data-urlencode 'match[]={__name__="rhods_aggregate_availability"}' --data-urlencode 'match[]={__name__="rhods_total_users"}' --data-urlencode 'match[]={__name__="instance:etcd_disk_wal_fsync_duration_seconds:histogram_quantile",quantile="0.99"}' --data-urlencode 'match[]={__name__="instance:etcd_mvcc_db_total_size_in_bytes:sum"}' --data-urlencode 'match[]={__name__="instance:etcd_network_peer_round_trip_time_seconds:histogram_quantile",quantile="0.99"}' --data-urlencode 'match[]={__name__="instance:etcd_mvcc_db_total_size_in_use_in_bytes:sum"}' --data-urlencode 'match[]={__name__="instance:etcd_disk_backend_commit_duration_seconds:histogram_quantile",quantile="0.99"}' --data-urlencode 'match[]={__name__="jaeger_operator_instances_storage_types"}' --data-urlencode 'match[]={__name__="jaeger_operator_instances_strategies"}' --data-urlencode 'match[]={__name__="jaeger_operator_instances_agent_strategies"}' --data-urlencode 'match[]={__name__="appsvcs:cores_by_product:sum"}' --data-urlencode 'match[]={__name__="nto_custom_profiles:count"}' --data-urlencode 'match[]={__name__="openshift_csi_share_configmap"}' --data-urlencode 'match[]={__name__="openshift_csi_share_secret"}' --data-urlencode 'match[]={__name__="openshift_csi_share_mount_failures_total"}' --data-urlencode 'match[]={__name__="openshift_csi_share_mount_requests_total"}' --data-urlencode 'match[]={__name__="cluster:velero_backup_total:max"}' --data-urlencode 'match[]={__name__="cluster:velero_restore_total:max"}' --data-urlencode 'match[]={__name__="eo_es_storage_info"}' --data-urlencode 'match[]={__name__="eo_es_redundancy_policy_info"}' --data-urlencode 'match[]={__name__="eo_es_defined_delete_namespaces_total"}' --data-urlencode 'match[]={__name__="eo_es_misconfigured_memory_resources_info"}' --data-urlencode 'match[]={__name__="cluster:eo_es_data_nodes_total:max"}' --data-urlencode 'match[]={__name__="cluster:eo_es_documents_created_total:sum"}' --data-urlencode 'match[]={__name__="cluster:eo_es_documents_deleted_total:sum"}' --data-urlencode 'match[]={__name__="pod:eo_es_shards_total:max"}' --data-urlencode 'match[]={__name__="eo_es_cluster_management_state_info"}' --data-urlencode 'match[]={__name__="imageregistry:imagestreamtags_count:sum"}' --data-urlencode 'match[]={__name__="imageregistry:operations_count:sum"}' --data-urlencode 'match[]={__name__="log_logging_info"}' --data-urlencode 'match[]={__name__="log_collector_error_count_total"}' --data-urlencode 'match[]={__name__="log_forwarder_pipeline_info"}' --data-urlencode 'match[]={__name__="log_forwarder_input_info"}' --data-urlencode 'match[]={__name__="log_forwarder_output_info"}' --data-urlencode 'match[]={__name__="cluster:log_collected_bytes_total:sum"}' --data-urlencode 'match[]={__name__="cluster:log_logged_bytes_total:sum"}' --data-urlencode 'match[]={__name__="cluster:kata_monitor_running_shim_count:sum"}' --data-urlencode 'match[]={__name__="platform:hypershift_hostedclusters:max"}' --data-urlencode 'match[]={__name__="platform:hypershift_nodepools:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_unhealthy_bucket_claims:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_buckets_claims:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_unhealthy_namespace_resources:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_namespace_resources:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_unhealthy_namespace_buckets:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_namespace_buckets:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_accounts:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_usage:max"}' --data-urlencode 'match[]={__name__="namespace:noobaa_system_health_status:max"}' --data-urlencode 'match[]={__name__="ocs_advanced_feature_usage"}' --data-urlencode 'match[]={__name__="os_image_url_override:sum"}'
INSIGHTS_OPERATOR_POD=USD(oc get pods --namespace=openshift-insights -o custom-columns=:metadata.name --no-headers --field-selector=status.phase=Running)
oc cp openshift-insights/USDINSIGHTS_OPERATOR_POD:/var/lib/insights-operator ./insights-data
oc extract secret/pull-secret -n openshift-config --to=.
"cloud.openshift.com":{"auth":"<hash>","email":"<email_address>"}
oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' ><pull_secret_location> 1
oc registry login --registry="<registry>" \ 1 --auth-basic="<username>:<password>" \ 2 --to=<pull_secret_location> 3
oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=<pull_secret_location> 1
{ "auths": { "cloud.openshift.com": { "auth": " <your_token> ", "email": " <email_address> " } } }
oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' > pull-secret
cp pull-secret pull-secret-backup
set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=pull-secret
oc get pods --namespace=openshift-insights -o custom-columns=:metadata.name --no-headers --field-selector=status.phase=Running
oc cp openshift-insights/<insights_operator_pod_name>:/var/lib/insights-operator ./insights-data 1
{ "name": "clusterconfig/authentication", "duration_in_ms": 730, 1 "records_count": 1, "errors": null, "panic": null }
apiVersion: config.openshift.io/v1alpha1 kind: InsightsDataGather metadata: . spec: 1 gatherConfig: disabledGatherers: - all 2
spec: gatherConfig: disabledGatherers: - clusterconfig/container_images 1 - clusterconfig/host_subnets - workloads/workload_info
apiVersion: config.openshift.io/v1alpha1 kind: InsightsDataGather metadata: . spec: gatherConfig: 1 disabledGatherers: all
spec: gatherConfig: disabledGatherers: - clusterconfig/container_images 1 - clusterconfig/host_subnets - workloads/workload_info
apiVersion: insights.openshift.io/v1alpha1 kind: DataGather metadata: name: <your_data_gather> 1 spec: gatherers: 2 - name: workloads state: Disabled
oc apply -f <your_datagather_definition>.yaml
apiVersion: insights.openshift.io/v1alpha1 kind: DataGather metadata: name: <your_data_gather> 1 spec: gatherers: 2 - name: workloads state: Disabled
apiVersion: batch/v1 kind: Job metadata: name: insights-operator-job annotations: config.openshift.io/inject-proxy: insights-operator spec: backoffLimit: 6 ttlSecondsAfterFinished: 600 template: spec: restartPolicy: OnFailure serviceAccountName: operator nodeSelector: beta.kubernetes.io/os: linux node-role.kubernetes.io/master: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Exists - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 900 - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 900 volumes: - name: snapshots emptyDir: {} - name: service-ca-bundle configMap: name: service-ca-bundle optional: true initContainers: - name: insights-operator image: quay.io/openshift/origin-insights-operator:latest terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: snapshots mountPath: /var/lib/insights-operator - name: service-ca-bundle mountPath: /var/run/configmaps/service-ca-bundle readOnly: true ports: - containerPort: 8443 name: https resources: requests: cpu: 10m memory: 70Mi args: - gather - -v=4 - --config=/etc/insights-operator/server.yaml containers: - name: sleepy image: quay.io/openshift/origin-base:latest args: - /bin/sh - -c - sleep 10m volumeMounts: [{name: snapshots, mountPath: /var/lib/insights-operator}]
oc get -n openshift-insights deployment insights-operator -o yaml
apiVersion: apps/v1 kind: Deployment metadata: name: insights-operator namespace: openshift-insights spec: template: spec: containers: - args: image: registry.ci.openshift.org/ocp/4.15-2023-10-12-212500@sha256:a0aa581400805ad0... 1
apiVersion: batch/v1 kind: Job metadata: name: insights-operator-job spec: template: spec: initContainers: - name: insights-operator image: image: registry.ci.openshift.org/ocp/4.15-2023-10-12-212500@sha256:a0aa581400805ad0... 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts:
oc apply -n openshift-insights -f gather-job.yaml
oc describe -n openshift-insights job/insights-operator-job
Name: insights-operator-job Namespace: openshift-insights Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal SuccessfulCreate 7m18s job-controller Created pod: insights-operator-job-<your_job>
oc logs -n openshift-insights insights-operator-job-<your_job> insights-operator
I0407 11:55:38.192084 1 diskrecorder.go:34] Wrote 108 records to disk in 33ms
oc cp openshift-insights/insights-operator-job- <your_job> :/var/lib/insights-operator ./insights-data
oc delete -n openshift-insights job insights-operator-job
oc extract secret/pull-secret -n openshift-config --to=.
{ "auths": { "cloud.openshift.com": { "auth": " <your_token> ", "email": "asd@redhat.com" } }
curl -v -H "User-Agent: insights-operator/one10time200gather184a34f6a168926d93c330 cluster/ <cluster_id> " -H "Authorization: Bearer <your_token> " -F "upload=@ <path_to_archive> ; type=application/vnd.redhat.openshift.periodic+tar" https://console.redhat.com/api/ingress/v1/upload
* Connection #0 to host console.redhat.com left intact {"request_id":"393a7cf1093e434ea8dd4ab3eb28884c","upload":{"account_number":"6274079"}}%
oc adm must-gather --image=registry.redhat.io/container-native-virtualization/cnv-must-gather-rhel9:v4.14.11
oc adm must-gather -- /usr/bin/gather_audit_logs
NAMESPACE NAME READY STATUS RESTARTS AGE openshift-must-gather-5drcj must-gather-bklx4 2/2 Running 0 72s openshift-must-gather-5drcj must-gather-s8sdh 2/2 Running 0 72s
oc adm must-gather --run-namespace <namespace> --image=registry.redhat.io/container-native-virtualization/cnv-must-gather-rhel9:v4.14.11
oc import-image is/must-gather -n openshift
oc adm must-gather
tar cvaf must-gather.tar.gz must-gather.local.5421342344627712289/ 1
oc adm must-gather --image-stream=openshift/must-gather \ 1 --image=registry.redhat.io/container-native-virtualization/cnv-must-gather-rhel9:v4.14.11 2
oc adm must-gather --image=USD(oc -n openshift-logging get deployment.apps/cluster-logging-operator -o jsonpath='{.spec.template.spec.containers[?(@.name == "cluster-logging-operator")].image}')
├── cluster-logging │ ├── clo │ │ ├── cluster-logging-operator-74dd5994f-6ttgt │ │ ├── clusterlogforwarder_cr │ │ ├── cr │ │ ├── csv │ │ ├── deployment │ │ └── logforwarding_cr │ ├── collector │ │ ├── fluentd-2tr64 │ ├── eo │ │ ├── csv │ │ ├── deployment │ │ └── elasticsearch-operator-7dc7d97b9d-jb4r4 │ ├── es │ │ ├── cluster-elasticsearch │ │ │ ├── aliases │ │ │ ├── health │ │ │ ├── indices │ │ │ ├── latest_documents.json │ │ │ ├── nodes │ │ │ ├── nodes_stats.json │ │ │ └── thread_pool │ │ ├── cr │ │ ├── elasticsearch-cdm-lp8l38m0-1-794d6dd989-4jxms │ │ └── logs │ │ ├── elasticsearch-cdm-lp8l38m0-1-794d6dd989-4jxms │ ├── install │ │ ├── co_logs │ │ ├── install_plan │ │ ├── olmo_logs │ │ └── subscription │ └── kibana │ ├── cr │ ├── kibana-9d69668d4-2rkvz ├── cluster-scoped-resources │ └── core │ ├── nodes │ │ ├── ip-10-0-146-180.eu-west-1.compute.internal.yaml │ └── persistentvolumes │ ├── pvc-0a8d65d9-54aa-4c44-9ecc-33d9381e41c1.yaml ├── event-filter.html ├── gather-debug.log └── namespaces ├── openshift-logging │ ├── apps │ │ ├── daemonsets.yaml │ │ ├── deployments.yaml │ │ ├── replicasets.yaml │ │ └── statefulsets.yaml │ ├── batch │ │ ├── cronjobs.yaml │ │ └── jobs.yaml │ ├── core │ │ ├── configmaps.yaml │ │ ├── endpoints.yaml │ │ ├── events │ │ │ ├── elasticsearch-im-app-1596020400-gm6nl.1626341a296c16a1.yaml │ │ │ ├── elasticsearch-im-audit-1596020400-9l9n4.1626341a2af81bbd.yaml │ │ │ ├── elasticsearch-im-infra-1596020400-v98tk.1626341a2d821069.yaml │ │ │ ├── elasticsearch-im-app-1596020400-cc5vc.1626341a3019b238.yaml │ │ │ ├── elasticsearch-im-audit-1596020400-s8d5s.1626341a31f7b315.yaml │ │ │ ├── elasticsearch-im-infra-1596020400-7mgv8.1626341a35ea59ed.yaml │ │ ├── events.yaml │ │ ├── persistentvolumeclaims.yaml │ │ ├── pods.yaml │ │ ├── replicationcontrollers.yaml │ │ ├── secrets.yaml │ │ └── services.yaml │ ├── openshift-logging.yaml │ ├── pods │ │ ├── cluster-logging-operator-74dd5994f-6ttgt │ │ │ ├── cluster-logging-operator │ │ │ │ └── cluster-logging-operator │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ └── cluster-logging-operator-74dd5994f-6ttgt.yaml │ │ ├── cluster-logging-operator-registry-6df49d7d4-mxxff │ │ │ ├── cluster-logging-operator-registry │ │ │ │ └── cluster-logging-operator-registry │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ ├── cluster-logging-operator-registry-6df49d7d4-mxxff.yaml │ │ │ └── mutate-csv-and-generate-sqlite-db │ │ │ └── mutate-csv-and-generate-sqlite-db │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ │ ├── elasticsearch-cdm-lp8l38m0-1-794d6dd989-4jxms │ │ ├── elasticsearch-im-app-1596030300-bpgcx │ │ │ ├── elasticsearch-im-app-1596030300-bpgcx.yaml │ │ │ └── indexmanagement │ │ │ └── indexmanagement │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ │ ├── fluentd-2tr64 │ │ │ ├── fluentd │ │ │ │ └── fluentd │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ ├── fluentd-2tr64.yaml │ │ │ └── fluentd-init │ │ │ └── fluentd-init │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ │ ├── kibana-9d69668d4-2rkvz │ │ │ ├── kibana │ │ │ │ └── kibana │ │ │ │ └── logs │ │ │ │ ├── current.log │ │ │ │ ├── previous.insecure.log │ │ │ │ └── previous.log │ │ │ ├── kibana-9d69668d4-2rkvz.yaml │ │ │ └── kibana-proxy │ │ │ └── kibana-proxy │ │ │ └── logs │ │ │ ├── current.log │ │ │ ├── previous.insecure.log │ │ │ └── previous.log │ └── route.openshift.io │ └── routes.yaml └── openshift-operators-redhat ├──
oc adm must-gather --image-stream=openshift/must-gather \ 1 --image=quay.io/kubevirt/must-gather 2
tar cvaf must-gather.tar.gz must-gather.local.5421342344627712289/ 1
oc adm must-gather -- gather_network_logs
tar cvaf must-gather.tar.gz must-gather.local.472290403699006248 1
oc get clusterversion -o jsonpath='{.items[].spec.clusterID}{"\n"}'
oc get nodes
oc debug node/my-cluster-node
oc new-project dummy
oc patch namespace dummy --type=merge -p '{"metadata": {"annotations": { "scheduler.alpha.kubernetes.io/defaultTolerations": "[{\"operator\": \"Exists\"}]"}}}'
oc debug node/my-cluster-node
chroot /host
toolbox
sos report -k crio.all=on -k crio.logs=on -k podman.all=on -k podman.logs=on 1
sos report --all-logs
Your sosreport has been generated and saved in: /host/var/tmp/sosreport-my-cluster-node-01234567-2020-05-28-eyjknxt.tar.xz 1 The checksum is: 382ffc167510fd71b4f12a4f40b97a4e
oc debug node/my-cluster-node -- bash -c 'cat /host/var/tmp/sosreport-my-cluster-node-01234567-2020-05-28-eyjknxt.tar.xz' > /tmp/sosreport-my-cluster-node-01234567-2020-05-28-eyjknxt.tar.xz 1
ssh core@<bootstrap_fqdn> journalctl -b -f -u bootkube.service
ssh core@<bootstrap_fqdn> 'for pod in USD(sudo podman ps -a -q); do sudo podman logs USDpod; done'
oc adm node-logs --role=master -u kubelet 1
oc adm node-logs --role=master --path=openshift-apiserver
oc adm node-logs --role=master --path=openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/openshift-apiserver/audit.log
oc adm must-gather --dest-dir /tmp/captures \ <.> --source-dir '/tmp/tcpdump/' \ <.> --image registry.redhat.io/openshift4/network-tools-rhel8:latest \ <.> --node-selector 'node-role.kubernetes.io/worker' \ <.> --host-network=true \ <.> --timeout 30s \ <.> -- tcpdump -i any \ <.> -w /tmp/tcpdump/%Y-%m-%dT%H:%M:%S.pcap -W 1 -G 300
tmp/captures ├── event-filter.html ├── ip-10-0-192-217-ec2-internal 1 │ └── registry-redhat-io-openshift4-network-tools-rhel8-sha256-bca │ └── 2022-01-13T19:31:31.pcap ├── ip-10-0-201-178-ec2-internal 2 │ └── registry-redhat-io-openshift4-network-tools-rhel8-sha256-bca │ └── 2022-01-13T19:31:30.pcap ├── ip- └── timestamp
oc get nodes
oc debug node/my-cluster-node
chroot /host
ip ad
toolbox
tcpdump -nn -s 0 -i ens5 -w /host/var/tmp/my-cluster-node_USD(date +%d_%m_%Y-%H_%M_%S-%Z).pcap 1
chroot /host crictl ps
chroot /host crictl inspect --output yaml a7fe32346b120 | grep 'pid' | awk '{print USD2}'
nsenter -n -t 49628 -- tcpdump -nn -i ens5 -w /host/var/tmp/my-cluster-node-my-container_USD(date +%d_%m_%Y-%H_%M_%S-%Z).pcap 1
oc debug node/my-cluster-node -- bash -c 'cat /host/var/tmp/my-tcpdump-capture-file.pcap' > /tmp/my-tcpdump-capture-file.pcap 1
oc debug node/my-cluster-node -- bash -c 'cat /host/var/tmp/my-diagnostic-data.tar.gz' > /var/tmp/my-diagnostic-data.tar.gz 1
chroot /host
toolbox
dnf install -y <package_name>
chroot /host
vi ~/.toolboxrc
REGISTRY=quay.io 1 IMAGE=fedora/fedora:33-x86_64 2 TOOLBOX_NAME=toolbox-fedora-33 3
toolbox
oc get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.13.8 True False 8h Cluster version is 4.13.8
oc describe clusterversion
Name: version Namespace: Labels: <none> Annotations: <none> API Version: config.openshift.io/v1 Kind: ClusterVersion Image: quay.io/openshift-release-dev/ocp-release@sha256:a956488d295fe5a59c8663a4d9992b9b5d0950f510a7387dbbfb8d20fc5970ce URL: https://access.redhat.com/errata/RHSA-2023:4456 Version: 4.13.8 History: Completion Time: 2023-08-17T13:20:21Z Image: quay.io/openshift-release-dev/ocp-release@sha256:a956488d295fe5a59c8663a4d9992b9b5d0950f510a7387dbbfb8d20fc5970ce Started Time: 2023-08-17T12:59:45Z State: Completed Verified: false Version: 4.13.8
ssh <user_name>@<load_balancer> systemctl status haproxy
ssh <user_name>@<load_balancer> netstat -nltupe | grep -E ':80|:443|:6443|:22623'
ssh <user_name>@<load_balancer> ss -nltupe | grep -E ':80|:443|:6443|:22623'
dig <wildcard_fqdn> @<dns_server>
./openshift-install --dir <installation_directory> wait-for bootstrap-complete --log-level debug 1
./openshift-install create ignition-configs --dir=./install_dir
tail -f ~/<installation_directory>/.openshift_install.log
ssh core@<bootstrap_fqdn> journalctl -b -f -u bootkube.service
oc adm node-logs --role=master -u kubelet
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=master -u crio
ssh core@master-N.cluster_name.sub_domain.domain journalctl -b -f -u crio.service
curl -I http://<http_server_fqdn>:<port>/bootstrap.ign 1
grep -is 'bootstrap.ign' /var/log/httpd/access_log
ssh core@<bootstrap_fqdn> journalctl -b -f -u bootkube.service
ssh core@<bootstrap_fqdn> 'for pod in USD(sudo podman ps -a -q); do sudo podman logs USDpod; done'
curl -I http://<http_server_fqdn>:<port>/master.ign 1
grep -is 'master.ign' /var/log/httpd/access_log
oc get nodes
oc describe node <master_node>
oc get daemonsets -n openshift-sdn
oc get pods -n openshift-sdn
oc logs <sdn_pod> -n openshift-sdn
oc get network.config.openshift.io cluster -o yaml
./openshift-install create manifests
oc get pods -n openshift-network-operator
oc logs pod/<network_operator_pod_name> -n openshift-network-operator
oc adm node-logs --role=master -u kubelet
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=master -u crio
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u crio.service
oc adm node-logs --role=master --path=openshift-apiserver
oc adm node-logs --role=master --path=openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl ps -a
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
curl https://api-int.<cluster_name>:22623/config/master
dig api-int.<cluster_name> @<dns_server>
dig -x <load_balancer_mco_ip_address> @<dns_server>
ssh core@<bootstrap_fqdn> curl https://api-int.<cluster_name>:22623/config/master
ssh core@<node>.<cluster_name>.<base_domain> chronyc tracking
openssl s_client -connect api-int.<cluster_name>:22623 | openssl x509 -noout -text
oc get pods -n openshift-etcd
oc get pods -n openshift-etcd-operator
oc describe pod/<pod_name> -n <namespace>
oc logs pod/<pod_name> -n <namespace>
oc logs pod/<pod_name> -c <container_name> -n <namespace>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl pods --name=etcd-
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspectp <pod_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl ps | grep '<pod_id>'
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspect <container_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
oc adm node-logs --role=master -u kubelet
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=master -u kubelet | grep -is 'x509: certificate has expired'
ssh core@<master-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service | grep -is 'x509: certificate has expired'
curl -I http://<http_server_fqdn>:<port>/worker.ign 1
grep -is 'worker.ign' /var/log/httpd/access_log
oc get nodes
oc describe node <worker_node>
oc get pods -n openshift-machine-api
oc describe pod/<machine_api_operator_pod_name> -n openshift-machine-api
oc logs pod/<machine_api_operator_pod_name> -n openshift-machine-api -c machine-api-operator
oc logs pod/<machine_api_operator_pod_name> -n openshift-machine-api -c kube-rbac-proxy
oc adm node-logs --role=worker -u kubelet
ssh core@<worker-node>.<cluster_name>.<base_domain> journalctl -b -f -u kubelet.service
oc adm node-logs --role=worker -u crio
ssh core@<worker-node>.<cluster_name>.<base_domain> journalctl -b -f -u crio.service
oc adm node-logs --role=worker --path=sssd
oc adm node-logs --role=worker --path=sssd/sssd.log
ssh core@<worker-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/sssd/sssd.log
ssh core@<worker-node>.<cluster_name>.<base_domain> sudo crictl ps -a
ssh core@<worker-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
curl https://api-int.<cluster_name>:22623/config/worker
dig api-int.<cluster_name> @<dns_server>
dig -x <load_balancer_mco_ip_address> @<dns_server>
ssh core@<bootstrap_fqdn> curl https://api-int.<cluster_name>:22623/config/worker
ssh core@<node>.<cluster_name>.<base_domain> chronyc tracking
openssl s_client -connect api-int.<cluster_name>:22623 | openssl x509 -noout -text
oc get clusteroperators
oc get csr
NAME AGE REQUESTOR CONDITION csr-8b2br 15m system:serviceaccount:openshift-machine-config-operator:node-bootstrapper Pending 1 csr-8vnps 15m system:serviceaccount:openshift-machine-config-operator:node-bootstrapper Pending csr-bfd72 5m26s system:node:ip-10-0-50-126.us-east-2.compute.internal Pending 2 csr-c57lv 5m26s system:node:ip-10-0-95-157.us-east-2.compute.internal Pending
oc adm certificate approve <csr_name> 1
oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve
oc describe clusteroperator <operator_name>
oc get pods -n <operator_namespace>
oc describe pod/<operator_pod_name> -n <operator_namespace>
oc logs pod/<operator_pod_name> -n <operator_namespace>
oc get pod -o "jsonpath={range .status.containerStatuses[*]}{.name}{'\t'}{.state}{'\t'}{.image}{'\n'}{end}" <operator_pod_name> -n <operator_namespace>
oc adm release info <image_path>:<tag> --commits
./openshift-install gather bootstrap --dir <installation_directory> 1
./openshift-install gather bootstrap --dir <installation_directory> \ 1 --bootstrap <bootstrap_address> \ 2 --master <master_1_address> \ 3 --master <master_2_address> \ 4 --master <master_3_address> 5
INFO Pulling debug logs from the bootstrap machine INFO Bootstrap gather logs captured here "<installation_directory>/log-bundle-<timestamp>.tar.gz"
oc get nodes
oc adm top nodes
oc adm top node my-node
oc debug node/my-node
chroot /host
systemctl is-active kubelet
systemctl status kubelet
oc adm node-logs --role=master -u kubelet 1
oc adm node-logs --role=master --path=openshift-apiserver
oc adm node-logs --role=master --path=openshift-apiserver/audit.log
ssh core@<master-node>.<cluster_name>.<base_domain> sudo tail -f /var/log/openshift-apiserver/audit.log
oc debug node/my-node
chroot /host
systemctl is-active crio
systemctl status crio.service
oc adm node-logs --role=master -u crio
oc adm node-logs <node_name> -u crio
ssh core@<node>.<cluster_name>.<base_domain> journalctl -b -f -u crio.service
Failed to create pod sandbox: rpc error: code = Unknown desc = failed to mount container XXX: error recreating the missing symlinks: error reading name of symlink for XXX: open /var/lib/containers/storage/overlay/XXX/link: no such file or directory
can't stat lower layer ... because it does not exist. Going through storage to recreate the missing symlinks.
oc adm cordon <node_name>
oc adm drain <node_name> --ignore-daemonsets --delete-emptydir-data
ssh core@node1.example.com sudo -i
systemctl stop kubelet
.. for pod in USD(crictl pods -q); do if [[ "USD(crictl inspectp USDpod | jq -r .status.linux.namespaces.options.network)" != "NODE" ]]; then crictl rmp -f USDpod; fi; done
crictl rmp -fa
systemctl stop crio
crio wipe -f
systemctl start crio systemctl start kubelet
oc get nodes
NAME STATUS ROLES AGE VERSION ci-ln-tkbxyft-f76d1-nvwhr-master-1 Ready, SchedulingDisabled master 133m v1.27.3
oc adm uncordon <node_name>
NAME STATUS ROLES AGE VERSION ci-ln-tkbxyft-f76d1-nvwhr-master-1 Ready master 133m v1.27.3
rpm-ostree kargs --append='crashkernel=256M'
systemctl enable kdump.service
systemctl reboot
variant: openshift version: 4.14.0 metadata: name: 99-worker-kdump 1 labels: machineconfiguration.openshift.io/role: worker 2 openshift: kernel_arguments: 3 - crashkernel=256M storage: files: - path: /etc/kdump.conf 4 mode: 0644 overwrite: true contents: inline: | path /var/crash core_collector makedumpfile -l --message-level 7 -d 31 - path: /etc/sysconfig/kdump 5 mode: 0644 overwrite: true contents: inline: | KDUMP_COMMANDLINE_REMOVE="hugepages hugepagesz slub_debug quiet log_buf_len swiotlb" KDUMP_COMMANDLINE_APPEND="irqpoll nr_cpus=1 reset_devices cgroup_disable=memory mce=off numa=off udev.children-max=2 panic=10 rootflags=nofail acpi_no_memhotplug transparent_hugepage=never nokaslr novmcoredd hest_disable" 6 KEXEC_ARGS="-s" KDUMP_IMG="vmlinuz" systemd: units: - name: kdump.service enabled: true
nfs server.example.com:/export/cores core_collector makedumpfile -l --message-level 7 -d 31 extra_modules nfs
butane 99-worker-kdump.bu -o 99-worker-kdump.yaml
oc create -f 99-worker-kdump.yaml
systemctl --failed
journalctl -u <unit>.service
NODEIP_HINT=192.0.2.1
echo -n 'NODEIP_HINT=192.0.2.1' | base64 -w0
Tk9ERUlQX0hJTlQ9MTkyLjAuMCxxxx==
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: master name: 99-nodeip-hint-master spec: config: ignition: version: 3.2.0 storage: files: - contents: source: data:text/plain;charset=utf-8;base64,<encoded_content> 1 mode: 0644 overwrite: true path: /etc/default/nodeip-configuration
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: 99-nodeip-hint-worker spec: config: ignition: version: 3.2.0 storage: files: - contents: source: data:text/plain;charset=utf-8;base64,<encoded_content> 1 mode: 0644 overwrite: true path: /etc/default/nodeip-configuration
[connection] id=eno1 type=ethernet interface-name=eno1 master=bond1 slave-type=bond autoconnect=true autoconnect-priority=20
[connection] id=eno2 type=ethernet interface-name=eno2 master=bond1 slave-type=bond autoconnect=true autoconnect-priority=20
[connection] id=bond1 type=bond interface-name=bond1 autoconnect=true connection.autoconnect-slaves=1 autoconnect-priority=20 [bond] mode=802.3ad miimon=100 xmit_hash_policy="layer3+4" [ipv4] method=auto
base64 <directory_path>/en01.config
base64 <directory_path>/eno2.config
base64 <directory_path>/bond1.config
export ROLE=<machine_role>
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: USD{worker} name: 12-USD{ROLE}-sec-bridge-cni spec: config: ignition: version: 3.2.0 storage: files: - contents: source: data:;base64,<base-64-encoded-contents-for-bond1.conf> path: /etc/NetworkManager/system-connections/bond1.nmconnection filesystem: root mode: 0600 - contents: source: data:;base64,<base-64-encoded-contents-for-eno1.conf> path: /etc/NetworkManager/system-connections/eno1.nmconnection filesystem: root mode: 0600 - contents: source: data:;base64,<base-64-encoded-contents-for-eno2.conf> path: /etc/NetworkManager/system-connections/eno2.nmconnection filesystem: root mode: 0600
oc create -f <machine_config_file_name>
bond1
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: USD{worker} name: 12-worker-extra-bridge spec: config: ignition: version: 3.2.0 storage: files: - path: /etc/ovnk/extra_bridge mode: 0420 overwrite: true contents: source: data:text/plain;charset=utf-8,bond1 filesystem: root
oc create -f <machine_config_file_name>
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: USD{worker} name: 12-worker-br-ex-override spec: config: ignition: version: 3.2.0 storage: files: - path: /var/lib/ovnk/iface_default_hint mode: 0420 overwrite: true contents: source: data:text/plain;charset=utf-8,bond0 1 filesystem: root
oc create -f <machine_config_file_name>
oc get nodes -o json | grep --color exgw-ip-addresses
"k8s.ovn.org/l3-gateway-config": \"exgw-ip-address\":\"172.xx.xx.yy/24\",\"next-hops\":[\"xx.xx.xx.xx\"],
oc debug node/<node_name> -- chroot /host sh -c "ip a | grep mtu | grep br-ex"
Starting pod/worker-1-debug To use host binaries, run `chroot /host` 5: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000 6: br-ex1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
oc debug node/<node_name> -- chroot /host sh -c "ip a | grep -A1 -E 'br-ex|bond0'
Starting pod/worker-1-debug To use host binaries, run `chroot /host` sh-5.1# ip a | grep -A1 -E 'br-ex|bond0' 2: bond0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master ovs-system state UP group default qlen 1000 link/ether fa:16:3e:47:99:98 brd ff:ff:ff:ff:ff:ff -- 5: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000 link/ether fa:16:3e:47:99:98 brd ff:ff:ff:ff:ff:ff inet 10.xx.xx.xx/21 brd 10.xx.xx.255 scope global dynamic noprefixroute br-ex
E0514 12:47:17.998892 2281 daemon.go:1350] content mismatch for file /etc/systemd/system/ovs-vswitchd.service: [Unit]
oc debug node/<node_name>
chroot /host
ovs-appctl vlog/list
console syslog file ------- ------ ------ backtrace OFF INFO INFO bfd OFF INFO INFO bond OFF INFO INFO bridge OFF INFO INFO bundle OFF INFO INFO bundles OFF INFO INFO cfm OFF INFO INFO collectors OFF INFO INFO command_line OFF INFO INFO connmgr OFF INFO INFO conntrack OFF INFO INFO conntrack_tp OFF INFO INFO coverage OFF INFO INFO ct_dpif OFF INFO INFO daemon OFF INFO INFO daemon_unix OFF INFO INFO dns_resolve OFF INFO INFO dpdk OFF INFO INFO
Restart=always ExecStartPre=-/bin/sh -c '/usr/bin/chown -R :USDUSD{OVS_USER_ID##*:} /var/lib/openvswitch' ExecStartPre=-/bin/sh -c '/usr/bin/chown -R :USDUSD{OVS_USER_ID##*:} /etc/openvswitch' ExecStartPre=-/bin/sh -c '/usr/bin/chown -R :USDUSD{OVS_USER_ID##*:} /run/openvswitch' ExecStartPost=-/usr/bin/ovs-appctl vlog/set syslog:dbg ExecReload=-/usr/bin/ovs-appctl vlog/set syslog:dbg
systemctl daemon-reload
systemctl restart ovs-vswitchd
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: master 1 name: 99-change-ovs-loglevel spec: config: ignition: version: 3.2.0 systemd: units: - dropins: - contents: | [Service] ExecStartPost=-/usr/bin/ovs-appctl vlog/set syslog:dbg 2 ExecReload=-/usr/bin/ovs-appctl vlog/set syslog:dbg name: 20-ovs-vswitchd-restart.conf name: ovs-vswitchd.service
oc apply -f 99-change-ovs-loglevel.yaml
oc adm node-logs <node_name> -u ovs-vswitchd
journalctl -b -f -u ovs-vswitchd.service
oc get subs -n <operator_namespace>
oc describe sub <subscription_name> -n <operator_namespace>
Name: cluster-logging Namespace: openshift-logging Labels: operators.coreos.com/cluster-logging.openshift-logging= Annotations: <none> API Version: operators.coreos.com/v1alpha1 Kind: Subscription Conditions: Last Transition Time: 2019-07-29T13:42:57Z Message: all available catalogsources are healthy Reason: AllCatalogSourcesHealthy Status: False Type: CatalogSourcesUnhealthy
oc get catalogsources -n openshift-marketplace
NAME DISPLAY TYPE PUBLISHER AGE certified-operators Certified Operators grpc Red Hat 55m community-operators Community Operators grpc Red Hat 55m example-catalog Example Catalog grpc Example Org 2m25s redhat-marketplace Red Hat Marketplace grpc Red Hat 55m redhat-operators Red Hat Operators grpc Red Hat 55m
oc describe catalogsource example-catalog -n openshift-marketplace
Name: example-catalog Namespace: openshift-marketplace Labels: <none> Annotations: operatorframework.io/managed-by: marketplace-operator target.workload.openshift.io/management: {"effect": "PreferredDuringScheduling"} API Version: operators.coreos.com/v1alpha1 Kind: CatalogSource Status: Connection State: Address: example-catalog.openshift-marketplace.svc:50051 Last Connect: 2021-09-09T17:07:35Z Last Observed State: TRANSIENT_FAILURE Registry Service: Created At: 2021-09-09T17:05:45Z Port: 50051 Protocol: grpc Service Name: example-catalog Service Namespace: openshift-marketplace
oc get pods -n openshift-marketplace
NAME READY STATUS RESTARTS AGE certified-operators-cv9nn 1/1 Running 0 36m community-operators-6v8lp 1/1 Running 0 36m marketplace-operator-86bfc75f9b-jkgbc 1/1 Running 0 42m example-catalog-bwt8z 0/1 ImagePullBackOff 0 3m55s redhat-marketplace-57p8c 1/1 Running 0 36m redhat-operators-smxx8 1/1 Running 0 36m
oc describe pod example-catalog-bwt8z -n openshift-marketplace
Name: example-catalog-bwt8z Namespace: openshift-marketplace Priority: 0 Node: ci-ln-jyryyg2-f76d1-ggdbq-worker-b-vsxjd/10.0.128.2 Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 48s default-scheduler Successfully assigned openshift-marketplace/example-catalog-bwt8z to ci-ln-jyryyf2-f76d1-fgdbq-worker-b-vsxjd Normal AddedInterface 47s multus Add eth0 [10.131.0.40/23] from openshift-sdn Normal BackOff 20s (x2 over 46s) kubelet Back-off pulling image "quay.io/example-org/example-catalog:v1" Warning Failed 20s (x2 over 46s) kubelet Error: ImagePullBackOff Normal Pulling 8s (x3 over 47s) kubelet Pulling image "quay.io/example-org/example-catalog:v1" Warning Failed 8s (x3 over 47s) kubelet Failed to pull image "quay.io/example-org/example-catalog:v1": rpc error: code = Unknown desc = reading manifest v1 in quay.io/example-org/example-catalog: unauthorized: access to the requested resource is not authorized Warning Failed 8s (x3 over 47s) kubelet Error: ErrImagePull
oc get clusteroperators
oc get pod -n <operator_namespace>
oc describe pod <operator_pod_name> -n <operator_namespace>
oc debug node/my-node
chroot /host
crictl ps
crictl ps --name network-operator
oc get pods -n <operator_namespace>
oc logs pod/<pod_name> -n <operator_namespace>
oc logs pod/<operator_pod_name> -c <container_name> -n <operator_namespace>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl pods
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspectp <operator_pod_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl ps --pod=<operator_pod_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl inspect <container_id>
ssh core@<master-node>.<cluster_name>.<base_domain> sudo crictl logs -f <container_id>
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool spec: paused: true 1
apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfigPool spec: paused: false 1
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/master
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/worker
oc get machineconfigpool/master --template='{{.spec.paused}}'
oc get machineconfigpool/worker --template='{{.spec.paused}}'
true
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING master rendered-master-33cf0a1254318755d7b48002c597bf91 True False worker rendered-worker-e405a5bdb0db1295acea08bcca33fa60 False False
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/master
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/worker
oc get machineconfigpool/master --template='{{.spec.paused}}'
oc get machineconfigpool/worker --template='{{.spec.paused}}'
false
oc get machineconfigpool
NAME CONFIG UPDATED UPDATING master rendered-master-546383f80705bd5aeaba93 True False worker rendered-worker-b4c51bb33ccaae6fc4a6a5 False True
ImagePullBackOff for Back-off pulling image "example.com/openshift4/ose-elasticsearch-operator-bundle@sha256:6d2587129c846ec28d384540322b40b05833e7e00b25cca584e004af9a1d292e"
rpc error: code = Unknown desc = error pinging docker registry example.com: Get "https://example.com/v2/": dial tcp: lookup example.com on 10.0.0.1:53: no such host
oc get sub,csv -n <namespace>
NAME PACKAGE SOURCE CHANNEL subscription.operators.coreos.com/elasticsearch-operator elasticsearch-operator redhat-operators 5.0 NAME DISPLAY VERSION REPLACES PHASE clusterserviceversion.operators.coreos.com/elasticsearch-operator.5.0.0-65 OpenShift Elasticsearch Operator 5.0.0-65 Succeeded
oc delete subscription <subscription_name> -n <namespace>
oc delete csv <csv_name> -n <namespace>
oc get job,configmap -n openshift-marketplace
NAME COMPLETIONS DURATION AGE job.batch/1de9443b6324e629ddf31fed0a853a121275806170e34c926d69e53a7fcbccb 1/1 26s 9m30s NAME DATA AGE configmap/1de9443b6324e629ddf31fed0a853a121275806170e34c926d69e53a7fcbccb 3 9m30s
oc delete job <job_name> -n openshift-marketplace
oc delete configmap <configmap_name> -n openshift-marketplace
oc get sub,csv,installplan -n <namespace>
message: 'Failed to delete all resource types, 1 remaining: Internal error occurred: error resolving resource'
oc get namespaces
operator-ns-1 Terminating
oc get crds
oc delete crd <crd_name>
oc get EtcdCluster -n <namespace_name>
oc get EtcdCluster --all-namespaces
oc delete <cr_name> <cr_instance_name> -n <namespace_name>
oc get namespace <namespace_name>
oc get sub,csv,installplan -n <namespace>
oc project <project_name>
oc get pods
oc status
skopeo inspect docker://<image_reference>
oc edit deployment/my-deployment
oc get pods -w
oc get events
oc logs <pod_name>
oc logs <pod_name> -c <container_name>
oc exec <pod_name> -- ls -alh /var/log
total 124K drwxr-xr-x. 1 root root 33 Aug 11 11:23 . drwxr-xr-x. 1 root root 28 Sep 6 2022 .. -rw-rw----. 1 root utmp 0 Jul 10 10:31 btmp -rw-r--r--. 1 root root 33K Jul 17 10:07 dnf.librepo.log -rw-r--r--. 1 root root 69K Jul 17 10:07 dnf.log -rw-r--r--. 1 root root 8.8K Jul 17 10:07 dnf.rpm.log -rw-r--r--. 1 root root 480 Jul 17 10:07 hawkey.log -rw-rw-r--. 1 root utmp 0 Jul 10 10:31 lastlog drwx------. 2 root root 23 Aug 11 11:14 openshift-apiserver drwx------. 2 root root 6 Jul 10 10:31 private drwxr-xr-x. 1 root root 22 Mar 9 08:05 rhsm -rw-rw-r--. 1 root utmp 0 Jul 10 10:31 wtmp
oc exec <pod_name> cat /var/log/<path_to_log>
2023-07-10T10:29:38+0000 INFO --- logging initialized --- 2023-07-10T10:29:38+0000 DDEBUG timer: config: 13 ms 2023-07-10T10:29:38+0000 DEBUG Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, groups-manager, needs-restarting, playground, product-id, repoclosure, repodiff, repograph, repomanage, reposync, subscription-manager, uploadprofile 2023-07-10T10:29:38+0000 INFO Updating Subscription Management repositories. 2023-07-10T10:29:38+0000 INFO Unable to read consumer identity 2023-07-10T10:29:38+0000 INFO Subscription Manager is operating in container mode. 2023-07-10T10:29:38+0000 INFO
oc exec <pod_name> -c <container_name> ls /var/log
oc exec <pod_name> -c <container_name> cat /var/log/<path_to_log>
oc project <namespace>
oc rsh <pod_name> 1
oc rsh -c <container_name> pod/<pod_name>
oc port-forward <pod_name> <host_port>:<pod_port> 1
oc get deployment -n <project_name>
oc debug deployment/my-deployment --as-root -n <project_name>
oc get deploymentconfigs -n <project_name>
oc debug deploymentconfig/my-deployment-configuration --as-root -n <project_name>
oc cp <local_path> <pod_name>:/<path> -c <container_name> 1
oc cp <pod_name>:/<path> -c <container_name> <local_path> 1
oc get pods -w 1
oc logs -f pod/<application_name>-<build_number>-build
oc logs -f pod/<application_name>-<build_number>-deploy
oc logs -f pod/<application_name>-<build_number>-<random_string>
oc describe pod/my-app-1-akdlg
oc logs -f pod/my-app-1-akdlg
oc exec my-app-1-akdlg -- cat /var/log/my-application.log
oc debug dc/my-deployment-configuration --as-root -- cat /var/log/my-application.log
oc exec -it my-app-1-akdlg /bin/bash
oc debug node/my-cluster-node
chroot /host
crictl ps
crictl inspect a7fe32346b120 --output yaml | grep 'pid:' | awk '{print USD2}'
nsenter -n -t 31150 -- ip ad
Unable to attach or mount volumes: unmounted volumes=[sso-mysql-pvol], unattached volumes=[sso-mysql-pvol default-token-x4rzc]: timed out waiting for the condition Multi-Attach error for volume "pvc-8837384d-69d7-40b2-b2e6-5df86943eef9" Volume is already used by pod(s) sso-mysql-1-ns6b4
oc delete pod <old_pod> --force=true --grace-period=0
oc logs -f deployment/windows-machine-config-operator -n openshift-windows-machine-config-operator
ssh -t -o StrictHostKeyChecking=no -o ProxyCommand='ssh -A -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -W %h:%p core@USD(oc get service --all-namespaces -l run=ssh-bastion -o go-template="{{ with (index (index .items 0).status.loadBalancer.ingress 0) }}{{ or .hostname .ip }}{{end}}")' <username>@<windows_node_internal_ip> 1 2
oc get nodes <node_name> -o jsonpath={.status.addresses[?\(@.type==\"InternalIP\"\)].address}
ssh -L 2020:<windows_node_internal_ip>:3389 \ 1 core@USD(oc get service --all-namespaces -l run=ssh-bastion -o go-template="{{ with (index (index .items 0).status.loadBalancer.ingress 0) }}{{ or .hostname .ip }}{{end}}")
oc get nodes <node_name> -o jsonpath={.status.addresses[?\(@.type==\"InternalIP\"\)].address}
C:\> net user <username> * 1
oc adm node-logs -l kubernetes.io/os=windows --path= /ip-10-0-138-252.us-east-2.compute.internal containers /ip-10-0-138-252.us-east-2.compute.internal hybrid-overlay /ip-10-0-138-252.us-east-2.compute.internal kube-proxy /ip-10-0-138-252.us-east-2.compute.internal kubelet /ip-10-0-138-252.us-east-2.compute.internal pods
oc adm node-logs -l kubernetes.io/os=windows --path=/kubelet/kubelet.log
oc adm node-logs -l kubernetes.io/os=windows --path=journal
oc adm node-logs -l kubernetes.io/os=windows --path=journal -u docker
C:\> powershell
C:\> Get-EventLog -LogName Application -Source Docker
oc -n ns1 get service prometheus-example-app -o yaml
labels: app: prometheus-example-app
oc -n ns1 get servicemonitor prometheus-example-monitor -o yaml
apiVersion: v1 kind: ServiceMonitor metadata: name: prometheus-example-monitor namespace: ns1 spec: endpoints: - interval: 30s port: web scheme: http selector: matchLabels: app: prometheus-example-app
oc -n openshift-user-workload-monitoring get pods
NAME READY STATUS RESTARTS AGE prometheus-operator-776fcbbd56-2nbfm 2/2 Running 0 132m prometheus-user-workload-0 5/5 Running 1 132m prometheus-user-workload-1 5/5 Running 1 132m thanos-ruler-user-workload-0 3/3 Running 0 132m thanos-ruler-user-workload-1 3/3 Running 0 132m
oc -n openshift-user-workload-monitoring logs prometheus-operator-776fcbbd56-2nbfm -c prometheus-operator
level=warn ts=2020-08-10T11:48:20.906739623Z caller=operator.go:1829 component=prometheusoperator msg="skipping servicemonitor" error="it accesses file system via bearer token file which Prometheus specification prohibits" servicemonitor=eagle/eagle namespace=openshift-user-workload-monitoring prometheus=user-workload
oc -n openshift-user-workload-monitoring edit configmap user-workload-monitoring-config
apiVersion: v1 kind: ConfigMap metadata: name: user-workload-monitoring-config namespace: openshift-user-workload-monitoring data: config.yaml: | prometheusOperator: logLevel: debug
oc -n openshift-user-workload-monitoring get deploy prometheus-operator -o yaml | grep "log-level"
- --log-level=debug
oc -n openshift-user-workload-monitoring get pods
topk(10, max by(namespace, job) (topk by(namespace, job) (1, scrape_samples_post_metric_relabeling)))
topk(10, sum by(namespace, job) (sum_over_time(scrape_series_added[1h])))
HOST=USD(oc -n openshift-monitoring get route prometheus-k8s -ojsonpath={.status.ingress[].host})
TOKEN=USD(oc whoami -t)
curl -H "Authorization: Bearer USDTOKEN" -k "https://USDHOST/api/v1/status/tsdb"
"status": "success","data":{"headStats":{"numSeries":507473, "numLabelPairs":19832,"chunkCount":946298,"minTime":1712253600010, "maxTime":1712257935346},"seriesCountByMetricName": [{"name":"etcd_request_duration_seconds_bucket","value":51840}, {"name":"apiserver_request_sli_duration_seconds_bucket","value":47718},
oc debug <prometheus_k8s_pod_name> -n openshift-monitoring \ 1 -c prometheus --image=USD(oc get po -n openshift-monitoring <prometheus_k8s_pod_name> \ 2 -o jsonpath='{.spec.containers[?(@.name=="prometheus")].image}') -- sh -c 'cd /prometheus/;du -hs USD(ls -dt */ | grep -Eo "[0-9|A-Z]{26}")'
308M 01HVKMPKQWZYWS8WVDAYQHNMW6 52M 01HVK64DTDA81799TBR9QDECEZ 102M 01HVK64DS7TRZRWF2756KHST5X 140M 01HVJS59K11FBVAPVY57K88Z11 90M 01HVH2A5Z58SKT810EM6B9AT50 152M 01HV8ZDVQMX41MKCN84S32RRZ1 354M 01HV6Q2N26BK63G4RYTST71FBF 156M 01HV664H9J9Z1FTZD73RD1563E 216M 01HTHXB60A7F239HN7S2TENPNS 104M 01HTHMGRXGS0WXA3WATRXHR36B
oc debug prometheus-k8s-0 -n openshift-monitoring -c prometheus --image=USD(oc get po -n openshift-monitoring prometheus-k8s-0 -o jsonpath='{.spec.containers[?(@.name=="prometheus")].image}') -- sh -c 'ls -latr /prometheus/ | egrep -o "[0-9|A-Z]{26}" | head -3 | while read BLOCK; do rm -r /prometheus/USDBLOCK; done'
oc debug <prometheus_k8s_pod_name> -n openshift-monitoring \ 1 --image=USD(oc get po -n openshift-monitoring <prometheus_k8s_pod_name> \ 2 -o jsonpath='{.spec.containers[?(@.name=="prometheus")].image}') -- df -h /prometheus/
Starting pod/prometheus-k8s-0-debug-j82w4 Filesystem Size Used Avail Use% Mounted on /dev/nvme0n1p4 40G 15G 40G 37% /prometheus Removing debug pod
oc <command> --loglevel <log_level>
oc whoami -t
sha256~RCV3Qcn7H-OEfqCGVI0CvnZ6