Hosted control planes OpenShift Container Platform 4.18 Using hosted control planes with OpenShift Container Platform Red Hat OpenShift Documentation Team

Commands:
oc get nodepool -A
oc get nodes --kubeconfig
oc get agents -A
apiVersion: v1 data: supported-versions: '{"versions":["4.18"]}' kind: ConfigMap metadata: labels: hypershift.openshift.io/supported-versions: "true" name: supported-versions namespace: hypershift
oc create -f <your-config-map-file.yaml>
kind: ConfigMap apiVersion: v1 metadata: name: hcp-sizing-baseline namespace: local-cluster data: incrementalCPUUsagePer1KQPS: "9.0" memoryRequestPerHCP: "18" minimumQPSPerHCP: "50.0"
oc delete deployment hypershift-addon-agent -n open-cluster-management-agent-addon
oc logs hypershift-addon-agent -n open-cluster-management-agent-addon
2024-01-05T19:41:05.392Z INFO agent.agent-reconciler agent/agent.go:793 setting cpuRequestPerHCP to 5 2024-01-05T19:41:05.392Z INFO agent.agent-reconciler agent/agent.go:802 setting memoryRequestPerHCP to 18 2024-01-05T19:53:54.070Z INFO agent.agent-reconciler agent/hcp_capacity_calculation.go:141 The worker nodes have 12.000000 vCPUs 2024-01-05T19:53:54.070Z INFO agent.agent-reconciler agent/hcp_capacity_calculation.go:142 The worker nodes have 49.173369 GB memory
2024-01-05T19:53:54.052Z ERROR agent.agent-reconciler agent/agent.go:788 failed to get configmap from the hub. Setting the HCP sizing baseline with default values. {"error": "configmaps \"hcp-sizing-baseline\" not found"}
oc login
oc edit addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: hypershift-addon-deploy-config namespace: multicluster-engine spec: customizedVariables: - name: hcMaxNumber value: "80" - name: hcThresholdNumber value: "60" - name: disableMetrics 1 value: "true"
oc apply -f <filename>.yaml
oc get ConsoleCLIDownload hcp-cli-download -o json | jq -r ".spec"
wget <hcp_cli_download_url> 1
tar xvzf hcp.tar.gz
chmod +x hcp
sudo mv hcp /usr/local/bin/.
hcp create cluster <platform> --help 1
tar xvzf hcp.tar.gz
chmod +x hcp
sudo mv hcp /usr/local/bin/.
hcp create cluster <platform> --help 1
tar xvzf hcp.tar.gz
chmod +x hcp
sudo mv hcp /usr/local/bin/.
hcp create cluster <platform> --help 1
oc label node/worker-1a node/worker-1b topology.kubernetes.io/zone=rack1
oc label node/worker-2a node/worker-2b topology.kubernetes.io/zone=rack2
spec: nodeSelector: node-role.kubernetes.io/infra: ""
spec: tolerations: - effect: NoSchedule key: kubernetes.io/custom operator: Exists
--toleration="key=kubernetes.io/custom,operator=Exists,effect=NoSchedule"
oc patch mce multiclusterengine --type=merge -p '{"spec":{"overrides":{"components":[{"name":"hypershift","enabled": true}]}}}' 1
oc get mce multiclusterengine -o yaml 1
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: hypershift enabled: true - name: hypershift-local-hosting enabled: true
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: hypershift-addon namespace: local-cluster spec: installNamespace: open-cluster-management-agent-addon
oc apply -f <filename>
oc get managedclusteraddons -n local-cluster hypershift-addon
NAME AVAILABLE DEGRADED PROGRESSING hypershift-addon True
oc get hostedcluster -A
oc patch mce multiclusterengine --type=merge -p \ 1 '{"spec":{"overrides":{"components":[{"name":"hypershift-local-hosting","enabled": false}]}}}'
oc patch mce multiclusterengine --type=merge -p \ 1 '{"spec":{"overrides":{"components":[{"name":"hypershift","enabled": false}]}}}'
oc get mce multiclusterengine -o yaml 1
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: hypershift enabled: false - name: hypershift-local-hosting enabled: false
oc get managedclusters local-cluster
aws s3api create-bucket --bucket <bucket_name> \ 1 --create-bucket-configuration LocationConstraint=<region> \ 2 --region <region> 3
aws s3api delete-public-access-block --bucket <bucket_name> 1
echo '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": "*", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::<bucket_name>/*" 1 } ] }' | envsubst > policy.json
aws s3api put-bucket-policy --bucket <bucket_name> \ 1 --policy file://policy.json
oc create secret generic <secret_name> --from-file=credentials=<path>/.aws/credentials --from-literal=bucket=<s3_bucket> --from-literal=region=<region> -n local-cluster
oc label secret hypershift-operator-oidc-provider-s3-credentials -n local-cluster cluster.open-cluster-management.io/backup=true
aws route53 create-hosted-zone --name <basedomain> \ 1 --caller-reference USD(whoami)-USD(date --rfc-3339=date)
aws sts get-caller-identity --query "Arn" --output text
arn:aws:iam::1234567890:user/<aws_username>
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": "<arn>" 1 }, "Action": "sts:AssumeRole" } ] }
aws iam create-role --role-name <name> \ 1 --assume-role-policy-document file://<file_name>.json \ 2 --query "Role.Arn"
arn:aws:iam::820196288204:role/myrole
{ "Version": "2012-10-17", "Statement": [ { "Sid": "EC2", "Effect": "Allow", "Action": [ "ec2:CreateDhcpOptions", "ec2:DeleteSubnet", "ec2:ReplaceRouteTableAssociation", "ec2:DescribeAddresses", "ec2:DescribeInstances", "ec2:DeleteVpcEndpoints", "ec2:CreateNatGateway", "ec2:CreateVpc", "ec2:DescribeDhcpOptions", "ec2:AttachInternetGateway", "ec2:DeleteVpcEndpointServiceConfigurations", "ec2:DeleteRouteTable", "ec2:AssociateRouteTable", "ec2:DescribeInternetGateways", "ec2:DescribeAvailabilityZones", "ec2:CreateRoute", "ec2:CreateInternetGateway", "ec2:RevokeSecurityGroupEgress", "ec2:ModifyVpcAttribute", "ec2:DeleteInternetGateway", "ec2:DescribeVpcEndpointConnections", "ec2:RejectVpcEndpointConnections", "ec2:DescribeRouteTables", "ec2:ReleaseAddress", "ec2:AssociateDhcpOptions", "ec2:TerminateInstances", "ec2:CreateTags", "ec2:DeleteRoute", "ec2:CreateRouteTable", "ec2:DetachInternetGateway", "ec2:DescribeVpcEndpointServiceConfigurations", "ec2:DescribeNatGateways", "ec2:DisassociateRouteTable", "ec2:AllocateAddress", "ec2:DescribeSecurityGroups", "ec2:RevokeSecurityGroupIngress", "ec2:CreateVpcEndpoint", "ec2:DescribeVpcs", "ec2:DeleteSecurityGroup", "ec2:DeleteDhcpOptions", "ec2:DeleteNatGateway", "ec2:DescribeVpcEndpoints", "ec2:DeleteVpc", "ec2:CreateSubnet", "ec2:DescribeSubnets" ], "Resource": "*" }, { "Sid": "ELB", "Effect": "Allow", "Action": [ "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DeleteTargetGroup" ], "Resource": "*" }, { "Sid": "IAMPassRole", "Effect": "Allow", "Action": "iam:PassRole", "Resource": "arn:*:iam::*:role/*-worker-role", "Condition": { "ForAnyValue:StringEqualsIfExists": { "iam:PassedToService": "ec2.amazonaws.com" } } }, { "Sid": "IAM", "Effect": "Allow", "Action": [ "iam:CreateInstanceProfile", "iam:DeleteInstanceProfile", "iam:GetRole", "iam:UpdateAssumeRolePolicy", "iam:GetInstanceProfile", "iam:TagRole", "iam:RemoveRoleFromInstanceProfile", "iam:CreateRole", "iam:DeleteRole", "iam:PutRolePolicy", "iam:AddRoleToInstanceProfile", "iam:CreateOpenIDConnectProvider", "iam:ListOpenIDConnectProviders", "iam:DeleteRolePolicy", "iam:UpdateRole", "iam:DeleteOpenIDConnectProvider", "iam:GetRolePolicy" ], "Resource": "*" }, { "Sid": "Route53", "Effect": "Allow", "Action": [ "route53:ListHostedZonesByVPC", "route53:CreateHostedZone", "route53:ListHostedZones", "route53:ChangeResourceRecordSets", "route53:ListResourceRecordSets", "route53:DeleteHostedZone", "route53:AssociateVPCWithHostedZone", "route53:ListHostedZonesByName" ], "Resource": "*" }, { "Sid": "S3", "Effect": "Allow", "Action": [ "s3:ListAllMyBuckets", "s3:ListBucket", "s3:DeleteObject", "s3:DeleteBucket" ], "Resource": "*" } ] }
aws iam put-role-policy --role-name <role_name> \ 1 --policy-name <policy_name> \ 2 --policy-document file://policy.json 3
aws sts get-session-token --output json > sts-creds.json
{ "Credentials": { "AccessKeyId": "ASIA1443CE0GN2ATHWJU", "SecretAccessKey": "XFLN7cZ5AP0d66KhyI4gd8Mu0UCQEDN9cfelW1", "SessionToken": "IQoJb3JpZ2luX2VjEEAaCXVzLWVhc3QtMiJHMEUCIDyipkM7oPKBHiGeI0pMnXst1gDLfs/TvfskXseKCbshAiEAnl1l/Html7Iq9AEIqf////KQburfkq4A3TuppHMr/9j1TgCj1z83SO261bHqlJUazKoy7vBFR/a6LHt55iMBqtKPEsIWjBgj/jSdRJI3j4Gyk1//luKDytcfF/tb9YrxDTPLrACS1lqAxSIFZ82I/jDhbDs=", "Expiration": "2025-05-16T04:19:32+00:00" } }
oc create secret generic <secret_name> --from-literal=aws-access-key-id=<aws_access_key_id> --from-literal=aws-secret-access-key=<aws_secret_access_key> --from-literal=region=<region> -n local-cluster
oc label secret hypershift-operator-private-link-credentials -n local-cluster cluster.open-cluster-management.io/backup=""
oc create secret generic <secret_name> --from-literal=provider=aws --from-literal=domain-filter=<domain_name> --from-file=credentials=<path_to_aws_credentials_file> -n local-cluster
oc label secret hypershift-operator-external-dns-credentials -n local-cluster cluster.open-cluster-management.io/backup=""
dig +short test.user-dest-public.aws.kerberos.com
192.168.1.1
hcp create cluster aws --name=<hosted_cluster_name> --endpoint-access=PublicAndPrivate --external-dns-domain=<public_hosted_zone> ... 1
platform: aws: endpointAccess: PublicAndPrivate services: - service: APIServer servicePublishingStrategy: route: hostname: api-example.service-provider-domain.com type: Route - service: OAuthServer servicePublishingStrategy: route: hostname: oauth-example.service-provider-domain.com type: Route - service: Konnectivity servicePublishingStrategy: type: Route - service: Ignition servicePublishingStrategy: type: Route
export KUBECONFIG=<path_to_management_cluster_kubeconfig>
oc get pod -n hypershift -lapp=external-dns
NAME READY STATUS RESTARTS AGE external-dns-7c89788c69-rn8gp 1/1 Running 0 40s
hcp create cluster aws --role-arn <arn_role> \ 1 --instance-type <instance_type> \ 2 --region <region> \ 3 --auto-repair --generate-ssh --name <hosted_cluster_name> \ 4 --namespace clusters --base-domain <service_consumer_domain> \ 5 --node-pool-replicas <node_replica_count> \ 6 --pull-secret <path_to_your_pull_secret> \ 7 --release-image quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 8 --external-dns-domain=<service_provider_domain> \ 9 --endpoint-access=PublicAndPrivate 10 --sts-creds <path_to_sts_credential_file> 11
hcp create cluster aws --name <hosted_cluster_name> \ 1 --infra-id <infra_id> \ 2 --base-domain <basedomain> \ 3 --sts-creds <path_to_sts_credential_file> \ 4 --pull-secret <path_to_pull_secret> \ 5 --region <region> \ 6 --generate-ssh --node-pool-replicas <node_pool_replica_count> \ 7 --namespace <hosted_cluster_namespace> \ 8 --role-arn <role_name> \ 9 --render-into <file_name>.yaml 10
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: name: <nodepool_name> 1 spec: platform: aws: placement: tenancy: "default" 2
oc get hostedclusters -n <hosted_cluster_namespace>
oc get nodepools --namespace <hosted_cluster_namespace>
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
hcp create kubeconfig --namespace <hosted_cluster_namespace> --name <hosted_cluster_name> > <hosted_cluster_name>.kubeconfig
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas=<node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --role-arn <arn_role> \ 5 --region <region> \ 6 --zones <zones> \ 7 --sts-creds <path_to_sts_credential_file> 8
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas <node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --sts-creds <path_to_sts_credential_file> \ 5 --region <region> \ 6 --role-arn <arn_role> 7
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas <node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --sts-creds <path_to_sts_credential_file> \ 5 --region <region> \ 6 --release-image quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 7 --role-arn <role_name> 8
hcp create nodepool aws --cluster-name <hosted_cluster_name> \ 1 --name <node_pool_name> \ 2 --node-count <node_pool_replica_count> \ 3 --arch <architecture> 4
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas=<node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --sts-creds <path_to_sts_credential_file> \ 5 --region <region> \ 6 --endpoint-access Private \ 7 --role-arn <role_name> 8
aws ec2 describe-instances --filter="Name=tag:kubernetes.io/cluster/<infra_id>,Values=owned" | jq '.Reservations[] | .Instances[] | select(.PublicDnsName=="") | .PrivateIpAddress'
hcp create kubeconfig > <hosted_cluster_kubeconfig>
ssh -o ProxyCommand="ssh ec2-user@<bastion_ip> -W %h:%p" core@<node_ip>
mv <path_to_kubeconfig_file> <new_file_name>
export KUBECONFIG=<path_to_kubeconfig_file>
oc get clusteroperators clusterversion
oc get managedclusters local-cluster
api.example.krnl.es. IN A 192.168.122.20 api.example.krnl.es. IN A 192.168.122.21 api.example.krnl.es. IN A 192.168.122.22 api-int.example.krnl.es. IN A 192.168.122.20 api-int.example.krnl.es. IN A 192.168.122.21 api-int.example.krnl.es. IN A 192.168.122.22 `*`.apps.example.krnl.es. IN A 192.168.122.23
api.example.krnl.es. IN A 2620:52:0:1306::5 api.example.krnl.es. IN A 2620:52:0:1306::6 api.example.krnl.es. IN A 2620:52:0:1306::7 api-int.example.krnl.es. IN A 2620:52:0:1306::5 api-int.example.krnl.es. IN A 2620:52:0:1306::6 api-int.example.krnl.es. IN A 2620:52:0:1306::7 `*`.apps.example.krnl.es. IN A 2620:52:0:1306::10
host-record=api-int.hub-dual.dns.base.domain.name,192.168.126.10 host-record=api.hub-dual.dns.base.domain.name,192.168.126.10 address=/apps.hub-dual.dns.base.domain.name/192.168.126.11 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,192.168.126.20 dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,192.168.126.21 dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,192.168.126.22 dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,192.168.126.25 dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,192.168.126.26 host-record=api-int.hub-dual.dns.base.domain.name,2620:52:0:1306::2 host-record=api.hub-dual.dns.base.domain.name,2620:52:0:1306::2 address=/apps.hub-dual.dns.base.domain.name/2620:52:0:1306::3 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,[2620:52:0:1306::5] dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,[2620:52:0:1306::6] dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,[2620:52:0:1306::7] dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,[2620:52:0:1306::8] dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,[2620:52:0:1306::9]
oc create ns <hosted_cluster_namespace>-<hosted_cluster_name>
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --etcd-storage-class=<etcd_storage_class> \ 6 --ssh-key <path_to_ssh_public_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --control-plane-availability-policy HighlyAvailable \ 9 --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 10 --node-pool-replicas <node_pool_replica_count> 11
oc -n <hosted_control_plane_namespace> get pods
NAME READY STATUS RESTARTS AGE capi-provider-7dcf5fc4c4-nr9sq 1/1 Running 0 4m32s catalog-operator-6cd867cc7-phb2q 2/2 Running 0 2m50s certified-operators-catalog-884c756c4-zdt64 1/1 Running 0 2m51s cluster-api-f75d86f8c-56wfz 1/1 Running 0 4m32s
- mirrors: - brew.registry.redhat.io source: registry.redhat.io - mirrors: - brew.registry.redhat.io source: registry.stage.redhat.io - mirrors: - brew.registry.redhat.io source: registry-proxy.engineering.redhat.com
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --image-content-sources icsp.yaml \ 6 --ssh-key <path_to_ssh_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> 9
oc extract -n <hosted-control-plane-namespace> secret/admin-kubeconfig --to=- > kubeconfig-<hosted-cluster-name>
oc get co --kubeconfig=kubeconfig-<hosted-cluster-name>
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console 4.10.26 True False False 2m38s dns 4.10.26 True False False 2m52s image-registry 4.10.26 True False False 2m8s ingress 4.10.26 True False False 22m
oc get pods -A --kubeconfig=kubeconfig-<hosted-cluster-name>
NAMESPACE NAME READY STATUS RESTARTS AGE kube-system konnectivity-agent-khlqv 0/1 Running 0 3m52s openshift-cluster-node-tuning-operator tuned-dhw5p 1/1 Running 0 109s openshift-cluster-storage-operator cluster-storage-operator-5f784969f5-vwzgz 1/1 Running 1 (113s ago) 20m openshift-cluster-storage-operator csi-snapshot-controller-6b7687b7d9-7nrfw 1/1 Running 0 3m8s openshift-console console-5cbf6c7969-6gk6z 1/1 Running 0 119s openshift-console downloads-7bcd756565-6wj5j 1/1 Running 0 4m3s openshift-dns-operator dns-operator-77d755cd8c-xjfbn 2/2 Running 0 21m openshift-dns dns-default-kfqnh 2/2 Running 0 113s
oc patch ingresscontroller -n openshift-ingress-operator default --type=json -p '[{ "op": "add", "path": "/spec/routeAdmission", "value": {wildcardPolicy: "WildcardsAllowed"}}]'
oc patch storageclass ocs-storagecluster-ceph-rbd -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
oc get managedclusters local-cluster
- lastTransitionTime: "2024-10-08T15:38:19Z" message: | 3 of 3 machines are not live migratable Machine user-np-ngst4-gw2hz: DisksNotLiveMigratable: user-np-ngst4-gw2hz is not a live migratable machine: cannot migrate VMI: PVC user-np-ngst4-gw2hz-rhcos is not shared, live migration requires that all PVCs must be shared (using ReadWriteMany access mode) Machine user-np-ngst4-npq7x: DisksNotLiveMigratable: user-np-ngst4-npq7x is not a live migratable machine: cannot migrate VMI: PVC user-np-ngst4-npq7x-rhcos is not shared, live migration requires that all PVCs must be shared (using ReadWriteMany access mode) Machine user-np-ngst4-q5nkb: DisksNotLiveMigratable: user-np-ngst4-q5nkb is not a live migratable machine: cannot migrate VMI: PVC user-np-ngst4-q5nkb-rhcos is not shared, live migration requires that all PVCs must be shared (using ReadWriteMany access mode) observedGeneration: 1 reason: DisksNotLiveMigratable status: "False" type: KubeVirtNodesLiveMigratable
- lastTransitionTime: "2024-10-08T15:38:19Z" message: "All is well" observedGeneration: 1 reason: AsExpected status: "True" type: KubeVirtNodesLiveMigratable
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <node_pool_replica_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <value_for_memory> \ 4 --cores <value_for_cpu> \ 5 --etcd-storage-class=<etcd_storage_class> 6
oc -n clusters-<hosted-cluster-name> get pods
NAME READY STATUS RESTARTS AGE capi-provider-5cc7b74f47-n5gkr 1/1 Running 0 3m catalog-operator-5f799567b7-fd6jw 2/2 Running 0 69s certified-operators-catalog-784b9899f9-mrp6p 1/1 Running 0 66s cluster-api-6bbc867966-l4dwl 1/1 Running 0 66s . . . redhat-operators-catalog-9d5fd4d44-z8qqk 1/1 Running 0 66s
oc get --namespace clusters hostedclusters
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters example <4.x.0> example-admin-kubeconfig Completed True False The hosted control plane is available
hcp create cluster kubevirt --name <hosted-cluster-name> \ 1 --node-pool-replicas <worker-count> \ 2 --pull-secret <path-to-pull-secret> \ 3 --memory <value-for-memory> \ 4 --cores <value-for-cpu> \ 5 --infra-namespace=<hosted-cluster-namespace>-<hosted-cluster-name> \ 6 --infra-kubeconfig-file=<path-to-external-infra-kubeconfig> 7
*.apps.mgmt-cluster.example.com
*.apps.guest.apps.mgmt-cluster.example.com
oc patch ingresscontroller -n openshift-ingress-operator default --type=json -p '[{ "op": "add", "path": "/spec/routeAdmission", "value": {wildcardPolicy: "WildcardsAllowed"}}]'
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <value_for_memory> \ 4 --cores <value_for_cpu> \ 5 --base-domain <basedomain> 6
oc get --namespace clusters hostedclusters
NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE example example-admin-kubeconfig Partial True False The hosted control plane is available
hcp create kubeconfig --name <hosted_cluster_name> > <hosted_cluster_name>-kubeconfig
oc --kubeconfig <hosted_cluster_name>-kubeconfig get co
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console <4.x.0> False False False 30m RouteHealthAvailable: failed to GET route (https://console-openshift-console.apps.example.hypershift.lab): Get "https://console-openshift-console.apps.example.hypershift.lab": dial tcp: lookup console-openshift-console.apps.example.hypershift.lab on 172.31.0.10:53: no such host ingress <4.x.0> True False True 28m The "default" ingress controller reports Degraded=True: DegradedConditions: One or more other status conditions indicate a degraded state: CanaryChecksSucceeding=False (CanaryChecksRepetitiveFailures: Canary route checks for the default ingress controller are failing)
oc --kubeconfig <hosted_cluster_name>-kubeconfig get services -n openshift-ingress router-nodeport-default -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}'
oc --kubeconfig <hosted_cluster_name>-kubeconfig get services -n openshift-ingress router-nodeport-default -o jsonpath='{.spec.ports[?(@.name=="https")].nodePort}'
apply -f - apiVersion: v1 kind: Service metadata: labels: app: <hosted_cluster_name> name: <hosted_cluster_name>-apps namespace: clusters-<hosted_cluster_name> spec: ports: - name: https-443 port: 443 protocol: TCP targetPort: <https_node_port> 1 - name: http-80 port: 80 protocol: TCP targetPort: <http-node-port> 2 selector: kubevirt.io: virt-launcher type: LoadBalancer
oc -n clusters-<hosted_cluster_name> get service <hosted-cluster-name>-apps -o jsonpath='{.status.loadBalancer.ingress[0].ip}'
192.168.20.30
*.apps.<hosted_cluster_name\>.<base_domain\>.
dig +short test.apps.example.hypershift.lab 192.168.20.30
oc get --namespace clusters hostedclusters
NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE example <4.x.0> example-admin-kubeconfig Completed True False The hosted control plane is available
apiVersion: metallb.io/v1beta1 kind: MetalLB metadata: name: metallb namespace: metallb-system
oc apply -f configure-metallb.yaml
metallb.metallb.io/metallb created
apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: metallb namespace: metallb-system spec: addresses: - 192.168.216.32-192.168.216.122 1
oc apply -f create-ip-address-pool.yaml
ipaddresspool.metallb.io/metallb created
apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: l2advertisement namespace: metallb-system spec: ipAddressPools: - metallb
oc apply -f l2advertisement.yaml
l2advertisement.metallb.io/metallb created
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --additional-network name:<namespace/name> \ 6 --additional-network name:<namespace/name>
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --attach-default-network false \ 6 --additional-network name:<namespace>/<network_name> 7
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --qos-class Guaranteed 6
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --vm-node-selector <label_key>=<label_value>,<label_key>=<label_value> 6
NODEPOOL_NAME=USD{CLUSTER_NAME}-work NODEPOOL_REPLICAS=5 oc scale nodepool/USDNODEPOOL_NAME --namespace clusters --replicas=USDNODEPOOL_REPLICAS
oc --kubeconfig USDCLUSTER_NAME-kubeconfig get nodes
NAME STATUS ROLES AGE VERSION example-9jvnf Ready worker 97s v1.27.4+18eadca example-n6prw Ready worker 116m v1.27.4+18eadca example-nc6g4 Ready worker 117m v1.27.4+18eadca example-thp29 Ready worker 4m17s v1.27.4+18eadca example-twxns Ready worker 88s v1.27.4+18eadca
export NODEPOOL_NAME=USD{CLUSTER_NAME}-extra-cpu export WORKER_COUNT="2" export MEM="6Gi" export CPU="4" export DISK="16" hcp create nodepool kubevirt --cluster-name USDCLUSTER_NAME --name USDNODEPOOL_NAME --node-count USDWORKER_COUNT --memory USDMEM --cores USDCPU --root-volume-size USDDISK
oc get nodepools --namespace clusters
NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE example example 5 5 False False <4.x.0> example-extra-cpu example 2 False False True True Minimum availability requires 2 replicas, current 0 available
oc --kubeconfig USDCLUSTER_NAME-kubeconfig get nodes
NAME STATUS ROLES AGE VERSION example-9jvnf Ready worker 97s v1.27.4+18eadca example-n6prw Ready worker 116m v1.27.4+18eadca example-nc6g4 Ready worker 117m v1.27.4+18eadca example-thp29 Ready worker 4m17s v1.27.4+18eadca example-twxns Ready worker 88s v1.27.4+18eadca example-extra-cpu-zh9l5 Ready worker 2m6s v1.27.4+18eadca example-extra-cpu-zr8mj Ready worker 102s v1.27.4+18eadca
oc get nodepools --namespace clusters
NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE example example 5 5 False False <4.x.0> example-extra-cpu example 2 2 False False <4.x.0>
oc get --namespace clusters hostedclusters <hosted_cluster_name>
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters example 4.12.2 example-admin-kubeconfig Completed True False The hosted control plane is available
hcp create kubeconfig --name <hosted_cluster_name> > <hosted_cluster_name>-kubeconfig
oc get co --kubeconfig=<hosted_cluster_name>-kubeconfig
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console 4.12.2 True False False 2m38s csi-snapshot-controller 4.12.2 True False False 4m3s dns 4.12.2 True False False 2m52s image-registry 4.12.2 True False False 2m8s ingress 4.12.2 True False False 22m kube-apiserver 4.12.2 True False False 23m kube-controller-manager 4.12.2 True False False 23m kube-scheduler 4.12.2 True False False 23m kube-storage-version-migrator 4.12.2 True False False 4m52s monitoring 4.12.2 True False False 69s network 4.12.2 True False False 4m3s node-tuning 4.12.2 True False False 2m22s openshift-apiserver 4.12.2 True False False 23m openshift-controller-manager 4.12.2 True False False 23m openshift-samples 4.12.2 True False False 2m15s operator-lifecycle-manager 4.12.2 True False False 22m operator-lifecycle-manager-catalog 4.12.2 True False False 23m operator-lifecycle-manager-packageserver 4.12.2 True False False 23m service-ca 4.12.2 True False False 4m41s storage 4.12.2 True False False 4m43s
oc get managedclusters local-cluster
api.example.krnl.es. IN A 192.168.122.20 api.example.krnl.es. IN A 192.168.122.21 api.example.krnl.es. IN A 192.168.122.22 api-int.example.krnl.es. IN A 192.168.122.20 api-int.example.krnl.es. IN A 192.168.122.21 api-int.example.krnl.es. IN A 192.168.122.22 `*`.apps.example.krnl.es. IN A 192.168.122.23
api.example.krnl.es. IN A 2620:52:0:1306::5 api.example.krnl.es. IN A 2620:52:0:1306::6 api.example.krnl.es. IN A 2620:52:0:1306::7 api-int.example.krnl.es. IN A 2620:52:0:1306::5 api-int.example.krnl.es. IN A 2620:52:0:1306::6 api-int.example.krnl.es. IN A 2620:52:0:1306::7 `*`.apps.example.krnl.es. IN A 2620:52:0:1306::10
host-record=api-int.hub-dual.dns.base.domain.name,192.168.126.10 host-record=api.hub-dual.dns.base.domain.name,192.168.126.10 address=/apps.hub-dual.dns.base.domain.name/192.168.126.11 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,192.168.126.20 dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,192.168.126.21 dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,192.168.126.22 dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,192.168.126.25 dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,192.168.126.26 host-record=api-int.hub-dual.dns.base.domain.name,2620:52:0:1306::2 host-record=api.hub-dual.dns.base.domain.name,2620:52:0:1306::2 address=/apps.hub-dual.dns.base.domain.name/2620:52:0:1306::3 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,[2620:52:0:1306::5] dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,[2620:52:0:1306::6] dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,[2620:52:0:1306::7] dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,[2620:52:0:1306::8] dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,[2620:52:0:1306::9]
oc create ns <hosted_cluster_namespace>-<hosted_cluster_name> 1
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --etcd-storage-class=<etcd_storage_class> \ 6 --ssh-key <path_to_ssh_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --control-plane-availability-policy HighlyAvailable \ 9 --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release> \ 10 --node-pool-replicas <node_pool_replica_count> 11
oc -n <hosted_control_plane_namespace> get pods
NAME READY STATUS RESTARTS AGE catalog-operator-6cd867cc7-phb2q 2/2 Running 0 2m50s control-plane-operator-f6b4c8465-4k5dh 1/1 Running 0 4m32s
- mirrors: - brew.registry.redhat.io source: registry.redhat.io - mirrors: - brew.registry.redhat.io source: registry.stage.redhat.io - mirrors: - brew.registry.redhat.io source: registry-proxy.engineering.redhat.com
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --image-content-sources icsp.yaml \ 6 --ssh-key <path_to_ssh_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> 9
oc extract -n <hosted_cluster_namespace> secret/<hosted_cluster_name>-admin-kubeconfig --to=- > kubeconfig-<hosted_cluster_name>
oc get co --kubeconfig=kubeconfig-<hosted_cluster_name>
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console 4.10.26 True False False 2m38s csi-snapshot-controller 4.10.26 True False False 4m3s dns 4.10.26 True False False 2m52s
oc get pods -A --kubeconfig=kubeconfig-<hosted_cluster_name>
NAMESPACE NAME READY STATUS RESTARTS AGE kube-system konnectivity-agent-khlqv 0/1 Running 0 3m52s openshift-cluster-samples-operator cluster-samples-operator-6b5bcb9dff-kpnbc 2/2 Running 0 20m openshift-monitoring alertmanager-main-0 6/6 Running 0 100s openshift-monitoring openshift-state-metrics-677b9fb74f-qqp6g 3/3 Running 0 104s
oc get managedclusters local-cluster
cat /var/named/<example.krnl.es.zone>
TTL 900 @ IN SOA bastion.example.krnl.es.com. hostmaster.example.krnl.es.com. ( 2019062002 1D 1H 1W 3H ) IN NS bastion.example.krnl.es.com. ; ; api IN A 1xx.2x.2xx.1xx 1 api-int IN A 1xx.2x.2xx.1xx ; ; *.apps IN A 1xx.2x.2xx.1xx ; ;EOF
compute-0 IN A 1xx.2x.2xx.1yy compute-1 IN A 1xx.2x.2xx.1yy
oc create ns <hosted_cluster_namespace>-<hosted_cluster_name>
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --etcd-storage-class=<etcd_storage_class> \ 6 --ssh-key <path_to_ssh_public_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --control-plane-availability-policy HighlyAvailable \ 9 --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 10 --node-pool-replicas <node_pool_replica_count> 11
oc -n <hosted_control_plane_namespace> get pods
NAME READY STATUS RESTARTS AGE capi-provider-7dcf5fc4c4-nr9sq 1/1 Running 0 4m32s catalog-operator-6cd867cc7-phb2q 2/2 Running 0 2m50s certified-operators-catalog-884c756c4-zdt64 1/1 Running 0 2m51s cluster-api-f75d86f8c-56wfz 1/1 Running 0 4m32s
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: <hosted_cluster_name> namespace: <hosted_control_plane_namespace> spec: cpuArchitecture: s390x pullSecretRef: name: pull-secret sshAuthorizedKey: <ssh_public_key>
oc apply -f infraenv-config.yaml
oc -n <hosted_control_plane_namespace> get InfraEnv <hosted_cluster_name> -o json
virt-install --name "<vm_name>" \ 1 --autostart --ram=16384 --cpu host --vcpus=4 --location "<path_to_kernel_initrd_image>,kernel=kernel.img,initrd=initrd.img" \ 2 --disk <qcow_image_path> \ 3 --network network:macvtap-net,mac=<mac_address> \ 4 --graphics none --noautoconsole --wait=-1 --extra-args "rd.neednet=1 nameserver=<nameserver> coreos.live.rootfs_url=http://<http_server>/rootfs.img random.trust_cpu=on rd.luks.options=discard ignition.firstboot ignition.platform.id=metal console=tty1 console=ttyS1,115200n8 coreos.inst.persistent-kargs=console=tty1 console=ttyS1,115200n8" 5
virt-install --name "<vm_name>" \ 1 --autostart --memory=16384 --cpu host --vcpus=4 --network network:macvtap-net,mac=<mac_address> \ 2 --cdrom "<path_to_image.iso>" \ 3 --disk <qcow_image_path> --graphics none --noautoconsole --os-variant <os_version> \ 4 --wait=-1
rd.neednet=1 cio_ignore=all,!condev console=ttysclp0 ignition.firstboot ignition.platform.id=metal coreos.live.rootfs_url=http://<http_server>/rhcos-<version>-live-rootfs.<architecture>.img \ 1 coreos.inst.persistent-kargs=console=ttysclp0 ip=<ip>::<gateway>:<netmask>:<hostname>::none nameserver=<dns> \ 2 rd.znet=qeth,<network_adaptor_range>,layer2=1 rd.<disk_type>=<adapter> \ 3 zfcp.allow_lun_scan=0 ai.ip_cfg_override=1 \ 4 random.trust_cpu=on rd.luks.options=discard
curl -k -L -o generic.ins "< url for ins-file >"
https://.../boot-artifacts/ins-file?arch=s390x&version=4.17.0
https://..../s390x-initrd-addrsize?api_key=<api-key>&arch=s390x&version=4.17.0
rd.neednet=1 cio_ignore=all,!condev console=ttysclp0 ignition.firstboot ignition.platform.id=metal coreos.live.rootfs_url=http://<http_server>/rhcos-<version>-live-rootfs.<architecture>.img \ 1 coreos.inst.persistent-kargs=console=ttysclp0 ip=<ip>::<gateway>:<netmask>:<hostname>::none nameserver=<dns> \ 2 rd.znet=qeth,<network_adaptor_range>,layer2=1 rd.<disk_type>=<adapter> \ 3 zfcp.allow_lun_scan=0 ai.ip_cfg_override=1 \ 4
vmur pun -r -u -N kernel.img USDINSTALLERKERNELLOCATION/<image name>
vmur pun -r -u -N generic.parm USDPARMFILELOCATION/paramfilename
vmur pun -r -u -N initrd.img USDINSTALLERINITRAMFSLOCATION/<image name>
cp ipl c
oc -n <hosted_control_plane_namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a auto-assign
oc -n <hosted_control_plane_namespace> patch agent 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d -p '{"spec":{"installation_disk_id":"/dev/sda","approved":true,"hostname":"worker-zvm-0.hostedn.example.com"}}' \ 1 --type merge
oc -n <hosted_control_plane_namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d true auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a true auto-assign
oc -n <clusters_namespace> scale nodepool <nodepool_name> --replicas 2
oc -n <hosted_control_plane_namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: Agent: 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d State: known-unbound BMH: Agent: 5e498cd3-542c-e54f-0c58-ed43e28b568a State: insufficient
oc -n <hosted_control_plane_namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d hosted-forwarder true auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a true auto-assign da503cf1-a347-44f2-875c-4960ddb04091 hosted-forwarder true auto-assign
hcp create kubeconfig --namespace <clusters_namespace> --name <hosted_cluster_namespace> > <hosted_cluster_name>.kubeconfig
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
NAME STATUS ROLES AGE VERSION worker-zvm-0.hostedn.example.com Ready worker 5m41s v1.24.0+3882f8f worker-zvm-1.hostedn.example.com Ready worker 6m3s v1.24.0+3882f8f
oc -n <hosted_control_plane_namespace> get machine.cluster.x-k8s.io
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hosted-forwarder-79558597ff-5tbqp hosted-forwarder-crqq5 worker-zvm-0.hostedn.example.com agent://50c23cda-cedc-9bbd-bcf1-9b3a5c75804d Running 41h 4.15.0 hosted-forwarder-79558597ff-lfjfk hosted-forwarder-crqq5 worker-zvm-1.hostedn.example.com agent://5e498cd3-542c-e54f-0c58-ed43e28b568a Running 41h 4.15.0
oc --kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.15.0-ec.2 True False 40h Cluster version is 4.15.0-ec.2
oc --kubeconfig <hosted_cluster_name>.kubeconfig get clusteroperators
oc get managedclusters local-cluster
cat /var/named/<example.krnl.es.zone>
TTL 900 @ IN SOA bastion.example.krnl.es.com. hostmaster.example.krnl.es.com. ( 2019062002 1D 1H 1W 3H ) IN NS bastion.example.krnl.es.com. ; ; api IN A 1xx.2x.2xx.1xx 1 api-int IN A 1xx.2x.2xx.1xx ; ; *.apps.<hosted-cluster-name>.<basedomain> IN A 1xx.2x.2xx.1xx ; ;EOF
compute-0 IN A 1xx.2x.2xx.1yy compute-1 IN A 1xx.2x.2xx.1yy
oc create ns <hosted_cluster_namespace>-<hosted_cluster_name>
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --etcd-storage-class=<etcd_storage_class> \ 6 --ssh-key <path_to_ssh_public_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --control-plane-availability-policy HighlyAvailable \ 9 --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 10 --node-pool-replicas <node_pool_replica_count> 11
oc -n <hosted_control_plane_namespace> get pods
NAME READY STATUS RESTARTS AGE capi-provider-7dcf5fc4c4-nr9sq 1/1 Running 0 4m32s catalog-operator-6cd867cc7-phb2q 2/2 Running 0 2m50s certified-operators-catalog-884c756c4-zdt64 1/1 Running 0 2m51s cluster-api-f75d86f8c-56wfz 1/1 Running 0 4m32s
endpointAccess: Public region: us-east-2 resourceTags: - key: kubernetes.io/cluster/example-cluster-bz4j5 value: owned rolesRef: controlPlaneOperatorARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-control-plane-operator imageRegistryARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-openshift-image-registry ingressARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-openshift-ingress kubeCloudControllerARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-cloud-controller networkARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-cloud-network-config-controller nodePoolManagementARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-node-pool storageARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-aws-ebs-csi-driver-controller type: AWS
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "elasticloadbalancing:DescribeLoadBalancers", "tag:GetResources", "route53:ListHostedZones" ], "Resource": "\*" }, { "Effect": "Allow", "Action": [ "route53:ChangeResourceRecordSets" ], "Resource": [ "arn:aws:route53:::PUBLIC_ZONE_ID", "arn:aws:route53:::PRIVATE_ZONE_ID" ] } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "s3:CreateBucket", "s3:DeleteBucket", "s3:PutBucketTagging", "s3:GetBucketTagging", "s3:PutBucketPublicAccessBlock", "s3:GetBucketPublicAccessBlock", "s3:PutEncryptionConfiguration", "s3:GetEncryptionConfiguration", "s3:PutLifecycleConfiguration", "s3:GetLifecycleConfiguration", "s3:GetBucketLocation", "s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucketMultipartUploads", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts" ], "Resource": "\*" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:AttachVolume", "ec2:CreateSnapshot", "ec2:CreateTags", "ec2:CreateVolume", "ec2:DeleteSnapshot", "ec2:DeleteTags", "ec2:DeleteVolume", "ec2:DescribeInstances", "ec2:DescribeSnapshots", "ec2:DescribeTags", "ec2:DescribeVolumes", "ec2:DescribeVolumesModifications", "ec2:DetachVolume", "ec2:ModifyVolume" ], "Resource": "\*" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:DescribeInstances", "ec2:DescribeInstanceStatus", "ec2:DescribeInstanceTypes", "ec2:UnassignPrivateIpAddresses", "ec2:AssignPrivateIpAddresses", "ec2:UnassignIpv6Addresses", "ec2:AssignIpv6Addresses", "ec2:DescribeSubnets", "ec2:DescribeNetworkInterfaces" ], "Resource": "\*" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Action": [ "ec2:DescribeInstances", "ec2:DescribeImages", "ec2:DescribeRegions", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVolumes", "ec2:CreateSecurityGroup", "ec2:CreateTags", "ec2:CreateVolume", "ec2:ModifyInstanceAttribute", "ec2:ModifyVolume", "ec2:AttachVolume", "ec2:AuthorizeSecurityGroupIngress", "ec2:CreateRoute", "ec2:DeleteRoute", "ec2:DeleteSecurityGroup", "ec2:DeleteVolume", "ec2:DetachVolume", "ec2:RevokeSecurityGroupIngress", "ec2:DescribeVpcs", "elasticloadbalancing:AddTags", "elasticloadbalancing:AttachLoadBalancerToSubnets", "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", "elasticloadbalancing:CreateLoadBalancer", "elasticloadbalancing:CreateLoadBalancerPolicy", "elasticloadbalancing:CreateLoadBalancerListeners", "elasticloadbalancing:ConfigureHealthCheck", "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:DeleteLoadBalancerListeners", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DetachLoadBalancerFromSubnets", "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", "elasticloadbalancing:ModifyLoadBalancerAttributes", "elasticloadbalancing:RegisterInstancesWithLoadBalancer", "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", "elasticloadbalancing:AddTags", "elasticloadbalancing:CreateListener", "elasticloadbalancing:CreateTargetGroup", "elasticloadbalancing:DeleteListener", "elasticloadbalancing:DeleteTargetGroup", "elasticloadbalancing:DescribeListeners", "elasticloadbalancing:DescribeLoadBalancerPolicies", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:ModifyListener", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:RegisterTargets", "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", "iam:CreateServiceLinkedRole", "kms:DescribeKey" ], "Resource": [ "\*" ], "Effect": "Allow" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Action": [ "ec2:AllocateAddress", "ec2:AssociateRouteTable", "ec2:AttachInternetGateway", "ec2:AuthorizeSecurityGroupIngress", "ec2:CreateInternetGateway", "ec2:CreateNatGateway", "ec2:CreateRoute", "ec2:CreateRouteTable", "ec2:CreateSecurityGroup", "ec2:CreateSubnet", "ec2:CreateTags", "ec2:DeleteInternetGateway", "ec2:DeleteNatGateway", "ec2:DeleteRouteTable", "ec2:DeleteSecurityGroup", "ec2:DeleteSubnet", "ec2:DeleteTags", "ec2:DescribeAccountAttributes", "ec2:DescribeAddresses", "ec2:DescribeAvailabilityZones", "ec2:DescribeImages", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", "ec2:DescribeNatGateways", "ec2:DescribeNetworkInterfaces", "ec2:DescribeNetworkInterfaceAttribute", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs", "ec2:DescribeVpcAttribute", "ec2:DescribeVolumes", "ec2:DetachInternetGateway", "ec2:DisassociateRouteTable", "ec2:DisassociateAddress", "ec2:ModifyInstanceAttribute", "ec2:ModifyNetworkInterfaceAttribute", "ec2:ModifySubnetAttribute", "ec2:ReleaseAddress", "ec2:RevokeSecurityGroupIngress", "ec2:RunInstances", "ec2:TerminateInstances", "tag:GetResources", "ec2:CreateLaunchTemplate", "ec2:CreateLaunchTemplateVersion", "ec2:DescribeLaunchTemplates", "ec2:DescribeLaunchTemplateVersions", "ec2:DeleteLaunchTemplate", "ec2:DeleteLaunchTemplateVersions" ], "Resource": [ "\*" ], "Effect": "Allow" }, { "Condition": { "StringLike": { "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" } }, "Action": [ "iam:CreateServiceLinkedRole" ], "Resource": [ "arn:*:iam::*:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing" ], "Effect": "Allow" }, { "Action": [ "iam:PassRole" ], "Resource": [ "arn:*:iam::*:role/*-worker-role" ], "Effect": "Allow" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:CreateVpcEndpoint", "ec2:DescribeVpcEndpoints", "ec2:ModifyVpcEndpoint", "ec2:DeleteVpcEndpoints", "ec2:CreateTags", "route53:ListHostedZones" ], "Resource": "\*" }, { "Effect": "Allow", "Action": [ "route53:ChangeResourceRecordSets", "route53:ListResourceRecordSets" ], "Resource": "arn:aws:route53:::%s" } ] }
--- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters spec: {} status: {} --- apiVersion: v1 data: .dockerconfigjson: xxxxxxxxxxx kind: Secret metadata: creationTimestamp: null labels: hypershift.openshift.io/safe-to-delete-with-cluster: "true" name: <pull_secret_name> 1 namespace: clusters --- apiVersion: v1 data: key: xxxxxxxxxxxxxxxxx kind: Secret metadata: creationTimestamp: null labels: hypershift.openshift.io/safe-to-delete-with-cluster: "true" name: <etcd_encryption_key_name> 2 namespace: clusters type: Opaque --- apiVersion: v1 data: id_rsa: xxxxxxxxx id_rsa.pub: xxxxxxxxx kind: Secret metadata: creationTimestamp: null labels: hypershift.openshift.io/safe-to-delete-with-cluster: "true" name: <ssh-key-name> 3 namespace: clusters --- apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: creationTimestamp: null name: <hosted_cluster_name> 4 namespace: clusters spec: autoscaling: {} configuration: {} controllerAvailabilityPolicy: SingleReplica dns: baseDomain: <dns_domain> 5 privateZoneID: xxxxxxxx publicZoneID: xxxxxxxx etcd: managed: storage: persistentVolume: size: 8Gi storageClassName: gp3-csi type: PersistentVolume managementType: Managed fips: false infraID: <infra_id> 6 issuerURL: <issuer_url> 7 networking: clusterNetwork: - cidr: 10.132.0.0/14 machineNetwork: - cidr: 10.0.0.0/16 networkType: OVNKubernetes serviceNetwork: - cidr: 172.31.0.0/16 olmCatalogPlacement: management platform: aws: cloudProviderConfig: subnet: id: <subnet_xxx> 8 vpc: <vpc_xxx> 9 zone: us-west-1b endpointAccess: Public multiArch: false region: us-west-1 rolesRef: controlPlaneOperatorARN: arn:aws:iam::820196288204:role/<infra_id>-control-plane-operator imageRegistryARN: arn:aws:iam::820196288204:role/<infra_id>-openshift-image-registry ingressARN: arn:aws:iam::820196288204:role/<infra_id>-openshift-ingress kubeCloudControllerARN: arn:aws:iam::820196288204:role/<infra_id>-cloud-controller networkARN: arn:aws:iam::820196288204:role/<infra_id>-cloud-network-config-controller nodePoolManagementARN: arn:aws:iam::820196288204:role/<infra_id>-node-pool storageARN: arn:aws:iam::820196288204:role/<infra_id>-aws-ebs-csi-driver-controller type: AWS pullSecret: name: <pull_secret_name> release: image: quay.io/openshift-release-dev/ocp-release:4.16-x86_64 secretEncryption: aescbc: activeKey: name: <etcd_encryption_key_name> type: aescbc services: - service: APIServer servicePublishingStrategy: type: LoadBalancer - service: OAuthServer servicePublishingStrategy: type: Route - service: Konnectivity servicePublishingStrategy: type: Route - service: Ignition servicePublishingStrategy: type: Route - service: OVNSbDb servicePublishingStrategy: type: Route sshKey: name: <ssh_key_name> status: controlPlaneEndpoint: host: "" port: 0 --- apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: creationTimestamp: null name: <node_pool_name> 10 namespace: clusters spec: arch: amd64 clusterName: <hosted_cluster_name> management: autoRepair: true upgradeType: Replace nodeDrainTimeout: 0s platform: aws: instanceProfile: <instance_profile_name> 11 instanceType: m6i.xlarge rootVolume: size: 120 type: gp3 subnet: id: <subnet_xxx> type: AWS release: image: quay.io/openshift-release-dev/ocp-release:4.16-x86_64 replicas: 2 status: replicas: 0
hcp create cluster aws --infra-id <infra_id> \ 1 --name <hosted_cluster_name> \ 2 --sts-creds <path_to_sts_credential_file> \ 3 --pull-secret <path_to_pull_secret> \ 4 --generate-ssh \ 5 --node-pool-replicas 3 --role-arn <role_name> 6
oc get hostedcluster/<hosted_cluster_name> \ 1 -o jsonpath='{.spec.release.image}'
quay.io/openshift-release-dev/ocp-release:<4.y.z>-x86_64 1
OCP_VERSION=USD(oc image info quay.io/openshift-release-dev/ocp-release@sha256:ac78ebf77f95ab8ff52847ecd22592b545415e1ff6c7ff7f66bf81f158ae4f5e -o jsonpath='{.config.config.Labels["io.openshift.release"]}')
MULTI_ARCH_TAG=USD(skopeo inspect docker://quay.io/openshift-release-dev/ocp-release@sha256:ac78ebf77f95ab8ff52847ecd22592b545415e1ff6c7ff7f66bf81f158ae4f5e | jq -r '.RepoTags' | sed 's/"//g' | sed 's/,//g' | grep -w "USDOCP_VERSION-multiUSD" | xargs)
IMAGE=quay.io/openshift-release-dev/ocp-release:USDMULTI_ARCH_TAG
oc image info USDIMAGE
OS DIGEST linux/amd64 sha256:b4c7a91802c09a5a748fe19ddd99a8ffab52d8a31db3a081a956a87f22a22ff8 linux/ppc64le sha256:66fda2ff6bd7704f1ba72be8bfe3e399c323de92262f594f8e482d110ec37388 linux/s390x sha256:b1c1072dc639aaa2b50ec99b530012e3ceac19ddc28adcbcdc9643f2dfd14f34 linux/arm64 sha256:7b046404572ac96202d82b6cb029b421dddd40e88c73bbf35f602ffc13017f21
oc patch -n clusters hostedclusters/<hosted_cluster_name> -p '{"spec":{"release":{"image":"quay.io/openshift-release-dev/ocp-release:<4.x.y>-multi"}}}' \ 1 --type=merge
oc get hostedcluster/<hosted_cluster_name> -o jsonpath='{.spec.release.image}'
oc get hostedcontrolplane -n <hosted_control_plane_namespace> -oyaml
# - lastTransitionTime: "2024-07-28T13:07:18Z" message: HostedCluster is deploying, upgrading, or reconfiguring observedGeneration: 5 reason: Progressing status: "True" type: Progressing #
oc get hostedcluster <hosted_cluster_name> -n <hosted_cluster_namespace> -oyaml
oc get hostedcontrolplane -n clusters-example -oyaml
# version: availableUpdates: null desired: image: quay.io/openshift-release-dev/ocp-release:<4.x.y>-multi 1 url: https://access.redhat.com/errata/RHBA-2024:4855 version: 4.16.5 history: - completionTime: "2024-07-28T13:10:58Z" image: quay.io/openshift-release-dev/ocp-release:<4.x.y>-multi startedTime: "2024-07-28T13:10:27Z" state: Completed verified: false version: <4.x.y>
hcp create nodepool aws --cluster-name <hosted_cluster_name> \ 1 --name <nodepool_name> \ 2 --node-count=<node_count> \ 3 --arch arm64
hcp create nodepool aws --cluster-name <hosted_cluster_name> \ 1 --name <nodepool_name> \ 2 --node-count=<node_count> \ 3 --arch amd64
oc get nodepool/<nodepool_name> -oyaml
# spec: arch: amd64 # release: image: quay.io/openshift-release-dev/ocp-release:<4.x.y>-multi 1
# spec: arch: arm64 # release: image: quay.io/openshift-release-dev/ocp-release:<4.x.y>-multi
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
hcp create kubeconfig --namespace <hosted_cluster_namespace> --name <hosted_cluster_name> > <hosted_cluster_name>.kubeconfig
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
oc -n <hosted_cluster_namespace> scale nodepool <nodepool_name> --replicas 2
oc -n <hosted_control_plane_namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 hypercluster1 true auto-assign d9198891-39f4-4930-a679-65fb142b108b true auto-assign da503cf1-a347-44f2-875c-4960ddb04091 hypercluster1 true auto-assign
oc -n <hosted_control_plane_namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: ocp-worker-2 Agent: 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 State: binding BMH: ocp-worker-0 Agent: d9198891-39f4-4930-a679-65fb142b108b State: known-unbound BMH: ocp-worker-1 Agent: da503cf1-a347-44f2-875c-4960ddb04091 State: insufficient
oc extract -n <hosted_cluster_namespace> secret/<hosted_cluster_name>-admin-kubeconfig --to=- > kubeconfig-<hosted_cluster_name>
oc --kubeconfig kubeconfig-<hosted_cluster_name> get nodes
NAME STATUS ROLES AGE VERSION ocp-worker-1 Ready worker 5m41s v1.24.0+3882f8f ocp-worker-2 Ready worker 6m3s v1.24.0+3882f8f
oc -n <hosted_control_plane_namespace> get machines
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hypercluster1-c96b6f675-m5vch hypercluster1-b2qhl ocp-worker-1 agent://da503cf1-a347-44f2-875c-4960ddb04091 Running 15m 4.x.z hypercluster1-c96b6f675-tl42p hypercluster1-b2qhl ocp-worker-2 agent://4dac1ab2-7dd5-4894-a220-6a3473b67ee6 Running 15m 4.x.z
oc --kubeconfig kubeconfig-<hosted_cluster_name> get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version False True 40m Unable to apply 4.x.z: the cluster operator console has not yet successfully rolled out NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.12z False False False 11m RouteHealthAvailable: failed to GET route (https://console-openshift-console.apps.hypercluster1.domain.com): Get "https://console-openshift-console.apps.hypercluster1.domain.com": dial tcp 10.19.3.29:443: connect: connection refused clusteroperator.config.openshift.io/csi-snapshot-controller 4.12z True False False 10m clusteroperator.config.openshift.io/dns 4.12z True False False 9m16s
hcp create nodepool agent --cluster-name <hosted_cluster_name> \ 1 --name <nodepool_name> \ 2 --node-count <worker_node_count> \ 3 --agentLabelSelector size=medium 4
oc get nodepools --namespace clusters
oc extract -n <hosted_control_plane_namespace> secret/admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
oc --kubeconfig ./hostedcluster-secrets get nodes
oc get nodepools --namespace clusters
oc -n <hosted_cluster_namespace> patch nodepool <hosted_cluster_name> --type=json -p '[{"op": "remove", "path": "/spec/replicas"},{"op":"add", "path": "/spec/autoScaling", "value": { "max": 5, "min": 2 }}]'
apiVersion: apps/v1 kind: Deployment metadata: creationTimestamp: null labels: app: reversewords name: reversewords namespace: default spec: replicas: 40 selector: matchLabels: app: reversewords strategy: {} template: metadata: creationTimestamp: null labels: app: reversewords spec: containers: - image: quay.io/mavazque/reversewords:latest name: reversewords resources: requests: memory: 2Gi status: {}
oc apply -f workload-config.yaml
oc extract -n <hosted_cluster_namespace> secret/<hosted_cluster_name>-admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
oc --kubeconfig ./hostedcluster-secrets get nodes
oc --kubeconfig ./hostedcluster-secrets -n <namespace> delete deployment <deployment_name>
oc --kubeconfig ./hostedcluster-secrets get nodes
oc -n <hosted_cluster_namespace> patch nodepool <hosted_cluster_name> --type=json -p '[\{"op":"remove", "path": "/spec/autoScaling"}, \{"op": "add", "path": "/spec/replicas", "value": <specify_value_to_scale_replicas>]'
apiVersion: v1 kind: Namespace metadata: name: metallb labels: openshift.io/cluster-monitoring: "true" annotations: workload.openshift.io/allowed: management --- apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: metallb-operator-operatorgroup namespace: metallb --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: metallb-operator namespace: metallb spec: channel: "stable" name: metallb-operator source: redhat-operators sourceNamespace: openshift-marketplace
oc apply -f metallb-operator-config.yaml
apiVersion: metallb.io/v1beta1 kind: MetalLB metadata: name: metallb namespace: metallb
oc apply -f metallb-instance-config.yaml
apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: namespace: metallb name: <ip_address_pool_name> 1 spec: addresses: - <ingress_ip>-<ingress_ip> 2 autoAssign: false
oc apply -f ipaddresspool.yaml
apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: <l2_advertisement_name> 1 namespace: metallb spec: ipAddressPools: - <ip_address_pool_name> 2
oc apply -f l2advertisement.yaml
kind: Service apiVersion: v1 metadata: annotations: metallb.io/address-pool: ingress-public-ip name: metallb-ingress namespace: openshift-ingress spec: ports: - name: http protocol: TCP port: 80 targetPort: 80 - name: https protocol: TCP port: 443 targetPort: 443 selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller: default type: LoadBalancer
oc apply -f metallb-loadbalancer-service.yaml
curl -kI https://console-openshift-console.apps.example.krnl.es
HTTP/1.1 200 OK
oc --kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.x.y True False 3m32s Cluster version is 4.x.y NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.x.y True False False 3m50s clusteroperator.config.openshift.io/ingress 4.x.y True False False 53m
oc get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep nodeDrainTimeout
nodeDrainTimeout: 30s
oc patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec":{"nodeDrainTimeout": "30m"}}' --type=merge
oc patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":true}}}' --type=merge
oc get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
oc patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":false}}}' --type=merge
oc get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
hcp create kubeconfig --namespace <hosted_cluster_namespace> --name <hosted_cluster_name> > <hosted_cluster_name>.kubeconfig
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
oc -n <hosted_cluster_namespace> patch nodepool <hosted_cluster_name> --type=json -p '[{"op": "remove", "path": "/spec/replicas"},{"op":"add", "path": "/spec/autoScaling", "value": { "max": 5, "min": 2 }}]'
apiVersion: apps/v1 kind: Deployment metadata: creationTimestamp: null labels: app: reversewords name: reversewords namespace: default spec: replicas: 40 selector: matchLabels: app: reversewords strategy: {} template: metadata: creationTimestamp: null labels: app: reversewords spec: containers: - image: quay.io/mavazque/reversewords:latest name: reversewords resources: requests: memory: 2Gi status: {}
oc apply -f workload-config.yaml
oc extract -n <hosted_cluster_namespace> secret/<hosted_cluster_name>-admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
oc --kubeconfig ./hostedcluster-secrets get nodes
oc --kubeconfig ./hostedcluster-secrets -n <namespace> delete deployment <deployment_name>
oc --kubeconfig ./hostedcluster-secrets get nodes
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class> \ 6
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class> \ 6 --infra-volumesnapshot-class-mapping=<infrastructure_volume_snapshot_class>/<hosted_volume_snapshot_class> 7
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class>,group=<group_name> \ 6 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class>,group=<group_name> --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class>,group=<group_name> --infra-volumesnapshot-class-mapping=<infrastructure_volume_snapshot_class>/<hosted_volume_snapshot_class>,group=<group_name> \ 7 --infra-volumesnapshot-class-mapping=<infrastructure_volume_snapshot_class>/<hosted_volume_snapshot_class>,group=<group_name>
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --root-volume-storage-class <root_volume_storage_class> \ 6 --root-volume-size <volume_size> 7
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --root-volume-cache-strategy=PVC 6
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --etcd-storage-class=<etcd_storage_class_name> 6
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --host-device-name="<gpu_device_name>,count:<value>" 6
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: name: <hosted_cluster_name> 1 namespace: <hosted_cluster_namespace> 2 spec: arch: amd64 clusterName: <hosted_cluster_name> management: autoRepair: false upgradeType: Replace nodeDrainTimeout: 0s nodeVolumeDetachTimeout: 0s platform: kubevirt: attachDefaultNetwork: true compute: cores: <cpu> 3 memory: <memory> 4 hostDevices: 5 - count: <count> 6 deviceName: <gpu_device_name> 7 networkInterfaceMultiqueue: Enable rootVolume: persistent: size: 32Gi type: Persistent type: KubeVirt replicas: <worker_node_count> 8
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: name: <hosted_cluster_name> namespace: <hosted_cluster_namespace> spec: arch: amd64 clusterName: <hosted_cluster_name> management: autoRepair: false upgradeType: Replace nodeDrainTimeout: 0s nodeVolumeDetachTimeout: 0s platform: kubevirt: attachDefaultNetwork: true compute: cores: <cpu> memory: <memory> hostDevices: - count: <count> deviceName: <gpu_device_name> - count: <count> deviceName: <gpu_device_name> - count: <count> deviceName: <gpu_device_name> - count: <count> deviceName: <gpu_device_name> networkInterfaceMultiqueue: Enable rootVolume: persistent: size: 32Gi type: Persistent type: KubeVirt replicas: <worker_node_count>
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
hcp create kubeconfig --namespace <hosted_cluster_namespace> --name <hosted_cluster_name> > <hosted_cluster_name>.kubeconfig
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
oc -n <hosted_cluster_namespace> scale nodepool <nodepool_name> --replicas 2
oc -n <hosted_control_plane_namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 hypercluster1 true auto-assign d9198891-39f4-4930-a679-65fb142b108b true auto-assign da503cf1-a347-44f2-875c-4960ddb04091 hypercluster1 true auto-assign
oc -n <hosted_control_plane_namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: ocp-worker-2 Agent: 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 State: binding BMH: ocp-worker-0 Agent: d9198891-39f4-4930-a679-65fb142b108b State: known-unbound BMH: ocp-worker-1 Agent: da503cf1-a347-44f2-875c-4960ddb04091 State: insufficient
oc extract -n <hosted_cluster_namespace> secret/<hosted_cluster_name>-admin-kubeconfig --to=- > kubeconfig-<hosted_cluster_name>
oc --kubeconfig kubeconfig-<hosted_cluster_name> get nodes
NAME STATUS ROLES AGE VERSION ocp-worker-1 Ready worker 5m41s v1.24.0+3882f8f ocp-worker-2 Ready worker 6m3s v1.24.0+3882f8f
oc -n <hosted_control_plane_namespace> get machines
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hypercluster1-c96b6f675-m5vch hypercluster1-b2qhl ocp-worker-1 agent://da503cf1-a347-44f2-875c-4960ddb04091 Running 15m 4.x.z hypercluster1-c96b6f675-tl42p hypercluster1-b2qhl ocp-worker-2 agent://4dac1ab2-7dd5-4894-a220-6a3473b67ee6 Running 15m 4.x.z
oc --kubeconfig kubeconfig-<hosted_cluster_name> get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version False True 40m Unable to apply 4.x.z: the cluster operator console has not yet successfully rolled out NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.12z False False False 11m RouteHealthAvailable: failed to GET route (https://console-openshift-console.apps.hypercluster1.domain.com): Get "https://console-openshift-console.apps.hypercluster1.domain.com": dial tcp 10.19.3.29:443: connect: connection refused clusteroperator.config.openshift.io/csi-snapshot-controller 4.12z True False False 10m clusteroperator.config.openshift.io/dns 4.12z True False False 9m16s
hcp create nodepool agent --cluster-name <hosted_cluster_name> \ 1 --name <nodepool_name> \ 2 --node-count <worker_node_count> \ 3 --agentLabelSelector size=medium 4
oc get nodepools --namespace clusters
oc extract -n <hosted_control_plane_namespace> secret/admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
oc --kubeconfig ./hostedcluster-secrets get nodes
oc get nodepools --namespace clusters
oc -n <hosted_cluster_namespace> patch nodepool <hosted_cluster_name> --type=json -p '[{"op": "remove", "path": "/spec/replicas"},{"op":"add", "path": "/spec/autoScaling", "value": { "max": 5, "min": 2 }}]'
apiVersion: apps/v1 kind: Deployment metadata: creationTimestamp: null labels: app: reversewords name: reversewords namespace: default spec: replicas: 40 selector: matchLabels: app: reversewords strategy: {} template: metadata: creationTimestamp: null labels: app: reversewords spec: containers: - image: quay.io/mavazque/reversewords:latest name: reversewords resources: requests: memory: 2Gi status: {}
oc apply -f workload-config.yaml
oc extract -n <hosted_cluster_namespace> secret/<hosted_cluster_name>-admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
oc --kubeconfig ./hostedcluster-secrets get nodes
oc --kubeconfig ./hostedcluster-secrets -n <namespace> delete deployment <deployment_name>
oc --kubeconfig ./hostedcluster-secrets get nodes
oc -n <hosted_cluster_namespace> patch nodepool <hosted_cluster_name> --type=json -p '[\{"op":"remove", "path": "/spec/autoScaling"}, \{"op": "add", "path": "/spec/replicas", "value": <specify_value_to_scale_replicas>]'
apiVersion: v1 kind: Namespace metadata: name: metallb labels: openshift.io/cluster-monitoring: "true" annotations: workload.openshift.io/allowed: management --- apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: metallb-operator-operatorgroup namespace: metallb --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: metallb-operator namespace: metallb spec: channel: "stable" name: metallb-operator source: redhat-operators sourceNamespace: openshift-marketplace
oc apply -f metallb-operator-config.yaml
apiVersion: metallb.io/v1beta1 kind: MetalLB metadata: name: metallb namespace: metallb
oc apply -f metallb-instance-config.yaml
apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: namespace: metallb name: <ip_address_pool_name> 1 spec: addresses: - <ingress_ip>-<ingress_ip> 2 autoAssign: false
oc apply -f ipaddresspool.yaml
apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: <l2_advertisement_name> 1 namespace: metallb spec: ipAddressPools: - <ip_address_pool_name> 2
oc apply -f l2advertisement.yaml
kind: Service apiVersion: v1 metadata: annotations: metallb.io/address-pool: ingress-public-ip name: metallb-ingress namespace: openshift-ingress spec: ports: - name: http protocol: TCP port: 80 targetPort: 80 - name: https protocol: TCP port: 443 targetPort: 443 selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller: default type: LoadBalancer
oc apply -f metallb-loadbalancer-service.yaml
curl -kI https://console-openshift-console.apps.example.krnl.es
HTTP/1.1 200 OK
oc --kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.x.y True False 3m32s Cluster version is 4.x.y NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.x.y True False False 3m50s clusteroperator.config.openshift.io/ingress 4.x.y True False False 53m
oc get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep nodeDrainTimeout
nodeDrainTimeout: 30s
oc patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec":{"nodeDrainTimeout": "30m"}}' --type=merge
oc patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":true}}}' --type=merge
oc get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
oc patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":false}}}' --type=merge
oc get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: <hosted_cluster_name> \ 1 namespace: <hosted_control_plane_namespace> \ 2 spec: cpuArchitecture: ppc64le pullSecretRef: name: pull-secret sshAuthorizedKey: <path_to_ssh_public_key> 3
oc apply -f infraenv-config.yaml
oc -n <hosted_control_plane_namespace> get InfraEnv <hosted_cluster_name> -o json
oc -n <hosted_control_plane_namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 86f7ac75-4fc4-4b36-8130-40fa12602218 auto-assign e57a637f-745b-496e-971d-1abbf03341ba auto-assign
oc -n <hosted_control_plane_namespace> patch agent <agent_name> -p '{"spec":{"installation_disk_id":"<installation_disk_id>","approved":true}}' --type merge
oc -n <hosted_control_plane_namespace> patch agent <agent_name> -p '{"spec":{"hostname":"<hostname>","approved":true}}' --type merge
oc -n <hosted_control_plane_namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 86f7ac75-4fc4-4b36-8130-40fa12602218 true auto-assign e57a637f-745b-496e-971d-1abbf03341ba true auto-assign
oc -n <hosted_cluster_namespace> scale nodepool <nodepool_name> --replicas 2
oc -n <hosted_control_plane_namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: Agent: 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d State: known-unbound BMH: Agent: 5e498cd3-542c-e54f-0c58-ed43e28b568a State: insufficient
oc -n <hosted_control_plane_namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d hosted-forwarder true auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a true auto-assign da503cf1-a347-44f2-875c-4960ddb04091 hosted-forwarder true auto-assign
hcp create kubeconfig --namespace <hosted_cluster_namespace> --name <hosted_cluster_name> > <hosted_cluster_name>.kubeconfig
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
NAME STATUS ROLES AGE VERSION worker-zvm-0.hostedn.example.com Ready worker 5m41s v1.24.0+3882f8f worker-zvm-1.hostedn.example.com Ready worker 6m3s v1.24.0+3882f8f
oc -n <hosted_control_plane_namespace> get machine.cluster.x-k8s.io
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hosted-forwarder-79558597ff-5tbqp hosted-forwarder-crqq5 worker-zvm-0.hostedn.example.com agent://50c23cda-cedc-9bbd-bcf1-9b3a5c75804d Running 41h 4.15.0 hosted-forwarder-79558597ff-lfjfk hosted-forwarder-crqq5 worker-zvm-1.hostedn.example.com agent://5e498cd3-542c-e54f-0c58-ed43e28b568a Running 41h 4.15.0
oc --kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.15.0 True False 40h Cluster version is 4.15.0
oc --kubeconfig <hosted_cluster_name>.kubeconfig get clusteroperators
apiVersion: mirror.openshift.io/v2alpha1 kind: ImageSetConfiguration mirror: platform: channels: - name: candidate-4.18 minVersion: <4.x.y-build> 1 maxVersion: <4.x.y-build> 2 type: ocp kubeVirtContainer: true 3 graph: true additionalImages: 4 - name: quay.io/karmab/origin-keepalived-ipfailover:latest - name: quay.io/karmab/kubectl:latest - name: quay.io/karmab/haproxy:latest - name: quay.io/karmab/mdns-publisher:latest - name: quay.io/karmab/origin-coredns:latest - name: quay.io/karmab/curl:latest - name: quay.io/karmab/kcli:latest - name: quay.io/user-name/trbsht:latest - name: quay.io/user-name/hypershift:BMSelfManage-v4.18 - name: registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.18 packages: - name: lvms-operator - name: local-storage-operator - name: odf-csi-addons-operator - name: odf-operator - name: mcg-operator - name: ocs-operator - name: metallb-operator - name: kubevirt-hyperconverged 5
oc-mirror --v2 --config imagesetconfig.yaml --workspace file://mirror-file docker://<registry>
apiVersion: mirror.openshift.io/v2alpha1 kind: ImageSetConfiguration mirror: platform: graph: true release: registry.ci.openshift.org/ocp/release:<4.x.y-build> 1 kubeVirtContainer: true 2
oc mirror -c imagesetconfig.yaml --workspace file://<file_path> docker://<mirror_registry_url> --v2
oc mirror -c imagesetconfig.yaml file://<file_path> --v2
oc mirror -c imagesetconfig.yaml --from file://<file_path> docker://<mirror_registry_url> --v2
oc get packagemanifest
oc apply -f oc-mirror-workspace/results-XXXXXX/imageContentSourcePolicy.yaml
oc apply -f catalogSource-XXXXXXXX-index.yaml
apiVersion: v1 kind: ConfigMap metadata: name: <config_map_name> 1 namespace: <config_map_namespace> 2 data: 3 <registry_name>..<port>: | 4 -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- <registry_name>..<port>: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- <registry_name>..<port>: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE-----
spec: additionalTrustedCA: - name: registry-config
spec: additionalTrustBundle: - name: user-ca-bundle 1
apiVersion: v1 data: ca-bundle.crt: | // Registry1 CA -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- // Registry2 CA -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- // Registry3 CA -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- kind: ConfigMap metadata: name: user-ca-bundle namespace: <hosted_cluster_namespace> 1
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <node_pool_replica_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <value_for_memory> \ 4 --cores <value_for_cpu> \ 5 --etcd-storage-class=<etcd_storage_class> 6
oc -n clusters-<hosted-cluster-name> get pods
NAME READY STATUS RESTARTS AGE capi-provider-5cc7b74f47-n5gkr 1/1 Running 0 3m catalog-operator-5f799567b7-fd6jw 2/2 Running 0 69s certified-operators-catalog-784b9899f9-mrp6p 1/1 Running 0 66s cluster-api-6bbc867966-l4dwl 1/1 Running 0 66s . . . redhat-operators-catalog-9d5fd4d44-z8qqk 1/1 Running 0 66s
oc get --namespace clusters hostedclusters
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters example <4.x.0> example-admin-kubeconfig Completed True False The hosted control plane is available
*.apps.mgmt-cluster.example.com
*.apps.guest.apps.mgmt-cluster.example.com
oc patch ingresscontroller -n openshift-ingress-operator default --type=json -p '[{ "op": "add", "path": "/spec/routeAdmission", "value": {wildcardPolicy: "WildcardsAllowed"}}]'
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <value_for_memory> \ 4 --cores <value_for_cpu> \ 5 --base-domain <basedomain> 6
oc get --namespace clusters hostedclusters
NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE example example-admin-kubeconfig Partial True False The hosted control plane is available
hcp create kubeconfig --name <hosted_cluster_name> > <hosted_cluster_name>-kubeconfig
oc --kubeconfig <hosted_cluster_name>-kubeconfig get co
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console <4.x.0> False False False 30m RouteHealthAvailable: failed to GET route (https://console-openshift-console.apps.example.hypershift.lab): Get "https://console-openshift-console.apps.example.hypershift.lab": dial tcp: lookup console-openshift-console.apps.example.hypershift.lab on 172.31.0.10:53: no such host ingress <4.x.0> True False True 28m The "default" ingress controller reports Degraded=True: DegradedConditions: One or more other status conditions indicate a degraded state: CanaryChecksSucceeding=False (CanaryChecksRepetitiveFailures: Canary route checks for the default ingress controller are failing)
oc --kubeconfig <hosted_cluster_name>-kubeconfig get services -n openshift-ingress router-nodeport-default -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}'
oc --kubeconfig <hosted_cluster_name>-kubeconfig get services -n openshift-ingress router-nodeport-default -o jsonpath='{.spec.ports[?(@.name=="https")].nodePort}'
apply -f - apiVersion: v1 kind: Service metadata: labels: app: <hosted_cluster_name> name: <hosted_cluster_name>-apps namespace: clusters-<hosted_cluster_name> spec: ports: - name: https-443 port: 443 protocol: TCP targetPort: <https_node_port> 1 - name: http-80 port: 80 protocol: TCP targetPort: <http-node-port> 2 selector: kubevirt.io: virt-launcher type: LoadBalancer
oc -n clusters-<hosted_cluster_name> get service <hosted-cluster-name>-apps -o jsonpath='{.status.loadBalancer.ingress[0].ip}'
192.168.20.30
*.apps.<hosted_cluster_name\>.<base_domain\>.
dig +short test.apps.example.hypershift.lab 192.168.20.30
oc get --namespace clusters hostedclusters
NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE example <4.x.0> example-admin-kubeconfig Completed True False The hosted control plane is available
export KUBECONFIG=/root/.kcli/clusters/hub-ipv4/auth/kubeconfig
watch "oc get pod -n hypershift;echo;echo; oc get pod -n clusters-hosted-ipv4;echo;echo; oc get bmh -A;echo;echo; oc get agent -A;echo;echo; oc get infraenv -A;echo;echo; oc get hostedcluster -A;echo;echo; oc get nodepool -A;echo;echo;"
oc get secret -n clusters-hosted-ipv4 admin-kubeconfig -o jsonpath='{.data.kubeconfig}' | base64 -d > /root/hc_admin_kubeconfig.yaml
export KUBECONFIG=/root/hc_admin_kubeconfig.yaml
watch "oc get clusterversion,nodes,co"
apiVersion: v1 kind: ConfigMap metadata: name: custom-registries namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- # -----END CERTIFICATE----- registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/openshift4" mirror-by-digest-only = true [[registry.mirror]] location = "registry.ocp-edge-cluster-0.qe.lab.redhat.com:5000/openshift4" [[registry]] prefix = "" location = "registry.redhat.io/rhacm2" mirror-by-digest-only = true
oc adm release info <tagged_openshift_release_image> | grep "Pull From"
Pull From: quay.io/openshift-release-dev/ocp-release@sha256:69d1292f64a2b67227c5592c1a7d499c7d00376e498634ff8e1946bc9ccdddfe
sudo dnf install dnsmasq radvd vim golang podman bind-utils net-tools httpd-tools tree htop strace tmux -y
systemctl enable --now podman
sudo yum -y install libvirt libvirt-daemon-driver-qemu qemu-kvm
sudo usermod -aG qemu,libvirt USD(id -un)
sudo newgrp libvirt
sudo systemctl enable --now libvirtd
sudo dnf -y copr enable karmab/kcli
sudo dnf -y install kcli
sudo kcli create pool -p /var/lib/libvirt/images default
kcli create host kvm -H 127.0.0.1 local
sudo setfacl -m u:USD(id -un):rwx /var/lib/libvirt/images
kcli create network -c 192.168.122.0/24 default
#!/bin/bash export IP="192.168.126.1" 1 export BASE_RESOLV_CONF="/run/NetworkManager/resolv.conf" if ! [[ `grep -q "USDIP" /etc/resolv.conf` ]]; then export TMP_FILE=USD(mktemp /etc/forcedns_resolv.conf.XXXXXX) cp USDBASE_RESOLV_CONF USDTMP_FILE chmod --reference=USDBASE_RESOLV_CONF USDTMP_FILE sed -i -e "s/dns.base.domain.name//" -e "s/search /& dns.base.domain.name /" -e "0,/nameserver/s/nameserver/& USDIP\n&/" USDTMP_FILE 2 mv USDTMP_FILE /etc/resolv.conf fi echo "ok"
chmod 755 /etc/NetworkManager/dispatcher.d/forcedns
sudo dnf install python3-pyOpenSSL.noarch python3-cherrypy -y
kcli create sushy-service --ssl --ipv6 --port 9000
sudo systemctl daemon-reload
systemctl enable --now ksushy
systemctl status ksushy
sed -i s/^SELINUX=.*USD/SELINUX=permissive/ /etc/selinux/config; setenforce 0
systemctl disable --now firewalld
systemctl restart libvirtd
systemctl enable --now libvirtd
api.example.krnl.es. IN A 192.168.122.20 api.example.krnl.es. IN A 192.168.122.21 api.example.krnl.es. IN A 192.168.122.22 api-int.example.krnl.es. IN A 192.168.122.20 api-int.example.krnl.es. IN A 192.168.122.21 api-int.example.krnl.es. IN A 192.168.122.22 `*`.apps.example.krnl.es. IN A 192.168.122.23
api.example.krnl.es. IN A 2620:52:0:1306::5 api.example.krnl.es. IN A 2620:52:0:1306::6 api.example.krnl.es. IN A 2620:52:0:1306::7 api-int.example.krnl.es. IN A 2620:52:0:1306::5 api-int.example.krnl.es. IN A 2620:52:0:1306::6 api-int.example.krnl.es. IN A 2620:52:0:1306::7 `*`.apps.example.krnl.es. IN A 2620:52:0:1306::10
host-record=api-int.hub-dual.dns.base.domain.name,192.168.126.10 host-record=api.hub-dual.dns.base.domain.name,192.168.126.10 address=/apps.hub-dual.dns.base.domain.name/192.168.126.11 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,192.168.126.20 dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,192.168.126.21 dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,192.168.126.22 dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,192.168.126.25 dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,192.168.126.26 host-record=api-int.hub-dual.dns.base.domain.name,2620:52:0:1306::2 host-record=api.hub-dual.dns.base.domain.name,2620:52:0:1306::2 address=/apps.hub-dual.dns.base.domain.name/2620:52:0:1306::3 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,[2620:52:0:1306::5] dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,[2620:52:0:1306::6] dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,[2620:52:0:1306::7] dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,[2620:52:0:1306::8] dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,[2620:52:0:1306::9]
#!/usr/bin/env bash set -euo pipefail PRIMARY_NIC=USD(ls -1 /sys/class/net | grep -v podman | head -1) export PATH=/root/bin:USDPATH export PULL_SECRET="/root/baremetal/hub/openshift_pull.json" 1 if [[ ! -f USDPULL_SECRET ]];then echo "Pull Secret not found, exiting..." exit 1 fi dnf -y install podman httpd httpd-tools jq skopeo libseccomp-devel export IP=USD(ip -o addr show USDPRIMARY_NIC | head -1 | awk '{print USD4}' | cut -d'/' -f1) REGISTRY_NAME=registry.USD(hostname --long) REGISTRY_USER=dummy REGISTRY_PASSWORD=dummy KEY=USD(echo -n USDREGISTRY_USER:USDREGISTRY_PASSWORD | base64) echo "{\"auths\": {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\", \"email\": \"sample-email@domain.ltd\"}}}" > /root/disconnected_pull.json mv USD{PULL_SECRET} /root/openshift_pull.json.old jq ".auths += {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\",\"email\": \"sample-email@domain.ltd\"}}" < /root/openshift_pull.json.old > USDPULL_SECRET mkdir -p /opt/registry/{auth,certs,data,conf} cat <<EOF > /opt/registry/conf/config.yml version: 0.1 log: fields: service: registry storage: cache: blobdescriptor: inmemory filesystem: rootdirectory: /var/lib/registry delete: enabled: true http: addr: :5000 headers: X-Content-Type-Options: [nosniff] health: storagedriver: enabled: true interval: 10s threshold: 3 compatibility: schema1: enabled: true EOF openssl req -newkey rsa:4096 -nodes -sha256 -keyout /opt/registry/certs/domain.key -x509 -days 3650 -out /opt/registry/certs/domain.crt -subj "/C=US/ST=Madrid/L=San Bernardo/O=Karmalabs/OU=Guitar/CN=USDREGISTRY_NAME" -addext "subjectAltName=DNS:USDREGISTRY_NAME" cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/ update-ca-trust extract htpasswd -bBc /opt/registry/auth/htpasswd USDREGISTRY_USER USDREGISTRY_PASSWORD create --name registry --net host --security-opt label=disable --replace -v /opt/registry/data:/var/lib/registry:z -v /opt/registry/auth:/auth:z -v /opt/registry/conf/config.yml:/etc/docker/registry/config.yml -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry" -e "REGISTRY_HTTP_SECRET=ALongRandomSecretForRegistry" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -v /opt/registry/certs:/certs:z -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key docker.io/library/registry:latest [ "USD?" == "0" ] || !! systemctl enable --now registry
chmod u+x USD{HOME}/registry.sh
USD{HOME}/registry.sh
systemctl status
systemctl start
systemctl stop
kcli create network -c 192.168.126.0/24 -P dhcp=false -P dns=false -d 2620:52:0:1306::0/64 --domain dns.base.domain.name --nodhcp dual
kcli list network Listing Networks +---------+--------+---------------------+-------+------------------+------+ | Network | Type | Cidr | Dhcp | Domain | Mode | +---------+--------+---------------------+-------+------------------+------+ | default | routed | 192.168.122.0/24 | True | default | nat | | ipv4 | routed | 2620:52:0:1306::/64 | False | dns.base.domain.name | nat | | ipv4 | routed | 192.168.125.0/24 | False | dns.base.domain.name | nat | | ipv6 | routed | 2620:52:0:1305::/64 | False | dns.base.domain.name | nat | +---------+--------+---------------------+-------+------------------+------+
kcli info network ipv6 Providing information about network ipv6 cidr: 2620:52:0:1306::/64 dhcp: false domain: dns.base.domain.name mode: nat plan: kvirt type: routed
plan: hub-dual force: true version: stable tag: "<4.x.y>-x86_64" 1 cluster: "hub-dual" dualstack: true domain: dns.base.domain.name api_ip: 192.168.126.10 ingress_ip: 192.168.126.11 service_networks: - 172.30.0.0/16 - fd02::/112 cluster_networks: - 10.132.0.0/14 - fd01::/48 disconnected_url: registry.dns.base.domain.name:5000 disconnected_update: true disconnected_user: dummy disconnected_password: dummy disconnected_operators_version: v4.14 disconnected_operators: - name: metallb-operator - name: lvms-operator channels: - name: stable-4.14 disconnected_extra_images: - quay.io/user-name/trbsht:latest - quay.io/user-name/hypershift:BMSelfManage-v4.14-rc-v3 - registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 dualstack: true disk_size: 200 extra_disks: [200] memory: 48000 numcpus: 16 ctlplanes: 3 workers: 0 manifests: extra-manifests metal3: true network: dual users_dev: developer users_devpassword: developer users_admin: admin users_adminpassword: admin metallb_pool: dual-virtual-network metallb_ranges: - 192.168.126.150-192.168.126.190 metallb_autoassign: true apps: - users - lvms-operator - metallb-operator vmrules: - hub-bootstrap: nets: - name: ipv6 mac: aa:aa:aa:aa:10:07 - hub-ctlplane-0: nets: - name: ipv6 mac: aa:aa:aa:aa:10:01 - hub-ctlplane-1: nets: - name: ipv6 mac: aa:aa:aa:aa:10:02 - hub-ctlplane-2: nets: - name: ipv6 mac: aa:aa:aa:aa:10:03
kcli create cluster openshift --pf mgmt-compact-hub-dual.yaml
oc adm -a USD{LOCAL_SECRET_JSON} release extract --command=openshift-install "USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}"
#!/bin/bash WEBSRV_FOLDER=/opt/srv ROOTFS_IMG_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.pxe.rootfs.location')" 1 LIVE_ISO_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.iso.disk.location')" 2 mkdir -p USD{WEBSRV_FOLDER}/images curl -Lk USD{ROOTFS_IMG_URL} -o USD{WEBSRV_FOLDER}/images/USD{ROOTFS_IMG_URL##*/} curl -Lk USD{LIVE_ISO_URL} -o USD{WEBSRV_FOLDER}/images/USD{LIVE_ISO_URL##*/} chmod -R 755 USD{WEBSRV_FOLDER}/* ## Run Webserver ps --noheading | grep -q websrv-ai if [[ USD? == 0 ]];then echo "Launching Registry pod..." /usr/bin/podman run --name websrv-ai --net host -v /opt/srv:/usr/local/apache2/htdocs:z quay.io/alosadag/httpd:p8080 fi
apiVersion: mirror.openshift.io/v2alpha1 kind: ImageSetConfiguration mirror: platform: channels: - name: candidate-4.18 minVersion: <4.x.y-build> 1 maxVersion: <4.x.y-build> 2 type: ocp kubeVirtContainer: true 3 graph: true additionalImages: 4 - name: quay.io/karmab/origin-keepalived-ipfailover:latest - name: quay.io/karmab/kubectl:latest - name: quay.io/karmab/haproxy:latest - name: quay.io/karmab/mdns-publisher:latest - name: quay.io/karmab/origin-coredns:latest - name: quay.io/karmab/curl:latest - name: quay.io/karmab/kcli:latest - name: quay.io/user-name/trbsht:latest - name: quay.io/user-name/hypershift:BMSelfManage-v4.18 - name: registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.18 packages: - name: lvms-operator - name: local-storage-operator - name: odf-csi-addons-operator - name: odf-operator - name: mcg-operator - name: ocs-operator - name: metallb-operator - name: kubevirt-hyperconverged 5
oc-mirror --v2 --config imagesetconfig.yaml --workspace file://mirror-file docker://<registry>
apiVersion: mirror.openshift.io/v2alpha1 kind: ImageSetConfiguration mirror: platform: graph: true release: registry.ci.openshift.org/ocp/release:<4.x.y-build> 1 kubeVirtContainer: true 2
oc mirror -c imagesetconfig.yaml --workspace file://<file_path> docker://<mirror_registry_url> --v2
oc mirror -c imagesetconfig.yaml file://<file_path> --v2
oc mirror -c imagesetconfig.yaml --from file://<file_path> docker://<mirror_registry_url> --v2
oc get packagemanifest
oc apply -f oc-mirror-workspace/results-XXXXXX/imageContentSourcePolicy.yaml
oc apply -f catalogSource-XXXXXXXX-index.yaml
apiVersion: v1 kind: ConfigMap metadata: name: custom-registries namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/openshift4" mirror-by-digest-only = true [[registry.mirror]] location = "registry.dns.base.domain.name:5000/openshift4" 1 [[registry]] prefix = "" location = "registry.redhat.io/rhacm2" mirror-by-digest-only = true # #
apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: annotations: unsupported.agent-install.openshift.io/assisted-service-configmap: assisted-service-config 1 name: agent namespace: multicluster-engine spec: mirrorRegistryRef: name: custom-registries 2 databaseStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 10Gi filesystemStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 20Gi osImages: 3 - cpuArchitecture: x86_64 4 openshiftVersion: "4.14" rootFSUrl: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live-rootfs.x86_64.img 5 url: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live.x86_64.iso version: 414.92.202308281054-0 - cpuArchitecture: x86_64 openshiftVersion: "4.15" rootFSUrl: http://registry.dns.base.domain.name:8080/images/rhcos-415.92.202403270524-0-live-rootfs.x86_64.img url: http://registry.dns.base.domain.name:8080/images/rhcos-415.92.202403270524-0-live.x86_64.iso version: 415.92.202403270524-0
oc apply -f agentServiceConfig.yaml
assisted-image-service-0 1/1 Running 2 11d 1 assisted-service-668b49548-9m7xw 2/2 Running 5 11d 2
apiVersion: v1 kind: ConfigMap metadata: name: <config_map_name> 1 namespace: <config_map_namespace> 2 data: 3 <registry_name>..<port>: | 4 -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- <registry_name>..<port>: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- <registry_name>..<port>: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE-----
spec: additionalTrustedCA: - name: registry-config
spec: additionalTrustBundle: - name: user-ca-bundle 1
apiVersion: v1 data: ca-bundle.crt: | // Registry1 CA -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- // Registry2 CA -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- // Registry3 CA -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- kind: ConfigMap metadata: name: user-ca-bundle namespace: <hosted_cluster_namespace> 1
--- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: <hosted_cluster_namespace>-<hosted_cluster_name> 1 spec: {} status: {} --- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: <hosted_cluster_namespace> 2 spec: {} status: {}
--- apiVersion: v1 data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- kind: ConfigMap metadata: name: user-ca-bundle namespace: <hosted_cluster_namespace> 1 --- apiVersion: v1 data: .dockerconfigjson: xxxxxxxxx kind: Secret metadata: creationTimestamp: null name: <hosted_cluster_name>-pull-secret 2 namespace: <hosted_cluster_namespace> 3 --- apiVersion: v1 kind: Secret metadata: name: sshkey-cluster-<hosted_cluster_name> 4 namespace: <hosted_cluster_namespace> 5 stringData: id_rsa.pub: ssh-rsa xxxxxxxxx --- apiVersion: v1 data: key: nTPtVBEt03owkrKhIdmSW8jrWRxU57KO/fnZa8oaG0Y= kind: Secret metadata: creationTimestamp: null name: <hosted_cluster_name>-etcd-encryption-key 6 namespace: <hosted_cluster_namespace> 7 type: Opaque
apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: creationTimestamp: null name: capi-provider-role namespace: <hosted_cluster_namespace>-<hosted_cluster_name> 1 2 rules: - apiGroups: - agent-install.openshift.io resources: - agents verbs: - '*'
apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: name: <hosted_cluster_name> 1 namespace: <hosted_cluster_namespace> 2 spec: additionalTrustBundle: name: "user-ca-bundle" olmCatalogPlacement: guest imageContentSources: 3 - source: quay.io/openshift-release-dev/ocp-v4.0-art-dev mirrors: - registry.<dns.base.domain.name>:5000/openshift/release 4 - source: quay.io/openshift-release-dev/ocp-release mirrors: - registry.<dns.base.domain.name>:5000/openshift/release-images 5 - mirrors: autoscaling: {} controllerAvailabilityPolicy: SingleReplica dns: baseDomain: <dns.base.domain.name> 6 etcd: managed: storage: persistentVolume: size: 8Gi restoreSnapshotURL: null type: PersistentVolume managementType: Managed fips: false networking: clusterNetwork: - cidr: 10.132.0.0/14 - cidr: fd01::/48 networkType: OVNKubernetes serviceNetwork: - cidr: 172.31.0.0/16 - cidr: fd02::/112 platform: agent: agentNamespace: <hosted_cluster_namespace>-<hosted_cluster_name> 7 8 type: Agent pullSecret: name: <hosted_cluster_name>-pull-secret 9 release: image: registry.<dns.base.domain.name>:5000/openshift/release-images:<4.x.y>-x86_64 10 11 secretEncryption: aescbc: activeKey: name: <hosted_cluster_name>-etcd-encryption-key 12 type: aescbc services: - service: APIServer servicePublishingStrategy: type: LoadBalancer - service: OAuthServer servicePublishingStrategy: type: Route - service: OIDC servicePublishingStrategy: type: Route - service: Konnectivity servicePublishingStrategy: type: Route - service: Ignition servicePublishingStrategy: type: Route sshKey: name: sshkey-cluster-<hosted_cluster_name> 13 status: controlPlaneEndpoint: host: "" port: 0
oc adm release info registry.<dns.base.domain.name>:5000/openshift-release-dev/ocp-release:<4.x.y>-x86_64 | grep hypershift
hypershift sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.<dns.base.domain.name>:5000/openshift-release-dev/ocp-v4.0-art-dev@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Trying to pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Getting image source signatures Copying blob d8190195889e skipped: already exists Copying blob c71d2589fba7 skipped: already exists Copying blob d4dc6e74b6ce skipped: already exists Copying blob 97da74cc6d8f skipped: already exists Copying blob b70007a560c9 done Copying config 3a62961e6e done Writing manifest to image destination Storing signatures 3a62961e6ed6edab46d5ec8429ff1f41d6bb68de51271f037c6cb8941a007fde
oc apply -f 01-4.14-hosted_cluster-nodeport.yaml
NAME READY STATUS RESTARTS AGE capi-provider-5b57dbd6d5-pxlqc 1/1 Running 0 3m57s catalog-operator-9694884dd-m7zzv 2/2 Running 0 93s cluster-api-f98b9467c-9hfrq 1/1 Running 0 3m57s cluster-autoscaler-d7f95dd5-d8m5d 1/1 Running 0 93s cluster-image-registry-operator-5ff5944b4b-648ht 1/2 Running 0 93s cluster-network-operator-77b896ddc-wpkq8 1/1 Running 0 94s cluster-node-tuning-operator-84956cd484-4hfgf 1/1 Running 0 94s cluster-policy-controller-5fd8595d97-rhbwf 1/1 Running 0 95s cluster-storage-operator-54dcf584b5-xrnts 1/1 Running 0 93s cluster-version-operator-9c554b999-l22s7 1/1 Running 0 95s control-plane-operator-6fdc9c569-t7hr4 1/1 Running 0 3m57s csi-snapshot-controller-785c6dc77c-8ljmr 1/1 Running 0 77s csi-snapshot-controller-operator-7c6674bc5b-d9dtp 1/1 Running 0 93s csi-snapshot-webhook-5b8584875f-2492j 1/1 Running 0 77s dns-operator-6874b577f-9tc6b 1/1 Running 0 94s etcd-0 3/3 Running 0 3m39s hosted-cluster-config-operator-f5cf5c464-4nmbh 1/1 Running 0 93s ignition-server-6b689748fc-zdqzk 1/1 Running 0 95s ignition-server-proxy-54d4bb9b9b-6zkg7 1/1 Running 0 95s ingress-operator-6548dc758b-f9gtg 1/2 Running 0 94s konnectivity-agent-7767cdc6f5-tw782 1/1 Running 0 95s kube-apiserver-7b5799b6c8-9f5bp 4/4 Running 0 3m7s kube-controller-manager-5465bc4dd6-zpdlk 1/1 Running 0 44s kube-scheduler-5dd5f78b94-bbbck 1/1 Running 0 2m36s machine-approver-846c69f56-jxvfr 1/1 Running 0 92s oauth-openshift-79c7bf44bf-j975g 2/2 Running 0 62s olm-operator-767f9584c-4lcl2 2/2 Running 0 93s openshift-apiserver-5d469778c6-pl8tj 3/3 Running 0 2m36s openshift-controller-manager-6475fdff58-hl4f7 1/1 Running 0 95s openshift-oauth-apiserver-dbbc5cc5f-98574 2/2 Running 0 95s openshift-route-controller-manager-5f6997b48f-s9vdc 1/1 Running 0 95s packageserver-67c87d4d4f-kl7qh 2/2 Running 0 93s
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters hosted-dual hosted-admin-kubeconfig Partial True False The hosted control plane is available
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: creationTimestamp: null name: <hosted_cluster_name> \ 1 namespace: <hosted_cluster_namespace> \ 2 spec: arch: amd64 clusterName: <hosted_cluster_name> management: autoRepair: false \ 3 upgradeType: InPlace \ 4 nodeDrainTimeout: 0s platform: type: Agent release: image: registry.<dns.base.domain.name>:5000/openshift/release-images:4.x.y-x86_64 \ 5 replicas: 2 6 status: replicas: 2
oc apply -f 02-nodepool.yaml
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted-dual hosted 0 False False 4.x.y-x86_64
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: <hosted_cluster_name> namespace: <hosted-cluster-namespace>-<hosted_cluster_name> 1 2 spec: pullSecretRef: 3 name: pull-secret sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDk7ICaUE+/k4zTpxLk4+xFdHi4ZuDi5qjeF52afsNkw0w/glILHhwpL5gnp5WkRuL8GwJuZ1VqLC9EKrdmegn4MrmUlq7WTsP0VFOZFBfq2XRUxo1wrRdor2z0Bbh93ytR+ZsDbbLlGngXaMa0Vbt+z74FqlcajbHTZ6zBmTpBVq5RHtDPgKITdpE1fongp7+ZXQNBlkaavaqv8bnyrP4BWahLP4iO9/xJF9lQYboYwEEDzmnKLMW1VtCE6nJzEgWCufACTbxpNS7GvKtoHT/OVzw8ArEXhZXQUS1UY8zKsX2iXwmyhw5Sj6YboA8WICs4z+TrFP89LmxXY0j6536TQFyRz1iB4WWvCbH5n6W+ABV2e8ssJB1AmEy8QYNwpJQJNpSxzoKBjI73XxvPYYC/IjPFMySwZqrSZCkJYqQ023ySkaQxWZT7in4KeMu7eS2tC+Kn4deJ7KwwUycx8n6RHMeD8Qg9flTHCv3gmab8JKZJqN3hW1D378JuvmIX4V0= 4
oc apply -f 03-infraenv.yaml
NAMESPACE NAME ISO CREATED AT clusters-hosted-dual hosted 2023-09-11T15:14:10Z
kcli delete plan <hosted_cluster_name> 1
kcli create vm -P start=False \ 1 -P uefi_legacy=true \ 2 -P plan=<hosted_cluster_name> \ 3 -P memory=8192 -P numcpus=16 \ 4 -P disks=[200,200] \ 5 -P nets=["{\"name\": \"<network>\", \"mac\": \"aa:aa:aa:aa:11:01\"}"] \ 6 -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa1101 -P name=<hosted_cluster_name>-worker0 7
kcli create vm -P start=False \ 1 -P uefi_legacy=true \ 2 -P plan=<hosted_cluster_name> \ 3 -P memory=8192 -P numcpus=16 \ 4 -P disks=[200,200] \ 5 -P nets=["{\"name\": \"<network>\", \"mac\": \"aa:aa:aa:aa:11:02\"}"] \ 6 -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa1102 -P name=<hosted_cluster_name>-worker1 7
kcli create vm -P start=False \ 1 -P uefi_legacy=true \ 2 -P plan=<hosted_cluster_name> \ 3 -P memory=8192 -P numcpus=16 \ 4 -P disks=[200,200] \ 5 -P nets=["{\"name\": \"<network>\", \"mac\": \"aa:aa:aa:aa:11:03\"}"] \ 6 -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa1103 -P name=<hosted_cluster_name>-worker2 7
systemctl restart ksushy
+---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | Name | Status | Ip | Source | Plan | Profile | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | hosted-worker0 | down | | | hosted-dual | kvirt | | hosted-worker1 | down | | | hosted-dual | kvirt | | hosted-worker2 | down | | | hosted-dual | kvirt | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+
apiVersion: v1 kind: Secret metadata: name: <hosted_cluster_name>-worker0-bmc-secret \ 1 namespace: <hosted_cluster_namespace>-<hosted_cluster_name> \ 2 data: password: YWRtaW4= \ 3 username: YWRtaW4= \ 4 type: Opaque apiVersion: metal3.io/v1alpha1 kind: BareMetalHost metadata: name: <hosted_cluster_name>-worker0 namespace: <hosted_cluster_namespace>-<hosted_cluster_name> \ 5 labels: infraenvs.agent-install.openshift.io: <hosted_cluster_name> \ 6 annotations: inspect.metal3.io: disabled bmac.agent-install.openshift.io/hostname: <hosted_cluster_name>-worker0 \ 7 spec: automatedCleaningMode: disabled \ 8 bmc: disableCertificateVerification: true \ 9 address: redfish-virtualmedia://[192.168.126.1]:9000/redfish/v1/Systems/local/<hosted_cluster_name>-worker0 \ 10 credentialsName: <hosted_cluster_name>-worker0-bmc-secret \ 11 bootMACAddress: aa:aa:aa:aa:02:11 \ 12 online: true 13
oc apply -f 04-bmh.yaml
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 registering true 2s clusters-hosted hosted-worker1 registering true 2s clusters-hosted hosted-worker2 registering true 2s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioning true 16s clusters-hosted hosted-worker1 provisioning true 16s clusters-hosted hosted-worker2 provisioning true 16s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioned true 67s clusters-hosted hosted-worker1 provisioned true 67s clusters-hosted hosted-worker2 provisioned true 67s
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0411 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0412 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0413 true auto-assign
oc -n <hosted_cluster_namespace> scale nodepool <hosted_cluster_name> --replicas 3
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0411 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0412 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0413 hosted true auto-assign
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted hosted 3 False False <4.x.y>-x86_64 Minimum availability requires 3 replicas, current 0 available
oc apply -f registry-config.yaml
apiVersion: v1 kind: ConfigMap metadata: name: registry-config namespace: openshift-config data: <mirror_registry>: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE-----
spec: additionalTrustedCA: - name: registry-config
oc get secret/pull-secret -n openshift-config -o json | jq -r '.data.".dockerconfigjson"' | base64 -d > authfile
"auths": { "<mirror_registry>": { 1 "auth": "<credentials>", 2 "email": "you@example.com" } },
oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=authfile
apiVersion: v1 kind: ConfigMap metadata: name: mirror-config namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registries.conf: | [[registry]] location = "registry.stage.redhat.io" insecure = false blocked = false mirror-by-digest-only = true prefix = "" [[registry.mirror]] location = "<mirror_registry>" insecure = false [[registry]] location = "registry.redhat.io/multicluster-engine" insecure = false blocked = false mirror-by-digest-only = true prefix = "" [[registry.mirror]] location = "<mirror_registry>/multicluster-engine" 1 insecure = false
spec: mirrorRegistryRef: name: mirror-config
cat icsp.yaml - mirrors: - <mirror_registry>/openshift/release source: quay.io/openshift-release-dev/ocp-v4.0-art-dev - mirrors: - <mirror_registry>/openshift/release-images source: quay.io/openshift-release-dev/ocp-release
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> --etcd-storage-class=<etcd_storage_class> \ 5 --ssh-key <path_to_ssh_public_key> \ 6 --namespace <hosted_cluster_namespace> \ 7 --control-plane-availability-policy SingleReplica --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 8 --additional-trust-bundle <path for cert> \ 9 --image-content-sources icsp.yaml
oc get events -n hypershift
LAST SEEN TYPE REASON OBJECT MESSAGE 4m46s Warning ReconcileError deployment/operator Failed to ensure UWM telemetry remote write: cannot get telemeter client secret: Secret "telemeter-client" not found
kind: ConfigMap apiVersion: v1 metadata: name: hypershift-operator-install-flags namespace: local-cluster data: installFlagsToAdd: "" installFlagsToRemove: "--enable-uwm-telemetry-remote-write"
oc apply -f <filename>.yaml
oc patch mce <multiclusterengine> --type=merge -p '{"spec":{"overrides":{"components":[{"name":"hypershift","enabled": true}]}}}'
oc get managedclusteraddons -n local-cluster hypershift-addon
NAME AVAILABLE DEGRADED PROGRESSING hypershift-addon True False
oc wait --for=condition=Degraded=True managedclusteraddons/hypershift-addon -n local-cluster --timeout=5m
oc wait --for=condition=Available=True managedclusteraddons/hypershift-addon -n local-cluster --timeout=5m
oc edit addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: hypershift-addon-deploy-config namespace: multicluster-engine spec: nodePlacement: nodeSelector: node-role.kubernetes.io/infra: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/infra operator: Exists
spec: autoscaling: {} channel: stable-4.y 1 clusterID: d6d42268-7dff-4d37-92cf-691bd2d42f41 configuration: {} controllerAvailabilityPolicy: SingleReplica dns: baseDomain: dev11.red-chesterfield.com privateZoneID: Z0180092I0DQRKL55LN0 publicZoneID: Z00206462VG6ZP0H2QLWK
oc get -n <hosted_cluster_namespace> hostedcluster <hosted_cluster_name> -o yaml
version: availableUpdates: - channels: - candidate-4.16 - candidate-4.17 - eus-4.16 - fast-4.16 - stable-4.16 image: quay.io/openshift-release-dev/ocp-release@sha256:b7517d13514c6308ae16c5fd8108133754eb922cd37403ed27c846c129e67a9a url: https://access.redhat.com/errata/RHBA-2024:6401 version: 4.16.11 - channels: - candidate-4.16 - candidate-4.17 - eus-4.16 - fast-4.16 - stable-4.16 image: quay.io/openshift-release-dev/ocp-release@sha256:d08e7c8374142c239a07d7b27d1170eae2b0d9f00ccf074c3f13228a1761c162 url: https://access.redhat.com/errata/RHSA-2024:6004 version: 4.16.10 - channels: - candidate-4.16 - candidate-4.17 - eus-4.16 - fast-4.16 - stable-4.16 image: quay.io/openshift-release-dev/ocp-release@sha256:6a80ac72a60635a313ae511f0959cc267a21a89c7654f1c15ee16657aafa41a0 url: https://access.redhat.com/errata/RHBA-2024:5757 version: 4.16.9 - channels: - candidate-4.16 - candidate-4.17 - eus-4.16 - fast-4.16 - stable-4.16 image: quay.io/openshift-release-dev/ocp-release@sha256:ea624ae7d91d3f15094e9e15037244679678bdc89e5a29834b2ddb7e1d9b57e6 url: https://access.redhat.com/errata/RHSA-2024:5422 version: 4.16.8 - channels: - candidate-4.16 - candidate-4.17 - eus-4.16 - fast-4.16 - stable-4.16 image: quay.io/openshift-release-dev/ocp-release@sha256:e4102eb226130117a0775a83769fe8edb029f0a17b6cbca98a682e3f1225d6b7 url: https://access.redhat.com/errata/RHSA-2024:4965 version: 4.16.6 - channels: - candidate-4.16 - candidate-4.17 - eus-4.16 - fast-4.16 - stable-4.16 image: quay.io/openshift-release-dev/ocp-release@sha256:f828eda3eaac179e9463ec7b1ed6baeba2cd5bd3f1dd56655796c86260db819b url: https://access.redhat.com/errata/RHBA-2024:4855 version: 4.16.5 conditionalUpdates: - conditions: - lastTransitionTime: "2024-09-23T22:33:38Z" message: |- Could not evaluate exposure to update risk SRIOVFailedToConfigureVF (creating PromQL round-tripper: unable to load specified CA cert /etc/tls/service-ca/service-ca.crt: open /etc/tls/service-ca/service-ca.crt: no such file or directory) SRIOVFailedToConfigureVF description: OCP Versions 4.14.34, 4.15.25, 4.16.7 and ALL subsequent versions include kernel datastructure changes which are not compatible with older versions of the SR-IOV operator. Please update SR-IOV operator to versions dated 20240826 or newer before updating OCP. SRIOVFailedToConfigureVF URL: https://issues.redhat.com/browse/NHE-1171 reason: EvaluationFailed status: Unknown type: Recommended release: channels: - candidate-4.16 - candidate-4.17 - eus-4.16 - fast-4.16 - stable-4.16 image: quay.io/openshift-release-dev/ocp-release@sha256:fb321a3f50596b43704dbbed2e51fdefd7a7fd488ee99655d03784d0cd02283f url: https://access.redhat.com/errata/RHSA-2024:5107 version: 4.16.7 risks: - matchingRules: - promql: promql: | group(csv_succeeded{_id="d6d42268-7dff-4d37-92cf-691bd2d42f41", name=~"sriov-network-operator[.].*"}) or 0 * group(csv_count{_id="d6d42268-7dff-4d37-92cf-691bd2d42f41"}) type: PromQL message: OCP Versions 4.14.34, 4.15.25, 4.16.7 and ALL subsequent versions include kernel datastructure changes which are not compatible with older versions of the SR-IOV operator. Please update SR-IOV operator to versions dated 20240826 or newer before updating OCP. name: SRIOVFailedToConfigureVF url: https://issues.redhat.com/browse/NHE-1171
apiVersion: v1 data: server-version: 2f6cfe21a0861dea3130f3bed0d3ae5553b8c28b supported-versions: '{"versions":["4.17","4.16","4.15","4.14"]}' kind: ConfigMap metadata: creationTimestamp: "2024-06-20T07:12:31Z" labels: hypershift.openshift.io/supported-versions: "true" name: supported-versions namespace: hypershift resourceVersion: "927029" uid: f6336f91-33d3-472d-b747-94abae725f70
Client Version: openshift/hypershift: fe67b47fb60e483fe60e4755a02b3be393256343. Latest supported OCP: 4.17.0 Server Version: 05864f61f24a8517731664f8091cedcfc5f9b60d Server Supports OCP Versions: 4.17, 4.16, 4.15, 4.14
oc patch nodepool <node_pool_name> -n <hosted_cluster_namespace> \ 1 --type=merge -p '{"spec":{"nodeDrainTimeout":"60s","release":{"image":"<openshift_release_image>"}}}' 2
oc get -n <hosted_cluster_namespace> nodepool <node_pool_name> -o yaml
status: conditions: - lastTransitionTime: "2024-05-20T15:00:40Z" message: 'Using release image: quay.io/openshift-release-dev/ocp-release:4.y.z-x86_64' 1 reason: AsExpected status: "True" type: ValidReleaseImage
oc annotate hostedcluster -n <hosted_cluster_namespace> <hosted_cluster_name> \ 1 "hypershift.openshift.io/force-upgrade-to=<openshift_release_image>" \ 2 --overwrite
oc patch hostedcluster <hosted_cluster_name> -n <hosted_cluster_namespace> --type=merge -p '{"spec":{"release":{"image":"<openshift_release_image>"}}}'
oc get -n <hosted_cluster_namespace> hostedcluster <hosted_cluster_name> -o yaml
status: conditions: - lastTransitionTime: "2024-05-20T15:01:01Z" message: Payload loaded version="4.y.z" image="quay.io/openshift-release-dev/ocp-release:4.y.z-x86_64" 1 status: "True" type: ClusterVersionReleaseAccepted # version: availableUpdates: null desired: image: quay.io/openshift-release-dev/ocp-release:4.y.z-x86_64 2 version: 4.y.z
oc rsh -n openshift-etcd -c etcd <etcd_pod_name>
sh-4.4# etcdctl endpoint status -w table
+------------------------------+-----------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | +------------------------------+-----------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+ | https://192.168.1xxx.20:2379 | 8fxxxxxxxxxx | 3.5.12 | 123 MB | false | false | 10 | 180156 | 180156 | | | https://192.168.1xxx.21:2379 | a5xxxxxxxxxx | 3.5.12 | 122 MB | false | false | 10 | 180156 | 180156 | | | https://192.168.1xxx.22:2379 | 7cxxxxxxxxxx | 3.5.12 | 124 MB | true | false | 10 | 180156 | 180156 | | +-----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
oc get pods -l app=etcd -n openshift-etcd
NAME READY STATUS RESTARTS AGE etcd-0 2/2 Running 0 64m etcd-1 2/2 Running 0 45m etcd-2 1/2 CrashLoopBackOff 1 (5s ago) 64m
oc delete pods etcd-2 -n openshift-etcd
oc get pods -l app=etcd -n openshift-etcd
NAME READY STATUS RESTARTS AGE etcd-0 2/2 Running 0 67m etcd-1 2/2 Running 0 48m etcd-2 2/2 Running 0 2m2s
CLUSTER_NAME=my-cluster
HOSTED_CLUSTER_NAMESPACE=clusters
CONTROL_PLANE_NAMESPACE="USD{HOSTED_CLUSTER_NAMESPACE}-USD{CLUSTER_NAME}"
oc patch -n USD{HOSTED_CLUSTER_NAMESPACE} hostedclusters/USD{CLUSTER_NAME} -p '{"spec":{"pausedUntil":"true"}}' --type=merge
oc get -n USD{CONTROL_PLANE_NAMESPACE} pods -l app=etcd
ETCD_POD=etcd-0
oc exec -n USD{CONTROL_PLANE_NAMESPACE} -c etcd -t USD{ETCD_POD} -- env ETCDCTL_API=3 /usr/bin/etcdctl --cacert /etc/etcd/tls/etcd-ca/ca.crt --cert /etc/etcd/tls/client/etcd-client.crt --key /etc/etcd/tls/client/etcd-client.key --endpoints=https://localhost:2379 snapshot save /var/lib/snapshot.db
oc exec -n USD{CONTROL_PLANE_NAMESPACE} -c etcd -t USD{ETCD_POD} -- env ETCDCTL_API=3 /usr/bin/etcdctl -w table snapshot status /var/lib/snapshot.db
oc cp -c etcd USD{CONTROL_PLANE_NAMESPACE}/USD{ETCD_POD}:/var/lib/snapshot.db /tmp/etcd.snapshot.db
oc get -n USD{CONTROL_PLANE_NAMESPACE} pods -l app=etcd
oc cp -c etcd USD{CONTROL_PLANE_NAMESPACE}/USD{ETCD_POD}:/var/lib/data/member/snap/db /tmp/etcd.snapshot.db
oc scale -n USD{CONTROL_PLANE_NAMESPACE} statefulset/etcd --replicas=0
oc delete -n USD{CONTROL_PLANE_NAMESPACE} pvc/data-etcd-1 pvc/data-etcd-2
ETCD_IMAGE=USD(oc get -n USD{CONTROL_PLANE_NAMESPACE} statefulset/etcd -o jsonpath='{ .spec.template.spec.containers[0].image }')
cat << EOF | oc apply -n USD{CONTROL_PLANE_NAMESPACE} -f - apiVersion: apps/v1 kind: Deployment metadata: name: etcd-data spec: replicas: 1 selector: matchLabels: app: etcd-data template: metadata: labels: app: etcd-data spec: containers: - name: access image: USDETCD_IMAGE volumeMounts: - name: data mountPath: /var/lib command: - /usr/bin/bash args: - -c - |- while true; do sleep 1000 done volumes: - name: data persistentVolumeClaim: claimName: data-etcd-0 EOF
oc get -n USD{CONTROL_PLANE_NAMESPACE} pods -l app=etcd-data
DATA_POD=USD(oc get -n USD{CONTROL_PLANE_NAMESPACE} pods --no-headers -l app=etcd-data -o name | cut -d/ -f2)
oc cp /tmp/etcd.snapshot.db USD{CONTROL_PLANE_NAMESPACE}/USD{DATA_POD}:/var/lib/restored.snap.db
oc exec -n USD{CONTROL_PLANE_NAMESPACE} USD{DATA_POD} -- rm -rf /var/lib/data
oc exec -n USD{CONTROL_PLANE_NAMESPACE} USD{DATA_POD} -- mkdir -p /var/lib/data
oc exec -n USD{CONTROL_PLANE_NAMESPACE} USD{DATA_POD} -- etcdutl snapshot restore /var/lib/restored.snap.db --data-dir=/var/lib/data --skip-hash-check --name etcd-0 --initial-cluster-token=etcd-cluster --initial-cluster etcd-0=https://etcd-0.etcd-discovery.USD{CONTROL_PLANE_NAMESPACE}.svc:2380,etcd-1=https://etcd-1.etcd-discovery.USD{CONTROL_PLANE_NAMESPACE}.svc:2380,etcd-2=https://etcd-2.etcd-discovery.USD{CONTROL_PLANE_NAMESPACE}.svc:2380 --initial-advertise-peer-urls https://etcd-0.etcd-discovery.USD{CONTROL_PLANE_NAMESPACE}.svc:2380
oc exec -n USD{CONTROL_PLANE_NAMESPACE} USD{DATA_POD} -- rm /var/lib/restored.snap.db
oc delete -n USD{CONTROL_PLANE_NAMESPACE} deployment/etcd-data
oc scale -n USD{CONTROL_PLANE_NAMESPACE} statefulset/etcd --replicas=3
oc get -n USD{CONTROL_PLANE_NAMESPACE} pods -l app=etcd -w
oc patch -n USD{HOSTED_CLUSTER_NAMESPACE} hostedclusters/USD{CLUSTER_NAME} -p '{"spec":{"pausedUntil":"null"}}' --type=merge
oc patch -n clusters hostedclusters/<hosted_cluster_name> -p '{"spec":{"pausedUntil":"true"}}' --type=merge
oc scale deployment -n <hosted_cluster_namespace> --replicas=0 kube-apiserver openshift-apiserver openshift-oauth-apiserver
oc exec -it <etcd_pod_name> -n <hosted_cluster_namespace> -- env ETCDCTL_API=3 /usr/bin/etcdctl --cacert /etc/etcd/tls/etcd-ca/ca.crt --cert /etc/etcd/tls/client/etcd-client.crt --key /etc/etcd/tls/client/etcd-client.key --endpoints=localhost:2379 snapshot save /var/lib/data/snapshot.db
oc exec -it <etcd_pod_name> -n <hosted_cluster_namespace> -- env ETCDCTL_API=3 /usr/bin/etcdctl -w table snapshot status /var/lib/data/snapshot.db
BUCKET_NAME=somebucket CLUSTER_NAME=cluster_name FILEPATH="/USD{BUCKET_NAME}/USD{CLUSTER_NAME}-snapshot.db" CONTENT_TYPE="application/x-compressed-tar" DATE_VALUE=`date -R` SIGNATURE_STRING="PUT\n\nUSD{CONTENT_TYPE}\nUSD{DATE_VALUE}\nUSD{FILEPATH}" ACCESS_KEY=accesskey SECRET_KEY=secret SIGNATURE_HASH=`echo -en USD{SIGNATURE_STRING} | openssl sha1 -hmac USD{SECRET_KEY} -binary | base64` HOSTED_CLUSTER_NAMESPACE=hosted_cluster_namespace exec -it etcd-0 -n USD{HOSTED_CLUSTER_NAMESPACE} -- curl -X PUT -T "/var/lib/data/snapshot.db" -H "Host: USD{BUCKET_NAME}.s3.amazonaws.com" -H "Date: USD{DATE_VALUE}" -H "Content-Type: USD{CONTENT_TYPE}" -H "Authorization: AWS USD{ACCESS_KEY}:USD{SIGNATURE_HASH}" https://USD{BUCKET_NAME}.s3.amazonaws.com/USD{CLUSTER_NAME}-snapshot.db
oc get hostedcluster <hosted_cluster_name> -o=jsonpath='{.spec.secretEncryption.aescbc}' {"activeKey":{"name":"<hosted_cluster_name>-etcd-encryption-key"}}
oc get secret <hosted_cluster_name>-etcd-encryption-key -o=jsonpath='{.data.key}'
oc scale deployment -n <control_plane_namespace> --replicas=3 kube-apiserver openshift-apiserver openshift-oauth-apiserver
oc patch -n <hosted_cluster_namespace> -p '[\{"op": "remove", "path": "/spec/pausedUntil"}]' --type=json
ETCD_SNAPSHOT=USD{ETCD_SNAPSHOT:-"s3://USD{BUCKET_NAME}/USD{CLUSTER_NAME}-snapshot.db"} ETCD_SNAPSHOT_URL=USD(aws s3 presign USD{ETCD_SNAPSHOT})
spec: etcd: managed: storage: persistentVolume: size: 4Gi type: PersistentVolume restoreSnapshotURL: - "USD{ETCD_SNAPSHOT_URL}" managementType: Managed
apiVersion: velero.io/v1 kind: Backup metadata: name: hc-clusters-hosted-backup namespace: openshift-adp labels: velero.io/storage-location: default spec: includedNamespaces: 1 - clusters - clusters-hosted includedResources: - sa - role - rolebinding - deployment - statefulset - pv - pvc - bmh - configmap - infraenv - priorityclasses - pdb - hostedcluster - nodepool - secrets - hostedcontrolplane - cluster - datavolume - service - route excludedResources: [ ] labelSelector: 2 matchExpressions: - key: 'hypershift.openshift.io/is-kubevirt-rhcos' operator: 'DoesNotExist' storageLocation: default preserveNodePorts: true ttl: 4h0m0s snapshotMoveData: true 3 datamover: "velero" 4 defaultVolumesToFsBackup: false 5
oc apply -f <backup_file_name>.yaml
watch "oc get backups.velero.io -n openshift-adp <backup_file_name> -o jsonpath='{.status}' | jq"
oc logs -n openshift-adp -ldeploy=velero -f
apiVersion: velero.io/v1 kind: Restore metadata: name: hc-clusters-hosted-restore namespace: openshift-adp spec: backupName: hc-clusters-hosted-backup restorePVs: true 1 existingResourcePolicy: update 2 excludedResources: - nodes - events - events.events.k8s.io - backups.velero.io - restores.velero.io - resticrepositories.velero.io
oc apply -f <restore_resource_file_name>.yaml
watch "oc get restores.velero.io -n openshift-adp <backup_file_name> -o jsonpath='{.status}' | jq"
oc logs -n openshift-adp -ldeploy=velero -f
--external-dns-provider=aws --external-dns-credentials=<path_to_aws_credentials_file> --external-dns-domain-filter=<basedomain>
oc create configmap mgmt-parent-cluster -n default --from-literal=from=USD{MGMT_CLUSTER_NAME}
PAUSED_UNTIL="true" oc patch -n USD{HC_CLUSTER_NS} hostedclusters/USD{HC_CLUSTER_NAME} -p '{"spec":{"pausedUntil":"'USD{PAUSED_UNTIL}'"}}' --type=merge oc scale deployment -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --replicas=0 kube-apiserver openshift-apiserver openshift-oauth-apiserver control-plane-operator
PAUSED_UNTIL="true" oc patch -n USD{HC_CLUSTER_NS} hostedclusters/USD{HC_CLUSTER_NAME} -p '{"spec":{"pausedUntil":"'USD{PAUSED_UNTIL}'"}}' --type=merge oc patch -n USD{HC_CLUSTER_NS} nodepools/USD{NODEPOOLS} -p '{"spec":{"pausedUntil":"'USD{PAUSED_UNTIL}'"}}' --type=merge oc scale deployment -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --replicas=0 kube-apiserver openshift-apiserver openshift-oauth-apiserver control-plane-operator
ETCD Backup ETCD_PODS="etcd-0" if [ "USD{CONTROL_PLANE_AVAILABILITY_POLICY}" = "HighlyAvailable" ]; then ETCD_PODS="etcd-0 etcd-1 etcd-2" fi for POD in USD{ETCD_PODS}; do # Create an etcd snapshot oc exec -it USD{POD} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -- env ETCDCTL_API=3 /usr/bin/etcdctl --cacert /etc/etcd/tls/client/etcd-client-ca.crt --cert /etc/etcd/tls/client/etcd-client.crt --key /etc/etcd/tls/client/etcd-client.key --endpoints=localhost:2379 snapshot save /var/lib/data/snapshot.db oc exec -it USD{POD} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -- env ETCDCTL_API=3 /usr/bin/etcdctl -w table snapshot status /var/lib/data/snapshot.db FILEPATH="/USD{BUCKET_NAME}/USD{HC_CLUSTER_NAME}-USD{POD}-snapshot.db" CONTENT_TYPE="application/x-compressed-tar" DATE_VALUE=`date -R` SIGNATURE_STRING="PUT\n\nUSD{CONTENT_TYPE}\nUSD{DATE_VALUE}\nUSD{FILEPATH}" set +x ACCESS_KEY=USD(grep aws_access_key_id USD{AWS_CREDS} | head -n1 | cut -d= -f2 | sed "s/ //g") SECRET_KEY=USD(grep aws_secret_access_key USD{AWS_CREDS} | head -n1 | cut -d= -f2 | sed "s/ //g") SIGNATURE_HASH=USD(echo -en USD{SIGNATURE_STRING} | openssl sha1 -hmac "USD{SECRET_KEY}" -binary | base64) set -x # FIXME: this is pushing to the OIDC bucket oc exec -it etcd-0 -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -- curl -X PUT -T "/var/lib/data/snapshot.db" -H "Host: USD{BUCKET_NAME}.s3.amazonaws.com" -H "Date: USD{DATE_VALUE}" -H "Content-Type: USD{CONTENT_TYPE}" -H "Authorization: AWS USD{ACCESS_KEY}:USD{SIGNATURE_HASH}" https://USD{BUCKET_NAME}.s3.amazonaws.com/USD{HC_CLUSTER_NAME}-USD{POD}-snapshot.db done
mkdir -p USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS} USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} chmod 700 USD{BACKUP_DIR}/namespaces/ HostedCluster echo "Backing Up HostedCluster Objects:" oc get hc USD{HC_CLUSTER_NAME} -n USD{HC_CLUSTER_NS} -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/hc-USD{HC_CLUSTER_NAME}.yaml echo "--> HostedCluster" sed -i '' -e '/^status:USD/,USDd' USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/hc-USD{HC_CLUSTER_NAME}.yaml NodePool oc get np USD{NODEPOOLS} -n USD{HC_CLUSTER_NS} -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/np-USD{NODEPOOLS}.yaml echo "--> NodePool" sed -i '' -e '/^status:USD/,USD d' USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/np-USD{NODEPOOLS}.yaml Secrets in the HC Namespace echo "--> HostedCluster Secrets:" for s in USD(oc get secret -n USD{HC_CLUSTER_NS} | grep "^USD{HC_CLUSTER_NAME}" | awk '{print USD1}'); do oc get secret -n USD{HC_CLUSTER_NS} USDs -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/secret-USD{s}.yaml done Secrets in the HC Control Plane Namespace echo "--> HostedCluster ControlPlane Secrets:" for s in USD(oc get secret -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} | egrep -v "docker|service-account-token|oauth-openshift|NAME|token-USD{HC_CLUSTER_NAME}" | awk '{print USD1}'); do oc get secret -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USDs -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/secret-USD{s}.yaml done Hosted Control Plane echo "--> HostedControlPlane:" oc get hcp USD{HC_CLUSTER_NAME} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/hcp-USD{HC_CLUSTER_NAME}.yaml Cluster echo "--> Cluster:" CL_NAME=USD(oc get hcp USD{HC_CLUSTER_NAME} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o jsonpath={.metadata.labels.\*} | grep USD{HC_CLUSTER_NAME}) oc get cluster USD{CL_NAME} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/cl-USD{HC_CLUSTER_NAME}.yaml AWS Cluster echo "--> AWS Cluster:" oc get awscluster USD{HC_CLUSTER_NAME} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/awscl-USD{HC_CLUSTER_NAME}.yaml AWS MachineTemplate echo "--> AWS Machine Template:" oc get awsmachinetemplate USD{NODEPOOLS} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/awsmt-USD{HC_CLUSTER_NAME}.yaml AWS Machines echo "--> AWS Machine:" CL_NAME=USD(oc get hcp USD{HC_CLUSTER_NAME} -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o jsonpath={.metadata.labels.\*} | grep USD{HC_CLUSTER_NAME}) for s in USD(oc get awsmachines -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --no-headers | grep USD{CL_NAME} | cut -f1 -d\ ); do oc get -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} awsmachines USDs -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/awsm-USD{s}.yaml done MachineDeployments echo "--> HostedCluster MachineDeployments:" for s in USD(oc get machinedeployment -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o name); do mdp_name=USD(echo USD{s} | cut -f 2 -d /) oc get -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USDs -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/machinedeployment-USD{mdp_name}.yaml done MachineSets echo "--> HostedCluster MachineSets:" for s in USD(oc get machineset -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o name); do ms_name=USD(echo USD{s} | cut -f 2 -d /) oc get -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USDs -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/machineset-USD{ms_name}.yaml done Machines echo "--> HostedCluster Machine:" for s in USD(oc get machine -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o name); do m_name=USD(echo USD{s} | cut -f 2 -d /) oc get -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USDs -o yaml > USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/machine-USD{m_name}.yaml done
oc delete routes -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --all
function clean_routes() { if [[ -z "USD{1}" ]];then echo "Give me the NS where to clean the routes" exit 1 fi # Constants if [[ -z "USD{2}" ]];then echo "Give me the Route53 zone ID" exit 1 fi ZONE_ID=USD{2} ROUTES=10 timeout=40 count=0 # This allows us to remove the ownership in the AWS for the API route oc delete route -n USD{1} --all while [ USD{ROUTES} -gt 2 ] do echo "Waiting for ExternalDNS Operator to clean the DNS Records in AWS Route53 where the zone id is: USD{ZONE_ID}..." echo "Try: (USD{count}/USD{timeout})" sleep 10 if [[ USDcount -eq timeout ]];then echo "Timeout waiting for cleaning the Route53 DNS records" exit 1 fi count=USD((count+1)) ROUTES=USD(aws route53 list-resource-record-sets --hosted-zone-id USD{ZONE_ID} --max-items 10000 --output json | grep -c USD{EXTERNAL_DNS_DOMAIN}) done } SAMPLE: clean_routes "<HC ControlPlane Namespace>" "<AWS_ZONE_ID>" clean_routes "USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}" "USD{AWS_ZONE_ID}"
Just in case export KUBECONFIG=USD{MGMT2_KUBECONFIG} BACKUP_DIR=USD{HC_CLUSTER_DIR}/backup Namespace deletion in the destination Management cluster oc delete ns USD{HC_CLUSTER_NS} || true oc delete ns USD{HC_CLUSTER_NS}-{HC_CLUSTER_NAME} || true
Namespace creation oc new-project USD{HC_CLUSTER_NS} oc new-project USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}
oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/secret-*
Secrets oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/secret-* Cluster oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/hcp-* oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/cl-*
AWS oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/awscl-* oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/awsmt-* oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/awsm-* Machines oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/machinedeployment-* oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/machineset-* oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME}/machine-*
ETCD_PODS="etcd-0" if [ "USD{CONTROL_PLANE_AVAILABILITY_POLICY}" = "HighlyAvailable" ]; then ETCD_PODS="etcd-0 etcd-1 etcd-2" fi HC_RESTORE_FILE=USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/hc-USD{HC_CLUSTER_NAME}-restore.yaml HC_BACKUP_FILE=USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/hc-USD{HC_CLUSTER_NAME}.yaml HC_NEW_FILE=USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/hc-USD{HC_CLUSTER_NAME}-new.yaml cat USD{HC_BACKUP_FILE} > USD{HC_NEW_FILE} cat > USD{HC_RESTORE_FILE} <<EOF restoreSnapshotURL: EOF for POD in USD{ETCD_PODS}; do # Create a pre-signed URL for the etcd snapshot ETCD_SNAPSHOT="s3://USD{BUCKET_NAME}/USD{HC_CLUSTER_NAME}-USD{POD}-snapshot.db" ETCD_SNAPSHOT_URL=USD(AWS_DEFAULT_REGION=USD{MGMT2_REGION} aws s3 presign USD{ETCD_SNAPSHOT}) # FIXME no CLI support for restoreSnapshotURL yet cat >> USD{HC_RESTORE_FILE} <<EOF - "USD{ETCD_SNAPSHOT_URL}" EOF done cat USD{HC_RESTORE_FILE} if ! grep USD{HC_CLUSTER_NAME}-snapshot.db USD{HC_NEW_FILE}; then sed -i '' -e "/type: PersistentVolume/r USD{HC_RESTORE_FILE}" USD{HC_NEW_FILE} sed -i '' -e '/pausedUntil:/d' USD{HC_NEW_FILE} fi HC=USD(oc get hc -n USD{HC_CLUSTER_NS} USD{HC_CLUSTER_NAME} -o name || true) if [[ USD{HC} == "" ]];then echo "Deploying HC Cluster: USD{HC_CLUSTER_NAME} in USD{HC_CLUSTER_NS} namespace" oc apply -f USD{HC_NEW_FILE} else echo "HC Cluster USD{HC_CLUSTER_NAME} already exists, avoiding step" fi
oc apply -f USD{BACKUP_DIR}/namespaces/USD{HC_CLUSTER_NS}/np-*
timeout=40 count=0 NODE_STATUS=USD(oc get nodes --kubeconfig=USD{HC_KUBECONFIG} | grep -v NotReady | grep -c "worker") || NODE_STATUS=0 while [ USD{NODE_POOL_REPLICAS} != USD{NODE_STATUS} ] do echo "Waiting for Nodes to be Ready in the destination MGMT Cluster: USD{MGMT2_CLUSTER_NAME}" echo "Try: (USD{count}/USD{timeout})" sleep 30 if [[ USDcount -eq timeout ]];then echo "Timeout waiting for Nodes in the destination MGMT Cluster" exit 1 fi count=USD((count+1)) NODE_STATUS=USD(oc get nodes --kubeconfig=USD{HC_KUBECONFIG} | grep -v NotReady | grep -c "worker") || NODE_STATUS=0 done
Just in case export KUBECONFIG=USD{MGMT_KUBECONFIG} Scale down deployments oc scale deployment -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --replicas=0 --all oc scale statefulset.apps -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --replicas=0 --all sleep 15
NODEPOOLS=USD(oc get nodepools -n USD{HC_CLUSTER_NS} -o=jsonpath='{.items[?(@.spec.clusterName=="'USD{HC_CLUSTER_NAME}'")].metadata.name}') if [[ ! -z "USD{NODEPOOLS}" ]];then oc patch -n "USD{HC_CLUSTER_NS}" nodepool USD{NODEPOOLS} --type=json --patch='[ { "op":"remove", "path": "/metadata/finalizers" }]' oc delete np -n USD{HC_CLUSTER_NS} USD{NODEPOOLS} fi
Machines for m in USD(oc get machines -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o name); do oc patch -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USD{m} --type=json --patch='[ { "op":"remove", "path": "/metadata/finalizers" }]' || true oc delete -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USD{m} || true done oc delete machineset -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --all || true
Cluster C_NAME=USD(oc get cluster -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o name) oc patch -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USD{C_NAME} --type=json --patch='[ { "op":"remove", "path": "/metadata/finalizers" }]' oc delete cluster.cluster.x-k8s.io -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --all
AWS Machines for m in USD(oc get awsmachine.infrastructure.cluster.x-k8s.io -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} -o name) do oc patch -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USD{m} --type=json --patch='[ { "op":"remove", "path": "/metadata/finalizers" }]' || true oc delete -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} USD{m} || true done
Delete HCP and ControlPlane HC NS oc patch -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} hostedcontrolplane.hypershift.openshift.io USD{HC_CLUSTER_NAME} --type=json --patch='[ { "op":"remove", "path": "/metadata/finalizers" }]' oc delete hostedcontrolplane.hypershift.openshift.io -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} --all oc delete ns USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} || true
Delete HC and HC Namespace oc -n USD{HC_CLUSTER_NS} patch hostedclusters USD{HC_CLUSTER_NAME} -p '{"metadata":{"finalizers":null}}' --type merge || true oc delete hc -n USD{HC_CLUSTER_NS} USD{HC_CLUSTER_NAME} || true oc delete ns USD{HC_CLUSTER_NS} || true
Validations export KUBECONFIG=USD{MGMT2_KUBECONFIG} oc get hc -n USD{HC_CLUSTER_NS} oc get np -n USD{HC_CLUSTER_NS} oc get pod -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} oc get machines -n USD{HC_CLUSTER_NS}-USD{HC_CLUSTER_NAME} Inside the HostedCluster export KUBECONFIG=USD{HC_KUBECONFIG} oc get clusterversion oc get nodes
oc delete pod -n openshift-ovn-kubernetes --all
oc --kubeconfig <management_cluster_kubeconfig_file> patch hostedcluster -n <hosted_cluster_namespace> <hosted_cluster_name> --type json -p '[{"op": "add", "path": "/spec/pausedUntil", "value": "true"}]'
oc get hostedcluster -n local-cluster <hosted_cluster_name> -o=jsonpath="{.spec.infraID}"
oc --kubeconfig <management_cluster_kubeconfig_file> patch cluster.cluster.x-k8s.io -n local-cluster-<hosted_cluster_name> <hosted_cluster_infra_id> --type json -p '[{"op": "add", "path": "/spec/paused", "value": true}]'
oc --kubeconfig <management_cluster_kubeconfig_file> patch nodepool -n <hosted_cluster_namespace> <node_pool_name> --type json -p '[{"op": "add", "path": "/spec/pausedUntil", "value": "true"}]'
oc --kubeconfig <management_cluster_kubeconfig_file> annotate agentcluster -n <hosted_control_plane_namespace> cluster.x-k8s.io/paused=true --all'
oc --kubeconfig <management_cluster_kubeconfig_file> annotate agentmachine -n <hosted_control_plane_namespace> cluster.x-k8s.io/paused=true --all'
oc --kubeconfig <management_cluster_kubeconfig_file> annotate hostedcluster -n <hosted_cluster_namespace> <hosted_cluster_name> hypershift.openshift.io/skip-delete-hosted-controlplane-namespace=true
apiVersion: velero.io/v1 kind: Backup metadata: name: <backup_resource_name> 1 namespace: openshift-adp labels: velero.io/storage-location: default spec: hooks: {} includedNamespaces: 2 - <hosted_cluster_namespace> 3 - <hosted_control_plane_namespace> 4 includedResources: - sa - role - rolebinding - pod - pvc - pv - bmh - configmap - infraenv 5 - priorityclasses - pdb - agents - hostedcluster - nodepool - secrets - hostedcontrolplane - cluster - agentcluster - agentmachinetemplate - agentmachine - machinedeployment - machineset - machine excludedResources: [] storageLocation: default ttl: 2h0m0s snapshotMoveData: true 6 datamover: "velero" 7 defaultVolumesToFsBackup: true 8
oc apply -f backup-control-plane.yaml
oc get backups.velero.io <backup_resource_name> -n openshift-adp -o jsonpath='{.status.phase}'
oc get pod pvc -n <hosted_control_plane_namespace>
No resources found
apiVersion: velero.io/v1 kind: Restore metadata: name: <restore_resource_name> 1 namespace: openshift-adp spec: backupName: <backup_resource_name> 2 restorePVs: true 3 existingResourcePolicy: update 4 excludedResources: - nodes - events - events.events.k8s.io - backups.velero.io - restores.velero.io - resticrepositories.velero.io
oc apply -f restore-hosted-cluster.yaml
oc get hostedcluster <hosted_cluster_name> -n <hosted_cluster_namespace> -o jsonpath='{.status.phase}'
oc --kubeconfig <management_cluster_kubeconfig_file> patch hostedcluster -n <hosted_cluster_namespace> <hosted_cluster_name> --type json -p '[{"op": "add", "path": "/spec/pausedUntil", "value": "false"}]'
oc --kubeconfig <management_cluster_kubeconfig_file> patch nodepool -n <hosted_cluster_namespace> <node_pool_name> --type json -p '[{"op": "add", "path": "/spec/pausedUntil", "value": "false"}]'
oc --kubeconfig <management_cluster_kubeconfig_file> annotate agentcluster -n <hosted_control_plane_namespace> cluster.x-k8s.io/paused- --overwrite=true --all
oc --kubeconfig <management_cluster_kubeconfig_file> annotate agentmachine -n <hosted_control_plane_namespace> cluster.x-k8s.io/paused- --overwrite=true --all
oc --kubeconfig <management_cluster_kubeconfig_file> annotate hostedcluster -n <hosted_cluster_namespace> <hosted_cluster_name> hypershift.openshift.io/skip-delete-hosted-controlplane-namespace- --overwrite=true --all
oc --kubeconfig <management_cluster_kubeconfig_file> scale nodepool -n <hosted_cluster_namespace> <node_pool_name> --replicas <replica_count> 1
watch "oc get backups.velero.io -n openshift-adp <backup_resource_name> -o jsonpath='{.status}'"
watch "oc get restores.velero.io -n openshift-adp <backup_resource_name> -o jsonpath='{.status}'"
oc logs -n openshift-adp -ldeploy=velero -f
watch "echo BackupRepositories:;echo;oc get backuprepositories.velero.io -A;echo; echo BackupStorageLocations: ;echo; oc get backupstoragelocations.velero.io -A;echo;echo DataUploads: ;echo;oc get datauploads.velero.io -A;echo;echo DataDownloads: ;echo;oc get datadownloads.velero.io -n openshift-adp; echo;echo VolumeSnapshotLocations: ;echo;oc get volumesnapshotlocations.velero.io -A;echo;echo Backups:;echo;oc get backup -A; echo;echo Restores:;echo;oc get restore -A"
alias velero='oc -n openshift-adp exec deployment/velero -c velero -it -- ./velero'
velero restore describe <restore_resource_name> --details 1
velero restore describe <backup_resource_name> --details 1
oc edit <hosted_cluster_name> -n <hosted_cluster_namespace>
apiVersion: hypershift.openshift.io/v1alpha1 kind: HostedCluster metadata: name: <hosted_cluster_name> 1 namespace: <hosted_cluster_namespace> 2 spec: configuration: oauth: identityProviders: - openID: 3 claims: email: 4 - <email_address> name: 5 - <display_name> preferredUsername: 6 - <preferred_username> clientID: <client_id> 7 clientSecret: name: <client_id_secret_name> 8 issuer: https://example.com/identity 9 mappingMethod: lookup 10 name: IAM type: OpenID
spec: configuration: oauth: identityProviders: - openID: 1 claims: email: 2 - <email_address> name: 3 - <display_name> preferredUsername: 4 - <preferred_username> clientID: <client_id> 5 clientSecret: name: <client_id_secret_name> 6 issuer: https://example.com/identity 7 mappingMethod: lookup 8 name: IAM type: OpenID
oc get cloudcredentials <hosted_cluster_name> -n <hosted_cluster_namespace> -o=jsonpath={.spec.credentialsMode}
Manual
oc get authentication cluster --kubeconfig <hosted_cluster_name>.kubeconfig -o jsonpath --template '{.spec.serviceAccountIssuer }'
https://aos-hypershift-ci-oidc-29999.s3.us-east-2.amazonaws.com/hypershift-ci-29999
install: spec: clusterPermissions: - rules: - apiGroups: - "cloudcredential.openshift.io" resources: - credentialsrequests verbs: - create - delete - get - list - patch - update - watch
metadata: annotations: features.operators.openshift.io/token-auth-aws: "true"
// Get ENV var roleARN := os.Getenv("ROLEARN") setupLog.Info("getting role ARN", "role ARN = ", roleARN) webIdentityTokenPath := "/var/run/secrets/openshift/serviceaccount/token"
import ( minterv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var in = minterv1.AWSProviderSpec{ StatementEntries: []minterv1.StatementEntry{ { Action: []string{ "s3:*", }, Effect: "Allow", Resource: "arn:aws:s3:*:*:*", }, }, STSIAMRoleARN: "<role_arn>", } var codec = minterv1.Codec var ProviderSpec, _ = codec.EncodeProviderSpec(in.DeepCopyObject()) const ( name = "<credential_request_name>" namespace = "<namespace_name>" ) var CredentialsRequestTemplate = &minterv1.CredentialsRequest{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "openshift-cloud-credential-operator", }, Spec: minterv1.CredentialsRequestSpec{ ProviderSpec: ProviderSpec, SecretRef: corev1.ObjectReference{ Name: "<secret_name>", Namespace: namespace, }, ServiceAccountNames: []string{ "<service_account_name>", }, CloudTokenPath: "", }, }
// CredentialsRequest is a struct that represents a request for credentials type CredentialsRequest struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` Metadata struct { Name string `yaml:"name"` Namespace string `yaml:"namespace"` } `yaml:"metadata"` Spec struct { SecretRef struct { Name string `yaml:"name"` Namespace string `yaml:"namespace"` } `yaml:"secretRef"` ProviderSpec struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` StatementEntries []struct { Effect string `yaml:"effect"` Action []string `yaml:"action"` Resource string `yaml:"resource"` } `yaml:"statementEntries"` STSIAMRoleARN string `yaml:"stsIAMRoleARN"` } `yaml:"providerSpec"` // added new field CloudTokenPath string `yaml:"cloudTokenPath"` } `yaml:"spec"` } // ConsumeCredsRequestAddingTokenInfo is a function that takes a YAML filename and two strings as arguments // It unmarshals the YAML file to a CredentialsRequest object and adds the token information. func ConsumeCredsRequestAddingTokenInfo(fileName, tokenString, tokenPath string) (*CredentialsRequest, error) { // open a file containing YAML form of a CredentialsRequest file, err := os.Open(fileName) if err != nil { return nil, err } defer file.Close() // create a new CredentialsRequest object cr := &CredentialsRequest{} // decode the yaml file to the object decoder := yaml.NewDecoder(file) err = decoder.Decode(cr) if err != nil { return nil, err } // assign the string to the existing field in the object cr.Spec.CloudTokenPath = tokenPath // return the modified object return cr, nil }
// apply CredentialsRequest on install credReq := credreq.CredentialsRequestTemplate credReq.Spec.CloudTokenPath = webIdentityTokenPath c := mgr.GetClient() if err := c.Create(context.TODO(), credReq); err != nil { if !errors.IsAlreadyExists(err) { setupLog.Error(err, "unable to create CredRequest") os.Exit(1) } }
// WaitForSecret is a function that takes a Kubernetes client, a namespace, and a v1 "k8s.io/api/core/v1" name as arguments // It waits until the secret object with the given name exists in the given namespace // It returns the secret object or an error if the timeout is exceeded func WaitForSecret(client kubernetes.Interface, namespace, name string) (*v1.Secret, error) { // set a timeout of 10 minutes timeout := time.After(10 * time.Minute) 1 // set a polling interval of 10 seconds ticker := time.NewTicker(10 * time.Second) // loop until the timeout or the secret is found for { select { case <-timeout: // timeout is exceeded, return an error return nil, fmt.Errorf("timed out waiting for secret %s in namespace %s", name, namespace) // add to this error with a pointer to instructions for following a manual path to a Secret that will work on STS case <-ticker.C: // polling interval is reached, try to get the secret secret, err := client.CoreV1().Secrets(namespace).Get(context.Background(), name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { // secret does not exist yet, continue waiting continue } else { // some other error occurred, return it return nil, err } } else { // secret is found, return it return secret, nil } } } }
func SharedCredentialsFileFromSecret(secret *corev1.Secret) (string, error) { var data []byte switch { case len(secret.Data["credentials"]) > 0: data = secret.Data["credentials"] default: return "", errors.New("invalid secret for aws credentials") } f, err := ioutil.TempFile("", "aws-shared-credentials") if err != nil { return "", errors.Wrap(err, "failed to create file for shared credentials") } defer f.Close() if _, err := f.Write(data); err != nil { return "", errors.Wrapf(err, "failed to write credentials to %s", f.Name()) } return f.Name(), nil }
sharedCredentialsFile, err := SharedCredentialsFileFromSecret(secret) if err != nil { // handle error } options := session.Options{ SharedConfigState: session.SharedConfigEnable, SharedConfigFiles: []string{sharedCredentialsFile}, }
apiVersion: v1 kind: ConfigMap metadata: name: <configmap_name> namespace: clusters data: config: | apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: labels: machineconfiguration.openshift.io/role: worker name: <machineconfig_name> spec: config: ignition: version: 3.2.0 storage: files: - contents: source: data: mode: 420 overwrite: true path: USD{PATH} 1
oc edit nodepool <nodepool_name> --namespace <hosted_cluster_namespace>
apiVersion: hypershift.openshift.io/v1alpha1 kind: NodePool metadata: name: nodepool-1 namespace: clusters spec: config: - name: <configmap_name> 1
apiVersion: v1 kind: ConfigMap metadata: name: <configmap_name> 1 namespace: clusters data: config: | apiVersion: machineconfiguration.openshift.io/v1 kind: KubeletConfig metadata: name: <kubeletconfig_name> 2 spec: kubeletConfig: registerWithTaints: - key: "example.sh/unregistered" value: "true" effect: "NoExecute"
oc edit nodepool <nodepool_name> --namespace clusters 1
apiVersion: hypershift.openshift.io/v1alpha1 kind: NodePool metadata: name: nodepool-1 namespace: clusters spec: config: - name: <configmap_name> 1
apiVersion: v1 kind: ConfigMap metadata: name: tuned-1 namespace: clusters data: tuning: | apiVersion: tuned.openshift.io/v1 kind: Tuned metadata: name: tuned-1 namespace: openshift-cluster-node-tuning-operator spec: profile: - data: | [main] summary=Custom OpenShift profile include=openshift-node [sysctl] vm.dirty_ratio="55" name: tuned-1-profile recommend: - priority: 20 profile: tuned-1-profile
oc --kubeconfig="USDMGMT_KUBECONFIG" create -f tuned-1.yaml
apiVersion: hypershift.openshift.io/v1alpha1 kind: NodePool metadata: name: nodepool-1 namespace: clusters spec: tuningConfig: - name: tuned-1 status:
oc --kubeconfig="USDHC_KUBECONFIG" get tuned.tuned.openshift.io -n openshift-cluster-node-tuning-operator
NAME AGE default 7m36s rendered 7m36s tuned-1 65s
oc --kubeconfig="USDHC_KUBECONFIG" get profile.tuned.openshift.io -n openshift-cluster-node-tuning-operator
NAME TUNED APPLIED DEGRADED AGE nodepool-1-worker-1 tuned-1-profile True False 7m43s nodepool-1-worker-2 tuned-1-profile True False 7m14s
oc --kubeconfig="USDHC_KUBECONFIG" debug node/nodepool-1-worker-1 -- chroot /host sysctl vm.dirty_ratio
vm.dirty_ratio = 55
apiVersion: v1 kind: Namespace metadata: name: openshift-sriov-network-operator --- apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: sriov-network-operators namespace: openshift-sriov-network-operator spec: targetNamespaces: - openshift-sriov-network-operator
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: sriov-network-operator-subsription namespace: openshift-sriov-network-operator spec: channel: stable name: sriov-network-operator config: nodeSelector: node-role.kubernetes.io/worker: "" source: s/qe-app-registry/redhat-operators sourceNamespace: openshift-marketplace
oc get csv -n openshift-sriov-network-operator
NAME DISPLAY VERSION REPLACES PHASE sriov-network-operator.4.18.0-202211021237 SR-IOV Network Operator 4.18.0-202211021237 sriov-network-operator.4.18.0-202210290517 Succeeded
oc get pods -n openshift-sriov-network-operator
oc edit <hosted_cluster_name> -n <hosted_cluster_namespace>
apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: name: <hosted_cluster_name> 1 namespace: <hosted_cluster_namespace> 2 spec: configuration: featureGate: featureSet: TechPreviewNoUpgrade 3
oc get featuregate cluster -o yaml
oc set env -n hypershift deployment/operator METRICS_SET=All
kubeAPIServer: - action: "drop" regex: "etcd_(debugging|disk|server).*" sourceLabels: ["__name__"] - action: "drop" regex: "apiserver_admission_controller_admission_latencies_seconds_.*" sourceLabels: ["__name__"] - action: "drop" regex: "apiserver_admission_step_admission_latencies_seconds_.*" sourceLabels: ["__name__"] - action: "drop" regex: "scheduler_(e2e_scheduling_latency_microseconds|scheduling_algorithm_predicate_evaluation|scheduling_algorithm_priority_evaluation|scheduling_algorithm_preemption_evaluation|scheduling_algorithm_latency_microseconds|binding_latency_microseconds|scheduling_latency_seconds)" sourceLabels: ["__name__"] - action: "drop" regex: "apiserver_(request_count|request_latencies|request_latencies_summary|dropped_requests|storage_data_key_generation_latencies_microseconds|storage_transformation_failures_total|storage_transformation_latencies_microseconds|proxy_tunnel_sync_latency_secs)" sourceLabels: ["__name__"] - action: "drop" regex: "docker_(operations|operations_latency_microseconds|operations_errors|operations_timeout)" sourceLabels: ["__name__"] - action: "drop" regex: "reflector_(items_per_list|items_per_watch|list_duration_seconds|lists_total|short_watches_total|watch_duration_seconds|watches_total)" sourceLabels: ["__name__"] - action: "drop" regex: "etcd_(helper_cache_hit_count|helper_cache_miss_count|helper_cache_entry_count|request_cache_get_latencies_summary|request_cache_add_latencies_summary|request_latencies_summary)" sourceLabels: ["__name__"] - action: "drop" regex: "transformation_(transformation_latencies_microseconds|failures_total)" sourceLabels: ["__name__"] - action: "drop" regex: "network_plugin_operations_latency_microseconds|sync_proxy_rules_latency_microseconds|rest_client_request_latency_seconds" sourceLabels: ["__name__"] - action: "drop" regex: "apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)" sourceLabels: ["__name__", "le"] kubeControllerManager: - action: "drop" regex: "etcd_(debugging|disk|request|server).*" sourceLabels: ["__name__"] - action: "drop" regex: "rest_client_request_latency_seconds_(bucket|count|sum)" sourceLabels: ["__name__"] - action: "drop" regex: "root_ca_cert_publisher_sync_duration_seconds_(bucket|count|sum)" sourceLabels: ["__name__"] openshiftAPIServer: - action: "drop" regex: "etcd_(debugging|disk|server).*" sourceLabels: ["__name__"] - action: "drop" regex: "apiserver_admission_controller_admission_latencies_seconds_.*" sourceLabels: ["__name__"] - action: "drop" regex: "apiserver_admission_step_admission_latencies_seconds_.*" sourceLabels: ["__name__"] - action: "drop" regex: "apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)" sourceLabels: ["__name__", "le"] openshiftControllerManager: - action: "drop" regex: "etcd_(debugging|disk|request|server).*" sourceLabels: ["__name__"] openshiftRouteControllerManager: - action: "drop" regex: "etcd_(debugging|disk|request|server).*" sourceLabels: ["__name__"] olm: - action: "drop" regex: "etcd_(debugging|disk|server).*" sourceLabels: ["__name__"] catalogOperator: - action: "drop" regex: "etcd_(debugging|disk|server).*" sourceLabels: ["__name__"] cvo: - action: drop regex: "etcd_(debugging|disk|server).*" sourceLabels: ["__name__"]
kind: ConfigMap apiVersion: v1 metadata: name: hypershift-operator-install-flags namespace: local-cluster data: installFlagsToAdd: "--monitoring-dashboards" installFlagsToRemove: ""
- name: MONITORING_DASHBOARDS value: "1"
oc adm must-gather --image=registry.redhat.io/multicluster-engine/must-gather-rhel9:v<mce_version> /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME --dest-dir=NAME ; tar -cvzf NAME.tgz NAME
REGISTRY=registry.example.com:5000 IMAGE=USDREGISTRY/multicluster-engine/must-gather-rhel8@sha256:ff9f37eb400dc1f7d07a9b6f2da9064992934b69847d17f59e385783c071b9d8 oc adm must-gather --image=USDIMAGE /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME --dest-dir=./data
oc get vm -n <namespace>
oc get pods -n <namespace> | grep "import"
oc get pods -n <namespace> -l kubevirt.io=virt-launcher
oc get nodes -o yaml
oc get pods -A --field-selector=status.phase!=Running,status,phase!=Succeeded
oc patch ingresscontroller -n openshift-ingress-operator default --type=json -p '[{ "op": "add", "path": "/spec/routeAdmission", "value": {wildcardPolicy: "WildcardsAllowed"}}]'
oc get pods -n <hosted_control_plane_namespace> -l app=cloud-controller-manager
oc get pods -n openshift-cluster-csi-drivers -o wide -l app=kubevirt-csi-driver
oc get pods -n <hcp namespace> -l app=kubevirt-csi-driver
oc get imagecontentsourcepolicy -o json | jq -r '.items[].spec.repositoryDigestMirrors[0].mirrors[0]'
oc get clusterversion version -ojsonpath='{.status.desired.image}'
oc image extract --file /release-manifests/0000_50_installer_coreos-bootimages.yaml <payload_image> --confirm
cat 0000_50_installer_coreos-bootimages.yaml | yq -r .data.stream | jq -r '.architectures.x86_64.images.kubevirt."digest-ref"'
oc image mirror <rhcos_image> <internal_registry>
apiVersion: config.openshift.io/v1 kind: ImageDigestMirrorSet metadata: name: rhcos-boot-kubevirt spec: repositoryDigestMirrors: - mirrors: - <rhcos_image_no_digest> 1 source: virthost.ostest.test.metalkube.org:5000/localimages/ocp-v4.0-art-dev 2
oc apply -f rhcos-boot-kubevirt.yaml
oc logs -n multicluster-engine <assisted_service_pod_name> 1
error="failed to get pull secret for update: invalid pull secret data in secret pull-secret"
pull secret must contain auth for \"registry.redhat.io\"
oc annotate hostedcluster -n <hosted_cluster_namespace> <hosted_cluster_name> hypershift.openshift.io/restart-date=USD(date --iso-8601=seconds) 1
oc patch -n <hosted_cluster_namespace> hostedclusters/<hosted_cluster_name> -p '{"spec":{"pausedUntil":"<timestamp>"}}' --type=merge 1
oc patch -n <hosted_cluster_namespace> hostedclusters/<hosted_cluster_name> -p '{"spec":{"pausedUntil":"true"}}' --type=merge
oc patch -n <hosted_cluster_namespace> hostedclusters/<hosted_cluster_name> -p '{"spec":{"pausedUntil":null}}' --type=merge
export KUBECONFIG=<install_directory>/auth/kubeconfig
oc get nodepool --namespace <hosted_cluster_namespace>
oc edit nodepool <nodepool_name> --namespace <hosted_cluster_namespace>
apiVersion: hypershift.openshift.io/v1alpha1 kind: NodePool metadata: name: nodepool-1 namespace: clusters spec: arch: amd64 clusterName: clustername 1 management: autoRepair: false replace: rollingUpdate: maxSurge: 1 maxUnavailable: 0 strategy: RollingUpdate upgradeType: Replace nodeDrainTimeout: 0s 2
oc scale nodepool/<nodepool_name> --namespace <hosted_cluster_namespace> --replicas=0
oc scale nodepool/<nodepool_name> --namespace <hosted_cluster_namespace> --replicas=1
oc get nodepool -n <hosted_cluster_namespace> <nodepool_name> -ojsonpath='{.spec.nodeDrainTimeout}'
oc delete managedcluster <hosted_cluster_name> 1
hcp destroy cluster aws --name <hosted_cluster_name> \ 1 --infra-id <infra_id> \ 2 --role-arn <arn_role> \ 3 --sts-creds <path_to_sts_credential_file> \ 4 --base-domain <basedomain> 5
aws sts get-session-token --output json > sts-creds.json
hcp destroy cluster agent --name <hosted_cluster_name> 1
oc delete managedcluster <hosted_cluster_name>
hcp destroy cluster kubevirt --name <hosted_cluster_name>
oc -n <hosted_cluster_namespace> scale nodepool <nodepool_name> --replicas 0
oc --kubeconfig <hosted_cluster_name>.kubeconfig delete node <compute_node_name>
oc --kubeconfig <hosted_cluster_name>.kubeconfig get nodes
oc -n <hosted_control_plane_namespace> delete agent <agent_name>
hcp destroy cluster agent --name <hosted_cluster_name> --namespace <hosted_cluster_namespace>
hcp destroy cluster agent --name=<hosted_cluster_name> \ 1 --namespace=<hosted_cluster_namespace> \ 2 --cluster-grace-period <duration> 3
hcp destroy cluster agent --name <hosted_cluster_name> 1
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: import.open-cluster-management.io/hosting-cluster-name: local-cluster import.open-cluster-management.io/klusterlet-deploy-mode: Hosted open-cluster-management/created-via: hypershift labels: cloud: auto-detect cluster.open-cluster-management.io/clusterset: default name: <hosted_cluster_name> 1 vendor: OpenShift name: <hosted_cluster_name> spec: hubAcceptsClient: true leaseDurationSeconds: 60
oc apply -f <file_name> 1
apiVersion: agent.open-cluster-management.io/v1 kind: KlusterletAddonConfig metadata: name: <hosted_cluster_name> 1 namespace: <hosted_cluster_namespace> 2 spec: clusterName: <hosted_cluster_name> clusterNamespace: <hosted_cluster_namespace> clusterLabels: cloud: auto-detect vendor: auto-detect applicationManager: enabled: true certPolicyController: enabled: true iamPolicyController: enabled: true policyController: enabled: true searchCollector: enabled: false
oc apply -f <file_name> 1
oc get managedcluster <hosted_cluster_name>
oc edit addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: hypershift-addon-deploy-config namespace: multicluster-engine spec: customizedVariables: - name: hcMaxNumber value: "80" - name: hcThresholdNumber value: "60" - name: autoImportDisabled value: "true"