Clusters Red Hat Advanced Cluster Management for Kubernetes 2.11 Cluster management

Commands:
delete csv -n openshift-operators volsync-product.v0.6.0
E0203 07:10:38.266841 1 reflector.go:138] sigs.k8s.io/controller-runtime/pkg/cache/internal/informers_map.go:224: Failed to watch *v1.ClusterClaim: failed to list *v1.ClusterClaim: v1.ClusterClaimList.Items: []v1.ClusterClaim: v1.ClusterClaim.v1.ClusterClaim.Spec: v1.ClusterClaimSpec.Lifetime: unmarshalerDecoder: time: unknown unit "w" in duration "1w", error found in #10 byte of ...|time":"1w"}},{"apiVe|..., bigger context ...|clusterPoolName":"policy-aas-hubs","lifetime":"1w"}},{"apiVersion":"hive.openshift.io/v1","kind":"Cl|
delete deployment multiclusterhub-repo -n <namespace>
get bmh -n <cluster_provisioning_namespace>
describe bmh -n <cluster_provisioning_namespace> <bmh_name>
Status: Error Count: 1 Error Message: Image provisioning failed: ... [Errno 36] File name too long
edit mch -n open-cluster-management multiclusterhub
delete managedcluster local-cluster
edit mch -n open-cluster-management multiclusterhub
edit clusterdeployment/<mycluster> -n <namespace>
delete ns <namespace>
compute: - hyperthreading: Enabled name: 'worker' replicas: 3 platform: azure: type: Standard_D2s_v3 osDisk: diskSizeGB: 128 vmNetworkingType: 'Basic'
status: agentLabelSelector: matchLabels: infraenvs.agent-install.openshift.io: qe2 bootArtifacts: initrd: https://assisted-image-service-multicluster-engine.redhat.com/images/0000/pxe-initrd?api_key=0000000&arch=x86_64&version=4.11 ipxeScript: https://assisted-service-multicluster-engine.redhat.com/api/assisted-install/v2/infra-envs/00000/downloads/files?api_key=000000000&file_name=ipxe-script kernel: https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/4.12/latest/rhcos-live-kernel-x86_64 rootfs: https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/4.12/latest/rhcos-live-rootfs.x86_64.img
for artifact in oc get infraenv qe2 -ojsonpath="{.status.bootArtifacts}" | jq ". | keys[]" | sed "s/\"//g" do curl -k oc get infraenv qe2 -ojsonpath="{.status.bootArtifacts.USD{artifact}}"` -o USDartifact
patch clusterdeployment <clusterdeployment-name> -p '{"metadata":{"finalizers":null}}' --type=merge
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: cluster namespace: cluster spec: ignitionConfigOverride: '{"ignition":{"version":"3.2.0"},"storage":{"files":[{"path":"/etc/containers/policy.json","mode":420,"overwrite":true,"contents":{"source":"data:text/plain;charset=utf-8;base64,ewogICAgImRlZmF1bHQiOiBbCiAgICAgICAgewogICAgICAgICAgICAidHlwZSI6ICJpbnNlY3VyZUFjY2VwdEFueXRoaW5nIgogICAgICAgIH0KICAgIF0sCiAgICAidHJhbnNwb3J0cyI6CiAgICAgICAgewogICAgICAgICAgICAiZG9ja2VyLWRhZW1vbiI6CiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgIiI6IFt7InR5cGUiOiJpbnNlY3VyZUFjY2VwdEFueXRoaW5nIn1dCiAgICAgICAgICAgICAgICB9CiAgICAgICAgfQp9"}}]}}'
{ "default": [ { "type": "insecureAcceptAnything" } ], "transports": { "docker-daemon": { "": [ { "type": "insecureAcceptAnything" } ] } } }
apiVersion: v1 kind: ConfigMap metadata: name: my-assisted-service-config namespace: multicluster-engine data: ALLOW_CONVERGED_FLOW: "false" 1
annotate --overwrite AgentServiceConfig agent unsupported.agent-install.openshift.io/assisted-service-configmap=my-assisted-service-config
Error querying resource logs: Service unavailable
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv spec: osImageVersion: 4.15
get nodepool -A
get nodes --kubeconfig
get agents -A
create clusterrolebinding <role-binding-name> --clusterrole=open-cluster-management:cluster-manager-admin --user=<username>
create clusterrolebinding (role-binding-name) --clusterrole=open-cluster-management:admin:<cluster-name> --user=<username>
create rolebinding <role-binding-name> -n <cluster-name> --clusterrole=admin --user=<username>
create clusterrolebinding <role-binding-name> --clusterrole=open-cluster-management:view:<cluster-name> --user=<username>
create rolebinding <role-binding-name> -n <cluster-name> --clusterrole=view --user=<username>
get managedclusters.clusterview.open-cluster-management.io
get managedclustersets.clusterview.open-cluster-management.io
adm policy add-cluster-role-to-group open-cluster-management:clusterset-admin:server-foundation-clusterset server-foundation-team-admin
adm policy add-cluster-role-to-group open-cluster-management:clusterset-view:server-foundation-clusterset server-foundation-team-user
adm new-project server-foundation-clusterpool adm policy add-role-to-group admin server-foundation-team-admin --namespace server-foundation-clusterpool
-n openshift-console get route console
console console-openshift-console.apps.new-coral.purple-chesterfield.com console https reencrypt/Redirect None
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: {}
create namespace <namespace>
project <namespace>
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: <default> namespace: <namespace> spec: targetNamespaces: - <namespace>
apply -f <path-to-file>/<operator-group>.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: multicluster-engine spec: sourceNamespace: openshift-marketplace source: redhat-operators channel: stable-2.6 installPlanApproval: Automatic name: multicluster-engine
apply -f <path-to-file>/<subscription>.yaml
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: {}
apply -f <path-to-file>/<custom-resource>.yaml
error: unable to recognize "./mce.yaml": no matches for kind "MultiClusterEngine" in version "operator.multicluster-engine.io/v1"
get mce -o=jsonpath='{.items[0].status.phase}'
metadata: labels: node-role.kubernetes.io/infra: "" spec: taints: - effect: NoSchedule key: node-role.kubernetes.io/infra
spec: config: nodeSelector: node-role.kubernetes.io/infra: "" tolerations: - key: node-role.kubernetes.io/infra effect: NoSchedule operator: Exists
spec: nodeSelector: node-role.kubernetes.io/infra: ""
-n openshift-console get route console
console console-openshift-console.apps.new-name.purple-name.com console https reencrypt/Redirect None
additionalTrustBundle: | -----BEGIN CERTIFICATE----- certificate_content -----END CERTIFICATE----- sshKey: >-
- mirrors: - <your_registry>/rhacm2 source: registry.redhat.io/rhacm2 - mirrors: - <your_registry>/quay source: registry.redhat.io/quay - mirrors: - <your_registry>/compliance source: registry.redhat.io/compliance
apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: mce-repo spec: repositoryDigestMirrors: - mirrors: - mirror.registry.com:5000/multicluster-engine source: registry.redhat.io/multicluster-engine
apply -f mce-policy.yaml
UPSTREAM_REGISTRY=quay.io PRODUCT_REPO=openshift-release-dev RELEASE_NAME=ocp-release OCP_RELEASE=4.15.2-x86_64 LOCAL_REGISTRY=USD(hostname):5000 LOCAL_SECRET_JSON=<pull-secret> adm -a USD{LOCAL_SECRET_JSON} release mirror --from=USD{UPSTREAM_REGISTRY}/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE} --to=USD{LOCAL_REGISTRY}/ocp4 --to-release-image=USD{LOCAL_REGISTRY}/ocp4/release:USD{OCP_RELEASE}
git clone https://github.com/openshift/cincinnati-graph-data
build -f <docker-path> -t <graph-path>:latest
push <graph-path>:latest --authfile=<pull-secret>.json
apiVersion: v1 kind: ConfigMap metadata: name: trusted-ca data: updateservice-registry: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE-----
patch image.config.openshift.io cluster -p '{"spec":{"additionalTrustedCA":{"name":"<trusted_ca>"}}}' --type merge
apiVersion: update-service.openshift.io/v1beta2 kind: update-service metadata: name: openshift-cincinnati-instance namespace: openshift-update-service spec: registry: <registry-host-name>:<port> 1 replicas: 1 repository: USD{LOCAL_REGISTRY}/ocp4/release graphDataImage: '<host-name>:<port>/cincinnati-graph-data-container' 2
apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-mirror namespace: default spec: disabled: false remediationAction: enforce policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-image-content-source-policy spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: <your-local-mirror-name> 1 spec: repositoryDigestMirrors: - mirrors: - <your-registry> 2 source: registry.redhat.io --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-mirror namespace: default placementRef: name: placement-policy-mirror kind: PlacementRule apiGroup: apps.open-cluster-management.io subjects: - name: policy-mirror kind: Policy apiGroup: policy.open-cluster-management.io --- apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: placement-policy-mirror namespace: default spec: clusterConditions: - status: "True" type: ManagedClusterConditionAvailable clusterSelector: matchExpressions: [] 3
apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-catalog namespace: default spec: disabled: false remediationAction: enforce policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-catalog spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: config.openshift.io/v1 kind: OperatorHub metadata: name: cluster spec: disableAllDefaultSources: true - complianceType: musthave objectDefinition: apiVersion: operators.coreos.com/v1alpha1 kind: CatalogSource metadata: name: my-operator-catalog namespace: openshift-marketplace spec: sourceType: grpc image: '<registry_host_name>:<port>/olm/redhat-operators:v1' 1 displayName: My Operator Catalog publisher: grpc --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-catalog namespace: default placementRef: name: placement-policy-catalog kind: PlacementRule apiGroup: apps.open-cluster-management.io subjects: - name: policy-catalog kind: Policy apiGroup: policy.open-cluster-management.io --- apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: placement-policy-catalog namespace: default spec: clusterConditions: - status: "True" type: ManagedClusterConditionAvailable clusterSelector: matchExpressions: [] 2
get clusterversion -o yaml
get routes
apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-cluster-version namespace: default annotations: policy.open-cluster-management.io/standards: null policy.open-cluster-management.io/categories: null policy.open-cluster-management.io/controls: null spec: disabled: false remediationAction: enforce policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-cluster-version spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: config.openshift.io/v1 kind: ClusterVersion metadata: name: version spec: channel: stable-4.4 upstream: >- https://example-cincinnati-policy-engine-uri/api/upgrades_info/v1/graph 1 --- apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-cluster-version namespace: default placementRef: name: placement-policy-cluster-version kind: PlacementRule apiGroup: apps.open-cluster-management.io subjects: - name: policy-cluster-version kind: Policy apiGroup: policy.open-cluster-management.io --- apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: name: placement-policy-cluster-version namespace: default spec: clusterConditions: - status: "True" type: ManagedClusterConditionAvailable clusterSelector: matchExpressions: [] 2
get clusterversion -o yaml
apiVersion: v1 items: - apiVersion: config.openshift.io/v1 kind: ClusterVersion [..] spec: channel: stable-4.4 upstream: https://<hub-cincinnati-uri>/api/upgrades_info/v1/graph
apiVersion: operator.open-cluster-management.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: <name> 1 enabled: true
patch MultiClusterEngine <multiclusterengine-name> --type=json -p='[{"op": "add", "path": "/spec/overrides/components/-","value":{"name":"<name>","enabled":true}}]'
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: local-cluster enabled: false
create secret generic <secret> -n <namespace> --from-file=.dockerconfigjson=<path-to-pull-secret> --type=kubernetes.io/dockerconfigjson
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: imagePullSecret: <secret>
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: targetNamespace: <target>
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: availabilityConfig: "Basic"
spec: nodeSelector: node-role.kubernetes.io/infra: ""
spec: tolerations: - key: node-role.kubernetes.io/infra effect: NoSchedule operator: Exists
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: managedserviceaccount enabled: true
patch MultiClusterEngine <multiclusterengine-name> --type=json -p='[{"op": "add", "path": "/spec/overrides/components/-","value":{"name":"managedserviceaccount","enabled":true}}]'
Cannot delete MultiClusterEngine resource because ManagedCluster resource(s) exist
project <namespace>
delete multiclusterengine --all
get multiclusterengine -o yaml
❯ oc get csv NAME DISPLAY VERSION REPLACES PHASE multicluster-engine.v2.0.0 multicluster engine for Kubernetes 2.0.0 Succeeded ❯ oc delete clusterserviceversion multicluster-engine.v2.0.0 ❯ oc delete sub multicluster-engine
#!/bin/bash delete apiservice v1.admission.cluster.open-cluster-management.io v1.admission.work.open-cluster-management.io delete validatingwebhookconfiguration multiclusterengines.multicluster.openshift.io delete mce --all
curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: addonDeploymentConfig metadata: name: addon-ns-config namespace: multicluster-engine spec: agentInstallNamespace: open-cluster-management-agent-addon-discovery
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: work-manager spec: addonMeta: displayName: work-manager installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All type: Placements
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: work-manager spec: addonMeta: displayName: work-manager installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All configs: - group: addon.open-cluster-management.io name: addon-ns-config namespace: multicluster-engine resource: addondeploymentconfigs type: Placements
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: managed-serviceaccount spec: addonMeta: displayName: managed-serviceaccount installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All configs: - group: addon.open-cluster-management.io name: addon-ns-config namespace: multicluster-engine resource: addondeploymentconfigs type: Placements
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: cluster-proxy spec: addonMeta: displayName: cluster-proxy installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All configs: - group: addon.open-cluster-management.io name: addon-ns-config namespace: multicluster-engine resource: addondeploymentconfigs type: Placements
get deployment -n open-cluster-management-agent-addon-discovery
NAME READY UP-TO-DATE AVAILABLE AGE cluster-proxy-proxy-agent 1/1 1 1 24h klusterlet-addon-workmgr 1/1 1 1 24h managed-serviceaccount-addon-agent 1/1 1 1 24h
kind: KlusterletConfig apiVersion: config.open-cluster-management.io/v1alpha1 metadata: name: mce-import-klusterlet-config spec: installMode: type: noOperator noOperator: postfix: mce-import
label addondeploymentconfig addon-ns-config -n multicluster-engine cluster.open-cluster-management.io/backup=true
label addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine cluster.open-cluster-management.io/backup=true
label clustermanagementaddon work-manager cluster.open-cluster-management.io/backup=true
label clustermanagementaddon cluster-proxy cluster.open-cluster-management.io/backup=true
label clustermanagementaddon managed-serviceaccount cluster.open-cluster-management.io/backup=true
label KlusterletConfig mce-import-klusterlet-config cluster.open-cluster-management.io/backup=true
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: agent.open-cluster-management.io/klusterlet-config: mce-import-klusterlet-config 1 labels: cloud: auto-detect vendor: auto-detect name: mce-a 2 spec: hubAcceptsClient: true leaseDurationSeconds: 60
get managedcluster
NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE local-cluster true https://<api.acm-hub.com:port> True True 44h mce-a true https://<api.mce-a.com:port> True True 27s
patch addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine --type=merge -p '{"spec":{"agentInstallNamespace":"open-cluster-management-agent-addon-discovery"}}'
patch addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine --type=merge -p '{"spec":{"customizedVariables":[{"name":"disableMetrics","value": "true"},{"name":"disableHOManagement","value": "true"}]}}'
clusteradm addon enable --names hypershift-addon --clusters <managed-cluster-names>
get managedcluster
get deployment -n open-cluster-management-agent-addon-discovery
NAME READY UP-TO-DATE AVAILABLE AGE cluster-proxy-proxy-agent 1/1 1 1 24h klusterlet-addon-workmgr 1/1 1 1 24h hypershift-addon-agent 1/1 1 1 24h managed-serviceaccount-addon-agent 1/1 1 1 24h
apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: creationTimestamp: "2024-05-30T23:05:39Z" generation: 1 labels: hypershift.open-cluster-management.io/hc-name: hosted-cluster-1 hypershift.open-cluster-management.io/hc-namespace: clusters name: hosted-cluster-1 namespace: mce-1 resourceVersion: "1740725" uid: b4c36dca-a0c4-49f9-9673-f561e601d837 spec: apiUrl: https://a43e6fe6dcef244f8b72c30426fb6ae3-ea3fec7b113c88da.elb.us-west-1.amazonaws.com:6443 cloudProvider: aws creationTimestamp: "2024-05-30T23:02:45Z" credential: {} displayName: mce-1-hosted-cluster-1 importAsManagedCluster: false isManagedCluster: false name: hosted-cluster-1 openshiftVersion: 0.0.0 status: Active type: MultiClusterEngineHCP
apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-mce-hcp-autoimport namespace: open-cluster-management-global-set annotations: policy.open-cluster-management.io/standards: NIST SP 800-53 policy.open-cluster-management.io/categories: CM Configuration Management policy.open-cluster-management.io/controls: CM-2 Baseline Configuration policy.open-cluster-management.io/description: Discovered clusters that are of type MultiClusterEngineHCP can be automatically imported into ACM as managed clusters. This policy configure those discovered clusters so they are automatically imported. Fine tuning MultiClusterEngineHCP clusters to be automatically imported can be done by configure filters at the configMap or add annotation to the discoverd cluster. spec: disabled: false policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: mce-hcp-autoimport-config spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: v1 kind: ConfigMap metadata: name: discovery-config namespace: open-cluster-management-global-set data: rosa-filter: "" remediationAction: enforce 1 severity: low - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-mce-hcp-autoimport spec: remediationAction: enforce severity: low object-templates-raw: | {{- /* find the MultiClusterEngineHCP DiscoveredClusters */ -}} {{- range USDdc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }} {{- /* Check for the flag that indicates the import should be skipped */ -}} {{- USDskip := "false" -}} {{- range USDkey, USDvalue := USDdc.metadata.annotations }} {{- if and (eq USDkey "discovery.open-cluster-management.io/previously-auto-imported") (eq USDvalue "true") }} {{- USDskip = "true" }} {{- end }} {{- end }} {{- /* if the type is MultiClusterEngineHCP and the status is Active */ -}} {{- if and (eq USDdc.spec.status "Active") (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "mce-hcp-filter") USDdc.spec.displayName) (eq USDdc.spec.type "MultiClusterEngineHCP") (eq USDskip "false") }} - complianceType: musthave objectDefinition: apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: {{ USDdc.metadata.name }} namespace: {{ USDdc.metadata.namespace }} spec: importAsManagedCluster: true 2 {{- end }} {{- end }}
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: policy-mce-hcp-autoimport-placement namespace: open-cluster-management-global-set spec: tolerations: - key: cluster.open-cluster-management.io/unreachable operator: Exists - key: cluster.open-cluster-management.io/unavailable operator: Exists clusterSets: - global predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: local-cluster operator: In values: - "true"
apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: policy-mce-hcp-autoimport-placement-binding namespace: open-cluster-management-global-set placementRef: name: policy-mce-hcp-autoimport-placement apiGroup: cluster.open-cluster-management.io kind: Placement subjects: - name: policy-mce-hcp-autoimport apiGroup: policy.open-cluster-management.io kind: Policy
get policy policy-mce-hcp-autoimport -n <namespace>
annotations: discovery.open-cluster-management.io/previously-auto-imported: "true"
apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-rosa-autoimport annotations: policy.open-cluster-management.io/standards: NIST SP 800-53 policy.open-cluster-management.io/categories: CM Configuration Management policy.open-cluster-management.io/controls: CM-2 Baseline Configuration policy.open-cluster-management.io/description: OpenShift Service on AWS discovered clusters can be automatically imported into Red Hat Advanced Cluster Management as managed clusters with this policy. You can select and configure those managed clusters so you can import. Configure filters or add an annotation if you do not want all of your OpenShift Service on AWS clusters to be automatically imported. spec: remediationAction: inform 1 disabled: false policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: rosa-autoimport-config spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: v1 kind: ConfigMap metadata: name: discovery-config namespace: open-cluster-management-global-set data: rosa-filter: "" 2 remediationAction: enforce severity: low - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-rosa-autoimport spec: remediationAction: enforce severity: low object-templates-raw: | {{- /* find the ROSA DiscoveredClusters */ -}} {{- range USDdc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }} {{- /* Check for the flag that indicates the import should be skipped */ -}} {{- USDskip := "false" -}} {{- range USDkey, USDvalue := USDdc.metadata.annotations }} {{- if and (eq USDkey "discovery.open-cluster-management.io/previously-auto-imported") (eq USDvalue "true") }} {{- USDskip = "true" }} {{- end }} {{- end }} {{- /* if the type is ROSA and the status is Active */ -}} {{- if and (eq USDdc.spec.status "Active") (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "rosa-filter") USDdc.spec.displayName) (eq USDdc.spec.type "ROSA") (eq USDskip "false") }} - complianceType: musthave objectDefinition: apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: {{ USDdc.metadata.name }} namespace: {{ USDdc.metadata.namespace }} spec: importAsManagedCluster: true {{- end }} {{- end }} - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-rosa-managedcluster-status spec: remediationAction: enforce severity: low object-templates-raw: | {{- /* Use the same DiscoveredCluster list to check ManagedCluster status */ -}} {{- range USDdc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }} {{- /* Check for the flag that indicates the import should be skipped */ -}} {{- USDskip := "false" -}} {{- range USDkey, USDvalue := USDdc.metadata.annotations }} {{- if and (eq USDkey "discovery.open-cluster-management.io/previously-auto-imported") (eq USDvalue "true") }} {{- USDskip = "true" }} {{- end }} {{- end }} {{- /* if the type is ROSA and the status is Active */ -}} {{- if and (eq USDdc.spec.status "Active") (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "rosa-filter") USDdc.spec.displayName) (eq USDdc.spec.type "ROSA") (eq USDskip "false") }} - complianceType: musthave objectDefinition: apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: {{ USDdc.spec.displayName }} namespace: {{ USDdc.spec.displayName }} status: conditions: - type: ManagedClusterConditionAvailable status: "True" {{- end }} {{- end }}
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement-openshift-plus-hub spec: predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: name operator: In values: - local-cluster
apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-rosa-autoimport placementRef: apiGroup: cluster.open-cluster-management.io kind: Placement name: placement-policy-rosa-autoimport subjects: - apiGroup: policy.open-cluster-management.io kind: Policy name: policy-rosa-autoimport
get policy policy-rosa-autoimport -n <namespace>
kind: Secret metadata: name: <managed-cluster-name>-aws-creds namespace: <managed-cluster-namespace> type: Opaque data: aws_access_key_id: USD(echo -n "USD{AWS_KEY}" | base64 -w0) aws_secret_access_key: USD(echo -n "USD{AWS_SECRET}" | base64 -w0)
label secret hypershift-operator-oidc-provider-s3-credentials -n local-cluster "cluster.open-cluster-management.io/type=awss3" label secret hypershift-operator-oidc-provider-s3-credentials -n local-cluster "cluster.open-cluster-management.io/credentials=credentials="
az ad sp create-for-rbac --role Contributor --name <service_principal> --scopes <subscription_path>
az ad sp create-for-rbac --role Contributor --name <service_principal> --scopes <subscription_path>
az account show
az account show
kind: Secret metadata: name: <managed-cluster-name>-azure-creds namespace: <managed-cluster-namespace> type: Opaque data: baseDomainResourceGroupName: USD(echo -n "USD{azure_resource_group_name}" | base64 -w0) osServicePrincipal.json: USD(base64 -w0 "USD{AZURE_CRED_JSON}")
kind: Secret metadata: name: <managed-cluster-name>-gcp-creds namespace: <managed-cluster-namespace> type: Opaque data: osServiceAccount.json: USD(base64 -w0 "USD{GCP_CRED_JSON}")
- mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release-nightly - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-v4.0-art-dev
- mirrors: - registry.example.com:5000/rhacm2 source: registry.redhat.io/rhacm2
kind: Secret metadata: name: <managed-cluster-name>-vsphere-creds namespace: <managed-cluster-namespace> type: Opaque data: username: USD(echo -n "USD{VMW_USERNAME}" | base64 -w0) password.json: USD(base64 -w0 "USD{VMW_PASSWORD}")
- mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release-nightly - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-release - mirrors: - registry.example.com:5000/ocp4 source: quay.io/openshift-release-dev/ocp-v4.0-art-dev
- mirrors: - registry.example.com:5000/rhacm2 source: registry.redhat.io/rhacm2
kind: Secret metadata: name: <managed-cluster-name>-osp-creds namespace: <managed-cluster-namespace> type: Opaque data: clouds.yaml: USD(base64 -w0 "USD{OSP_CRED_YAML}") cloud: USD(echo -n "openstack" | base64 -w0)
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast visible: 'true' name: img4.x.1-x86-64-appsub spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.x.1-x86_64
get clusterimageset
quay.io/openshift-release-dev/ocp-release:4.6.8-x86_64
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast visible: 'true' name: img4.x.0-multi-appsub spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.x.0-multi
pull quay.io/openshift-release-dev/ocp-release:4.x.1-x86_64 pull quay.io/openshift-release-dev/ocp-release:4.x.1-ppc64le pull quay.io/openshift-release-dev/ocp-release:4.x.1-s390x pull quay.io/openshift-release-dev/ocp-release:4.x.1-aarch64
login <private-repo>
push quay.io/openshift-release-dev/ocp-release:4.x.1-x86_64 <private-repo>/ocp-release:4.x.1-x86_64 push quay.io/openshift-release-dev/ocp-release:4.x.1-ppc64le <private-repo>/ocp-release:4.x.1-ppc64le push quay.io/openshift-release-dev/ocp-release:4.x.1-s390x <private-repo>/ocp-release:4.x.1-s390x push quay.io/openshift-release-dev/ocp-release:4.x.1-aarch64 <private-repo>/ocp-release:4.x.1-aarch64
manifest create mymanifest
manifest add mymanifest <private-repo>/ocp-release:4.x.1-x86_64 manifest add mymanifest <private-repo>/ocp-release:4.x.1-ppc64le manifest add mymanifest <private-repo>/ocp-release:4.x.1-s390x manifest add mymanifest <private-repo>/ocp-release:4.x.1-aarch64
manifest push mymanifest docker://<private-repo>/ocp-release:4.x.1
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast visible: "true" name: img4.x.1-appsub spec: releaseImage: <private-repo>/ocp-release:4.x.1
apply -f <file-name>.yaml
apiVersion: v1 kind: ConfigMap metadata: name: cluster-image-set-git-repo namespace: multicluster-engine data: gitRepoUrl: <forked acm-hive-openshift-releases repository URL> gitRepoBranch: backplane-<2.x> gitRepoPath: clusterImageSets channel: <fast or stable>
get clusterImageSets delete clusterImageSet <clusterImageSet_NAME>
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: fast name: img<4.x.x>-x86-64-appsub spec: releaseImage: IMAGE_REGISTRY_IPADDRESS_or__DNSNAME/REPO_PATH/ocp-release@sha256:073a4e46289be25e2a05f5264c8f1d697410db66b960c9ceeddebd1c61e58717
adm release info <tagged_openshift_release_image> | grep "Pull From"
Pull From: quay.io/openshift-release-dev/ocp-release@sha256:69d1292f64a2b67227c5592c1a7d499c7d00376e498634ff8e1946bc9ccdddfe
create -f <clusterImageSet_FILE>
create -f img4.11.9-x86_64.yaml
delete -f subscribe/subscription-fast
make subscribe-candidate
make subscribe-fast
make subscribe-stable
delete -f subscribe/subscription-fast
get crd baremetalhosts.metal3.io
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "baremetalhosts.metal3.io" not found
apply -f
get provisioning
patch provisioning provisioning-configuration --type merge -p '{"spec":{"watchAllNamespaces": true }}'
apiVersion: metal3.io/v1alpha1 kind: Provisioning metadata: name: provisioning-configuration spec: provisioningNetwork: "Disabled" watchAllNamespaces: true
apply -f
apiVersion: v1 kind: ConfigMap metadata: name: <mirror-config> namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | <certificate-content> registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/multicluster-engine" mirror-by-digest-only = true [[registry.mirror]] location = "mirror.registry.com:5000/multicluster-engine"
Authorization: Basic xyz
api_key: myexampleapikey
create secret generic -n multicluster-engine os-images-http-auth --from-file=./query_params --from-file=./headers
-n multicluster-engine create configmap image-service-additional-ca --from-file=tls.crt
apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: name: agent spec: databaseStorage: accessModes: - ReadWriteOnce resources: requests: storage: <db_volume_size> filesystemStorage: accessModes: - ReadWriteOnce resources: requests: storage: <fs_volume_size> mirrorRegistryRef: name: <mirror_config> 1 unauthenticatedRegistries: - <unauthenticated_registry> 2 imageStorage: accessModes: - ReadWriteOnce resources: requests: storage: <img_volume_size> 3 OSImageAdditionalParamsRef: name: os-images-http-auth OSImageCACertRef: name: image-service-additional-ca osImages: - openshiftVersion: "<ocp_version>" 4 version: "<ocp_release_version>" 5 url: "<iso_url>" 6 cpuArchitecture: "x86_64"
apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: name: agent spec: databaseStorage: accessModes: - ReadWriteOnce resources: requests: storage: <db_volume_size> 1 filesystemStorage: accessModes: - ReadWriteOnce resources: requests: storage: <fs_volume_size> 2 imageStorage: accessModes: - ReadWriteOnce resources: requests: storage: <img_volume_size> 3
login
apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: annotations: agent-install.openshift.io/service-image-base: el8
get routes --all-namespaces | grep assisted-image-service
apiVersion: operator.openshift.io/v1 kind: IngressController metadata: name: ingress-controller-with-nlb namespace: openshift-ingress-operator spec: domain: nlb-apps.<domain>.com routeSelector: matchLabels: router-type: nlb endpointPublishingStrategy: type: LoadBalancerService loadBalancer: scope: External providerParameters: type: AWS aws: type: NLB
apply -f ingresscontroller.yaml
get ingresscontroller -n openshift-ingress-operator
edit ingresscontroller <name> -n openshift-ingress-operator
edit route assisted-image-service -n <namespace>
metadata: labels: router-type: nlb name: assisted-image-service
assisted-image-service-multicluster-engine.apps.<yourdomain>.com
get pods -n multicluster-engine | grep assist
login
apiVersion: v1 kind: Namespace metadata: name: <your_namespace> 1
apply -f namespace.yaml
apiVersion: v1 kind: Secret type: kubernetes.io/dockerconfigjson metadata: name: pull-secret 1 namespace: <your_namespace> stringData: .dockerconfigjson: <your_pull_secret> 2
apply -f pull-secret.yaml
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: myinfraenv namespace: <your_namespace> spec: proxy: httpProxy: <http://user:password@ipaddr:port> httpsProxy: <http://user:password@ipaddr:port> noProxy: additionalNTPSources: sshAuthorizedKey: pullSecretRef: name: <name> agentLabels: <key>: <value> nmStateConfigLabelSelector: matchLabels: <key>: <value> clusterRef: name: <cluster_name> namespace: <project_name> ignitionConfigOverride: '{"ignition": {"version": "3.1.0"}, ...}' cpuArchitecture: x86_64 ipxeScriptType: DiscoveryImageAlways kernelArguments: - operation: append value: audit=0 additionalTrustBundle: <bundle> osImageVersion: <version>
apply -f infra-env.yaml
describe infraenv myinfraenv -n <your_namespace>
apiVersion: agent-install.openshift.io/v1beta1 kind: NMStateConfig metadata: name: mynmstateconfig namespace: <your-infraenv-namespace> labels: some-key: <some-value> spec: config: interfaces: - name: eth0 type: ethernet state: up mac-address: 02:00:00:80:12:14 ipv4: enabled: true address: - ip: 192.168.111.30 prefix-length: 24 dhcp: false - name: eth1 type: ethernet state: up mac-address: 02:00:00:80:12:15 ipv4: enabled: true address: - ip: 192.168.140.30 prefix-length: 24 dhcp: false dns-resolver: config: server: - 192.168.126.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.1 next-hop-interface: eth1 table-id: 254 - destination: 0.0.0.0/0 next-hop-address: 192.168.140.1 next-hop-interface: eth1 table-id: 254 interfaces: - name: "eth0" macAddress: "02:00:00:80:12:14" - name: "eth1" macAddress: "02:00:00:80:12:15"
apply -f nmstateconfig.yaml
You now see a URL to download the ISO file. Booted hosts appear in the host inventory table. Hosts might take a few minutes to appear.
*Note:* By default, the ISO that is provided is a _minimal_ ISO. The minimal ISO does not contain the root file system, `RootFS`. The `RootFS` is downloaded later. To display full ISO, replace `minimal.iso` in the URL with `full.iso`.
get infraenv -n <infra env namespace> <infra env name> -o jsonpath='{.status.isoDownloadURL}'
https://assisted-image-service-assisted-installer.apps.example-acm-hub.com/byapikey/eyJhbGciOiJFUzI1NiIsInC93XVCJ9.eyJpbmZyYV9lbnZfaWQcTA0Y38sWVjYi02MTA0LTQ4NDMtODasdkOGIxYTZkZGM5ZTUifQ.3ydTpHaXJmTasd7uDp2NvGUFRKin3Z9Qct3lvDky1N-5zj3KsRePhAM48aUccBqmucGt3g/4.16/x86_64/minimal.iso
get agent -n <infra env namespace>
NAME CLUSTER APPROVED ROLE STAGE 24a92a6f-ea35-4d6f-9579-8f04c0d3591e false auto-assign
patch agent -n <infra env namespace> <agent name> -p '{"spec":{"approved":true}}' --type merge
get agent -n <infra env namespace>
NAME CLUSTER APPROVED ROLE STAGE 173e3a84-88e2-4fe1-967f-1a9242503bec true auto-assign
apiVersion: v1 kind: Secret metadata: name: <bmc-secret-name> namespace: <your_infraenv_namespace> 1 type: Opaque data: username: <username> password: <password>
apiVersion: metal3.io/v1alpha1 kind: BareMetalHost metadata: name: <bmh-name> namespace: <your-infraenv-namespace> 1 annotations: inspect.metal3.io: disabled bmac.agent-install.openshift.io/hostname: <hostname> 2 bmac.agent-install.openshift.io/role: <role> 3 labels: infraenvs.agent-install.openshift.io: <your-infraenv> 4 spec: online: true automatedCleaningMode: disabled 5 bootMACAddress: <your-mac-address> 6 bmc: address: <machine-address> 7 credentialsName: <bmc-secret-name> 8 rootDeviceHints: deviceName: /dev/sda 9
apiVersion: v1 kind: ConfigMap metadata: name: my-assisted-service-config namespace: multicluster-engine data: ALLOW_CONVERGED_FLOW: "false"
annotate --overwrite AgentServiceConfig agent unsupported.agent-install.openshift.io/assisted-service-configmap=my-assisted-service-config
bmac.agent-install.openshift.io/remove-agent-and-node-on-delete: true
delete bmh <bmh-name>
--append-karg
ip=192.0.2.2::192.0.2.254:255.255.255.0:core0.example.com:enp1s0:none
--save-partindex
4
{"ignition": "version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}
git clone https://github.com/stolostron/acm-hive-openshift-releases.git cd acm-hive-openshift-releases git checkout origin/backplane-<2.x>
find clusterImageSets/fast -type d -exec oc apply -f {} \; 2> /dev/null
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: labels: channel: stable visible: 'true' name: img4.x.47-x86-64-appsub spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.x.47-x86_64
kind: ConfigMap apiVersion: v1 metadata: name: <my-baremetal-cluster-install-manifests> namespace: <mynamespace> data: 99_metal3-config.yaml: | kind: ConfigMap apiVersion: v1 metadata: name: metal3-config namespace: openshift-machine-api data: http_port: "6180" provisioning_interface: "enp1s0" provisioning_ip: "172.00.0.3/24" dhcp_range: "172.00.0.10,172.00.0.100" deploy_kernel_url: "http://172.00.0.3:6180/images/ironic-python-agent.kernel" deploy_ramdisk_url: "http://172.00.0.3:6180/images/ironic-python-agent.initramfs" ironic_endpoint: "http://172.00.0.3:6385/v1/" ironic_inspector_endpoint: "http://172.00.0.3:5150/v1/" cache_url: "http://192.168.111.1/images" rhcos_image_url: "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.3/43.81.201911192044.0/x86_64/rhcos-43.81.201911192044.0-openstack.x86_64.qcow2.gz"
apply -f <filename>.yaml
apiVersion: hive.openshift.io/v1 kind: ClusterDeployment metadata: name: <my-baremetal-cluster> namespace: <mynamespace> annotations: hive.openshift.io/try-install-once: "true" spec: baseDomain: test.example.com clusterName: <my-baremetal-cluster> controlPlaneConfig: servingCertificates: {} platform: baremetal: libvirtSSHPrivateKeySecretRef: name: provisioning-host-ssh-private-key provisioning: installConfigSecretRef: name: <my-baremetal-cluster-install-config> sshPrivateKeySecretRef: name: <my-baremetal-hosts-ssh-private-key> manifestsConfigMapRef: name: <my-baremetal-cluster-install-manifests> imageSetRef: name: <my-clusterimageset> sshKnownHosts: - "10.1.8.90 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXvVVVKUYVkuyvkuygkuyTCYTytfkufTYAAAAIbmlzdHAyNTYAAABBBKWjJRzeUVuZs4yxSy4eu45xiANFIIbwE3e1aPzGD58x/NX7Yf+S8eFKq4RrsfSaK2hVJyJjvVIhUsU9z2sBJP8=" pullSecretRef: name: <my-baremetal-cluster-pull-secret>
apply -f <filename>.yaml
vpc-1 (us-gov-east-1) : 10.0.0.0/20 subnet-11 (us-gov-east-1a): 10.0.0.0/23 subnet-12 (us-gov-east-1b): 10.0.2.0/23 subnet-13 (us-gov-east-1c): 10.0.4.0/23 subnet-12 (us-gov-east-1d): 10.0.8.0/23 subnet-12 (us-gov-east-1e): 10.0.10.0/23 subnet-12 (us-gov-east-1f): 10.0.12.0/2
vpc-2 (us-gov-east-1) : 10.0.16.0/20 subnet-21 (us-gov-east-1a): 10.0.16.0/23 subnet-22 (us-gov-east-1b): 10.0.18.0/23 subnet-23 (us-gov-east-1c): 10.0.20.0/23 subnet-24 (us-gov-east-1d): 10.0.22.0/23 subnet-25 (us-gov-east-1e): 10.0.24.0/23 subnet-26 (us-gov-east-1f): 10.0.28.0/23
ec2:CreateVpcEndpointServiceConfiguration ec2:DescribeVpcEndpointServiceConfigurations ec2:ModifyVpcEndpointServiceConfiguration ec2:DescribeVpcEndpointServicePermissions ec2:ModifyVpcEndpointServicePermissions ec2:DeleteVpcEndpointServiceConfigurations
ec2:DescribeVpcEndpointServices ec2:DescribeVpcEndpoints ec2:CreateVpcEndpoint ec2:CreateTags ec2:DescribeNetworkInterfaces ec2:DescribeVPCs ec2:DeleteVpcEndpoints route53:CreateHostedZone route53:GetHostedZone route53:ListHostedZonesByVPC route53:AssociateVPCWithHostedZone route53:DisassociateVPCFromHostedZone route53:CreateVPCAssociationAuthorization route53:DeleteVPCAssociationAuthorization route53:ListResourceRecordSets route53:ChangeResourceRecordSets route53:DeleteHostedZone
route53:AssociateVPCWithHostedZone route53:DisassociateVPCFromHostedZone ec2:DescribeVPCs
spec: awsPrivateLink: ## The list of inventory of VPCs that can be used to create VPC ## endpoints by the controller. endpointVPCInventory: - region: us-east-1 vpcID: vpc-1 subnets: - availabilityZone: us-east-1a subnetID: subnet-11 - availabilityZone: us-east-1b subnetID: subnet-12 - availabilityZone: us-east-1c subnetID: subnet-13 - availabilityZone: us-east-1d subnetID: subnet-14 - availabilityZone: us-east-1e subnetID: subnet-15 - availabilityZone: us-east-1f subnetID: subnet-16 - region: us-east-1 vpcID: vpc-2 subnets: - availabilityZone: us-east-1a subnetID: subnet-21 - availabilityZone: us-east-1b subnetID: subnet-22 - availabilityZone: us-east-1c subnetID: subnet-23 - availabilityZone: us-east-1d subnetID: subnet-24 - availabilityZone: us-east-1e subnetID: subnet-25 - availabilityZone: us-east-1f subnetID: subnet-26 ## The credentialsSecretRef references a secret with permissions to create. ## The resources in the account where the inventory of VPCs exist. credentialsSecretRef: name: <hub-account-credentials-secret-name> ## A list of VPC where various mce clusters exists. associatedVPCs: - region: region-mce1 vpcID: vpc-mce1 credentialsSecretRef: name: <credentials-that-have-access-to-account-where-MCE1-VPC-exists> - region: region-mce2 vpcID: vpc-mce2 credentialsSecretRef: name: <credentials-that-have-access-to-account-where-MCE2-VPC-exists>
api.<cluster_name>.<base_domain>
*.apps.<cluster_name>.<base_domain>
api.<cluster_name>.<base_domain>
*.apps.<cluster_name>.<base_domain>
apiVersion: v1 kind: Secret type: Opaque metadata: name: ocp3-openstack-trust namespace: ocp3 stringData: ca.crt: | -----BEGIN CERTIFICATE----- <Base64 certificate contents here> -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- <Base64 certificate contents here> -----END CERTIFICATE----
platform: openstack: certificatesSecretRef: name: ocp3-openstack-trust credentialsSecretRef: name: ocp3-openstack-creds cloud: openstack
api.<cluster_name>.<base_domain>
*.apps.<cluster_name>.<base_domain>
apiVersion: v1 kind: Namespace metadata: name: sample-namespace
apiVersion: v1 kind: Secret type: kubernetes.io/dockerconfigjson metadata: name: <pull-secret> namespace: sample-namespace stringData: .dockerconfigjson: 'your-pull-secret-json' 1
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: name: openshift-v4.14.0 spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.14.0-rc.0-x86_64
apiVersion: hive.openshift.io/v1 kind: ClusterDeployment metadata: name: single-node namespace: demo-worker4 spec: baseDomain: hive.example.com clusterInstallRef: group: extensions.hive.openshift.io kind: AgentClusterInstall name: test-agent-cluster-install 1 version: v1beta1 clusterName: test-cluster controlPlaneConfig: servingCertificates: {} platform: agentBareMetal: agentSelector: matchLabels: location: internal pullSecretRef: name: <pull-secret> 2
apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall metadata: name: test-agent-cluster-install namespace: demo-worker4 spec: platformType: BareMetal 1 clusterDeploymentRef: name: single-node 2 imageSetRef: name: openshift-v4.14.0 3 networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 machineNetwork: - cidr: 192.168.111.0/24 serviceNetwork: - 172.30.0.0/16 provisionRequirements: controlPlaneAgents: 1 sshPublicKey: ssh-rsa <your-public-key-here> 4
apiVersion: agent-install.openshift.io/v1beta1 kind: NMStateConfig metadata: name: <mynmstateconfig> namespace: <demo-worker4> labels: demo-nmstate-label: <value> spec: config: interfaces: - name: eth0 type: ethernet state: up mac-address: 02:00:00:80:12:14 ipv4: enabled: true address: - ip: 192.168.111.30 prefix-length: 24 dhcp: false - name: eth1 type: ethernet state: up mac-address: 02:00:00:80:12:15 ipv4: enabled: true address: - ip: 192.168.140.30 prefix-length: 24 dhcp: false dns-resolver: config: server: - 192.168.126.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.1 next-hop-interface: eth1 table-id: 254 - destination: 0.0.0.0/0 next-hop-address: 192.168.140.1 next-hop-interface: eth1 table-id: 254 interfaces: - name: "eth0" macAddress: "02:00:00:80:12:14" - name: "eth1" macAddress: "02:00:00:80:12:15"
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: myinfraenv namespace: demo-worker4 spec: clusterRef: name: single-node 1 namespace: demo-worker4 2 pullSecretRef: name: pull-secret sshAuthorizedKey: <your_public_key_here> nmStateConfigLabelSelector: matchLabels: demo-nmstate-label: value proxy: httpProxy: http://USERNAME:PASSWORD@proxy.example.com:PORT httpsProxy: https://USERNAME:PASSWORD@proxy.example.com:PORT noProxy: .example.com,172.22.0.0/24,10.10.0.0/24
curl --insecure -o image.iso USD(kubectl -n sample-namespace get infraenvs.agent-install.openshift.io myinfraenv -o=jsonpath="{.status.isoDownloadURL}")
-n sample-namespace patch agents.agent-install.openshift.io 07e80ea9-200c-4f82-aff4-4932acb773d4 -p '{"spec":{"approved":true}}' --type merge
apiVersion: v1 kind: Proxy baseDomain: <domain> proxy: httpProxy: http://<username>:<password>@<proxy.example.com>:<port> httpsProxy: https://<username>:<password>@<proxy.example.com>:<port> noProxy: <wildcard-of-domain>,<provisioning-network/CIDR>,<BMC-address-range/CIDR>
apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall spec: proxy: httpProxy: http://<username>:<password>@<proxy.example.com>:<port> 1 httpsProxy: https://<username>:<password>@<proxy.example.com>:<port> 2 noProxy: <wildcard-of-domain>,<provisioning-network/CIDR>,<BMC-address-range/CIDR> 3
create secret generic pull-secret -n <open-cluster-management> --from-file=.dockerconfigjson=<path-to-pull-secret> --type=kubernetes.io/dockerconfigjson
edit managedcluster <cluster-name>
spec: hubAcceptsClient: true managedClusterClientConfigs: - url: <https://api.new-managed.dev.redhat.com> 1
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: open-cluster-management/nodeSelector: '{"dedicated":"acm"}' open-cluster-management/tolerations: '[{"key":"dedicated","operator":"Equal","value":"acm","effect":"NoSchedule"}]'
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: <klusterletconfigName> spec: nodePlacement: nodeSelector: dedicated: acm tolerations: - key: dedicated operator: Equal value: acm effect: NoSchedule
delete po -n open-cluster-management `oc get pod -n open-cluster-management | grep multiclusterhub-operator| cut -d' ' -f1`
login
new-project <cluster_name>
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: <cluster_name> labels: cloud: auto-detect vendor: auto-detect spec: hubAcceptsClient: true
cloud: Amazon vendor: OpenShift
apply -f managed-cluster.yaml
apiVersion: v1 kind: Secret metadata: name: auto-import-secret namespace: <cluster_name> stringData: autoImportRetry: "5" # If you are using the kubeconfig file, add the following value for the kubeconfig file # that has the current context set to the cluster to import: kubeconfig: |- <kubeconfig_file> # If you are using the token/server pair, add the following two values instead of # the kubeconfig file: token: <Token to access the cluster> server: <cluster_api_url> type: Opaque
apply -f auto-import-secret.yaml
-n <cluster_name> annotate secrets auto-import-secret managedcluster-import-controller.open-cluster-management.io/keeping-auto-import-secret=""
get managedcluster <cluster_name>
login
get pod -n open-cluster-management-agent
get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.crds\\.yaml} | base64 --decode > klusterlet-crd.yaml
get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.import\\.yaml} | base64 --decode > import.yaml
login
apply -f klusterlet-crd.yaml
apply -f import.yaml
get managedcluster <cluster_name>
apiVersion: agent.open-cluster-management.io/v1 kind: KlusterletAddonConfig metadata: name: <cluster_name> namespace: <cluster_name> spec: applicationManager: enabled: true certPolicyController: enabled: true policyController: enabled: true searchCollector: enabled: true
apply -f klusterlet-addon-config.yaml
get pod -n open-cluster-management-agent-addon
delete managedcluster <cluster_name>
export agent_registration_host=USD(oc get route -n multicluster-engine agent-registration -o=jsonpath="{.spec.host}")
get configmap -n kube-system kube-root-ca.crt -o=jsonpath="{.data['ca\.crt']}" > ca.crt_
apiVersion: v1 kind: ServiceAccount metadata: name: managed-cluster-import-agent-registration-sa namespace: multicluster-engine --- apiVersion: v1 kind: Secret type: kubernetes.io/service-account-token metadata: name: managed-cluster-import-agent-registration-sa-token namespace: multicluster-engine annotations: kubernetes.io/service-account.name: "managed-cluster-import-agent-registration-sa" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: managedcluster-import-controller-agent-registration-client rules: - nonResourceURLs: ["/agent-registration/*"] verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: managed-cluster-import-agent-registration roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: managedcluster-import-controller-agent-registration-client subjects: - kind: ServiceAccount name: managed-cluster-import-agent-registration-sa namespace: multicluster-engine
export token=USD(oc get secret -n multicluster-engine managed-cluster-import-agent-registration-sa-token -o=jsonpath='{.data.token}' | base64 -d)
patch clustermanager cluster-manager --type=merge -p '{"spec":{"registrationConfiguration":{"featureGates":[ {"feature": "ManagedClusterAutoApproval", "mode": "Enable"}], "autoApproveUsers":["system:serviceaccount:multicluster-engine:agent-registration-bootstrap"]}}}'
curl --cacert ca.crt -H "Authorization: Bearer USDtoken" https://USDagent_registration_host/agent-registration/crds/v1 | oc apply -f -
curl --cacert ca.crt -H "Authorization: Bearer USDtoken" https://USDagent_registration_host/agent-registration/manifests/<clusterName>?klusterletconfig=<klusterletconfigName> | oc apply -f -
apiVersion: v1 kind: Namespace metadata: name: managed-cluster
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: name: openshift-v4.11.18 spec: releaseImage: quay.io/openshift-release-dev/ocp-release@sha256:22e149142517dfccb47be828f012659b1ccf71d26620e6f62468c264a7ce7863
apiVersion: v1 kind: Secret type: kubernetes.io/dockerconfigjson metadata: name: pull-secret namespace: managed-cluster stringData: .dockerconfigjson: <pull-secret-json> 1
get secret -n openshift-kube-apiserver node-kubeconfigs -ojson | jq '.data["lb-ext.kubeconfig"]' --raw-output | base64 -d > /tmp/kubeconfig.some-other-cluster
-n managed-cluster create secret generic some-other-cluster-admin-kubeconfig --from-file=kubeconfig=/tmp/kubeconfig.some-other-cluster
apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall metadata: name: <your-cluster-name> 1 namespace: <managed-cluster> spec: networking: userManagedNetworking: true clusterDeploymentRef: name: <your-cluster> imageSetRef: name: openshift-v4.11.18 provisionRequirements: controlPlaneAgents: 2 sshPublicKey: <""> 3
apiVersion: hive.openshift.io/v1 kind: ClusterDeployment metadata: name: <your-cluster-name> 1 namespace: managed-cluster spec: baseDomain: <redhat.com> 2 installed: <true> 3 clusterMetadata: adminKubeconfigSecretRef: name: <your-cluster-name-admin-kubeconfig> 4 clusterID: <""> 5 infraID: <""> 6 clusterInstallRef: group: extensions.hive.openshift.io kind: AgentClusterInstall name: your-cluster-name-install version: v1beta1 clusterName: your-cluster-name platform: agentBareMetal: pullSecretRef: name: pull-secret
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: your-infraenv namespace: managed-cluster spec: clusterRef: name: your-cluster-name namespace: managed-cluster pullSecretRef: name: pull-secret sshAuthorizedKey: ""
get infraenv -n managed-cluster some-other-infraenv -ojson | jq ".status.<url>" --raw-output | xargs curl -k -o /storage0/isos/some-other.iso
apiVersion: imageregistry.open-cluster-management.io/v1alpha1 kind: ManagedClusterImageRegistry metadata: name: <imageRegistryName> namespace: <namespace> spec: placementRef: group: cluster.open-cluster-management.io resource: placements name: <placementName> 1 pullSecret: name: <pullSecretName> 2 registries: 3 - mirror: <mirrored-image-registry-address> source: <image-registry-address> - mirror: <mirrored-image-registry-address> source: <image-registry-address>
registries: - mirror: localhost:5000/rhacm2/ source: registry.redhat.io/rhacm2 - mirror: localhost:5000/multicluster-engine source: registry.redhat.io/multicluster-engine
registries: - mirror: localhost:5000/rhacm2-registration-rhel8-operator source: registry.redhat.io/rhacm2/registration-rhel8-operator
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: <klusterletconfigName> spec: pullSecret: namespace: <pullSecretNamespace> name: <pullSecretName> registries: - mirror: <mirrored-image-registry-address> source: <image-registry-address> - mirror: <mirrored-image-registry-address> source: <image-registry-address>
kubectl create secret docker-registry myPullSecret --docker-server=<your-registry-server> --docker-username=<my-name> --docker-password=<my-password>
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: myPlacement namespace: myNamespace spec: clusterSets: - myClusterSet tolerations: - key: "cluster.open-cluster-management.io/unreachable" operator: Exists
apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSet metadata: name: myClusterSet --- apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSetBinding metadata: name: myClusterSet namespace: myNamespace spec: clusterSet: myClusterSet
apiVersion: imageregistry.open-cluster-management.io/v1alpha1 kind: ManagedClusterImageRegistry metadata: name: myImageRegistry namespace: myNamespace spec: placementRef: group: cluster.open-cluster-management.io resource: placements name: myPlacement pullSecret: name: myPullSecret registry: myRegistryAddress
get machinepools -n <managed-cluster-namespace>
edit machinepool <MachinePool-resource-name> -n <managed-cluster-namespace>
get machinepools -n <managed-cluster-namespace>
edit machinepool <name-of-MachinePool-resource> -n <namespace-of-managed-cluster>
get machinepools -n <managed-cluster-namespace>
edit machinepool <MachinePool-resource-name> -n <managed-cluster-namespace>
export <kubeconfig_name>=USD(oc get cd USD<cluster_name> -o "jsonpath={.spec.clusterMetadata.adminKubeconfigSecretRef.name}") extract secret/USD<kubeconfig_name> --keys=kubeconfig --to=- > original-kubeconfig --kubeconfig=original-kubeconfig get node
Unable to connect to the server: tls: failed to verify certificate: x509: certificate signed by unknown authority
echo <base64 encoded blob> | base64 --decode > decoded-existing-certs.pem
cp original-kubeconfig <new_kubeconfig_name>
cat decoded-existing-certs.pem new-ca-certificate.pem | openssl base64 -A
KUBECONFIG=<new_kubeconfig_name> oc get nodes
patch secret USDoriginal-kubeconfig --type='json' -p="[{'op': 'replace', 'path': '/data/kubeconfig', 'value': 'USD(openssl base64 -A -in <new_kubeconfig_name>)'},{'op': 'replace', 'path': '/data/raw-kubeconfig', 'value': 'USD(openssl base64 -A -in <new_kubeconfig_name>)'}]"
watch -n 5 "oc get agent -n managed-cluster"
get agent -n managed-cluster -ojson | jq -r '.items[] | select(.spec.approved==false) |select(.spec.clusterDeploymentName==null) | .metadata.name'| xargs oc -n managed-cluster patch -p '{"spec":{"clusterDeploymentName":{"name":"some-other-cluster","namespace":"managed-cluster"}}}' --type merge agent
get agent -n managed-cluster -ojson | jq -r '.items[] | select(.spec.approved==false) | .metadata.name'| xargs oc -n managed-cluster patch -p '{"spec":{"approved":true}}' --type merge agent
patch agent <AGENT-NAME> -p '{"spec":{"role": "master"}}' --type=merge
bmac.agent-install.openshift.io/role: master
patch agent <AGENT-NAME> -p '{"spec":{"role": "master"}}' --type=merge
edit clusterdeployment <name-of-cluster> -n <namespace-of-cluster>
get clusterdeployment <name-of-cluster> -n <namespace-of-cluster> -o yaml
edit clusterdeployment <name-of-cluster> -n <namespace-of-cluster>
get clusterdeployment <name-of-cluster> -n <namespace-of-cluster> -o yaml
UPSTREAM_REGISTRY=quay.io PRODUCT_REPO=openshift-release-dev RELEASE_NAME=ocp-release OCP_RELEASE=4.12.2-x86_64 LOCAL_REGISTRY=USD(hostname):5000 LOCAL_SECRET_JSON=/path/to/pull/secret 1 adm -a USD{LOCAL_SECRET_JSON} release mirror --from=USD{UPSTREAM_REGISTRY}/USD{PRODUCT_REPO}/USD{RELEASE_NAME}:USD{OCP_RELEASE} --to=USD{LOCAL_REGISTRY}/ocp4 --to-release-image=USD{LOCAL_REGISTRY}/ocp4/release:USD{OCP_RELEASE}
git clone https://github.com/openshift/cincinnati-graph-data
FROM registry.access.redhat.com/ubi8/ubi:8.1 1 RUN curl -L -o cincinnati-graph-data.tar.gz https://github.com/openshift/cincinnati-graph-data/archive/master.tar.gz 2 RUN mkdir -p /var/lib/cincinnati/graph-data/ 3 CMD exec /bin/bash -c "tar xvzf cincinnati-graph-data.tar.gz -C /var/lib/ cincinnati/graph-data/ --strip-components=1" 4
build -f <path_to_Dockerfile> -t <USD{DISCONNECTED_REGISTRY}/cincinnati/cincinnati-graph-data-container>:latest 1 2 push <USD{DISCONNECTED_REGISTRY}/cincinnati/cincinnati-graph-data-container><2>:latest --authfile=</path/to/pull_secret>.json 3
apiVersion: v1 kind: ConfigMap metadata: name: trusted-ca data: updateservice-registry: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE-----
patch image.config.openshift.io cluster -p '{"spec":{"additionalTrustedCA":{"name":"trusted-ca"}}}' --type merge
apiVersion: cincinnati.openshift.io/v1beta2 kind: Cincinnati metadata: name: openshift-update-service-instance namespace: openshift-cincinnati spec: registry: <registry_host_name>:<port> 1 replicas: 1 repository: USD{LOCAL_REGISTRY}/ocp4/release graphDataImage: '<host_name>:<port>/cincinnati-graph-data-container' 2
apiVersion: operator.openshift.io/v1alpha1 kind: ImageContentSourcePolicy metadata: name: <your-local-mirror-name> 1 spec: repositoryDigestMirrors: - mirrors: - <your-registry> 2 source: registry.redhat.io
apply -f mirror.yaml
apiVersion: config.openshift.io/v1 kind: OperatorHub metadata: name: cluster spec: disableAllDefaultSources: true --- apiVersion: operators.coreos.com/v1alpha1 kind: CatalogSource metadata: name: my-operator-catalog namespace: openshift-marketplace spec: sourceType: grpc image: '<registry_host_name>:<port>/olm/redhat-operators:v1' 1 displayName: My Operator Catalog publisher: grpc
apply -f source.yaml
get clusterversion -o yaml
apiVersion: v1 items: - apiVersion: config.openshift.io/v1 kind: ClusterVersion [..] spec: channel: stable-4.x upstream: https://api.openshift.com/api/upgrades_info/v1/graph
get routes
edit clusterversion version
get routes -A
get clusterversion -o yaml
apiVersion: v1 items: - apiVersion: config.openshift.io/v1 kind: ClusterVersion [..] spec: channel: stable-4.x upstream: https://<hub-cincinnati-uri>/api/upgrades_info/v1/graph
export KUBECONFIG=<managed-cluster-kubeconfig>
create role -n default test-role --verb=list,get --resource=pods create rolebinding -n default test-rolebinding --serviceaccount=default:default --role=test-role
get secret -n default | grep <default-token>
export MANAGED_CLUSTER_TOKEN=USD(kubectl -n default get secret <default-token> -o jsonpath={.data.token} | base64 -d)
config view --minify --raw=true > cluster-proxy.kubeconfig
export TARGET_MANAGED_CLUSTER=<managed-cluster-name> export NEW_SERVER=https://USD(oc get route -n multicluster-engine cluster-proxy-addon-user -o=jsonpath='{.spec.host}')/USDTARGET_MANAGED_CLUSTER sed -i'' -e '/server:/c\ server: '"USDNEW_SERVER"'' cluster-proxy.kubeconfig export CADATA=USD(oc get configmap -n openshift-service-ca kube-root-ca.crt -o=go-template='{{index .data "ca.crt"}}' | base64) sed -i'' -e '/certificate-authority-data:/c\ certificate-authority-data: '"USDCADATA"'' cluster-proxy.kubeconfig
sed -i'' -e '/client-certificate-data/d' cluster-proxy.kubeconfig sed -i'' -e '/client-key-data/d' cluster-proxy.kubeconfig sed -i'' -e '/token/d' cluster-proxy.kubeconfig
sed -i'' -e 'USDa\ token: '"USDMANAGED_CLUSTER_TOKEN"'' cluster-proxy.kubeconfig
get pods --kubeconfig=cluster-proxy.kubeconfig -n <default>
export PROMETHEUS_TOKEN=USD(kubectl get secret -n openshift-monitoring USD(kubectl get serviceaccount -n openshift-monitoring prometheus-k8s -o=jsonpath='{.secrets[0].name}') -o=jsonpath='{.data.token}' | base64 -d)
get configmap kube-root-ca.crt -o=jsonpath='{.data.ca\.crt}' > hub-ca.crt
export SERVICE_NAMESPACE=openshift-monitoring export SERVICE_NAME=prometheus-k8s export SERVICE_PORT=9091 export SERVICE_PATH="api/v1/query?query=machine_cpu_sockets" curl --cacert hub-ca.crt USDNEW_SERVER/api/v1/namespaces/USDSERVICE_NAMESPACE/services/USDSERVICE_NAME:USDSERVICE_PORT/proxy-service/USDSERVICE_PATH -H "Authorization: Bearer USDPROMETHEUS_TOKEN"
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: <name> 1 namespace: <namespace> 2 spec: agentInstallNamespace: open-cluster-managment-agent-addon proxyConfig: httpsProxy: "http://<username>:<password>@<ip>:<port>" 3 noProxy: ".cluster.local,.svc,172.30.0.1" 4 caBundle: <value> 5
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: cluster-proxy namespace: <namespace> 1 spec: installNamespace: open-cluster-managment-addon configs: group: addon.open-cluster-management.io resource: AddonDeploymentConfig name: <name> 2 namespace: <namespace> 3
operation: retryPosthook: installPosthook
operation: retryPosthook: upgradePosthook
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: test-inno namespace: test-inno spec: desiredCuration: upgrade destroy: {} install: {} scale: {} upgrade: channel: stable-4.x desiredUpdate: 4.x.1 monitorTimeout: 150 posthook: - extra_vars: {} clusterName: test-inno type: post_check name: ACM Upgrade Checks prehook: - extra_vars: {} clusterName: test-inno type: pre_check name: ACM Upgrade Checks towerAuthSecret: awx inventory: Demo Inventory
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: cluster1 {{{} namespace: cluster1 labels: test1: test1 test2: test2 {}}}spec: desiredCuration: install install: jobMonitorTimeout: 5 posthook: - extra_vars: {} name: Demo Job Template type: Job prehook: - extra_vars: {} name: Demo Job Template type: Job towerAuthSecret: toweraccess
spec: desiredCuration: upgrade upgrade: intermediateUpdate: 4.13.x desiredUpdate: 4.14.x monitorTimeout: 120
posthook: - extra_vars: {} name: Unpause machinepool type: Job prehook: - extra_vars: {} name: Pause machinepool type: Job
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: annotations: cluster.open-cluster-management.io/upgrade-clusterversion-backoff-limit: "10" name: your-name namespace: your-namespace spec: desiredCuration: upgrade upgrade: intermediateUpdate: 4.13.x desiredUpdate: 4.14.x monitorTimeout: 120 posthook: - extra_vars: {} name: Unpause machinepool type: Job prehook: - extra_vars: {} name: Pause machinepool type: Job
apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: name: my-cluster namespace: clusters spec: pausedUntil: 'true'
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: name: my-cluster-us-east-2 namespace: clusters spec: pausedUntil: 'true'
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: my-cluster namespace: clusters labels: open-cluster-management: curator spec: desiredCuration: install install: jobMonitorTimeout: 5 prehook: - name: Demo Job Template extra_vars: variable1: something-interesting variable2: 2 - name: Demo Job Template posthook: - name: Demo Job Template towerAuthSecret: toweraccess
apiVersion: v1 kind: Secret metadata: name: toweraccess namespace: clusters stringData: host: https://my-tower-domain.io token: ANSIBLE_TOKEN_FOR_admin
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: my-cluster namespace: clusters labels: open-cluster-management: curator spec: desiredCuration: upgrade upgrade: desiredUpdate: 4.14.1 1 monitorTimeout: 120 prehook: - name: Demo Job Template extra_vars: variable1: something-interesting variable2: 2 - name: Demo Job Template posthook: - name: Demo Job Template towerAuthSecret: toweraccess
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: ClusterCurator metadata: name: my-cluster namespace: clusters labels: open-cluster-management: curator spec: desiredCuration: destroy destroy: jobMonitorTimeout: 5 prehook: - name: Demo Job Template extra_vars: variable1: something-interesting variable2: 2 - name: Demo Job Template posthook: - name: Demo Job Template towerAuthSecret: toweraccess
apiVersion: cluster.open-cluster-management.io/v1alpha1 kind: ClusterClaim metadata: name: id.openshift.io spec: value: 95f91f25-d7a2-4fc3-9237-2ef633d8451c
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: labels: cloud: Amazon clusterID: 95f91f25-d7a2-4fc3-9237-2ef633d8451c installer.name: multiclusterhub installer.namespace: open-cluster-management name: cluster1 vendor: OpenShift name: cluster1 spec: hubAcceptsClient: true leaseDurationSeconds: 60 status: allocatable: cpu: '15' memory: 65257Mi capacity: cpu: '18' memory: 72001Mi clusterClaims: - name: id.k8s.io value: cluster1 - name: kubeversion.open-cluster-management.io value: v1.18.3+6c42de8 - name: platform.open-cluster-management.io value: AWS - name: product.open-cluster-management.io value: OpenShift - name: id.openshift.io value: 95f91f25-d7a2-4fc3-9237-2ef633d8451c - name: consoleurl.openshift.io value: 'https://console-openshift-console.apps.xxxx.dev04.red-chesterfield.com' - name: version.openshift.io value: '4.x' conditions: - lastTransitionTime: '2020-10-26T07:08:49Z' message: Accepted by hub cluster admin reason: HubClusterAdminAccepted status: 'True' type: HubAcceptedManagedCluster - lastTransitionTime: '2020-10-26T07:09:18Z' message: Managed cluster joined reason: ManagedClusterJoined status: 'True' type: ManagedClusterJoined - lastTransitionTime: '2020-10-30T07:20:20Z' message: Managed cluster is available reason: ManagedClusterAvailable status: 'True' type: ManagedClusterConditionAvailable version: kubernetes: v1.18.3+6c42de8
apiVersion: cluster.open-cluster-management.io/v1alpha1 kind: ClusterClaim metadata: name: <custom_claim_name> spec: value: <custom_claim_value>
get clusterclaims.cluster.open-cluster-management.io
apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSetBinding metadata: name: global namespace: open-cluster-management-global-set spec: clusterSet: global
apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSet metadata: name: <cluster_set>
kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: clusterrole1 rules: - apiGroups: ["cluster.open-cluster-management.io"] resources: ["managedclustersets/join"] resourceNames: ["<cluster_set>"] verbs: ["create"]
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: <cluster_name> spec: hubAcceptsClient: true
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: <cluster_name> labels: cluster.open-cluster-management.io/clusterset: <cluster_set> spec: hubAcceptsClient: true
apiVersion: cluster.open-cluster-management.io/v1beta2 kind: ManagedClusterSetBinding metadata: namespace: <namespace> name: <cluster_set> spec: clusterSet: <cluster_set>
apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: <clusterrole> rules: - apiGroups: ["cluster.open-cluster-management.io"] resources: ["managedclustersets/bind"] resourceNames: ["<cluster_set>"] verbs: ["create"]
patch managedcluster <managed_cluster_name> -p '{"spec":{"taints":[{"key": "key", "value": "value", "effect": "NoSelect"}]}}' --type=merge
patch managedcluster <managed_cluster_name> --type='json' -p='[{"op": "add", "path": "/spec/taints/-", "value": {"key": "key", "value": "value", "effect": "NoSelect"}}]'
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unavailable timeAdded: '2022-02-21T08:11:54Z'
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unreachable timeAdded: '2022-02-21T08:11:06Z'
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: gpu value: "true" timeAdded: '2022-02-21T08:11:06Z'
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement1 namespace: default spec: tolerations: - key: gpu value: "true" operator: Equal
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unreachable timeAdded: '2022-02-21T08:11:06Z'
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: demo4 namespace: demo1 spec: tolerations: - key: cluster.open-cluster-management.io/unreachable operator: Exists tolerationSeconds: 300
get managedclusters -l cluster.open-cluster-management.io/clusterset=<cluster_set>
labels: cluster.open-cluster-management.io/clusterset: clusterset1
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchLabels: vendor: OpenShift
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: predicates: - requiredClusterSelector: claimSelector: matchExpressions: - key: region.open-cluster-management.io operator: In values: - us-west-1
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: clusterSets: - clusterset1 - clusterset2 predicates: - requiredClusterSelector: claimSelector: matchExpressions: - key: region.open-cluster-management.io operator: In values: - us-west-1
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 3 1 predicates: - requiredClusterSelector: labelSelector: matchLabels: vendor: OpenShift claimSelector: matchExpressions: - key: region.open-cluster-management.io operator: In values: - us-west-1
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: gpu value: "true" timeAdded: '2022-02-21T08:11:06Z'
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: tolerations: - key: gpu value: "true" operator: Equal
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: cluster1 spec: hubAcceptsClient: true taints: - effect: NoSelect key: cluster.open-cluster-management.io/unreachable timeAdded: '2022-02-21T08:11:06Z'
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: tolerations: - key: cluster.open-cluster-management.io/unreachable operator: Exists tolerationSeconds: 300 1
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 1 prioritizerPolicy: configurations: - scoreCoordinate: builtIn: ResourceAllocatableMemory
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 1 prioritizerPolicy: configurations: - scoreCoordinate: builtIn: ResourceAllocatableCPU weight: 2 - scoreCoordinate: builtIn: ResourceAllocatableMemory weight: 2
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement namespace: ns1 spec: numberOfClusters: 2 prioritizerPolicy: mode: Exact configurations: - scoreCoordinate: builtIn: Steady weight: 3 - scoreCoordinate: type: AddOn addOn: resourceName: default scoreName: cpuratio
feature.open-cluster-management.io/addon-application-manager: available
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement1 namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: feature.open-cluster-management.io/addon-application-manager operator: Exists
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement2 namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchLabels: "feature.open-cluster-management.io/addon-application-manager": "available"
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement3 namespace: ns1 spec: predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: feature.open-cluster-management.io/addon-application-manager operator: DoesNotExist
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: PlacementDecision metadata: labels: cluster.open-cluster-management.io/placement: placement1 name: placement1-kbc7q namespace: ns1 ownerReferences: - apiVersion: cluster.open-cluster-management.io/v1beta1 blockOwnerDeletion: true controller: true kind: Placement name: placement1 uid: 05441cf6-2543-4ecc-8389-1079b42fe63e status: decisions: - clusterName: cluster1 reason: '' - clusterName: cluster2 reason: '' - clusterName: cluster3 reason: ''
kind: ClusterClaim metadata: annotations: cluster.open-cluster-management.io/createmanagedcluster: "false" 1
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: managed-serviceaccount namespace: <target_managed_cluster> spec: installNamespace: open-cluster-management-agent-addon
apply -f -
apiVersion: authentication.open-cluster-management.io/v1alpha1 kind: ManagedServiceAccount metadata: name: <managedserviceaccount_name> namespace: <target_managed_cluster> spec: rotation: {}
get managedserviceaccount <managed_serviceaccount_name> -n <target_managed_cluster> -o yaml
get secret <managed_serviceaccount_name> -n <target_managed_cluster> -o yaml
apiVersion: config.openshift.io/v1 kind: APIServer metadata: name: cluster spec: audit: profile: Default servingCerts: namedCertificates: - names: - api.mycluster.example.com servingCertificate: name: old-cert-secret
cp old.crt combined.crt
cat new.crt >> combined.crt
create secret tls combined-certs-secret --cert=combined.crt --key=old.key -n openshift-config
apiVersion: config.openshift.io/v1 kind: APIServer metadata: name: cluster spec: audit: profile: Default servingCerts: namedCertificates: - names: - api.mycluster.example.com servingCertificate: name: combined-cert-secret
create secret tls new-cert-secret --cert=new.crt --key=new.key -n openshift-config {code}
apiVersion: config.openshift.io/v1 kind: APIServer metadata: name: cluster spec: audit: profile: Default servingCerts: namedCertificates: - names: - api.mycluster.example.com servingCertificate: name: new-cert-secret
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: http-proxy spec: hubKubeAPIServerProxyConfig: httpProxy: "http://<username>:<password>@<ip>:<port>"
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: https-proxy spec: hubKubeAPIServerProxyConfig: httpsProxy: "https://<username>:<password>@<ip>:<port>" caBundle: <user-ca-bundle>
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: agent.open-cluster-management.io/klusterlet-config: <klusterlet-config-name> name:<managed-cluster-name> spec: hubAcceptsClient: true leaseDurationSeconds: 60
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: open-cluster-management/nodeSelector: '{"dedicated":"acm"}' open-cluster-management/tolerations: '[{"key":"dedicated","operator":"Equal","value":"acm","effect":"NoSchedule"}]'
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: 1 spec: hubKubeAPIServerURL: "https://api.example.com:6443" 2 hubKubeAPIServerCABundle: "LS0tLS1CRU...LS0tCg==" 3
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: agent.open-cluster-management.io/klusterlet-config: 1 name: 2 spec: hubAcceptsClient: true leaseDurationSeconds: 60
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: test spec: hubKubeAPIServerURL: "example.test.com" - apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerURL: "example.global.com"
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: test spec: hubKubeAPIServerURL: "" - apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerURL: "example.global.com"
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: test - apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerURL: "example.global.com"
delete po -n open-cluster-management `oc get pod -n open-cluster-management | grep multiclusterhub-operator| cut -d' ' -f1`
delete managedcluster USDCLUSTER_NAME
delete clusterdeployment <CLUSTER_NAME> -n USDCLUSTER_NAME
delete po -n open-cluster-management `oc get pod -n open-cluster-management | grep multiclusterhub-operator| cut -d' ' -f1`
open-cluster-management-agent Active 10m open-cluster-management-agent-addon Active 10m
get klusterlet | grep klusterlet | awk '{print USD1}' | xargs oc patch klusterlet --type=merge -p '{"metadata":{"finalizers": []}}'
delete namespaces open-cluster-management-agent open-cluster-management-agent-addon --wait=false get crds | grep open-cluster-management.io | awk '{print USD1}' | xargs oc delete crds --wait=false get crds | grep open-cluster-management.io | awk '{print USD1}' | xargs oc patch crds --type=merge -p '{"metadata":{"finalizers": []}}'
get crds | grep open-cluster-management.io | awk '{print USD1}' get ns | grep open-cluster-management-agent
oc rsh -n openshift-etcd etcd-control-plane-0.example.com etcdctl endpoint status --cluster -w table
sh-4.4#etcdctl compact USD(etcdctl endpoint status --write-out="json" | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*' -m1)
compacted revision 158774421
apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveryConfig metadata: name: discovery namespace: <NAMESPACE_NAME> spec: credential: <SECRET_NAME> filters: lastActive: 7 openshiftVersions: - "4.14"
apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: fd51aafa-95a8-41f7-a992-6fb95eed3c8e namespace: <NAMESPACE_NAME> spec: activity_timestamp: "2021-04-19T21:06:14Z" cloudProvider: vsphere console: https://console-openshift-console.apps.qe1-vmware-pkt.dev02.red-chesterfield.com creation_timestamp: "2021-04-19T16:29:53Z" credential: apiVersion: v1 kind: Secret name: <SECRET_NAME> namespace: <NAMESPACE_NAME> display_name: qe1-vmware-pkt.dev02.red-chesterfield.com name: fd51aafa-95a8-41f7-a992-6fb95eed3c8e openshiftVersion: 4.14 status: Stale
apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: 28c17977-fc73-4050-b5cc-a5aa2d1d6892 namespace: discovery spec: openshiftVersion: <4.x.z> isManagedCluster: false cloudProvider: aws name: 28c17977-fc73-4050-b5cc-a5aa2d1d6892 displayName: rosa-dc status: Active importAsManagedCluster: true 1 type: <supported-type> 2
apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: annotations: discovery.open-cluster-management.io/previously-auto-imported: 'true'
2024-06-12T14:11:43.366Z INFO reconcile Skipped automatic import for DiscoveredCluster due to existing 'discovery.open-cluster-management.io/previously-auto-imported' annotation {"Name": "rosa-dc"}
patch discoveredcluster <name> -n <namespace> --type='json' -p='[{"op": "replace", "path": "/spec/importAsManagedCluster", "value": true}]'
get managedcluster <name>
rosa describe cluster --cluster=<cluster-name> | grep -o '^ID:.*
create -f <your-config-map-file.yaml>
kind: ConfigMap apiVersion: v1 metadata: name: hcp-sizing-baseline namespace: local-cluster data: incrementalCPUUsagePer1KQPS: "9.0" memoryRequestPerHCP: "18" minimumQPSPerHCP: "50.0"
delete deployment hypershift-addon-agent -n open-cluster-management-agent-addon
logs hypershift-addon-agent -n open-cluster-management-agent-addon
2024-01-05T19:41:05.392Z INFO agent.agent-reconciler agent/agent.go:793 setting cpuRequestPerHCP to 5 2024-01-05T19:41:05.392Z INFO agent.agent-reconciler agent/agent.go:802 setting memoryRequestPerHCP to 18 2024-01-05T19:53:54.070Z INFO agent.agent-reconciler agent/hcp_capacity_calculation.go:141 The worker nodes have 12.000000 vCPUs 2024-01-05T19:53:54.070Z INFO agent.agent-reconciler agent/hcp_capacity_calculation.go:142 The worker nodes have 49.173369 GB memory
2024-01-05T19:53:54.052Z ERROR agent.agent-reconciler agent/agent.go:788 failed to get configmap from the hub. Setting the HCP sizing baseline with default values. {"error": "configmaps \"hcp-sizing-baseline\" not found"}
login
edit addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: hypershift-addon-deploy-config namespace: multicluster-engine spec: customizedVariables: - name: hcMaxNumber value: "80" - name: hcThresholdNumber value: "60" - name: disableMetrics value: "true"
tar xvzf hcp.tar.gz
chmod +x hcp
sudo mv hcp /usr/local/bin/.
hcp create cluster <platform> --help 1
get ConsoleCLIDownload hcp-cli-download -o json | jq -r ".spec"
wget <hcp_cli_download_url> 1
tar xvzf hcp.tar.gz
chmod +x hcp
sudo mv hcp /usr/local/bin/.
tar xvzf hcp.tar.gz
chmod +x hcp
sudo mv hcp /usr/local/bin/.
hcp create cluster <platform> --help 1
label node/worker-1a node/worker-1b topology.kubernetes.io/zone=rack1 label node/worker-2a node/worker-2b topology.kubernetes.io/zone=rack2
spec: nodeSelector: role.kubernetes.io/infra: ""
get managedclusters local-cluster
aws s3api create-bucket --bucket <your-bucket-name> aws s3api delete-public-access-block --bucket <your-bucket-name> echo '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": "*", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::<your-bucket-name>/*" } ] }' | envsubst > policy.json aws s3api put-bucket-policy --bucket <your-bucket-name> --policy file://policy.json
aws s3api create-bucket --bucket <your-bucket-name> --create-bucket-configuration LocationConstraint=<region> --region <region> aws s3api delete-public-access-block --bucket <your-bucket-name> echo '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": "*", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::<your-bucket-name>/*" } ] }' | envsubst > policy.json aws s3api put-bucket-policy --bucket <your-bucket-name> --policy file://policy.json
create secret generic hypershift-operator-oidc-provider-s3-credentials --from-file=credentials=<path>/.aws/credentials --from-literal=bucket=<s3-bucket-for-hypershift> --from-literal=region=<region> -n local-cluster
label secret hypershift-operator-oidc-provider-s3-credentials -n local-cluster cluster.open-cluster-management.io/backup=true
aws route53 create-hosted-zone --name <your-basedomain> --caller-reference USD(whoami)-USD(date --rfc-3339=date)
aws sts get-caller-identity --query "Arn" --output text
arn:aws:iam::1234567890:user/<aws-username>
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": "<arn>" }, "Action": "sts:AssumeRole" } ] }
aws iam create-role --role-name <hcp-cli-role> --assume-role-policy-document file://trust-relationship.json --query "Role.Arn" --output json
arn:aws:iam::820196288204:role/myrole
{ "Version": "2012-10-17", "Statement": [ { "Sid": "EC2", "Effect": "Allow", "Action": [ "ec2:CreateDhcpOptions", "ec2:DeleteSubnet", "ec2:ReplaceRouteTableAssociation", "ec2:DescribeAddresses", "ec2:DescribeInstances", "ec2:DeleteVpcEndpoints", "ec2:CreateNatGateway", "ec2:CreateVpc", "ec2:DescribeDhcpOptions", "ec2:AttachInternetGateway", "ec2:DeleteVpcEndpointServiceConfigurations", "ec2:DeleteRouteTable", "ec2:AssociateRouteTable", "ec2:DescribeInternetGateways", "ec2:DescribeAvailabilityZones", "ec2:CreateRoute", "ec2:CreateInternetGateway", "ec2:RevokeSecurityGroupEgress", "ec2:ModifyVpcAttribute", "ec2:DeleteInternetGateway", "ec2:DescribeVpcEndpointConnections", "ec2:RejectVpcEndpointConnections", "ec2:DescribeRouteTables", "ec2:ReleaseAddress", "ec2:AssociateDhcpOptions", "ec2:TerminateInstances", "ec2:CreateTags", "ec2:DeleteRoute", "ec2:CreateRouteTable", "ec2:DetachInternetGateway", "ec2:DescribeVpcEndpointServiceConfigurations", "ec2:DescribeNatGateways", "ec2:DisassociateRouteTable", "ec2:AllocateAddress", "ec2:DescribeSecurityGroups", "ec2:RevokeSecurityGroupIngress", "ec2:CreateVpcEndpoint", "ec2:DescribeVpcs", "ec2:DeleteSecurityGroup", "ec2:DeleteDhcpOptions", "ec2:DeleteNatGateway", "ec2:DescribeVpcEndpoints", "ec2:DeleteVpc", "ec2:CreateSubnet", "ec2:DescribeSubnets" ], "Resource": "*" }, { "Sid": "ELB", "Effect": "Allow", "Action": [ "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DeleteTargetGroup" ], "Resource": "*" }, { "Sid": "IAMPassRole", "Effect": "Allow", "Action": "iam:PassRole", "Resource": "arn:*:iam::*:role/*-worker-role", "Condition": { "ForAnyValue:StringEqualsIfExists": { "iam:PassedToService": "ec2.amazonaws.com" } } }, { "Sid": "IAM", "Effect": "Allow", "Action": [ "iam:CreateInstanceProfile", "iam:DeleteInstanceProfile", "iam:GetRole", "iam:UpdateAssumeRolePolicy", "iam:GetInstanceProfile", "iam:TagRole", "iam:RemoveRoleFromInstanceProfile", "iam:CreateRole", "iam:DeleteRole", "iam:PutRolePolicy", "iam:AddRoleToInstanceProfile", "iam:CreateOpenIDConnectProvider", "iam:TagOpenIDConnectProvider", "iam:ListOpenIDConnectProviders", "iam:DeleteRolePolicy", "iam:UpdateRole", "iam:DeleteOpenIDConnectProvider", "iam:GetRolePolicy" ], "Resource": "*" }, { "Sid": "Route53", "Effect": "Allow", "Action": [ "route53:ListHostedZonesByVPC", "route53:CreateHostedZone", "route53:ListHostedZones", "route53:ChangeResourceRecordSets", "route53:ListResourceRecordSets", "route53:DeleteHostedZone", "route53:AssociateVPCWithHostedZone", "route53:ListHostedZonesByName" ], "Resource": "*" }, { "Sid": "S3", "Effect": "Allow", "Action": [ "s3:ListAllMyBuckets", "s3:ListBucket", "s3:DeleteObject", "s3:DeleteBucket" ], "Resource": "*" } ] }
aws iam put-role-policy --role-name <role_name> \ 1 --policy-name <policy_name> \ 2 --policy-document file://policy.json 3
aws sts get-session-token --output json > sts-creds.json
{ "Credentials": { "AccessKeyId": "ASIA1443CE0GN2ATHWJU", "SecretAccessKey": "XFLN7cZ5AP0d66KhyI4gd8Mu0UCQEDN9cfelW1", "SessionToken": "IQoJb3JpZ2luX2VjEEAaCXVzLWVhc3QtMiJHMEUCIDyipkM7oPKBHiGeI0pMnXst1gDLfs/TvfskXseKCbshAiEAnl1l/Html7Iq9AEIqf//////////KQburfkq4A3TuppHMr/9j1TgCj1z83SO261bHqlJUazKoy7vBFR/a6LHt55iMBqtKPEsIWjBgj/jSdRJI3j4Gyk1//luKDytcfF/tb9YrxDTPLrACS1lqAxSIFZ82I/jDhbDs=", "Expiration": "2025-05-16T04:19:32+00:00" } }
create secret generic hypershift-operator-external-dns-credentials --from-literal=provider=aws --from-literal=domain-filter=<domain_name> --from-file=credentials=<path_to_aws_credentials_file> -n local-cluster
label secret hypershift-operator-external-dns-credentials -n local-cluster cluster.open-cluster-management.io/backup=""
dig +short test.user-dest-public.aws.kerberos.com 192.168.1.1
hcp create cluster aws --name=<hosted_cluster_name> --endpoint-access=PublicAndPrivate --external-dns-domain=<public_hosted_zone>
platform: aws: endpointAccess: PublicAndPrivate services: - service: APIServer servicePublishingStrategy: route: hostname: api-example.service-provider-domain.com type: Route - service: OAuthServer servicePublishingStrategy: route: hostname: oauth-example.service-provider-domain.com type: Route - service: Konnectivity servicePublishingStrategy: type: Route - service: Ignition servicePublishingStrategy: type: Route
export KUBECONFIG=<path_to_management_cluster_kubeconfig>
get pod -n hypershift -lapp=external-dns
NAME READY STATUS RESTARTS AGE external-dns-7c89788c69-rn8gp 1/1 Running 0 40s
hcp create cluster aws --role-arn <arn_role> \ 1 --instance-type <instance_type> \ 2 --region <region> \ 3 --auto-repair --generate-ssh --name <hosted_cluster_name> \ 4 --namespace clusters --base-domain <service_consumer_domain> \ 5 --node-pool-replicas <node_replica_count> \ 6 --pull-secret <path_to_your_pull_secret> \ 7 --release-image quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 8 --external-dns-domain=<service_provider_domain> \ 9 --endpoint-access=PublicAndPrivate 10 --sts-creds <path_to_sts_credential_file> 11
create secret generic hypershift-operator-private-link-credentials --from-literal=aws-access-key-id=<aws-access-key-id> --from-literal=aws-secret-access-key=<aws-secret-access-key> --from-literal=region=<region> -n local-cluster
label secret hypershift-operator-private-link-credentials -n local-cluster cluster.open-cluster-management.io/backup=""
hcp create cluster aws --name <hosted_cluster_name> \ 1 --infra-id <infra_id> \ 2 --base-domain <basedomain> \ 3 --sts-creds <path_to_sts_credential_file> \ 4 --pull-secret <path_to_pull_secret> \ 5 --region <region> \ 6 --generate-ssh --node-pool-replicas <node_pool_replica_count> \ 7 --namespace <hosted_cluster_namespace> \ 8 --role-arn <role_name> 9
get hostedclusters -n <hosted_cluster_namespace>
get nodepools --namespace <hosted_cluster_namespace>
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas=<node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --role-arn <arn_role> \ 5 --region <region> \ 6 --zones <zones> 7 --sts-creds <path_to_sts_credential_file> 8
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas <node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --sts-creds <path_to_sts_credential_file> \ 5 --region <region> 6 --role-arn <arn_role> \ 7
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas <node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --sts-creds <path_to_sts_credential_file> \ 5 --region <region> \ 6 --release-image quay.io/openshift-release-dev/ocp-release:<ocp_release_image> \ 7 --role-arn <role_name> \ 8 --multi-arch 9
hcp create nodepool aws --cluster-name <hosted_cluster_name> \ 1 --name <nodepool_name> \ 2 --node-count <node_pool_replica_count> 3
hcp create nodepool aws --cluster-name <hosted_cluster_name> \ 1 --name <node_pool_name> \ 2 --node-count <node_pool_replica_count> 3 --arch <architecture> 4
--kubeconfig <hosted-cluster-name>.kubeconfig get nodes
hcp create kubeconfig --namespace <hosted-cluster-namespace> --name <hosted-cluster-name> > <hosted-cluster-name>.kubeconfig
--kubeconfig <hosted-cluster-name>.kubeconfig get nodes
hcp create cluster aws --name <hosted_cluster_name> \ 1 --node-pool-replicas=<node_pool_replica_count> \ 2 --base-domain <basedomain> \ 3 --pull-secret <path_to_pull_secret> \ 4 --sts-creds <path_to_sts_credential_file> \ 5 --region <region> \ 6 --endpoint-access Private 7 --role-arn <role_name> 8
aws ec2 describe-instances --filter="Name=tag:kubernetes.io/cluster/<infra_id>,Values=owned" | jq '.Reservations[] | .Instances[] | select(.PublicDnsName=="") | .PrivateIpAddress'
hcp create kubeconfig > <hosted_cluster_kubeconfig>
ssh -o ProxyCommand="ssh ec2-user@<bastion_ip> -W %h:%p" core@<node_ip>
mv <path_to_kubeconfig_file> <new_file_name>
export KUBECONFIG=<path_to_kubeconfig_file>
get clusteroperators clusterversion
endpointAccess: Public region: us-east-2 resourceTags: - key: kubernetes.io/cluster/example-cluster-bz4j5 value: owned rolesRef: controlPlaneOperatorARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-control-plane-operator imageRegistryARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-openshift-image-registry ingressARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-openshift-ingress kubeCloudControllerARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-cloud-controller networkARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-cloud-network-config-controller nodePoolManagementARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-node-pool storageARN: arn:aws:iam::820196288204:role/example-cluster-bz4j5-aws-ebs-csi-driver-controller type: AWS
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "elasticloadbalancing:DescribeLoadBalancers", "tag:GetResources", "route53:ListHostedZones" ], "Resource": "\*" }, { "Effect": "Allow", "Action": [ "route53:ChangeResourceRecordSets" ], "Resource": [ "arn:aws:route53:::PUBLIC_ZONE_ID", "arn:aws:route53:::PRIVATE_ZONE_ID" ] } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "s3:CreateBucket", "s3:DeleteBucket", "s3:PutBucketTagging", "s3:GetBucketTagging", "s3:PutBucketPublicAccessBlock", "s3:GetBucketPublicAccessBlock", "s3:PutEncryptionConfiguration", "s3:GetEncryptionConfiguration", "s3:PutLifecycleConfiguration", "s3:GetLifecycleConfiguration", "s3:GetBucketLocation", "s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucketMultipartUploads", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts" ], "Resource": "\*" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:AttachVolume", "ec2:CreateSnapshot", "ec2:CreateTags", "ec2:CreateVolume", "ec2:DeleteSnapshot", "ec2:DeleteTags", "ec2:DeleteVolume", "ec2:DescribeInstances", "ec2:DescribeSnapshots", "ec2:DescribeTags", "ec2:DescribeVolumes", "ec2:DescribeVolumesModifications", "ec2:DetachVolume", "ec2:ModifyVolume" ], "Resource": "\*" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:DescribeInstances", "ec2:DescribeInstanceStatus", "ec2:DescribeInstanceTypes", "ec2:UnassignPrivateIpAddresses", "ec2:AssignPrivateIpAddresses", "ec2:UnassignIpv6Addresses", "ec2:AssignIpv6Addresses", "ec2:DescribeSubnets", "ec2:DescribeNetworkInterfaces" ], "Resource": "\*" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Action": [ "ec2:DescribeInstances", "ec2:DescribeImages", "ec2:DescribeRegions", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVolumes", "ec2:CreateSecurityGroup", "ec2:CreateTags", "ec2:CreateVolume", "ec2:ModifyInstanceAttribute", "ec2:ModifyVolume", "ec2:AttachVolume", "ec2:AuthorizeSecurityGroupIngress", "ec2:CreateRoute", "ec2:DeleteRoute", "ec2:DeleteSecurityGroup", "ec2:DeleteVolume", "ec2:DetachVolume", "ec2:RevokeSecurityGroupIngress", "ec2:DescribeVpcs", "elasticloadbalancing:AddTags", "elasticloadbalancing:AttachLoadBalancerToSubnets", "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", "elasticloadbalancing:CreateLoadBalancer", "elasticloadbalancing:CreateLoadBalancerPolicy", "elasticloadbalancing:CreateLoadBalancerListeners", "elasticloadbalancing:ConfigureHealthCheck", "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:DeleteLoadBalancerListeners", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DetachLoadBalancerFromSubnets", "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", "elasticloadbalancing:ModifyLoadBalancerAttributes", "elasticloadbalancing:RegisterInstancesWithLoadBalancer", "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", "elasticloadbalancing:AddTags", "elasticloadbalancing:CreateListener", "elasticloadbalancing:CreateTargetGroup", "elasticloadbalancing:DeleteListener", "elasticloadbalancing:DeleteTargetGroup", "elasticloadbalancing:DescribeListeners", "elasticloadbalancing:DescribeLoadBalancerPolicies", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:ModifyListener", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:RegisterTargets", "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", "iam:CreateServiceLinkedRole", "kms:DescribeKey" ], "Resource": [ "\*" ], "Effect": "Allow" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Action": [ "ec2:AllocateAddress", "ec2:AssociateRouteTable", "ec2:AttachInternetGateway", "ec2:AuthorizeSecurityGroupIngress", "ec2:CreateInternetGateway", "ec2:CreateNatGateway", "ec2:CreateRoute", "ec2:CreateRouteTable", "ec2:CreateSecurityGroup", "ec2:CreateSubnet", "ec2:CreateTags", "ec2:DeleteInternetGateway", "ec2:DeleteNatGateway", "ec2:DeleteRouteTable", "ec2:DeleteSecurityGroup", "ec2:DeleteSubnet", "ec2:DeleteTags", "ec2:DescribeAccountAttributes", "ec2:DescribeAddresses", "ec2:DescribeAvailabilityZones", "ec2:DescribeImages", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", "ec2:DescribeNatGateways", "ec2:DescribeNetworkInterfaces", "ec2:DescribeNetworkInterfaceAttribute", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs", "ec2:DescribeVpcAttribute", "ec2:DescribeVolumes", "ec2:DetachInternetGateway", "ec2:DisassociateRouteTable", "ec2:DisassociateAddress", "ec2:ModifyInstanceAttribute", "ec2:ModifyNetworkInterfaceAttribute", "ec2:ModifySubnetAttribute", "ec2:ReleaseAddress", "ec2:RevokeSecurityGroupIngress", "ec2:RunInstances", "ec2:TerminateInstances", "tag:GetResources", "ec2:CreateLaunchTemplate", "ec2:CreateLaunchTemplateVersion", "ec2:DescribeLaunchTemplates", "ec2:DescribeLaunchTemplateVersions", "ec2:DeleteLaunchTemplate", "ec2:DeleteLaunchTemplateVersions" ], "Resource": [ "\*" ], "Effect": "Allow" }, { "Condition": { "StringLike": { "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" } }, "Action": [ "iam:CreateServiceLinkedRole" ], "Resource": [ "arn:*:iam::*:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing" ], "Effect": "Allow" }, { "Action": [ "iam:PassRole" ], "Resource": [ "arn:*:iam::*:role/*-worker-role" ], "Effect": "Allow" } ] }
{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:CreateVpcEndpoint", "ec2:DescribeVpcEndpoints", "ec2:ModifyVpcEndpoint", "ec2:DeleteVpcEndpoints", "ec2:CreateTags", "route53:ListHostedZones" ], "Resource": "\*" }, { "Effect": "Allow", "Action": [ "route53:ChangeResourceRecordSets", "route53:ListResourceRecordSets" ], "Resource": "arn:aws:route53:::%s" } ] }
--- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters spec: {} status: {} --- apiVersion: v1 data: .dockerconfigjson: xxxxxxxxxxx kind: Secret metadata: creationTimestamp: null labels: hypershift.openshift.io/safe-to-delete-with-cluster: "true" name: <pull_secret_name> 1 namespace: clusters --- apiVersion: v1 data: key: xxxxxxxxxxxxxxxxx kind: Secret metadata: creationTimestamp: null labels: hypershift.openshift.io/safe-to-delete-with-cluster: "true" name: <etcd_encryption_key_name> 2 namespace: clusters type: Opaque --- apiVersion: v1 data: id_rsa: xxxxxxxxx id_rsa.pub: xxxxxxxxx kind: Secret metadata: creationTimestamp: null labels: hypershift.openshift.io/safe-to-delete-with-cluster: "true" name: <ssh-key-name> 3 namespace: clusters --- apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: creationTimestamp: null name: <hosted_cluster_name> 4 namespace: clusters spec: autoscaling: {} configuration: {} controllerAvailabilityPolicy: SingleReplica dns: baseDomain: <dns_domain> 5 privateZoneID: xxxxxxxx publicZoneID: xxxxxxxx etcd: managed: storage: persistentVolume: size: 8Gi storageClassName: gp3-csi type: PersistentVolume managementType: Managed fips: false infraID: <infra_id> 6 issuerURL: <issuer_url> 7 networking: clusterNetwork: - cidr: 10.132.0.0/14 machineNetwork: - cidr: 10.0.0.0/16 networkType: OVNKubernetes serviceNetwork: - cidr: 172.31.0.0/16 olmCatalogPlacement: management platform: aws: cloudProviderConfig: subnet: id: <subnet_xxx> 8 vpc: <vpc_xxx> 9 zone: us-west-1b endpointAccess: Public multiArch: false region: us-west-1 rolesRef: controlPlaneOperatorARN: arn:aws:iam::820196288204:role/<infra_id>-control-plane-operator imageRegistryARN: arn:aws:iam::820196288204:role/<infra_id>-openshift-image-registry ingressARN: arn:aws:iam::820196288204:role/<infra_id>-openshift-ingress kubeCloudControllerARN: arn:aws:iam::820196288204:role/<infra_id>-cloud-controller networkARN: arn:aws:iam::820196288204:role/<infra_id>-cloud-network-config-controller nodePoolManagementARN: arn:aws:iam::820196288204:role/<infra_id>-node-pool storageARN: arn:aws:iam::820196288204:role/<infra_id>-aws-ebs-csi-driver-controller type: AWS pullSecret: name: <pull_secret_name> release: image: quay.io/openshift-release-dev/ocp-release:4.16-x86_64 secretEncryption: aescbc: activeKey: name: <etcd_encryption_key_name> type: aescbc services: - service: APIServer servicePublishingStrategy: type: LoadBalancer - service: OAuthServer servicePublishingStrategy: type: Route - service: Konnectivity servicePublishingStrategy: type: Route - service: Ignition servicePublishingStrategy: type: Route - service: OVNSbDb servicePublishingStrategy: type: Route sshKey: name: <ssh_key_name> status: controlPlaneEndpoint: host: "" port: 0 --- apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: creationTimestamp: null name: <node_pool_name> 10 namespace: clusters spec: arch: amd64 clusterName: <hosted_cluster_name> management: autoRepair: true upgradeType: Replace nodeDrainTimeout: 0s platform: aws: instanceProfile: <instance_profile_name> 11 instanceType: m6i.xlarge rootVolume: size: 120 type: gp3 subnet: id: <subnet_xxx> type: AWS release: image: quay.io/openshift-release-dev/ocp-release:4.16-x86_64 replicas: 2 status: replicas: 0
hcp create cluster aws --infra-id <infra_id> \ 1 --name <hosted_cluster_name> \ 2 --sts-creds <path_to_sts_credential_file> \ 3 --pull-secret <path_to_pull_secret> \ 4 --generate-ssh \ 5 --node-pool-replicas 3 --role-arn <role_name> 6
delete managedcluster <hosted_cluster_name>
hcp destroy cluster aws --name <hosted_cluster_name> \ 1 --infra-id <infra_id> \ 2 --role-arn <arn_role> \ 3 --sts-creds <path_to_sts_credential_file> \ 4 --base-domain <basedomain> 5
aws sts get-session-token --output json > sts-creds.json
get managedclusters local-cluster
api.example.krnl.es. IN A 192.168.122.20 api.example.krnl.es. IN A 192.168.122.21 api.example.krnl.es. IN A 192.168.122.22 api-int.example.krnl.es. IN A 192.168.122.20 api-int.example.krnl.es. IN A 192.168.122.21 api-int.example.krnl.es. IN A 192.168.122.22 `*`.apps.example.krnl.es. IN A 192.168.122.23
api.example.krnl.es. IN A 2620:52:0:1306::5 api.example.krnl.es. IN A 2620:52:0:1306::6 api.example.krnl.es. IN A 2620:52:0:1306::7 api-int.example.krnl.es. IN A 2620:52:0:1306::5 api-int.example.krnl.es. IN A 2620:52:0:1306::6 api-int.example.krnl.es. IN A 2620:52:0:1306::7 `*`.apps.example.krnl.es. IN A 2620:52:0:1306::10
host-record=api-int.hub-dual.dns.base.domain.name,192.168.126.10 host-record=api.hub-dual.dns.base.domain.name,192.168.126.10 address=/apps.hub-dual.dns.base.domain.name/192.168.126.11 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,192.168.126.20 dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,192.168.126.21 dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,192.168.126.22 dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,192.168.126.25 dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,192.168.126.26 host-record=api-int.hub-dual.dns.base.domain.name,2620:52:0:1306::2 host-record=api.hub-dual.dns.base.domain.name,2620:52:0:1306::2 address=/apps.hub-dual.dns.base.domain.name/2620:52:0:1306::3 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,[2620:52:0:1306::5] dhcp-host=aa:aa:aa:aa:10:02,ocp-master-1,[2620:52:0:1306::6] dhcp-host=aa:aa:aa:aa:10:03,ocp-master-2,[2620:52:0:1306::7] dhcp-host=aa:aa:aa:aa:10:06,ocp-installer,[2620:52:0:1306::8] dhcp-host=aa:aa:aa:aa:10:07,ocp-bootstrap,[2620:52:0:1306::9]
create ns <hosted_cluster_namespace>-<hosted_cluster_name>
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --etcd-storage-class=<etcd_storage_class> \ 6 --ssh-key <path_to_ssh_public_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --control-plane-availability-policy SingleReplica --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> 9
-n <hosted_control_plane_namespace> get pods
NAME READY STATUS RESTARTS AGE capi-provider-7dcf5fc4c4-nr9sq 1/1 Running 0 4m32s catalog-operator-6cd867cc7-phb2q 2/2 Running 0 2m50s certified-operators-catalog-884c756c4-zdt64 1/1 Running 0 2m51s cluster-api-f75d86f8c-56wfz 1/1 Running 0 4m32s
- mirrors: - brew.registry.redhat.io source: registry.redhat.io - mirrors: - brew.registry.redhat.io source: registry.stage.redhat.io - mirrors: - brew.registry.redhat.io source: registry-proxy.engineering.redhat.com
hcp create cluster agent --name=<hosted_cluster_name> \ 1 --pull-secret=<path_to_pull_secret> \ 2 --agent-namespace=<hosted_control_plane_namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted_cluster_name>.<basedomain> \ 5 --image-content-sources icsp.yaml \ 6 --ssh-key <path_to_ssh_key> \ 7 --namespace <hosted_cluster_namespace> \ 8 --release-image=quay.io/openshift-release-dev/ocp-release:<ocp_release_image> 9
extract -n <hosted-control-plane-namespace> secret/admin-kubeconfig --to=- > kubeconfig-<hosted-cluster-name>
get co --kubeconfig=kubeconfig-<hosted-cluster-name>
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console 4.10.26 True False False 2m38s dns 4.10.26 True False False 2m52s image-registry 4.10.26 True False False 2m8s ingress 4.10.26 True False False 22m
get pods -A --kubeconfig=kubeconfig-<hosted-cluster-name>
NAMESPACE NAME READY STATUS RESTARTS AGE kube-system konnectivity-agent-khlqv 0/1 Running 0 3m52s openshift-cluster-node-tuning-operator tuned-dhw5p 1/1 Running 0 109s openshift-cluster-storage-operator cluster-storage-operator-5f784969f5-vwzgz 1/1 Running 1 (113s ago) 20m openshift-cluster-storage-operator csi-snapshot-controller-6b7687b7d9-7nrfw 1/1 Running 0 3m8s openshift-console console-5cbf6c7969-6gk6z 1/1 Running 0 119s openshift-console downloads-7bcd756565-6wj5j 1/1 Running 0 4m3s openshift-dns-operator dns-operator-77d755cd8c-xjfbn 2/2 Running 0 21m openshift-dns dns-default-kfqnh 2/2 Running 0 113s
-n <hosted-cluster-namespace> scale nodepool <nodepool-name> --replicas 2
-n <hosted-control-plane-namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 hypercluster1 true auto-assign d9198891-39f4-4930-a679-65fb142b108b true auto-assign da503cf1-a347-44f2-875c-4960ddb04091 hypercluster1 true auto-assign
-n <hosted-control-plane-namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: ocp-worker-2 Agent: 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 State: binding BMH: ocp-worker-0 Agent: d9198891-39f4-4930-a679-65fb142b108b State: known-unbound BMH: ocp-worker-1 Agent: da503cf1-a347-44f2-875c-4960ddb04091 State: insufficient
extract -n <hosted-cluster-namespace> secret/<hosted-cluster-name>-admin-kubeconfig --to=- > kubeconfig-<hosted-cluster-name>
--kubeconfig kubeconfig-<hosted-cluster-name> get nodes
NAME STATUS ROLES AGE VERSION ocp-worker-1 Ready worker 5m41s v1.24.0+3882f8f ocp-worker-2 Ready worker 6m3s v1.24.0+3882f8f
-n <hosted-control-plane-namespace> get machines
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hypercluster1-c96b6f675-m5vch hypercluster1-b2qhl ocp-worker-1 agent://da503cf1-a347-44f2-875c-4960ddb04091 Running 15m 4.x.z hypercluster1-c96b6f675-tl42p hypercluster1-b2qhl ocp-worker-2 agent://4dac1ab2-7dd5-4894-a220-6a3473b67ee6 Running 15m 4.x.z
--kubeconfig kubeconfig-<hosted-cluster-name> get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version False True 40m Unable to apply 4.x.z: the cluster operator console has not yet successfully rolled out NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.12z False False False 11m RouteHealthAvailable: failed to GET route (https://console-openshift-console.apps.hypercluster1.domain.com): Get "https://console-openshift-console.apps.hypercluster1.domain.com": dial tcp 10.19.3.29:443: connect: connection refused clusteroperator.config.openshift.io/csi-snapshot-controller 4.12z True False False 10m clusteroperator.config.openshift.io/dns 4.12z True False False 9m16s
export NODEPOOL_NAME=USD{CLUSTER_NAME}-extra-cpu export WORKER_COUNT="2" hcp create nodepool agent --cluster-name USDCLUSTER_NAME --name USDNODEPOOL_NAME --node-count USDWORKER_COUNT --agentLabelSelector '{"matchLabels": {"size": "medium"}}' 1
get nodepools --namespace clusters
extract -n <hosted-control-plane-namespace> secret/admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
--kubeconfig ./hostedcluster-secrets get nodes
get nodepools --namespace clusters
apiVersion: v1 kind: Namespace metadata: name: metallb labels: openshift.io/cluster-monitoring: "true" annotations: workload.openshift.io/allowed: management --- apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: metallb-operator-operatorgroup namespace: metallb --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: metallb-operator namespace: metallb spec: channel: "stable" name: metallb-operator source: redhat-operators sourceNamespace: openshift-marketplace
apply -f metallb-operator-config.yaml
apiVersion: metallb.io/v1beta1 kind: MetalLB metadata: name: metallb namespace: metallb
apply -f metallb-instance-config.yaml
apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: <ip_address_pool_name> 1 namespace: metallb spec: protocol: layer2 autoAssign: false addresses: - <ingress_ip>-<ingress_ip> 2 --- apiVersion: metallb.io/v1beta1 kind: BGPAdvertisement metadata: name: <bgp_advertisement_name> 3 namespace: metallb spec: ipAddressPools: - <ip_address_pool_name> 4
apply -f ipaddresspool-bgpadvertisement-config.yaml
kind: Service apiVersion: v1 metadata: annotations: metallb.universe.tf/address-pool: ingress-public-ip name: metallb-ingress namespace: openshift-ingress spec: ports: - name: http protocol: TCP port: 80 targetPort: 80 - name: https protocol: TCP port: 443 targetPort: 443 selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller: default type: LoadBalancer
apply -f metallb-loadbalancer-service.yaml
curl -kI https://console-openshift-console.apps.example.krnl.es HTTP/1.1 200 OK
--kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.x.y True False 3m32s Cluster version is 4.x.y NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.x.y True False False 3m50s clusteroperator.config.openshift.io/ingress 4.x.y True False False 53m
-n <hosted-cluster-namespace> patch nodepool <hosted-cluster-name> --type=json -p '[{"op": "remove", "path": "/spec/replicas"},{"op":"add", "path": "/spec/autoScaling", "value": { "max": 5, "min": 2 }}]'
apiVersion: apps/v1 kind: Deployment metadata: creationTimestamp: null labels: app: reversewords name: reversewords namespace: default spec: replicas: 40 selector: matchLabels: app: reversewords strategy: {} template: metadata: creationTimestamp: null labels: app: reversewords spec: containers: - image: quay.io/mavazque/reversewords:latest name: reversewords resources: requests: memory: 2Gi status: {}
apply -f workload-config.yaml
extract -n <hosted-cluster-namespace> secret/<hosted-cluster-name>-admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
--kubeconfig ./hostedcluster-secrets get nodes
--kubeconfig ./hostedcluster-secrets -n default delete deployment reversewords
--kubeconfig ./hostedcluster-secrets get nodes
-n <hosted-cluster-namespace> patch nodepool <hosted-cluster-name> --type=json -p '[\{"op":"remove", "path": "/spec/autoScaling"}, \{"op": "add", "path": "/spec/replicas", "value": <specify-value-to-scale-replicas>]'
get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep nodeDrainTimeout
nodeDrainTimeout: 30s
patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec":{"nodeDrainTimeout": "30m"}}' --type=merge
patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":true}}}' --type=merge
get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":false}}}' --type=merge
get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
hcp destroy cluster agent --name <cluster_name>
get managedclusters local-cluster
api-int.example.krnl.es. IN A 192.168.122.22 `*`.apps.example.krnl.es. IN A 192.168.122.23
api-int.example.krnl.es. IN A 2620:52:0:1306::7 `*`.apps.example.krnl.es. IN A 2620:52:0:1306::10
host-record=api-int.hub-dual.dns.base.domain.name,2620:52:0:1306::2 address=/apps.hub-dual.dns.base.domain.name/2620:52:0:1306::3 dhcp-host=aa:aa:aa:aa:10:01,ocp-master-0,[2620:52:0:1306::5]
create ns <hosted-cluster-namespace>-<hosted-cluster-name>
hcp create cluster agent --name=<hosted-cluster-name> \ 1 --pull-secret=<path-to-pull-secret> \ 2 --agent-namespace=<hosted-control-plane-namespace> \ 3 --base-domain=<basedomain> \ 4 --api-server-address=api.<hosted-cluster-name>.<basedomain> \ 5 --etcd-storage-class=<etcd-storage-class> \ 6 --ssh-key <path-to-ssh-key> \ 7 --namespace <hosted-cluster-namespace> \ 8 --control-plane-availability-policy SingleReplica --release-image=quay.io/openshift-release-dev/ocp-release:<ocp-release> 9
-n <hosted-control-plane-namespace> get pods
NAME READY STATUS RESTARTS AGE catalog-operator-6cd867cc7-phb2q 2/2 Running 0 2m50s control-plane-operator-f6b4c8465-4k5dh 1/1 Running 0 4m32s
extract -n <hosted-cluster-namespace> secret/<hosted-cluster-name>-admin-kubeconfig --to=- > kubeconfig-<hosted-cluster-name>
get co --kubeconfig=kubeconfig-<hosted_cluster_name>
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console 4.10.26 True False False 2m38s csi-snapshot-controller 4.10.26 True False False 4m3s dns 4.10.26 True False False 2m52s
get pods -A --kubeconfig=kubeconfig-<hosted-cluster-name>
NAMESPACE NAME READY STATUS RESTARTS AGE kube-system konnectivity-agent-khlqv 0/1 Running 0 3m52s openshift-cluster-samples-operator cluster-samples-operator-6b5bcb9dff-kpnbc 2/2 Running 0 20m openshift-monitoring alertmanager-main-0 6/6 Running 0 100s openshift-monitoring openshift-state-metrics-677b9fb74f-qqp6g 3/3 Running 0 104s
-n <hosted-cluster-namespace> scale nodepool <nodepool-name> --replicas 2
-n <hosted-control-plane-namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 hypercluster1 true auto-assign
-n <hosted-control-plane-namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: ocp-worker-2 Agent: 4dac1ab2-7dd5-4894-a220-6a3473b67ee6 State: binding BMH: ocp-worker-1 Agent: da503cf1-a347-44f2-875c-4960ddb04091 State: insufficient
extract -n <hosted-cluster-namespace> secret/<hosted-cluster-name>-admin-kubeconfig --to=- > kubeconfig-<hosted-cluster-name>
--kubeconfig kubeconfig-<hosted-cluster-name> get nodes
NAME STATUS ROLES AGE VERSION ocp-worker-1 Ready worker 5m41s v1.24.0+3882f8f
-n <hosted-control-plane-namespace> get machines
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hypercluster1-c96b6f675-m5vch hypercluster1-b2qhl ocp-worker-1 agent://da503cf1-a347-44f2-875c-4960ddb04091 Running 15m 4.x.z
--kubeconfig kubeconfig-<hosted-cluster-name> get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version False True 40m Unable to apply 4.x.z: the cluster operator console has not yet successfully rolled out NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.x.z False False False 11m RouteHealthAvailable: failed to GET route (https://console-openshift-console.apps.hypercluster1.domain.com): Get "https://console-openshift-console.apps.hypercluster1.domain.com": dial tcp 10.19.3.29:443: connect: connection refused clusteroperator.config.openshift.io/csi-snapshot-controller 4.x.z True False False 10m clusteroperator.config.openshift.io/dns 4.x.z True False False 9m16s
export NODEPOOL_NAME=USD{CLUSTER_NAME}-extra-cpu export WORKER_COUNT="2" hcp create nodepool agent --cluster-name USDCLUSTER_NAME --name USDNODEPOOL_NAME --node-count USDWORKER_COUNT --agentLabelSelector '{"matchLabels": {"size": "medium"}}' 1
get nodepools --namespace clusters
extract -n <hosted-cluster-namespace> secret/<hosted-cluster-name>-admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
--kubeconfig ./hostedcluster-secrets get nodes
get nodepools --namespace clusters
apiVersion: v1 kind: Namespace metadata: name: metallb labels: openshift.io/cluster-monitoring: "true" annotations: workload.openshift.io/allowed: management --- apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: metallb-operator-operatorgroup namespace: metallb --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: metallb-operator namespace: metallb spec: channel: "stable" name: metallb-operator source: redhat-operators sourceNamespace: openshift-marketplace
apply -f metallb-operator-config.yaml
apiVersion: metallb.io/v1beta1 kind: MetalLB metadata: name: metallb namespace: metallb
apply -f metallb-instance-config.yaml
apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: <ip_address_pool_name> 1 namespace: metallb spec: protocol: layer2 autoAssign: false addresses: - <ingress_ip>-<ingress_ip> 2 --- apiVersion: metallb.io/v1beta1 kind: BGPAdvertisement metadata: name: <bgp_advertisement_name> 3 namespace: metallb spec: ipAddressPools: - <ip_address_pool_name> 4
apply -f ipaddresspool-bgpadvertisement-config.yaml
kind: Service apiVersion: v1 metadata: annotations: metallb.universe.tf/address-pool: ingress-public-ip name: metallb-ingress namespace: openshift-ingress spec: ports: - name: http protocol: TCP port: 80 targetPort: 80 - name: https protocol: TCP port: 443 targetPort: 443 selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller: default type: LoadBalancer
apply -f metallb-loadbalancer-service.yaml
curl -kI https://console-openshift-console.apps.example.krnl.es HTTP/1.1 200 OK
--kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.x.y True False 3m32s Cluster version is 4.x.y NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE clusteroperator.config.openshift.io/console 4.x.y True False False 3m50s clusteroperator.config.openshift.io/ingress 4.x.y True False False 53m
-n <hosted-cluster-namespace> patch nodepool <hosted-cluster-name> --type=json -p '[{"op": "remove", "path": "/spec/replicas"},{"op":"add", "path": "/spec/autoScaling", "value": { "max": 5, "min": 2 }}]'
apiVersion: apps/v1 kind: Deployment metadata: creationTimestamp: null labels: app: reversewords name: reversewords namespace: default spec: replicas: 40 selector: matchLabels: app: reversewords strategy: {} template: metadata: creationTimestamp: null labels: app: reversewords spec: containers: - image: quay.io/mavazque/reversewords:latest name: reversewords resources: requests: memory: 2Gi status: {}
apply -f workload-config.yaml
extract -n <hosted-cluster-namespace> secret/<hosted-cluster-name>admin-kubeconfig --to=./hostedcluster-secrets --confirm
hostedcluster-secrets/kubeconfig
--kubeconfig <hosted-cluster-name>.kubeconfig get nodes
--kubeconfig <hosted-cluster-name>.kubeconfig -n default delete deployment reversewords
--kubeconfig <hosted-cluster-name>.kubeconfig get nodes
-n <hosted-cluster-namespace> patch nodepool <hosted-cluster-name> --type=json -p '[\{"op":"remove", "path": "/spec/autoScaling"}, \{"op": "add", "path": "/spec/replicas", "value": <specify-value-to-scale-replicas>]'
get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep nodeDrainTimeout
nodeDrainTimeout: 30s
patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec":{"nodeDrainTimeout": "30m"}}' --type=merge
patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":true}}}' --type=merge
get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
patch nodepool -n <hosted_cluster_namespace> <nodepool_name> -p '{"spec": {"management": {"autoRepair":false}}}' --type=merge
get nodepool -n <hosted_cluster_namespace> <nodepool_name> -o yaml | grep autoRepair
hcp destroy cluster agent --name <hosted_cluster_name>
get managedclusters local-cluster
-n <hosted-control-plane-namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 86f7ac75-4fc4-4b36-8130-40fa12602218 auto-assign e57a637f-745b-496e-971d-1abbf03341ba auto-assign
-n <hosted-control-plane-namespace> patch agent 86f7ac75-4fc4-4b36-8130-40fa12602218 -p '{"spec":{"installation_disk_id":"/dev/sda","approved":true,"hostname":"worker-0.example.krnl.es"}}' --type merge -n <hosted-control-plane-namespace> patch agent 23d0c614-2caa-43f5-b7d3-0b3564688baa -p '{"spec":{"installation_disk_id":"/dev/sda","approved":true,"hostname":"worker-1.example.krnl.es"}}' --type merge
-n <hosted-control-plane-namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 86f7ac75-4fc4-4b36-8130-40fa12602218 true auto-assign e57a637f-745b-496e-971d-1abbf03341ba true auto-assign
cat /var/named/<example.krnl.es.zone>
TTL 900 @ IN SOA bastion.example.krnl.es.com. hostmaster.example.krnl.es.com. ( 2019062002 1D 1H 1W 3H ) IN NS bastion.example.krnl.es.com. ; ; api IN A 1xx.2x.2xx.1xx 1 api-int IN A 1xx.2x.2xx.1xx ; ; *.apps.<hosted-cluster-name>.<basedomain> IN A 1xx.2x.2xx.1xx ; ;EOF
compute-0 IN A 1xx.2x.2xx.1yy compute-1 IN A 1xx.2x.2xx.1yy
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: <hosted-cluster-name> namespace: <hosted-control-plane-namespace> spec: cpuArchitecture: ppc64le pullSecretRef: name: pull-secret sshAuthorizedKey: <ssh-public-key>
apply -f infraenv-config.yaml
-n <hosted-control-plane-namespace> get InfraEnv <hosted-cluster-name> -o json
-n <clusters_namespace> scale nodepool <nodepool_name> --replicas 2
-n <hosted_control_plane_namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: Agent: 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d State: known-unbound BMH: Agent: 5e498cd3-542c-e54f-0c58-ed43e28b568a State: insufficient
-n <hosted_control_plane_namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d hosted-forwarder true auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a true auto-assign da503cf1-a347-44f2-875c-4960ddb04091 hosted-forwarder true auto-assign
hcp create kubeconfig --namespace <clusters_namespace> --name <hosted_cluster_namespace> > <hosted_cluster_name>.kubeconfig
--kubeconfig <hosted_cluster_name>.kubeconfig get nodes
NAME STATUS ROLES AGE VERSION worker-zvm-0.hostedn.example.com Ready worker 5m41s v1.24.0+3882f8f worker-zvm-1.hostedn.example.com Ready worker 6m3s v1.24.0+3882f8f
-n <hosted_control_plane_namespace> get machine.cluster.x-k8s.io
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hosted-forwarder-79558597ff-5tbqp hosted-forwarder-crqq5 worker-zvm-0.hostedn.example.com agent://50c23cda-cedc-9bbd-bcf1-9b3a5c75804d Running 41h 4.15.0 hosted-forwarder-79558597ff-lfjfk hosted-forwarder-crqq5 worker-zvm-1.hostedn.example.com agent://5e498cd3-542c-e54f-0c58-ed43e28b568a Running 41h 4.15.0
--kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.15.0 True False 40h Cluster version is 4.15.0
--kubeconfig <hosted_cluster_name>.kubeconfig get clusteroperators
get managedclusters local-cluster
virt-install --name "<vm_name>" \ 1 --autostart --ram=16384 --cpu host --vcpus=4 --location "<path_to_kernel_initrd_image>,kernel=kernel.img,initrd=initrd.img" \ 2 --disk <qcow_image_path> \ 3 --network network:macvtap-net,mac=<mac_address> \ 4 --graphics none --noautoconsole --wait=-1 --extra-args "rd.neednet=1 nameserver=<nameserver> coreos.live.rootfs_url=http://<http_server>/rootfs.img random.trust_cpu=on rd.luks.options=discard ignition.firstboot ignition.platform.id=metal console=tty1 console=ttyS1,115200n8 coreos.inst.persistent-kargs=console=tty1 console=ttyS1,115200n8" 5
virt-install --name "<vm_name>" \ 1 --autostart --memory=16384 --cpu host --vcpus=4 --network network:macvtap-net,mac=<mac_address> \ 2 --cdrom "<path_to_image.iso>" \ 3 --disk <qcow_image_path> --graphics none --noautoconsole --os-variant <os_version> \ 4 --wait=-1
rd.neednet=1 cio_ignore=all,!condev console=ttysclp0 ignition.firstboot ignition.platform.id=metal coreos.live.rootfs_url=http://<http_server>/rhcos-<version>-live-rootfs.<architecture>.img \ 1 coreos.inst.persistent-kargs=console=ttysclp0 ip=<ip>::<gateway>:<netmask>:<hostname>::none nameserver=<dns> \ 2 rd.znet=qeth,<network_adaptor_range>,layer2=1 rd.<disk_type>=<adapter> \ 3 zfcp.allow_lun_scan=0 ai.ip_cfg_override=1 \ 4 random.trust_cpu=on rd.luks.options=discard
KERNEL_IMG_PATH='./kernel.img' INITRD_IMG_PATH='./initrd.img' CMDLINE_PATH='./generic.prm' kernel_size=USD(stat -c%s USDKERNEL_IMG_PATH ) initrd_size=USD(stat -c%s USDINITRD_IMG_PATH)
offset=USD(( (kernel_size + 1048575) / 1048576 * 1048576 ))
INITRD_IMG_NAME=USD(echo USDINITRD_IMG_PATH | rev | cut -d '/' -f 1 | rev) KERNEL_OFFSET=0x00000000 KERNEL_CMDLINE_OFFSET=0x00010480 INITRD_ADDR_SIZE_OFFSET=0x00010408 OFFSET_HEX=USD(printf '0x%08x\n' USDoffset)
printf "USD(printf '%016x\n' USDinitrd_size)" | xxd -r -p > temp_size.bin
cat temp_address.bin temp_size.bin > "USDINITRD_IMG_NAME.addrsize"
rm -rf temp_address.bin temp_size.bin
USDKERNEL_IMG_PATH USDKERNEL_OFFSET USDINITRD_IMG_PATH USDOFFSET_HEX USDINITRD_IMG_NAME.addrsize USDINITRD_ADDR_SIZE_OFFSET USDCMDLINE_PATH USDKERNEL_CMDLINE_OFFSET
rd.neednet=1 cio_ignore=all,!condev console=ttysclp0 ignition.firstboot ignition.platform.id=metal coreos.live.rootfs_url=http://<http_server>/rhcos-<version>-live-rootfs.<architecture>.img \ 1 coreos.inst.persistent-kargs=console=ttysclp0 ip=<ip>::<gateway>:<netmask>:<hostname>::none nameserver=<dns> \ 2 rd.znet=qeth,<network_adaptor_range>,layer2=1 rd.<disk_type>=<adapter> \ 3 zfcp.allow_lun_scan=0 ai.ip_cfg_override=1 \ 4
vmur pun -r -u -N kernel.img USDINSTALLERKERNELLOCATION/<image name>
vmur pun -r -u -N generic.parm USDPARMFILELOCATION/paramfilename
vmur pun -r -u -N initrd.img USDINSTALLERINITRAMFSLOCATION/<image name>
cp ipl c
-n <hosted_control_plane_namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a auto-assign
-n <hosted_control_plane_namespace> patch agent 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d -p '{"spec":{"installation_disk_id":"/dev/sda","approved":true,"hostname":"worker-zvm-0.hostedn.example.com"}}' --type merge
-n <hosted_control_plane_namespace> get agents
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d true auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a true auto-assign
cat /var/named/<example.krnl.es.zone>
TTL 900 @ IN SOA bastion.example.krnl.es.com. hostmaster.example.krnl.es.com. ( 2019062002 1D 1H 1W 3H ) IN NS bastion.example.krnl.es.com. ; ; api IN A 1xx.2x.2xx.1xx 1 api-int IN A 1xx.2x.2xx.1xx ; ; *.apps IN A 1xx.2x.2xx.1xx ; ;EOF
compute-0 IN A 1xx.2x.2xx.1yy compute-1 IN A 1xx.2x.2xx.1yy
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: <hosted-cluster-name> namespace: <hosted-control-plane-namespace> spec: cpuArchitecture: s390x pullSecretRef: name: pull-secret sshAuthorizedKey: <ssh-public-key>
apply -f infraenv-config.yaml
-n <hosted-control-plane-namespace> get InfraEnv <hosted-cluster-name> -o json
-n <clusters_namespace> scale nodepool <nodepool_name> --replicas 2
-n <hosted_control_plane_namespace> get agent -o jsonpath='{range .items[*]}BMH: {@.metadata.labels.agent-install\.openshift\.io/bmh} Agent: {@.metadata.name} State: {@.status.debugInfo.state}{"\n"}{end}'
BMH: Agent: 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d State: known-unbound BMH: Agent: 5e498cd3-542c-e54f-0c58-ed43e28b568a State: insufficient
-n <hosted_control_plane_namespace> get agent
NAME CLUSTER APPROVED ROLE STAGE 50c23cda-cedc-9bbd-bcf1-9b3a5c75804d hosted-forwarder true auto-assign 5e498cd3-542c-e54f-0c58-ed43e28b568a true auto-assign da503cf1-a347-44f2-875c-4960ddb04091 hosted-forwarder true auto-assign
hcp create kubeconfig --namespace <clusters_namespace> --name <hosted_cluster_namespace> > <hosted_cluster_name>.kubeconfig
--kubeconfig <hosted_cluster_name>.kubeconfig get nodes
NAME STATUS ROLES AGE VERSION worker-zvm-0.hostedn.example.com Ready worker 5m41s v1.24.0+3882f8f worker-zvm-1.hostedn.example.com Ready worker 6m3s v1.24.0+3882f8f
-n <hosted_control_plane_namespace> get machine.cluster.x-k8s.io
NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION hosted-forwarder-79558597ff-5tbqp hosted-forwarder-crqq5 worker-zvm-0.hostedn.example.com agent://50c23cda-cedc-9bbd-bcf1-9b3a5c75804d Running 41h 4.15.0 hosted-forwarder-79558597ff-lfjfk hosted-forwarder-crqq5 worker-zvm-1.hostedn.example.com agent://5e498cd3-542c-e54f-0c58-ed43e28b568a Running 41h 4.15.0
--kubeconfig <hosted_cluster_name>.kubeconfig get clusterversion,co
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS clusterversion.config.openshift.io/version 4.15.0-ec.2 True False 40h Cluster version is 4.15.0-ec.2
--kubeconfig <hosted_cluster_name>.kubeconfig get clusteroperators
-n <clusters_namespace> scale nodepool <nodepool_name> --replicas 0
--kubeconfig <hosted_clusted_name>.kubeconfig delete node <comopute_node_name>
--kubeconfig <hosted_cluster_name>.kubeconfig get nodes
-n <hosted_control_plane_namespace> delete agent <agent_name>
hcp destroy cluster agent --name <hosted_cluster_name> --namespace <clusters_namepsace>
patch ingresscontroller -n openshift-ingress-operator default --type=json -p '[{ "op": "add", "path": "/spec/routeAdmission", "value": {wildcardPolicy: "WildcardsAllowed"}}]'
patch storageclass ocs-storagecluster-ceph-rbd -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
get managedclusters local-cluster
hcp create cluster kubevirt --name <hosted-cluster-name> \ 1 --node-pool-replicas <worker-count> \ 2 --pull-secret <path-to-pull-secret> \ 3 --memory <value-for-memory> \ 4 --cores <value-for-cpu> \ 5 --etcd-storage-class=<etcd-storage-class> 6
-n clusters-<hosted-cluster-name> get pods
NAME READY STATUS RESTARTS AGE capi-provider-5cc7b74f47-n5gkr 1/1 Running 0 3m catalog-operator-5f799567b7-fd6jw 2/2 Running 0 69s certified-operators-catalog-784b9899f9-mrp6p 1/1 Running 0 66s cluster-api-6bbc867966-l4dwl 1/1 Running 0 66s . . . redhat-operators-catalog-9d5fd4d44-z8qqk 1/1 Running 0 66s
get --namespace clusters hostedclusters
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters example 4.x.0 example-admin-kubeconfig Completed True False The hosted control plane is available
hcp create cluster kubevirt --name <hosted-cluster-name> \ 1 --node-pool-replicas <worker-count> \ 2 --pull-secret <path-to-pull-secret> \ 3 --memory <value-for-memory> \ 4 --cores <value-for-cpu> \ 5 --infra-namespace=<hosted-cluster-namespace>-<hosted-cluster-name> \ 6 --infra-kubeconfig-file=<path-to-external-infra-kubeconfig> 7
*.apps.mgmt-cluster.example.com
*.apps.guest.apps.mgmt-cluster.example.com
patch ingresscontroller -n openshift-ingress-operator default --type=json -p '[{ "op": "add", "path": "/spec/routeAdmission", "value": {wildcardPolicy: "WildcardsAllowed"}}]'
hcp create cluster kubevirt --name <hosted-cluster-name> \ 1 --node-pool-replicas <worker-count> \ 2 --pull-secret <path-to-pull-secret> \ 3 --memory <value-for-memory> \ 4 --cores <value-for-cpu> \ 5 --base-domain <basedomain> 6
get --namespace clusters hostedclusters
NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE example example-admin-kubeconfig Partial True False The hosted control plane is available
hcp create kubeconfig --name <hosted-cluster-name> > <hosted-cluster-name>-kubeconfig
--kubeconfig <hosted-cluster-name>-kubeconfig get co
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console 4.x.0 False False False 30m RouteHealthAvailable: failed to GET route (https://console-openshift-console.apps.example.hypershift.lab): Get "https://console-openshift-console.apps.example.hypershift.lab": dial tcp: lookup console-openshift-console.apps.example.hypershift.lab on 172.31.0.10:53: no such host ingress 4.x.0 True False True 28m The "default" ingress controller reports Degraded=True: DegradedConditions: One or more other status conditions indicate a degraded state: CanaryChecksSucceeding=False (CanaryChecksRepetitiveFailures: Canary route checks for the default ingress controller are failing)
--kubeconfig <hosted-cluster-name>-kubeconfig get services -n openshift-ingress router-nodeport-default -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}'
--kubeconfig <hosted-cluster-name>-kubeconfig get services -n openshift-ingress router-nodeport-default -o jsonpath='{.spec.ports[?(@.name=="https")].nodePort}'
apply -f - apiVersion: v1 kind: Service metadata: labels: app: <hosted-cluster-name> name: <hosted-cluster-name>-apps namespace: clusters-<hosted-cluster-name> spec: ports: - name: https-443 port: 443 protocol: TCP targetPort: <https-node-port> 1 - name: http-80 port: 80 protocol: TCP targetPort: <http-node-port> 2 selector: kubevirt.io: virt-launcher type: LoadBalancer
-n clusters-<hosted-cluster-name> get service <hosted-cluster-name>-apps -o jsonpath='{.status.loadBalancer.ingress[0].ip}'
192.168.20.30
*.apps.<hosted-cluster-name\>.<base-domain\>.
dig +short test.apps.example.hypershift.lab 192.168.20.30
get --namespace clusters hostedclusters
NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE example 4.x.0 example-admin-kubeconfig Completed True False The hosted control plane is available
apiVersion: metallb.io/v1beta1 kind: MetalLB metadata: name: metallb namespace: metallb-system
apply -f configure-metallb.yaml
metallb.metallb.io/metallb created
apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: metallb namespace: metallb-system spec: addresses: - 192.168.216.32-192.168.216.122 1
apply -f create-ip-address-pool.yaml
ipaddresspool.metallb.io/metallb created
apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: l2advertisement namespace: metallb-system spec: ipAddressPools: - metallb
apply -f l2advertisement.yaml
l2advertisement.metallb.io/metallb created
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --additional-network name:<namespace/name> \ 6 --additional-network name:<namespace/name>
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --attach-default-network false \ 6 --additional-network name:<namespace>/<network_name> 7
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --qos-class Guaranteed 6
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --vm-node-selector <label_key>=<label_value>,<label_key>=<label_value> 6
NODEPOOL_NAME=USD{CLUSTER_NAME}-work NODEPOOL_REPLICAS=5 scale nodepool/USDNODEPOOL_NAME --namespace clusters --replicas=USDNODEPOOL_REPLICAS
--kubeconfig USDCLUSTER_NAME-kubeconfig get nodes
NAME STATUS ROLES AGE VERSION example-9jvnf Ready worker 97s v1.27.4+18eadca example-n6prw Ready worker 116m v1.27.4+18eadca example-nc6g4 Ready worker 117m v1.27.4+18eadca example-thp29 Ready worker 4m17s v1.27.4+18eadca example-twxns Ready worker 88s v1.27.4+18eadca
export NODEPOOL_NAME=USD{CLUSTER_NAME}-extra-cpu export WORKER_COUNT="2" export MEM="6Gi" export CPU="4" export DISK="16" hcp create nodepool kubevirt --cluster-name USDCLUSTER_NAME --name USDNODEPOOL_NAME --node-count USDWORKER_COUNT --memory USDMEM --cores USDCPU --root-volume-size USDDISK
get nodepools --namespace clusters
NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE example example 5 5 False False 4.x.0 example-extra-cpu example 2 False False True True Minimum availability requires 2 replicas, current 0 available
--kubeconfig USDCLUSTER_NAME-kubeconfig get nodes
NAME STATUS ROLES AGE VERSION example-9jvnf Ready worker 97s v1.27.4+18eadca example-n6prw Ready worker 116m v1.27.4+18eadca example-nc6g4 Ready worker 117m v1.27.4+18eadca example-thp29 Ready worker 4m17s v1.27.4+18eadca example-twxns Ready worker 88s v1.27.4+18eadca example-extra-cpu-zh9l5 Ready worker 2m6s v1.27.4+18eadca example-extra-cpu-zr8mj Ready worker 102s v1.27.4+18eadca
get nodepools --namespace clusters
NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE example example 5 5 False False 4.x.0 example-extra-cpu example 2 2 False False 4.x.0
get --namespace clusters hostedclusters USD{CLUSTER_NAME}
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters example 4.12.2 example-admin-kubeconfig Completed True False The hosted control plane is available
hcp create kubeconfig --name USDCLUSTER_NAME > USDCLUSTER_NAME-kubeconfig
get co --kubeconfig=USDCLUSTER_NAME-kubeconfig
NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE console 4.12.2 True False False 2m38s csi-snapshot-controller 4.12.2 True False False 4m3s dns 4.12.2 True False False 2m52s image-registry 4.12.2 True False False 2m8s ingress 4.12.2 True False False 22m kube-apiserver 4.12.2 True False False 23m kube-controller-manager 4.12.2 True False False 23m kube-scheduler 4.12.2 True False False 23m kube-storage-version-migrator 4.12.2 True False False 4m52s monitoring 4.12.2 True False False 69s network 4.12.2 True False False 4m3s node-tuning 4.12.2 True False False 2m22s openshift-apiserver 4.12.2 True False False 23m openshift-controller-manager 4.12.2 True False False 23m openshift-samples 4.12.2 True False False 2m15s operator-lifecycle-manager 4.12.2 True False False 22m operator-lifecycle-manager-catalog 4.12.2 True False False 23m operator-lifecycle-manager-packageserver 4.12.2 True False False 23m service-ca 4.12.2 True False False 4m41s storage 4.12.2 True False False 4m43s
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class> \ 6
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class> \ 6 --infra-volumesnapshot-class-mapping=<infrastructure_volume_snapshot_class>/<hosted_volume_snapshot_class> 7
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class>,group=<group_name> \ 6 --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class>,group=<group_name> --infra-storage-class-mapping=<infrastructure_storage_class>/<hosted_storage_class>,group=<group_name> --infra-volumesnapshot-class-mapping=<infrastructure_volume_snapshot_class>/<hosted_volume_snapshot_class>,group=<group_name> \ 7 --infra-volumesnapshot-class-mapping=<infrastructure_volume_snapshot_class>/<hosted_volume_snapshot_class>,group=<group_name>
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --root-volume-storage-class <root_volume_storage_class> \ 6 --root-volume-size <volume_size> 7
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --root-volume-cache-strategy=PVC 6
hcp create cluster kubevirt --name <hosted_cluster_name> \ 1 --node-pool-replicas <worker_node_count> \ 2 --pull-secret <path_to_pull_secret> \ 3 --memory <memory> \ 4 --cores <cpu> \ 5 --etcd-storage-class=<etcd_storage_class_name> 6
delete managedcluster <cluster_name>
hcp destroy cluster kubevirt --name USDCLUSTER_NAME
apiVersion: v1 kind: ConfigMap metadata: name: custom-registries namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/openshift4" mirror-by-digest-only = true [[registry.mirror]] location = "registry.ocp-edge-cluster-0.qe.lab.redhat.com:5000/openshift4" [[registry]] prefix = "" location = "registry.redhat.io/rhacm2" mirror-by-digest-only = true
adm release info <tagged_openshift_release_image> | grep "Pull From"
Pull From: quay.io/openshift-release-dev/ocp-release@sha256:69d1292f64a2b67227c5592c1a7d499c7d00376e498634ff8e1946bc9ccdddfe
get events -n hypershift
LAST SEEN TYPE REASON OBJECT MESSAGE 4m46s Warning ReconcileError deployment/operator Failed to ensure UWM telemetry remote write: cannot get telemeter client secret: Secret "telemeter-client" not found
kind: ConfigMap apiVersion: v1 metadata: name: hypershift-operator-install-flags namespace: local-cluster data: installFlagsToAdd: "" installFlagsToRemove: "--enable-uwm-telemetry-remote-write"
patch mce <multiclusterengine> --type=merge -p '{"spec":{"overrides":{"components":[{"name":"hypershift","enabled": true}]}}}'
get managedclusteraddons -n local-cluster hypershift-addon
NAME AVAILABLE DEGRADED PROGRESSING hypershift-addon True False
wait --for=condition=Degraded=True managedclusteraddons/hypershift-addon -n local-cluster --timeout=5m
wait --for=condition=Available=True managedclusteraddons/hypershift-addon -n local-cluster --timeout=5m
edit addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: hypershift-addon-deploy-config namespace: multicluster-engine spec: nodePlacement: nodeSelector: node-role.kubernetes.io/infra: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/infra operator: Exists
sudo dnf install dnsmasq radvd vim golang podman bind-utils net-tools httpd-tools tree htop strace tmux -y
systemctl enable --now podman
sudo yum -y install libvirt libvirt-daemon-driver-qemu qemu-kvm
sudo usermod -aG qemu,libvirt USD(id -un)
sudo newgrp libvirt
sudo systemctl enable --now libvirtd
sudo dnf -y copr enable karmab/kcli
sudo dnf -y install kcli
sudo kcli create pool -p /var/lib/libvirt/images default
kcli create host kvm -H 127.0.0.1 local
sudo setfacl -m u:USD(id -un):rwx /var/lib/libvirt/images
kcli create network -c 192.168.122.0/24 default
#!/bin/bash export IP="192.168.126.1" 1 export BASE_RESOLV_CONF="/run/NetworkManager/resolv.conf" if ! [[ `grep -q "USDIP" /etc/resolv.conf` ]]; then export TMP_FILE=USD(mktemp /etc/forcedns_resolv.conf.XXXXXX) cp USDBASE_RESOLV_CONF USDTMP_FILE chmod --reference=USDBASE_RESOLV_CONF USDTMP_FILE sed -i -e "s/dns.base.domain.name//" -e "s/search /& dns.base.domain.name /" -e "0,/nameserver/s/nameserver/& USDIP\n&/" USDTMP_FILE 2 mv USDTMP_FILE /etc/resolv.conf fi echo "ok"
chmod 755 /etc/NetworkManager/dispatcher.d/forcedns
sudo dnf install python3-pyOpenSSL.noarch python3-cherrypy -y
kcli create sushy-service --ssl --port 9000
sudo systemctl daemon-reload
systemctl enable --now ksushy
systemctl status ksushy
sed -i s/^SELINUX=.*USD/SELINUX=permissive/ /etc/selinux/config; setenforce 0
systemctl disable --now firewalld
systemctl restart libvirtd
systemctl enable --now libvirtd
#!/usr/bin/env bash set -euo pipefail PRIMARY_NIC=USD(ls -1 /sys/class/net | grep -v podman | head -1) export PATH=/root/bin:USDPATH export PULL_SECRET="/root/baremetal/hub/openshift_pull.json" 1 if [[ ! -f USDPULL_SECRET ]];then echo "Pull Secret not found, exiting..." exit 1 fi dnf -y install podman httpd httpd-tools jq skopeo libseccomp-devel export IP=USD(ip -o addr show USDPRIMARY_NIC | head -1 | awk '{print USD4}' | cut -d'/' -f1) REGISTRY_NAME=registry.USD(hostname --long) REGISTRY_USER=dummy REGISTRY_PASSWORD=dummy KEY=USD(echo -n USDREGISTRY_USER:USDREGISTRY_PASSWORD | base64) echo "{\"auths\": {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\", \"email\": \"sample-email@domain.ltd\"}}}" > /root/disconnected_pull.json mv USD{PULL_SECRET} /root/openshift_pull.json.old jq ".auths += {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\",\"email\": \"sample-email@domain.ltd\"}}" < /root/openshift_pull.json.old > USDPULL_SECRET mkdir -p /opt/registry/{auth,certs,data,conf} cat <<EOF > /opt/registry/conf/config.yml version: 0.1 log: fields: service: registry storage: cache: blobdescriptor: inmemory filesystem: rootdirectory: /var/lib/registry delete: enabled: true http: addr: :5000 headers: X-Content-Type-Options: [nosniff] health: storagedriver: enabled: true interval: 10s threshold: 3 compatibility: schema1: enabled: true EOF openssl req -newkey rsa:4096 -nodes -sha256 -keyout /opt/registry/certs/domain.key -x509 -days 3650 -out /opt/registry/certs/domain.crt -subj "/C=US/ST=Madrid/L=San Bernardo/O=Karmalabs/OU=Guitar/CN=USDREGISTRY_NAME" -addext "subjectAltName=DNS:USDREGISTRY_NAME" cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/ update-ca-trust extract htpasswd -bBc /opt/registry/auth/htpasswd USDREGISTRY_USER USDREGISTRY_PASSWORD create --name registry --net host --security-opt label=disable --replace -v /opt/registry/data:/var/lib/registry:z -v /opt/registry/auth:/auth:z -v /opt/registry/conf/config.yml:/etc/docker/registry/config.yml -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry" -e "REGISTRY_HTTP_SECRET=ALongRandomSecretForRegistry" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -v /opt/registry/certs:/certs:z -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key docker.io/library/registry:latest [ "USD?" == "0" ] || !! systemctl enable --now registry
chmod u+x USD{HOME}/registry.sh
USD{HOME}/registry.sh
systemctl status
systemctl start
systemctl stop
kcli create network -c 192.168.125.0/24 -P dhcp=false -P dns=false --domain dns.base.domain.name ipv4
kcli list network Listing Networks +---------+--------+---------------------+-------+------------------+------+ | Network | Type | Cidr | Dhcp | Domain | Mode | +---------+--------+---------------------+-------+------------------+------+ | default | routed | 192.168.122.0/24 | True | default | nat | | ipv4 | routed | 192.168.125.0/24 | False | dns.base.domain.name | nat | | ipv6 | routed | 2620:52:0:1306::/64 | False | dns.base.domain.name | nat | +---------+--------+---------------------+-------+------------------+------+
kcli info network ipv4 Providing information about network ipv4 cidr: 192.168.125.0/24 dhcp: false domain: dns.base.domain.name mode: nat plan: kvirt type: routed
plan: hub-ipv4 force: true version: nightly tag: "4.x.y-x86_64" 1 cluster: "hub-ipv4" domain: dns.base.domain.name api_ip: 192.168.125.10 ingress_ip: 192.168.125.11 disconnected_url: registry.dns.base.domain.name:5000 disconnected_update: true disconnected_user: dummy disconnected_password: dummy disconnected_operators_version: v4.14 disconnected_operators: - name: metallb-operator - name: lvms-operator channels: - name: stable-4.14 disconnected_extra_images: - quay.io/user-name/trbsht:latest - quay.io/user-name/hypershift:BMSelfManage-v4.14-rc-v3 - registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 dualstack: false disk_size: 200 extra_disks: [200] memory: 48000 numcpus: 16 ctlplanes: 3 workers: 0 manifests: extra-manifests metal3: true network: ipv4 users_dev: developer users_devpassword: developer users_admin: admin users_adminpassword: admin metallb_pool: ipv4-virtual-network metallb_ranges: - 192.168.125.150-192.168.125.190 metallb_autoassign: true apps: - users - lvms-operator - metallb-operator vmrules: - hub-bootstrap: nets: - name: ipv4 mac: aa:aa:aa:aa:02:10 - hub-ctlplane-0: nets: - name: ipv4 mac: aa:aa:aa:aa:02:01 - hub-ctlplane-1: nets: - name: ipv4 mac: aa:aa:aa:aa:02:02 - hub-ctlplane-2: nets: - name: ipv4 mac: aa:aa:aa:aa:02:03
kcli create cluster openshift --pf mgmt-compact-hub-ipv4.yaml
adm -a USD{LOCAL_SECRET_JSON} release extract --command=openshift-install "USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}"
#!/bin/bash WEBSRV_FOLDER=/opt/srv ROOTFS_IMG_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.pxe.rootfs.location')" 1 LIVE_ISO_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.iso.disk.location')" 2 mkdir -p USD{WEBSRV_FOLDER}/images curl -Lk USD{ROOTFS_IMG_URL} -o USD{WEBSRV_FOLDER}/images/USD{ROOTFS_IMG_URL##*/} curl -Lk USD{LIVE_ISO_URL} -o USD{WEBSRV_FOLDER}/images/USD{LIVE_ISO_URL##*/} chmod -R 755 USD{WEBSRV_FOLDER}/* ## Run Webserver ps --noheading | grep -q websrv-ai if [[ USD? == 0 ]];then echo "Launching Registry pod..." /usr/bin/podman run --name websrv-ai --net host -v /opt/srv:/usr/local/apache2/htdocs:z quay.io/alosadag/httpd:p8080 fi
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: registry: imageURL: registry.dns.base.domain.name:5000/openshift/release/metadata:latest 1 mirror: platform: channels: - name: candidate-4.14 minVersion: 4.x.y-x86_64 2 maxVersion: 4.x.y-x86_64 type: ocp graph: true additionalImages: - name: quay.io/karmab/origin-keepalived-ipfailover:latest - name: quay.io/karmab/kubectl:latest - name: quay.io/karmab/haproxy:latest - name: quay.io/karmab/mdns-publisher:latest - name: quay.io/karmab/origin-coredns:latest - name: quay.io/karmab/curl:latest - name: quay.io/karmab/kcli:latest - name: quay.io/user-name/trbsht:latest - name: quay.io/user-name/hypershift:BMSelfManage-v4.14-rc-v3 - name: registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.14 packages: - name: lvms-operator - name: local-storage-operator - name: odf-csi-addons-operator - name: odf-operator - name: mcg-operator - name: ocs-operator - name: metallb-operator - name: kubevirt-hyperconverged
oc-mirror --source-skip-tls --config imagesetconfig.yaml docker://USD{REGISTRY}
REGISTRY=registry.USD(hostname --long):5000 adm release mirror --from=registry.ci.openshift.org/ocp/release:4.x.y-x86_64 --to=USD{REGISTRY}/openshift/release --to-release-image=USD{REGISTRY}/openshift/release-images:4.x.y-x86_64
get packagemanifest
apply -f oc-mirror-workspace/results-XXXXXX/imageContentSourcePolicy.yaml
apply -f catalogSource-XXXXXXXX-index.yaml
--- apiVersion: v1 kind: ConfigMap metadata: name: custom-registries namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/openshift4" mirror-by-digest-only = true [[registry.mirror]] location = "registry.dns.base.domain.name:5000/openshift4" 1 [[registry]] prefix = "" location = "registry.redhat.io/rhacm2" mirror-by-digest-only = true
--- apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: annotations: unsupported.agent-install.openshift.io/assisted-service-configmap: assisted-service-config 1 name: agent namespace: multicluster-engine spec: mirrorRegistryRef: name: custom-registries 2 databaseStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 10Gi filesystemStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 20Gi osImages: 3 - cpuArchitecture: x86_64 openshiftVersion: "4.14" rootFSUrl: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live-rootfs.x86_64.img 4 url: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live.x86_64.iso version: 414.92.202308281054-0
apply -f agentServiceConfig.yaml
assisted-image-service-0 1/1 Running 2 11d 1 assisted-service-668b49548-9m7xw 2/2 Running 5 11d 2
## REGISTRY_CERT_PATH=<PATH/TO/YOUR/CERTIFICATE/FILE> export REGISTRY_CERT_PATH=/opt/registry/certs/domain.crt create configmap user-ca-bundle -n openshift-config --from-file=ca-bundle.crt=USD{REGISTRY_CERT_PATH}
## REGISTRY_CERT_PATH=<PATH/TO/YOUR/CERTIFICATE/FILE> export REGISTRY_CERT_PATH=/opt/registry/certs/domain.crt export TMP_FILE=USD(mktemp) get cm -n openshift-config user-ca-bundle -ojsonpath='{.data.ca-bundle\.crt}' > USD{TMP_FILE} echo >> USD{TMP_FILE} echo \#registry.USD(hostname --long) >> USD{TMP_FILE} cat USD{REGISTRY_CERT_PATH} >> USD{TMP_FILE} create configmap user-ca-bundle -n openshift-config --from-file=ca-bundle.crt=USD{TMP_FILE} --dry-run=client -o yaml | kubectl apply -f -
--- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters-hosted-ipv4 spec: {} status: {} --- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters spec: {} status: {}
--- apiVersion: v1 data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- kind: ConfigMap metadata: name: user-ca-bundle namespace: clusters --- apiVersion: v1 data: .dockerconfigjson: xxxxxxxxx kind: Secret metadata: creationTimestamp: null name: hosted-ipv4-pull-secret namespace: clusters --- apiVersion: v1 kind: Secret metadata: name: sshkey-cluster-hosted-ipv4 namespace: clusters stringData: id_rsa.pub: ssh-rsa xxxxxxxxx --- apiVersion: v1 data: key: nTPtVBEt03owkrKhIdmSW8jrWRxU57KO/fnZa8oaG0Y= kind: Secret metadata: creationTimestamp: null name: hosted-ipv4-etcd-encryption-key namespace: clusters type: Opaque
apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: creationTimestamp: null name: capi-provider-role namespace: clusters-hosted-ipv4 rules: - apiGroups: - agent-install.openshift.io resources: - agents verbs: - '*'
apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: name: hosted-ipv4 namespace: clusters spec: additionalTrustBundle: name: "user-ca-bundle" olmCatalogPlacement: guest imageContentSources: 1 - source: quay.io/openshift-release-dev/ocp-v4.0-art-dev mirrors: - registry.dns.base.domain.name:5000/openshift/release - source: quay.io/openshift-release-dev/ocp-release mirrors: - registry.dns.base.domain.name:5000/openshift/release-images - mirrors: autoscaling: {} controllerAvailabilityPolicy: SingleReplica dns: baseDomain: dns.base.domain.name etcd: managed: storage: persistentVolume: size: 8Gi restoreSnapshotURL: null type: PersistentVolume managementType: Managed fips: false networking: clusterNetwork: - cidr: 10.132.0.0/14 networkType: OVNKubernetes serviceNetwork: - cidr: 172.31.0.0/16 platform: agent: agentNamespace: clusters-hosted-ipv4 type: Agent pullSecret: name: hosted-ipv4-pull-secret release: image: registry.dns.base.domain.name:5000/openshift/release-images:4.x.y-x86_64 secretEncryption: aescbc: activeKey: name: hosted-ipv4-etcd-encryption-key type: aescbc services: - service: APIServer servicePublishingStrategy: nodePort: address: api.hosted-ipv4.dns.base.domain.name type: NodePort - service: OAuthServer servicePublishingStrategy: nodePort: address: api.hosted-ipv4.dns.base.domain.name type: NodePort - service: OIDC servicePublishingStrategy: nodePort: address: api.hosted-ipv4.dns.base.domain.name type: NodePort - service: Konnectivity servicePublishingStrategy: nodePort: address: api.hosted-ipv4.dns.base.domain.name type: NodePort - service: Ignition servicePublishingStrategy: nodePort: address: api.hosted-ipv4.dns.base.domain.name type: NodePort sshKey: name: sshkey-cluster-hosted-ipv4 status: controlPlaneEndpoint: host: "" port: 0
adm release info registry.dns.base.domain.name:5000/openshift-release-dev/ocp-release:4.x.y-x86_64 | grep hypershift
hypershift sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.dns.base.domain.name:5000/openshift-release-dev/ocp-v4.0-art-dev@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Trying to pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Getting image source signatures Copying blob d8190195889e skipped: already exists Copying blob c71d2589fba7 skipped: already exists Copying blob d4dc6e74b6ce skipped: already exists Copying blob 97da74cc6d8f skipped: already exists Copying blob b70007a560c9 done Copying config 3a62961e6e done Writing manifest to image destination Storing signatures 3a62961e6ed6edab46d5ec8429ff1f41d6bb68de51271f037c6cb8941a007fde
apply -f 01-4.14-hosted_cluster-nodeport.yaml
NAME READY STATUS RESTARTS AGE capi-provider-5b57dbd6d5-pxlqc 1/1 Running 0 3m57s catalog-operator-9694884dd-m7zzv 2/2 Running 0 93s cluster-api-f98b9467c-9hfrq 1/1 Running 0 3m57s cluster-autoscaler-d7f95dd5-d8m5d 1/1 Running 0 93s cluster-image-registry-operator-5ff5944b4b-648ht 1/2 Running 0 93s cluster-network-operator-77b896ddc-wpkq8 1/1 Running 0 94s cluster-node-tuning-operator-84956cd484-4hfgf 1/1 Running 0 94s cluster-policy-controller-5fd8595d97-rhbwf 1/1 Running 0 95s cluster-storage-operator-54dcf584b5-xrnts 1/1 Running 0 93s cluster-version-operator-9c554b999-l22s7 1/1 Running 0 95s control-plane-operator-6fdc9c569-t7hr4 1/1 Running 0 3m57s csi-snapshot-controller-785c6dc77c-8ljmr 1/1 Running 0 77s csi-snapshot-controller-operator-7c6674bc5b-d9dtp 1/1 Running 0 93s csi-snapshot-webhook-5b8584875f-2492j 1/1 Running 0 77s dns-operator-6874b577f-9tc6b 1/1 Running 0 94s etcd-0 3/3 Running 0 3m39s hosted-cluster-config-operator-f5cf5c464-4nmbh 1/1 Running 0 93s ignition-server-6b689748fc-zdqzk 1/1 Running 0 95s ignition-server-proxy-54d4bb9b9b-6zkg7 1/1 Running 0 95s ingress-operator-6548dc758b-f9gtg 1/2 Running 0 94s konnectivity-agent-7767cdc6f5-tw782 1/1 Running 0 95s kube-apiserver-7b5799b6c8-9f5bp 4/4 Running 0 3m7s kube-controller-manager-5465bc4dd6-zpdlk 1/1 Running 0 44s kube-scheduler-5dd5f78b94-bbbck 1/1 Running 0 2m36s machine-approver-846c69f56-jxvfr 1/1 Running 0 92s oauth-openshift-79c7bf44bf-j975g 2/2 Running 0 62s olm-operator-767f9584c-4lcl2 2/2 Running 0 93s openshift-apiserver-5d469778c6-pl8tj 3/3 Running 0 2m36s openshift-controller-manager-6475fdff58-hl4f7 1/1 Running 0 95s openshift-oauth-apiserver-dbbc5cc5f-98574 2/2 Running 0 95s openshift-route-controller-manager-5f6997b48f-s9vdc 1/1 Running 0 95s packageserver-67c87d4d4f-kl7qh 2/2 Running 0 93s
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters hosted-ipv4 hosted-admin-kubeconfig Partial True False The hosted control plane is available
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: creationTimestamp: null name: hosted-ipv4 namespace: clusters spec: arch: amd64 clusterName: hosted-ipv4 management: autoRepair: false 1 upgradeType: InPlace 2 nodeDrainTimeout: 0s platform: type: Agent release: image: registry.dns.base.domain.name:5000/openshift/release-images:4.x.y-x86_64 3 replicas: 0 status: replicas: 0 4
apply -f 02-nodepool.yaml
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted-ipv4 hosted 0 False False 4.x.y-x86_64
--- apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: hosted-ipv4 namespace: clusters-hosted-ipv4 spec: pullSecretRef: 1 name: pull-secret sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDk7ICaUE+/k4zTpxLk4+xFdHi4ZuDi5qjeF52afsNkw0w/glILHhwpL5gnp5WkRuL8GwJuZ1VqLC9EKrdmegn4MrmUlq7WTsP0VFOZFBfq2XRUxo1wrRdor2z0Bbh93ytR+ZsDbbLlGngXaMa0Vbt+z74FqlcajbHTZ6zBmTpBVq5RHtDPgKITdpE1fongp7+ZXQNBlkaavaqv8bnyrP4BWahLP4iO9/xJF9lQYboYwEEDzmnKLMW1VtCE6nJzEgWCufACTbxpNS7GvKtoHT/OVzw8ArEXhZXQUS1UY8zKsX2iXwmyhw5Sj6YboA8WICs4z+TrFP89LmxXY0j6536TQFyRz1iB4WWvCbH5n6W+ABV2e8ssJB1AmEy8QYNwpJQJNpSxzoKBjI73XxvPYYC/IjPFMySwZqrSZCkJYqQ023ySkaQxWZT7in4KeMu7eS2tC+Kn4deJ7KwwUycx8n6RHMeD8Qg9flTHCv3gmab8JKZJqN3hW1D378JuvmIX4V0= 2
apply -f 03-infraenv.yaml
NAMESPACE NAME ISO CREATED AT clusters-hosted-ipv4 hosted 2023-09-11T15:14:10Z
kcli delete plan hosted-ipv4
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-ipv4 -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"ipv4\", \"mac\": \"aa:aa:aa:aa:02:11\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0211 -P name=hosted-ipv4-worker0
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-ipv4 -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"ipv4\", \"mac\": \"aa:aa:aa:aa:02:12\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0212 -P name=hosted-ipv4-worker1
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-ipv4 -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"ipv4\", \"mac\": \"aa:aa:aa:aa:02:13\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0213 -P name=hosted-ipv4-worker2
systemctl restart ksushy
+---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | Name | Status | Ip | Source | Plan | Profile | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | hosted-worker0 | down | | | hosted-ipv4 | kvirt | | hosted-worker1 | down | | | hosted-ipv4 | kvirt | | hosted-worker2 | down | | | hosted-ipv4 | kvirt | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+
--- apiVersion: v1 kind: Secret metadata: name: hosted-ipv4-worker0-bmc-secret namespace: clusters-hosted-ipv4 data: password: YWRtaW4= username: YWRtaW4= type: Opaque --- apiVersion: metal3.io/v1alpha1 kind: BareMetalHost metadata: name: hosted-ipv4-worker0 namespace: clusters-hosted-ipv4 labels: infraenvs.agent-install.openshift.io: hosted-ipv4 1 annotations: inspect.metal3.io: disabled bmac.agent-install.openshift.io/hostname: hosted-ipv4-worker0 2 spec: automatedCleaningMode: disabled 3 bmc: disableCertificateVerification: true 4 address: redfish-virtualmedia://[192.168.125.1]:9000/redfish/v1/Systems/local/hosted-ipv4-worker0 5 credentialsName: hosted-ipv4-worker0-bmc-secret 6 bootMACAddress: aa:aa:aa:aa:02:11 7 online: true 8
apply -f 04-bmh.yaml
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 registering true 2s clusters-hosted hosted-worker1 registering true 2s clusters-hosted hosted-worker2 registering true 2s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioning true 16s clusters-hosted hosted-worker1 provisioning true 16s clusters-hosted hosted-worker2 provisioning true 16s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioned true 67s clusters-hosted hosted-worker1 provisioned true 67s clusters-hosted hosted-worker2 provisioned true 67s
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0411 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0412 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0413 true auto-assign
-n clusters scale nodepool hosted-ipv4 --replicas 3
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0411 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0412 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0413 hosted true auto-assign
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted hosted 3 False False 4.x.y-x86_64 Minimum availability requires 3 replicas, current 0 available
export KUBECONFIG=/root/.kcli/clusters/hub-ipv4/auth/kubeconfig
watch "oc get pod -n hypershift;echo;echo;oc get pod -n clusters-hosted-ipv4;echo;echo;oc get bmh -A;echo;echo;oc get agent -A;echo;echo;oc get infraenv -A;echo;echo;oc get hostedcluster -A;echo;echo;oc get nodepool -A;echo;echo;"
get secret -n clusters-hosted-ipv4 admin-kubeconfig -o jsonpath='{.data.kubeconfig}' |base64 -d > /root/hc_admin_kubeconfig.yaml
export KUBECONFIG=/root/hc_admin_kubeconfig.yaml
watch "oc get clusterversion,nodes,co"
sudo dnf install dnsmasq radvd vim golang podman bind-utils net-tools httpd-tools tree htop strace tmux -y
systemctl enable --now podman
sudo yum -y install libvirt libvirt-daemon-driver-qemu qemu-kvm
sudo usermod -aG qemu,libvirt USD(id -un)
sudo newgrp libvirt
sudo systemctl enable --now libvirtd
sudo dnf -y copr enable karmab/kcli
sudo dnf -y install kcli
sudo kcli create pool -p /var/lib/libvirt/images default
kcli create host kvm -H 127.0.0.1 local
sudo setfacl -m u:USD(id -un):rwx /var/lib/libvirt/images
kcli create network -c 192.168.122.0/24 default
#!/bin/bash export IP="2620:52:0:1306::1" 1 export BASE_RESOLV_CONF="/run/NetworkManager/resolv.conf" if ! [[ `grep -q "USDIP" /etc/resolv.conf` ]]; then export TMP_FILE=USD(mktemp /etc/forcedns_resolv.conf.XXXXXX) cp USDBASE_RESOLV_CONF USDTMP_FILE chmod --reference=USDBASE_RESOLV_CONF USDTMP_FILE sed -i -e "s/dns.base.domain.name//" -e "s/search /& dns.base.domain.name /" -e "0,/nameserver/s/nameserver/& USDIP\n&/" USDTMP_FILE 2 mv USDTMP_FILE /etc/resolv.conf fi echo "ok"
chmod 755 /etc/NetworkManager/dispatcher.d/forcedns
sudo dnf install python3-pyOpenSSL.noarch python3-cherrypy -y
kcli create sushy-service --ssl --ipv6 --port 9000
sudo systemctl daemon-reload
systemctl enable --now ksushy
systemctl status ksushy
sed -i s/^SELINUX=.*USD/SELINUX=permissive/ /etc/selinux/config; setenforce 0
systemctl disable --now firewalld
systemctl restart libvirtd
systemctl enable --now libvirtd
#!/usr/bin/env bash set -euo pipefail PRIMARY_NIC=USD(ls -1 /sys/class/net | grep -v podman | head -1) export PATH=/root/bin:USDPATH export PULL_SECRET="/root/baremetal/hub/openshift_pull.json" 1 if [[ ! -f USDPULL_SECRET ]];then echo "Pull Secret not found, exiting..." exit 1 fi dnf -y install podman httpd httpd-tools jq skopeo libseccomp-devel export IP=USD(ip -o addr show USDPRIMARY_NIC | head -1 | awk '{print USD4}' | cut -d'/' -f1) REGISTRY_NAME=registry.USD(hostname --long) REGISTRY_USER=dummy REGISTRY_PASSWORD=dummy KEY=USD(echo -n USDREGISTRY_USER:USDREGISTRY_PASSWORD | base64) echo "{\"auths\": {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\", \"email\": \"sample-email@domain.ltd\"}}}" > /root/disconnected_pull.json mv USD{PULL_SECRET} /root/openshift_pull.json.old jq ".auths += {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\",\"email\": \"sample-email@domain.ltd\"}}" < /root/openshift_pull.json.old > USDPULL_SECRET mkdir -p /opt/registry/{auth,certs,data,conf} cat <<EOF > /opt/registry/conf/config.yml version: 0.1 log: fields: service: registry storage: cache: blobdescriptor: inmemory filesystem: rootdirectory: /var/lib/registry delete: enabled: true http: addr: :5000 headers: X-Content-Type-Options: [nosniff] health: storagedriver: enabled: true interval: 10s threshold: 3 compatibility: schema1: enabled: true EOF openssl req -newkey rsa:4096 -nodes -sha256 -keyout /opt/registry/certs/domain.key -x509 -days 3650 -out /opt/registry/certs/domain.crt -subj "/C=US/ST=Madrid/L=San Bernardo/O=Karmalabs/OU=Guitar/CN=USDREGISTRY_NAME" -addext "subjectAltName=DNS:USDREGISTRY_NAME" cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/ update-ca-trust extract htpasswd -bBc /opt/registry/auth/htpasswd USDREGISTRY_USER USDREGISTRY_PASSWORD create --name registry --net host --security-opt label=disable --replace -v /opt/registry/data:/var/lib/registry:z -v /opt/registry/auth:/auth:z -v /opt/registry/conf/config.yml:/etc/docker/registry/config.yml -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry" -e "REGISTRY_HTTP_SECRET=ALongRandomSecretForRegistry" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -v /opt/registry/certs:/certs:z -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key docker.io/library/registry:latest [ "USD?" == "0" ] || !! systemctl enable --now registry
chmod u+x USD{HOME}/registry.sh
USD{HOME}/registry.sh
systemctl status
systemctl start
systemctl stop
kcli create network -c 2620:52:0:1305::0/64 -P dhcp=false -P dns=false --domain dns.base.domain.name --nodhcp ipv6
kcli list network Listing Networks +---------+--------+---------------------+-------+------------------+------+ | Network | Type | Cidr | Dhcp | Domain | Mode | +---------+--------+---------------------+-------+------------------+------+ | default | routed | 192.168.122.0/24 | True | default | nat | | ipv4 | routed | 192.168.125.0/24 | False | dns.base.domain.name | nat | | ipv4 | routed | 2620:52:0:1305::/64 | False | dns.base.domain.name | nat | +---------+--------+---------------------+-------+------------------+------+
kcli info network ipv6 Providing information about network ipv6 cidr: 2620:52:0:1305::/64 dhcp: false domain: dns.base.domain.name mode: nat plan: kvirt type: routed
plan: hub-ipv6 force: true version: nightly tag: "4.x.y-x86_64" cluster: "hub-ipv6" ipv6: true domain: dns.base.domain.name api_ip: 2620:52:0:1305::2 ingress_ip: 2620:52:0:1305::3 disconnected_url: registry.dns.base.domain.name:5000 disconnected_update: true disconnected_user: dummy disconnected_password: dummy disconnected_operators_version: v4.14 disconnected_operators: - name: metallb-operator - name: lvms-operator channels: - name: stable-4.14 disconnected_extra_images: - quay.io/user-name/trbsht:latest - quay.io/user-name/hypershift:BMSelfManage-v4.14-rc-v3 - registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 dualstack: false disk_size: 200 extra_disks: [200] memory: 48000 numcpus: 16 ctlplanes: 3 workers: 0 manifests: extra-manifests metal3: true network: ipv6 users_dev: developer users_devpassword: developer users_admin: admin users_adminpassword: admin metallb_pool: ipv6-virtual-network metallb_ranges: - 2620:52:0:1305::150-2620:52:0:1305::190 metallb_autoassign: true apps: - users - lvms-operator - metallb-operator vmrules: - hub-bootstrap: nets: - name: ipv6 mac: aa:aa:aa:aa:03:10 - hub-ctlplane-0: nets: - name: ipv6 mac: aa:aa:aa:aa:03:01 - hub-ctlplane-1: nets: - name: ipv6 mac: aa:aa:aa:aa:03:02 - hub-ctlplane-2: nets: - name: ipv6 mac: aa:aa:aa:aa:03:03
kcli create cluster openshift --pf mgmt-compact-hub-ipv6.yaml
adm -a USD{LOCAL_SECRET_JSON} release extract --command=openshift-install "USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}"
#!/bin/bash WEBSRV_FOLDER=/opt/srv ROOTFS_IMG_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.pxe.rootfs.location')" 1 LIVE_ISO_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.iso.disk.location')" 2 mkdir -p USD{WEBSRV_FOLDER}/images curl -Lk USD{ROOTFS_IMG_URL} -o USD{WEBSRV_FOLDER}/images/USD{ROOTFS_IMG_URL##*/} curl -Lk USD{LIVE_ISO_URL} -o USD{WEBSRV_FOLDER}/images/USD{LIVE_ISO_URL##*/} chmod -R 755 USD{WEBSRV_FOLDER}/* ## Run Webserver ps --noheading | grep -q websrv-ai if [[ USD? == 0 ]];then echo "Launching Registry pod..." /usr/bin/podman run --name websrv-ai --net host -v /opt/srv:/usr/local/apache2/htdocs:z quay.io/alosadag/httpd:p8080 fi
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: registry: imageURL: registry.dns.base.domain.name:5000/openshift/release/metadata:latest 1 mirror: platform: channels: - name: candidate-4.14 minVersion: 4.x.y-x86_64 maxVersion: 4.x.y-x86_64 type: ocp graph: true additionalImages: - name: quay.io/karmab/origin-keepalived-ipfailover:latest - name: quay.io/karmab/kubectl:latest - name: quay.io/karmab/haproxy:latest - name: quay.io/karmab/mdns-publisher:latest - name: quay.io/karmab/origin-coredns:latest - name: quay.io/karmab/curl:latest - name: quay.io/karmab/kcli:latest - name: quay.io/user-name/trbsht:latest - name: quay.io/user-name/hypershift:BMSelfManage-v4.14-rc-v3 - name: registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.14 packages: - name: lvms-operator - name: local-storage-operator - name: odf-csi-addons-operator - name: odf-operator - name: mcg-operator - name: ocs-operator - name: metallb-operator
oc-mirror --source-skip-tls --config imagesetconfig.yaml docker://USD{REGISTRY}
REGISTRY=registry.USD(hostname --long):5000 adm release mirror --from=registry.ci.openshift.org/ocp/release:4.x.y-x86_64 --to=USD{REGISTRY}/openshift/release --to-release-image=USD{REGISTRY}/openshift/release-images:4.x.y-x86_64
get packagemanifest
apply -f oc-mirror-workspace/results-XXXXXX/imageContentSourcePolicy.yaml
apply -f catalogSource-XXXXXXXX-index.yaml
--- apiVersion: v1 kind: ConfigMap metadata: name: custom-registries namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/openshift4" mirror-by-digest-only = true [[registry.mirror]] location = "registry.dns.base.domain.name:5000/openshift4" 1 [[registry]] prefix = "" location = "registry.redhat.io/rhacm2" mirror-by-digest-only = true
--- apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: annotations: unsupported.agent-install.openshift.io/assisted-service-configmap: assisted-service-config 1 name: agent namespace: multicluster-engine spec: mirrorRegistryRef: name: custom-registries 2 databaseStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 10Gi filesystemStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 20Gi osImages: 3 - cpuArchitecture: x86_64 openshiftVersion: "4.14" rootFSUrl: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live-rootfs.x86_64.img 4 url: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live.x86_64.iso version: 414.92.202308281054-0
apply -f agentServiceConfig.yaml
assisted-image-service-0 1/1 Running 2 11d 1 assisted-service-668b49548-9m7xw 2/2 Running 5 11d 2
## REGISTRY_CERT_PATH=<PATH/TO/YOUR/CERTIFICATE/FILE> export REGISTRY_CERT_PATH=/opt/registry/certs/domain.crt create configmap user-ca-bundle -n openshift-config --from-file=ca-bundle.crt=USD{REGISTRY_CERT_PATH}
## REGISTRY_CERT_PATH=<PATH/TO/YOUR/CERTIFICATE/FILE> export REGISTRY_CERT_PATH=/opt/registry/certs/domain.crt export TMP_FILE=USD(mktemp) get cm -n openshift-config user-ca-bundle -ojsonpath='{.data.ca-bundle\.crt}' > USD{TMP_FILE} echo >> USD{TMP_FILE} echo \#registry.USD(hostname --long) >> USD{TMP_FILE} cat USD{REGISTRY_CERT_PATH} >> USD{TMP_FILE} create configmap user-ca-bundle -n openshift-config --from-file=ca-bundle.crt=USD{TMP_FILE} --dry-run=client -o yaml | kubectl apply -f -
--- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters-hosted-ipv6 spec: {} status: {} --- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters spec: {} status: {}
--- apiVersion: v1 data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- kind: ConfigMap metadata: name: user-ca-bundle namespace: clusters --- apiVersion: v1 data: .dockerconfigjson: xxxxxxxxx kind: Secret metadata: creationTimestamp: null name: hosted-ipv6-pull-secret namespace: clusters --- apiVersion: v1 kind: Secret metadata: name: sshkey-cluster-hosted-ipv6 namespace: clusters stringData: id_rsa.pub: ssh-rsa xxxxxxxxx --- apiVersion: v1 data: key: nTPtVBEt03owkrKhIdmSW8jrWRxU57KO/fnZa8oaG0Y= kind: Secret metadata: creationTimestamp: null name: hosted-ipv6-etcd-encryption-key namespace: clusters type: Opaque
apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: creationTimestamp: null name: capi-provider-role namespace: clusters-hosted-ipv6 rules: - apiGroups: - agent-install.openshift.io resources: - agents verbs: - '*'
apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: name: hosted-ipv6 namespace: clusters annotations: hypershift.openshift.io/control-plane-operator-image: registry.ocp-edge-cluster-0.qe.lab.redhat.com:5005/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 spec: additionalTrustBundle: name: "user-ca-bundle" olmCatalogPlacement: guest imageContentSources: 1 - source: quay.io/openshift-release-dev/ocp-v4.0-art-dev mirrors: - registry.dns.base.domain.name:5000/openshift/release - source: quay.io/openshift-release-dev/ocp-release mirrors: - registry.dns.base.domain.name:5000/openshift/release-images - mirrors: autoscaling: {} controllerAvailabilityPolicy: SingleReplica dns: baseDomain: dns.base.domain.name etcd: managed: storage: persistentVolume: size: 8Gi restoreSnapshotURL: null type: PersistentVolume managementType: Managed fips: false networking: clusterNetwork: - cidr: 10.132.0.0/14 networkType: OVNKubernetes serviceNetwork: - cidr: 172.31.0.0/16 platform: agent: agentNamespace: clusters-hosted-ipv6 type: Agent pullSecret: name: hosted-ipv6-pull-secret release: image: registry.dns.base.domain.name:5000/openshift/release-images:4.x.y-x86_64 secretEncryption: aescbc: activeKey: name: hosted-ipv6-etcd-encryption-key type: aescbc services: - service: APIServer servicePublishingStrategy: nodePort: address: api.hosted-ipv6.dns.base.domain.name type: NodePort - service: OAuthServer servicePublishingStrategy: nodePort: address: api.hosted-ipv6.dns.base.domain.name type: NodePort - service: OIDC servicePublishingStrategy: nodePort: address: api.hosted-ipv6.dns.base.domain.name type: NodePort - service: Konnectivity servicePublishingStrategy: nodePort: address: api.hosted-ipv6.dns.base.domain.name type: NodePort - service: Ignition servicePublishingStrategy: nodePort: address: api.hosted-ipv6.dns.base.domain.name type: NodePort sshKey: name: sshkey-cluster-hosted-ipv6 status: controlPlaneEndpoint: host: "" port: 0
adm release info registry.dns.base.domain.name:5000/openshift-release-dev/ocp-release:4.x.y-x86_64 | grep hypershift
hypershift sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.dns.base.domain.name:5000/openshift-release-dev/ocp-v4.0-art-dev@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Trying to pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Getting image source signatures Copying blob d8190195889e skipped: already exists Copying blob c71d2589fba7 skipped: already exists Copying blob d4dc6e74b6ce skipped: already exists Copying blob 97da74cc6d8f skipped: already exists Copying blob b70007a560c9 done Copying config 3a62961e6e done Writing manifest to image destination Storing signatures 3a62961e6ed6edab46d5ec8429ff1f41d6bb68de51271f037c6cb8941a007fde
apply -f 01-4.14-hosted_cluster-nodeport.yaml
NAME READY STATUS RESTARTS AGE capi-provider-5b57dbd6d5-pxlqc 1/1 Running 0 3m57s catalog-operator-9694884dd-m7zzv 2/2 Running 0 93s cluster-api-f98b9467c-9hfrq 1/1 Running 0 3m57s cluster-autoscaler-d7f95dd5-d8m5d 1/1 Running 0 93s cluster-image-registry-operator-5ff5944b4b-648ht 1/2 Running 0 93s cluster-network-operator-77b896ddc-wpkq8 1/1 Running 0 94s cluster-node-tuning-operator-84956cd484-4hfgf 1/1 Running 0 94s cluster-policy-controller-5fd8595d97-rhbwf 1/1 Running 0 95s cluster-storage-operator-54dcf584b5-xrnts 1/1 Running 0 93s cluster-version-operator-9c554b999-l22s7 1/1 Running 0 95s control-plane-operator-6fdc9c569-t7hr4 1/1 Running 0 3m57s csi-snapshot-controller-785c6dc77c-8ljmr 1/1 Running 0 77s csi-snapshot-controller-operator-7c6674bc5b-d9dtp 1/1 Running 0 93s csi-snapshot-webhook-5b8584875f-2492j 1/1 Running 0 77s dns-operator-6874b577f-9tc6b 1/1 Running 0 94s etcd-0 3/3 Running 0 3m39s hosted-cluster-config-operator-f5cf5c464-4nmbh 1/1 Running 0 93s ignition-server-6b689748fc-zdqzk 1/1 Running 0 95s ignition-server-proxy-54d4bb9b9b-6zkg7 1/1 Running 0 95s ingress-operator-6548dc758b-f9gtg 1/2 Running 0 94s konnectivity-agent-7767cdc6f5-tw782 1/1 Running 0 95s kube-apiserver-7b5799b6c8-9f5bp 4/4 Running 0 3m7s kube-controller-manager-5465bc4dd6-zpdlk 1/1 Running 0 44s kube-scheduler-5dd5f78b94-bbbck 1/1 Running 0 2m36s machine-approver-846c69f56-jxvfr 1/1 Running 0 92s oauth-openshift-79c7bf44bf-j975g 2/2 Running 0 62s olm-operator-767f9584c-4lcl2 2/2 Running 0 93s openshift-apiserver-5d469778c6-pl8tj 3/3 Running 0 2m36s openshift-controller-manager-6475fdff58-hl4f7 1/1 Running 0 95s openshift-oauth-apiserver-dbbc5cc5f-98574 2/2 Running 0 95s openshift-route-controller-manager-5f6997b48f-s9vdc 1/1 Running 0 95s packageserver-67c87d4d4f-kl7qh 2/2 Running 0 93s
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters hosted-ipv6 hosted-admin-kubeconfig Partial True False The hosted control plane is available
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: creationTimestamp: null name: hosted-ipv6 namespace: clusters spec: arch: amd64 clusterName: hosted-ipv6 management: autoRepair: false 1 upgradeType: InPlace 2 nodeDrainTimeout: 0s platform: type: Agent release: image: registry.dns.base.domain.name:5000/openshift/release-images:4.x.y-x86_64 3 replicas: 0 status: replicas: 0 4
apply -f 02-nodepool.yaml
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted-ipv6 hosted 0 False False 4.x.y-x86_64
--- apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: hosted-ipv6 namespace: clusters-hosted-ipv6 spec: pullSecretRef: 1 name: pull-secret sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDk7ICaUE+/k4zTpxLk4+xFdHi4ZuDi5qjeF52afsNkw0w/glILHhwpL5gnp5WkRuL8GwJuZ1VqLC9EKrdmegn4MrmUlq7WTsP0VFOZFBfq2XRUxo1wrRdor2z0Bbh93ytR+ZsDbbLlGngXaMa0Vbt+z74FqlcajbHTZ6zBmTpBVq5RHtDPgKITdpE1fongp7+ZXQNBlkaavaqv8bnyrP4BWahLP4iO9/xJF9lQYboYwEEDzmnKLMW1VtCE6nJzEgWCufACTbxpNS7GvKtoHT/OVzw8ArEXhZXQUS1UY8zKsX2iXwmyhw5Sj6YboA8WICs4z+TrFP89LmxXY0j6536TQFyRz1iB4WWvCbH5n6W+ABV2e8ssJB1AmEy8QYNwpJQJNpSxzoKBjI73XxvPYYC/IjPFMySwZqrSZCkJYqQ023ySkaQxWZT7in4KeMu7eS2tC+Kn4deJ7KwwUycx8n6RHMeD8Qg9flTHCv3gmab8JKZJqN3hW1D378JuvmIX4V0= 2
apply -f 03-infraenv.yaml
NAMESPACE NAME ISO CREATED AT clusters-hosted-ipv6 hosted 2023-09-11T15:14:10Z
kcli delete plan hosted-ipv6
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-ipv6 -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"ipv6\", \"mac\": \"aa:aa:aa:aa:02:11\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0211 -P name=hosted-ipv6-worker0
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-ipv6 -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"ipv6\", \"mac\": \"aa:aa:aa:aa:02:12\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0212 -P name=hosted-ipv6-worker1
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-ipv6 -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"ipv6\", \"mac\": \"aa:aa:aa:aa:02:13\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0213 -P name=hosted-ipv6-worker2
systemctl restart ksushy
+---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | Name | Status | Ip | Source | Plan | Profile | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | hosted-worker0 | down | | | hosted-ipv6 | kvirt | | hosted-worker1 | down | | | hosted-ipv6 | kvirt | | hosted-worker2 | down | | | hosted-ipv6 | kvirt | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+
--- apiVersion: v1 kind: Secret metadata: name: hosted-ipv6-worker0-bmc-secret namespace: clusters-hosted-ipv6 data: password: YWRtaW4= username: YWRtaW4= type: Opaque --- apiVersion: metal3.io/v1alpha1 kind: BareMetalHost metadata: name: hosted-ipv6-worker0 namespace: clusters-hosted-ipv6 labels: infraenvs.agent-install.openshift.io: hosted-ipv6 1 annotations: inspect.metal3.io: disabled bmac.agent-install.openshift.io/hostname: hosted-ipv6-worker0 2 spec: automatedCleaningMode: disabled 3 bmc: disableCertificateVerification: true 4 address: redfish-virtualmedia://[192.168.125.1]:9000/redfish/v1/Systems/local/hosted-ipv6-worker0 5 credentialsName: hosted-ipv6-worker0-bmc-secret 6 bootMACAddress: aa:aa:aa:aa:03:11 7 online: true 8
apply -f 04-bmh.yaml
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 registering true 2s clusters-hosted hosted-worker1 registering true 2s clusters-hosted hosted-worker2 registering true 2s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioning true 16s clusters-hosted hosted-worker1 provisioning true 16s clusters-hosted hosted-worker2 provisioning true 16s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioned true 67s clusters-hosted hosted-worker1 provisioned true 67s clusters-hosted hosted-worker2 provisioned true 67s
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0411 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0412 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0413 true auto-assign
-n clusters scale nodepool hosted-ipv6 --replicas 3
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0211 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0212 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0213 hosted true auto-assign
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted hosted 3 False False 4.x.y-x86_64 Minimum availability requires 3 replicas, current 0 available
export KUBECONFIG=/root/.kcli/clusters/hub-ipv4/auth/kubeconfig
watch "oc get pod -n hypershift;echo;echo;oc get pod -n clusters-hosted-ipv4;echo;echo;oc get bmh -A;echo;echo;oc get agent -A;echo;echo;oc get infraenv -A;echo;echo;oc get hostedcluster -A;echo;echo;oc get nodepool -A;echo;echo;"
get secret -n clusters-hosted-ipv4 admin-kubeconfig -o jsonpath='{.data.kubeconfig}' |base64 -d > /root/hc_admin_kubeconfig.yaml
export KUBECONFIG=/root/hc_admin_kubeconfig.yaml
watch "oc get clusterversion,nodes,co"
sudo dnf install dnsmasq radvd vim golang podman bind-utils net-tools httpd-tools tree htop strace tmux -y
systemctl enable --now podman
sudo yum -y install libvirt libvirt-daemon-driver-qemu qemu-kvm
sudo usermod -aG qemu,libvirt USD(id -un)
sudo newgrp libvirt
sudo systemctl enable --now libvirtd
sudo dnf -y copr enable karmab/kcli
sudo dnf -y install kcli
sudo kcli create pool -p /var/lib/libvirt/images default
kcli create host kvm -H 127.0.0.1 local
sudo setfacl -m u:USD(id -un):rwx /var/lib/libvirt/images
kcli create network -c 192.168.122.0/24 default
#!/bin/bash export IP="192.168.126.1" 1 export BASE_RESOLV_CONF="/run/NetworkManager/resolv.conf" if ! [[ `grep -q "USDIP" /etc/resolv.conf` ]]; then export TMP_FILE=USD(mktemp /etc/forcedns_resolv.conf.XXXXXX) cp USDBASE_RESOLV_CONF USDTMP_FILE chmod --reference=USDBASE_RESOLV_CONF USDTMP_FILE sed -i -e "s/dns.base.domain.name//" -e "s/search /& dns.base.domain.name /" -e "0,/nameserver/s/nameserver/& USDIP\n&/" USDTMP_FILE 2 mv USDTMP_FILE /etc/resolv.conf fi echo "ok"
chmod 755 /etc/NetworkManager/dispatcher.d/forcedns
sudo dnf install python3-pyOpenSSL.noarch python3-cherrypy -y
kcli create sushy-service --ssl --ipv6 --port 9000
sudo systemctl daemon-reload
systemctl enable --now ksushy
systemctl status ksushy
sed -i s/^SELINUX=.*USD/SELINUX=permissive/ /etc/selinux/config; setenforce 0
systemctl disable --now firewalld
systemctl restart libvirtd
systemctl enable --now libvirtd
#!/usr/bin/env bash set -euo pipefail PRIMARY_NIC=USD(ls -1 /sys/class/net | grep -v podman | head -1) export PATH=/root/bin:USDPATH export PULL_SECRET="/root/baremetal/hub/openshift_pull.json" 1 if [[ ! -f USDPULL_SECRET ]];then echo "Pull Secret not found, exiting..." exit 1 fi dnf -y install podman httpd httpd-tools jq skopeo libseccomp-devel export IP=USD(ip -o addr show USDPRIMARY_NIC | head -1 | awk '{print USD4}' | cut -d'/' -f1) REGISTRY_NAME=registry.USD(hostname --long) REGISTRY_USER=dummy REGISTRY_PASSWORD=dummy KEY=USD(echo -n USDREGISTRY_USER:USDREGISTRY_PASSWORD | base64) echo "{\"auths\": {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\", \"email\": \"sample-email@domain.ltd\"}}}" > /root/disconnected_pull.json mv USD{PULL_SECRET} /root/openshift_pull.json.old jq ".auths += {\"USDREGISTRY_NAME:5000\": {\"auth\": \"USDKEY\",\"email\": \"sample-email@domain.ltd\"}}" < /root/openshift_pull.json.old > USDPULL_SECRET mkdir -p /opt/registry/{auth,certs,data,conf} cat <<EOF > /opt/registry/conf/config.yml version: 0.1 log: fields: service: registry storage: cache: blobdescriptor: inmemory filesystem: rootdirectory: /var/lib/registry delete: enabled: true http: addr: :5000 headers: X-Content-Type-Options: [nosniff] health: storagedriver: enabled: true interval: 10s threshold: 3 compatibility: schema1: enabled: true EOF openssl req -newkey rsa:4096 -nodes -sha256 -keyout /opt/registry/certs/domain.key -x509 -days 3650 -out /opt/registry/certs/domain.crt -subj "/C=US/ST=Madrid/L=San Bernardo/O=Karmalabs/OU=Guitar/CN=USDREGISTRY_NAME" -addext "subjectAltName=DNS:USDREGISTRY_NAME" cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/ update-ca-trust extract htpasswd -bBc /opt/registry/auth/htpasswd USDREGISTRY_USER USDREGISTRY_PASSWORD create --name registry --net host --security-opt label=disable --replace -v /opt/registry/data:/var/lib/registry:z -v /opt/registry/auth:/auth:z -v /opt/registry/conf/config.yml:/etc/docker/registry/config.yml -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry" -e "REGISTRY_HTTP_SECRET=ALongRandomSecretForRegistry" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -v /opt/registry/certs:/certs:z -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key docker.io/library/registry:latest [ "USD?" == "0" ] || !! systemctl enable --now registry
chmod u+x USD{HOME}/registry.sh
USD{HOME}/registry.sh
systemctl status
systemctl start
systemctl stop
kcli create network -c 192.168.126.0/24 -P dhcp=false -P dns=false -d 2620:52:0:1306::0/64 --domain dns.base.domain.name --nodhcp dual
kcli list network Listing Networks +---------+--------+---------------------+-------+------------------+------+ | Network | Type | Cidr | Dhcp | Domain | Mode | +---------+--------+---------------------+-------+------------------+------+ | default | routed | 192.168.122.0/24 | True | default | nat | | ipv4 | routed | 2620:52:0:1306::/64 | False | dns.base.domain.name | nat | | ipv4 | routed | 192.168.125.0/24 | False | dns.base.domain.name | nat | | ipv6 | routed | 2620:52:0:1305::/64 | False | dns.base.domain.name | nat | +---------+--------+---------------------+-------+------------------+------+
kcli info network ipv6 Providing information about network ipv6 cidr: 2620:52:0:1306::/64 dhcp: false domain: dns.base.domain.name mode: nat plan: kvirt type: routed
plan: hub-dual force: true version: stable tag: "4.x.y-x86_64" 1 cluster: "hub-dual" dualstack: true domain: dns.base.domain.name api_ip: 192.168.126.10 ingress_ip: 192.168.126.11 service_networks: - 172.30.0.0/16 - fd02::/112 cluster_networks: - 10.132.0.0/14 - fd01::/48 disconnected_url: registry.dns.base.domain.name:5000 disconnected_update: true disconnected_user: dummy disconnected_password: dummy disconnected_operators_version: v4.14 disconnected_operators: - name: metallb-operator - name: lvms-operator channels: - name: stable-4.14 disconnected_extra_images: - quay.io/user-name/trbsht:latest - quay.io/user-name/hypershift:BMSelfManage-v4.14-rc-v3 - registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 dualstack: true disk_size: 200 extra_disks: [200] memory: 48000 numcpus: 16 ctlplanes: 3 workers: 0 manifests: extra-manifests metal3: true network: dual users_dev: developer users_devpassword: developer users_admin: admin users_adminpassword: admin metallb_pool: dual-virtual-network metallb_ranges: - 192.168.126.150-192.168.126.190 metallb_autoassign: true apps: - users - lvms-operator - metallb-operator vmrules: - hub-bootstrap: nets: - name: ipv6 mac: aa:aa:aa:aa:10:07 - hub-ctlplane-0: nets: - name: ipv6 mac: aa:aa:aa:aa:10:01 - hub-ctlplane-1: nets: - name: ipv6 mac: aa:aa:aa:aa:10:02 - hub-ctlplane-2: nets: - name: ipv6 mac: aa:aa:aa:aa:10:03
kcli create cluster openshift --pf mgmt-compact-hub-dual.yaml
adm -a USD{LOCAL_SECRET_JSON} release extract --command=openshift-install "USD{LOCAL_REGISTRY}/USD{LOCAL_REPOSITORY}:USD{OCP_RELEASE}-USD{ARCHITECTURE}"
#!/bin/bash WEBSRV_FOLDER=/opt/srv ROOTFS_IMG_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.pxe.rootfs.location')" 1 LIVE_ISO_URL="USD(./openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.iso.disk.location')" 2 mkdir -p USD{WEBSRV_FOLDER}/images curl -Lk USD{ROOTFS_IMG_URL} -o USD{WEBSRV_FOLDER}/images/USD{ROOTFS_IMG_URL##*/} curl -Lk USD{LIVE_ISO_URL} -o USD{WEBSRV_FOLDER}/images/USD{LIVE_ISO_URL##*/} chmod -R 755 USD{WEBSRV_FOLDER}/* ## Run Webserver ps --noheading | grep -q websrv-ai if [[ USD? == 0 ]];then echo "Launching Registry pod..." /usr/bin/podman run --name websrv-ai --net host -v /opt/srv:/usr/local/apache2/htdocs:z quay.io/alosadag/httpd:p8080 fi
apiVersion: mirror.openshift.io/v1alpha2 kind: ImageSetConfiguration storageConfig: registry: imageURL: registry.dns.base.domain.name:5000/openshift/release/metadata:latest mirror: platform: channels: - name: candidate-4.14 minVersion: 4.x.y-x86_64 1 maxVersion: 4.x.y-x86_64 type: ocp graph: true additionalImages: - name: quay.io/karmab/origin-keepalived-ipfailover:latest - name: quay.io/karmab/kubectl:latest - name: quay.io/karmab/haproxy:latest - name: quay.io/karmab/mdns-publisher:latest - name: quay.io/karmab/origin-coredns:latest - name: quay.io/karmab/curl:latest - name: quay.io/karmab/kcli:latest - name: quay.io/user-name/trbsht:latest - name: quay.io/user-name/hypershift:BMSelfManage-v4.14-rc-v3 - name: registry.redhat.io/openshift4/ose-kube-rbac-proxy:v4.10 operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.14 packages: - name: lvms-operator - name: local-storage-operator - name: odf-csi-addons-operator - name: odf-operator - name: mcg-operator - name: ocs-operator - name: metallb-operator
oc-mirror --source-skip-tls --config imagesetconfig.yaml docker://USD{REGISTRY}
REGISTRY=registry.USD(hostname --long):5000 adm release mirror --from=registry.ci.openshift.org/ocp/release:4.x.y-x86_64 --to=USD{REGISTRY}/openshift/release --to-release-image=USD{REGISTRY}/openshift/release-images:registry.dns.base.domain.name:5000/openshift/release-images:4.x.y-x86_64
get packagemanifest
apply -f oc-mirror-workspace/results-XXXXXX/imageContentSourcePolicy.yaml
apply -f catalogSource-XXXXXXXX-index.yaml
--- apiVersion: v1 kind: ConfigMap metadata: name: custom-registries namespace: multicluster-engine labels: app: assisted-service data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- registries.conf: | unqualified-search-registries = ["registry.access.redhat.com", "docker.io"] [[registry]] prefix = "" location = "registry.redhat.io/openshift4" mirror-by-digest-only = true [[registry.mirror]] location = "registry.dns.base.domain.name:5000/openshift4" 1 [[registry]] prefix = "" location = "registry.redhat.io/rhacm2" mirror-by-digest-only = true
--- apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: annotations: unsupported.agent-install.openshift.io/assisted-service-configmap: assisted-service-config 1 name: agent namespace: multicluster-engine spec: mirrorRegistryRef: name: custom-registries 2 databaseStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 10Gi filesystemStorage: storageClassName: lvms-vg1 accessModes: - ReadWriteOnce resources: requests: storage: 20Gi osImages: 3 - cpuArchitecture: x86_64 openshiftVersion: "4.14" rootFSUrl: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live-rootfs.x86_64.img 4 url: http://registry.dns.base.domain.name:8080/images/rhcos-414.92.202308281054-0-live.x86_64.iso version: 414.92.202308281054-0
apply -f agentServiceConfig.yaml
assisted-image-service-0 1/1 Running 2 11d 1 assisted-service-668b49548-9m7xw 2/2 Running 5 11d 2
## REGISTRY_CERT_PATH=<PATH/TO/YOUR/CERTIFICATE/FILE> export REGISTRY_CERT_PATH=/opt/registry/certs/domain.crt create configmap user-ca-bundle -n openshift-config --from-file=ca-bundle.crt=USD{REGISTRY_CERT_PATH}
## REGISTRY_CERT_PATH=<PATH/TO/YOUR/CERTIFICATE/FILE> export REGISTRY_CERT_PATH=/opt/registry/certs/domain.crt export TMP_FILE=USD(mktemp) get cm -n openshift-config user-ca-bundle -ojsonpath='{.data.ca-bundle\.crt}' > USD{TMP_FILE} echo >> USD{TMP_FILE} echo \#registry.USD(hostname --long) >> USD{TMP_FILE} cat USD{REGISTRY_CERT_PATH} >> USD{TMP_FILE} create configmap user-ca-bundle -n openshift-config --from-file=ca-bundle.crt=USD{TMP_FILE} --dry-run=client -o yaml | kubectl apply -f -
--- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters-hosted-dual spec: {} status: {} --- apiVersion: v1 kind: Namespace metadata: creationTimestamp: null name: clusters spec: {} status: {}
--- apiVersion: v1 data: ca-bundle.crt: | -----BEGIN CERTIFICATE----- -----END CERTIFICATE----- kind: ConfigMap metadata: name: user-ca-bundle namespace: clusters --- apiVersion: v1 data: .dockerconfigjson: xxxxxxxxx kind: Secret metadata: creationTimestamp: null name: hosted-dual-pull-secret namespace: clusters --- apiVersion: v1 kind: Secret metadata: name: sshkey-cluster-hosted-dual namespace: clusters stringData: id_rsa.pub: ssh-rsa xxxxxxxxx --- apiVersion: v1 data: key: nTPtVBEt03owkrKhIdmSW8jrWRxU57KO/fnZa8oaG0Y= kind: Secret metadata: creationTimestamp: null name: hosted-dual-etcd-encryption-key namespace: clusters type: Opaque
apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: creationTimestamp: null name: capi-provider-role namespace: clusters-hosted-dual rules: - apiGroups: - agent-install.openshift.io resources: - agents verbs: - '*'
apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: name: hosted-dual namespace: clusters spec: additionalTrustBundle: name: "user-ca-bundle" olmCatalogPlacement: guest imageContentSources: 1 - source: quay.io/openshift-release-dev/ocp-v4.0-art-dev mirrors: - registry.dns.base.domain.name:5000/openshift/release 2 - source: quay.io/openshift-release-dev/ocp-release mirrors: - registry.dns.base.domain.name:5000/openshift/release-images - mirrors: autoscaling: {} controllerAvailabilityPolicy: SingleReplica dns: baseDomain: dns.base.domain.name etcd: managed: storage: persistentVolume: size: 8Gi restoreSnapshotURL: null type: PersistentVolume managementType: Managed fips: false networking: clusterNetwork: - cidr: 10.132.0.0/14 - cidr: fd01::/48 networkType: OVNKubernetes serviceNetwork: - cidr: 172.31.0.0/16 - cidr: fd02::/112 platform: agent: agentNamespace: clusters-hosted-dual type: Agent pullSecret: name: hosted-dual-pull-secret release: image: registry.dns.base.domain.name:5000/openshift/release-images:4.x.y-x86_64 3 secretEncryption: aescbc: activeKey: name: hosted-dual-etcd-encryption-key type: aescbc services: - service: APIServer servicePublishingStrategy: nodePort: address: api.hosted-dual.dns.base.domain.name type: NodePort - service: OAuthServer servicePublishingStrategy: nodePort: address: api.hosted-dual.dns.base.domain.name type: NodePort - service: OIDC servicePublishingStrategy: nodePort: address: api.hosted-dual.dns.base.domain.name type: NodePort - service: Konnectivity servicePublishingStrategy: nodePort: address: api.hosted-dual.dns.base.domain.name type: NodePort - service: Ignition servicePublishingStrategy: nodePort: address: api.hosted-dual.dns.base.domain.name type: NodePort sshKey: name: sshkey-cluster-hosted-dual status: controlPlaneEndpoint: host: "" port: 0
adm release info registry.dns.base.domain.name:5000/openshift-release-dev/ocp-release:4.x.y-x86_64 | grep hypershift
hypershift sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.dns.base.domain.name:5000/openshift-release-dev/ocp-v4.0-art-dev@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8
pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Trying to pull registry.dns.base.domain.name:5000/openshift/release@sha256:31149e3e5f8c5e5b5b100ff2d89975cf5f7a73801b2c06c639bf6648766117f8 Getting image source signatures Copying blob d8190195889e skipped: already exists Copying blob c71d2589fba7 skipped: already exists Copying blob d4dc6e74b6ce skipped: already exists Copying blob 97da74cc6d8f skipped: already exists Copying blob b70007a560c9 done Copying config 3a62961e6e done Writing manifest to image destination Storing signatures 3a62961e6ed6edab46d5ec8429ff1f41d6bb68de51271f037c6cb8941a007fde
apply -f 01-4.14-hosted_cluster-nodeport.yaml
NAME READY STATUS RESTARTS AGE capi-provider-5b57dbd6d5-pxlqc 1/1 Running 0 3m57s catalog-operator-9694884dd-m7zzv 2/2 Running 0 93s cluster-api-f98b9467c-9hfrq 1/1 Running 0 3m57s cluster-autoscaler-d7f95dd5-d8m5d 1/1 Running 0 93s cluster-image-registry-operator-5ff5944b4b-648ht 1/2 Running 0 93s cluster-network-operator-77b896ddc-wpkq8 1/1 Running 0 94s cluster-node-tuning-operator-84956cd484-4hfgf 1/1 Running 0 94s cluster-policy-controller-5fd8595d97-rhbwf 1/1 Running 0 95s cluster-storage-operator-54dcf584b5-xrnts 1/1 Running 0 93s cluster-version-operator-9c554b999-l22s7 1/1 Running 0 95s control-plane-operator-6fdc9c569-t7hr4 1/1 Running 0 3m57s csi-snapshot-controller-785c6dc77c-8ljmr 1/1 Running 0 77s csi-snapshot-controller-operator-7c6674bc5b-d9dtp 1/1 Running 0 93s csi-snapshot-webhook-5b8584875f-2492j 1/1 Running 0 77s dns-operator-6874b577f-9tc6b 1/1 Running 0 94s etcd-0 3/3 Running 0 3m39s hosted-cluster-config-operator-f5cf5c464-4nmbh 1/1 Running 0 93s ignition-server-6b689748fc-zdqzk 1/1 Running 0 95s ignition-server-proxy-54d4bb9b9b-6zkg7 1/1 Running 0 95s ingress-operator-6548dc758b-f9gtg 1/2 Running 0 94s konnectivity-agent-7767cdc6f5-tw782 1/1 Running 0 95s kube-apiserver-7b5799b6c8-9f5bp 4/4 Running 0 3m7s kube-controller-manager-5465bc4dd6-zpdlk 1/1 Running 0 44s kube-scheduler-5dd5f78b94-bbbck 1/1 Running 0 2m36s machine-approver-846c69f56-jxvfr 1/1 Running 0 92s oauth-openshift-79c7bf44bf-j975g 2/2 Running 0 62s olm-operator-767f9584c-4lcl2 2/2 Running 0 93s openshift-apiserver-5d469778c6-pl8tj 3/3 Running 0 2m36s openshift-controller-manager-6475fdff58-hl4f7 1/1 Running 0 95s openshift-oauth-apiserver-dbbc5cc5f-98574 2/2 Running 0 95s openshift-route-controller-manager-5f6997b48f-s9vdc 1/1 Running 0 95s packageserver-67c87d4d4f-kl7qh 2/2 Running 0 93s
NAMESPACE NAME VERSION KUBECONFIG PROGRESS AVAILABLE PROGRESSING MESSAGE clusters hosted-dual hosted-admin-kubeconfig Partial True False The hosted control plane is available
apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: creationTimestamp: null name: hosted-dual namespace: clusters spec: arch: amd64 clusterName: hosted-dual management: autoRepair: false 1 upgradeType: InPlace 2 nodeDrainTimeout: 0s platform: type: Agent release: image: registry.dns.base.domain.name:5000/openshift/release-images:4.x.y-x86_64 3 replicas: 0 status: replicas: 0 4
apply -f 02-nodepool.yaml
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted-dual hosted 0 False False 4.x.y-x86_64
--- apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: hosted-dual namespace: clusters-hosted-dual spec: pullSecretRef: 1 name: pull-secret sshAuthorizedKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDk7ICaUE+/k4zTpxLk4+xFdHi4ZuDi5qjeF52afsNkw0w/glILHhwpL5gnp5WkRuL8GwJuZ1VqLC9EKrdmegn4MrmUlq7WTsP0VFOZFBfq2XRUxo1wrRdor2z0Bbh93ytR+ZsDbbLlGngXaMa0Vbt+z74FqlcajbHTZ6zBmTpBVq5RHtDPgKITdpE1fongp7+ZXQNBlkaavaqv8bnyrP4BWahLP4iO9/xJF9lQYboYwEEDzmnKLMW1VtCE6nJzEgWCufACTbxpNS7GvKtoHT/OVzw8ArEXhZXQUS1UY8zKsX2iXwmyhw5Sj6YboA8WICs4z+TrFP89LmxXY0j6536TQFyRz1iB4WWvCbH5n6W+ABV2e8ssJB1AmEy8QYNwpJQJNpSxzoKBjI73XxvPYYC/IjPFMySwZqrSZCkJYqQ023ySkaQxWZT7in4KeMu7eS2tC+Kn4deJ7KwwUycx8n6RHMeD8Qg9flTHCv3gmab8JKZJqN3hW1D378JuvmIX4V0= 2
apply -f 03-infraenv.yaml
NAMESPACE NAME ISO CREATED AT clusters-hosted-dual hosted 2023-09-11T15:14:10Z
kcli delete plan hosted-dual
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-dual -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"dual\", \"mac\": \"aa:aa:aa:aa:11:01\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa1101 -P name=hosted-dual-worker0
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-dual -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"dual\", \"mac\": \"aa:aa:aa:aa:11:02\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa1102 -P name=hosted-dual-worker1
kcli create vm -P start=False -P uefi_legacy=true -P plan=hosted-dual -P memory=8192 -P numcpus=16 -P disks=[200,200] -P nets=["{\"name\": \"dual\", \"mac\": \"aa:aa:aa:aa:11:03\"}"] -P uuid=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa1103 -P name=hosted-dual-worker2
systemctl restart ksushy
+---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | Name | Status | Ip | Source | Plan | Profile | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+ | hosted-worker0 | down | | | hosted-dual | kvirt | | hosted-worker1 | down | | | hosted-dual | kvirt | | hosted-worker2 | down | | | hosted-dual | kvirt | +---------------------+--------+-------------------+----------------------------------------------------+-------------+---------+
--- apiVersion: v1 kind: Secret metadata: name: hosted-dual-worker0-bmc-secret namespace: clusters-hosted-dual data: password: YWRtaW4= username: YWRtaW4= type: Opaque --- apiVersion: metal3.io/v1alpha1 kind: BareMetalHost metadata: name: hosted-dual-worker0 namespace: clusters-hosted-dual labels: infraenvs.agent-install.openshift.io: hosted-dual 1 annotations: inspect.metal3.io: disabled bmac.agent-install.openshift.io/hostname: hosted-dual-worker0 2 spec: automatedCleaningMode: disabled 3 bmc: disableCertificateVerification: true 4 address: redfish-virtualmedia://[192.168.126.1]:9000/redfish/v1/Systems/local/hosted-dual-worker0 5 credentialsName: hosted-dual-worker0-bmc-secret 6 bootMACAddress: aa:aa:aa:aa:02:11 7 online: true 8
apply -f 04-bmh.yaml
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 registering true 2s clusters-hosted hosted-worker1 registering true 2s clusters-hosted hosted-worker2 registering true 2s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioning true 16s clusters-hosted hosted-worker1 provisioning true 16s clusters-hosted hosted-worker2 provisioning true 16s
NAMESPACE NAME STATE CONSUMER ONLINE ERROR AGE clusters-hosted hosted-worker0 provisioned true 67s clusters-hosted hosted-worker1 provisioned true 67s clusters-hosted hosted-worker2 provisioned true 67s
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0411 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0412 true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0413 true auto-assign
-n clusters scale nodepool hosted-dual --replicas 3
NAMESPACE NAME CLUSTER APPROVED ROLE STAGE clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0411 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0412 hosted true auto-assign clusters-hosted aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaa0413 hosted true auto-assign
NAMESPACE NAME CLUSTER DESIRED NODES CURRENT NODES AUTOSCALING AUTOREPAIR VERSION UPDATINGVERSION UPDATINGCONFIG MESSAGE clusters hosted hosted 3 False False 4.x.y-x86_64 Minimum availability requires 3 replicas, current 0 available
export KUBECONFIG=/root/.kcli/clusters/hub-ipv4/auth/kubeconfig
watch "oc get pod -n hypershift;echo;echo;oc get pod -n clusters-hosted-ipv4;echo;echo;oc get bmh -A;echo;echo;oc get agent -A;echo;echo;oc get infraenv -A;echo;echo;oc get hostedcluster -A;echo;echo;oc get nodepool -A;echo;echo;"
get secret -n clusters-hosted-ipv4 admin-kubeconfig -o jsonpath='{.data.kubeconfig}' |base64 -d > /root/hc_admin_kubeconfig.yaml
export KUBECONFIG=/root/hc_admin_kubeconfig.yaml
watch "oc get clusterversion,nodes,co"
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: import.open-cluster-management.io/hosting-cluster-name: local-cluster import.open-cluster-management.io/klusterlet-deploy-mode: Hosted open-cluster-management/created-via: hypershift labels: cloud: auto-detect cluster.open-cluster-management.io/clusterset: default name: <cluster_name> vendor: OpenShift name: <cluster_name> spec: hubAcceptsClient: true leaseDurationSeconds: 60
apply -f <file_name>
apiVersion: agent.open-cluster-management.io/v1 kind: KlusterletAddonConfig metadata: name: <cluster_name> namespace: <cluster_name> spec: clusterName: <cluster_name> clusterNamespace: <cluster_name> clusterLabels: cloud: auto-detect vendor: auto-detect applicationManager: enabled: true certPolicyController: enabled: true policyController: enabled: true searchCollector: enabled: false
apply -f <file_name>
get managedcluster <cluster_name>
edit addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: hypershift-addon-deploy-config namespace: multicluster-engine spec: customizedVariables: - name: hcMaxNumber value: "80" - name: hcThresholdNumber value: "60" - name: autoImportDisabled value: "true"
patch mce multiclusterengine --type=merge -p '{"spec":{"overrides":{"components":[{"name":"hypershift","enabled": true}]}}}' 1
get mce multiclusterengine -o yaml 1
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: hypershift enabled: true - name: hypershift-local-hosting enabled: true
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: hypershift-addon namespace: local-cluster spec: installNamespace: open-cluster-management-agent-addon
apply -f <filename>
get managedclusteraddons -n local-cluster hypershift-addon
NAME AVAILABLE DEGRADED PROGRESSING hypershift-addon True
get hostedcluster -A
patch mce multiclusterengine --type=merge -p '{"spec":{"overrides":{"components":[{"name":"hypershift-local-hosting","enabled": false}]}}}' 1
patch mce multiclusterengine --type=merge -p '{"spec":{"overrides":{"components":[{"name":"hypershift","enabled": false}]}}}' 1
get mce multiclusterengine -o yaml 1
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: overrides: components: - name: hypershift enabled: false - name: hypershift-local-hosting enabled: false
GET /cluster.open-cluster-management.io/v1/managedclusters
POST /cluster.open-cluster-management.io/v1/managedclusters
{ "apiVersion" : "cluster.open-cluster-management.io/v1", "kind" : "ManagedCluster", "metadata" : { "labels" : { "vendor" : "OpenShift" }, "name" : "cluster1" }, "spec": { "hubAcceptsClient": true, "managedClusterClientConfigs": [ { "caBundle": "test", "url": "https://test.com" } ] }, "status" : { } }
GET /cluster.open-cluster-management.io/v1/managedclusters/{cluster_name}
DELETE /cluster.open-cluster-management.io/v1/managedclusters/{cluster_name}
"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?USD"
GET /cluster.open-cluster-management.io/v1beta2/managedclustersets
POST /cluster.open-cluster-management.io/v1beta2/managedclustersets
{ "apiVersion" : "cluster.open-cluster-management.io/v1beta2", "kind" : "ManagedClusterSet", "metadata" : { "name" : "clusterset1" }, "spec": { }, "status" : { } }
GET /cluster.open-cluster-management.io/v1beta2/managedclustersets/{clusterset_name}
DELETE /cluster.open-cluster-management.io/v1beta2/managedclustersets/{clusterset_name}
GET /cluster.open-cluster-management.io/v1beta2/namespaces/{namespace}/managedclustersetbindings
POST /cluster.open-cluster-management.io/v1beta2/namespaces/{namespace}/managedclustersetbindings
{ "apiVersion" : "cluster.open-cluster-management.io/v1", "kind" : "ManagedClusterSetBinding", "metadata" : { "name" : "clusterset1", "namespace" : "ns1" }, "spec": { "clusterSet": "clusterset1" }, "status" : { } }
GET /cluster.open-cluster-management.io/v1beta2/namespaces/{namespace}/managedclustersetbindings/{clustersetbinding_name}
DELETE /cluster.open-cluster-management.io/v1beta2/managedclustersetbindings/{clustersetbinding_name}
GET /managedclusters.clusterview.open-cluster-management.io
LIST /managedclusters.clusterview.open-cluster-management.io
{ "apiVersion" : "clusterview.open-cluster-management.io/v1alpha1", "kind" : "ClusterView", "metadata" : { "name" : "<user_ID>" }, "spec": { }, "status" : { } }
WATCH /managedclusters.clusterview.open-cluster-management.io
GET /managedclustersets.clusterview.open-cluster-management.io
LIST /managedclustersets.clusterview.open-cluster-management.io
WATCH /managedclustersets.clusterview.open-cluster-management.io
POST /authentication.open-cluster-management.io/v1beta1/managedserviceaccounts
apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.14.0 name: managedserviceaccounts.authentication.open-cluster-management.io spec: group: authentication.open-cluster-management.io names: kind: ManagedServiceAccount listKind: ManagedServiceAccountList plural: managedserviceaccounts singular: managedserviceaccount scope: Namespaced versions: - deprecated: true deprecationWarning: authentication.open-cluster-management.io/v1alpha1 ManagedServiceAccount is deprecated; use authentication.open-cluster-management.io/v1beta1 ManagedServiceAccount; version v1alpha1 will be removed in the next release name: v1alpha1 schema: openAPIV3Schema: description: ManagedServiceAccount is the Schema for the managedserviceaccounts API properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ManagedServiceAccountSpec defines the desired state of ManagedServiceAccount properties: rotation: description: Rotation is the policy for rotation the credentials. properties: enabled: default: true description: |- Enabled prescribes whether the ServiceAccount token will be rotated from the upstream type: boolean validity: default: 8640h0m0s description: Validity is the duration for which the signed ServiceAccount token is valid. type: string type: object ttlSecondsAfterCreation: description: |- ttlSecondsAfterCreation limits the lifetime of a ManagedServiceAccount. If the ttlSecondsAfterCreation field is set, the ManagedServiceAccount will be automatically deleted regardless of the ManagedServiceAccount's status. When the ManagedServiceAccount is deleted, its lifecycle guarantees (e.g. finalizers) will be honored. If this field is unset, the ManagedServiceAccount won't be automatically deleted. If this field is set to zero, the ManagedServiceAccount becomes eligible for deletion immediately after its creation. In order to use ttlSecondsAfterCreation, the EphemeralIdentity feature gate must be enabled. exclusiveMinimum: true format: int32 minimum: 0 type: integer required: - rotation type: object status: description: ManagedServiceAccountStatus defines the observed state of ManagedServiceAccount properties: conditions: description: Conditions is the condition list. items: description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t \ // other fields\n\t}" properties: lastTransitionTime: description: |- lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: description: |- message is a human readable message indicating details about the transition. This may be an empty string. maxLength: 32768 type: string observedGeneration: description: |- observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: description: |- reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?USD type: string status: description: status of the condition, one of True, False, Unknown. enum: - "True" - "False" - Unknown type: string type: description: |- type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])USD type: string required: - lastTransitionTime - message - reason - status - type type: object type: array expirationTimestamp: description: ExpirationTimestamp is the time when the token will expire. format: date-time type: string tokenSecretRef: description: |- TokenSecretRef is a reference to the corresponding ServiceAccount's Secret, which stores the CA certficate and token from the managed cluster. properties: lastRefreshTimestamp: description: |- LastRefreshTimestamp is the timestamp indicating when the token in the Secret is refreshed. format: date-time type: string name: description: Name is the name of the referenced secret. type: string required: - lastRefreshTimestamp - name type: object type: object type: object served: true storage: false subresources: status: {} - name: v1beta1 schema: openAPIV3Schema: description: ManagedServiceAccount is the Schema for the managedserviceaccounts API properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ManagedServiceAccountSpec defines the desired state of ManagedServiceAccount properties: rotation: description: Rotation is the policy for rotation the credentials. properties: enabled: default: true description: |- Enabled prescribes whether the ServiceAccount token will be rotated before it expires. Deprecated: All ServiceAccount tokens will be rotated before they expire regardless of this field. type: boolean validity: default: 8640h0m0s description: Validity is the duration of validity for requesting the signed ServiceAccount token. type: string type: object ttlSecondsAfterCreation: description: |- ttlSecondsAfterCreation limits the lifetime of a ManagedServiceAccount. If the ttlSecondsAfterCreation field is set, the ManagedServiceAccount will be automatically deleted regardless of the ManagedServiceAccount's status. When the ManagedServiceAccount is deleted, its lifecycle guarantees (e.g. finalizers) will be honored. If this field is unset, the ManagedServiceAccount won't be automatically deleted. If this field is set to zero, the ManagedServiceAccount becomes eligible for deletion immediately after its creation. In order to use ttlSecondsAfterCreation, the EphemeralIdentity feature gate must be enabled. exclusiveMinimum: true format: int32 minimum: 0 type: integer required: - rotation type: object status: description: ManagedServiceAccountStatus defines the observed state of ManagedServiceAccount properties: conditions: description: Conditions is the condition list. items: description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t \ // other fields\n\t}" properties: lastTransitionTime: description: |- lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: description: |- message is a human readable message indicating details about the transition. This may be an empty string. maxLength: 32768 type: string observedGeneration: description: |- observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: description: |- reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?USD type: string status: description: status of the condition, one of True, False, Unknown. enum: - "True" - "False" - Unknown type: string type: description: |- type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])USD type: string required: - lastTransitionTime - message - reason - status - type type: object type: array expirationTimestamp: description: ExpirationTimestamp is the time when the token will expire. format: date-time type: string tokenSecretRef: description: |- TokenSecretRef is a reference to the corresponding ServiceAccount's Secret, which stores the CA certficate and token from the managed cluster. properties: lastRefreshTimestamp: description: |- LastRefreshTimestamp is the timestamp indicating when the token in the Secret is refreshed. format: date-time type: string name: description: Name is the name of the referenced secret. type: string required: - lastRefreshTimestamp - name type: object type: object type: object served: true storage: true subresources: status: {}
GET /authentication.open-cluster-management.io/v1beta1/namespaces/{namespace}/managedserviceaccounts/{managedserviceaccount_name}
DELETE /authentication.open-cluster-management.io/v1beta1/namespaces/{namespace}/managedserviceaccounts/{managedserviceaccount_name}
POST /apis/multicluster.openshift.io/v1alpha1/multiclusterengines
{ "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": { "annotations": { "controller-gen.kubebuilder.io/version": "v0.4.1" }, "creationTimestamp": null, "name": "multiclusterengines.multicluster.openshift.io" }, "spec": { "group": "multicluster.openshift.io", "names": { "kind": "MultiClusterEngine", "listKind": "MultiClusterEngineList", "plural": "multiclusterengines", "shortNames": [ "mce" ], "singular": "multiclusterengine" }, "scope": "Cluster", "versions": [ { "additionalPrinterColumns": [ { "description": "The overall state of the MultiClusterEngine", "jsonPath": ".status.phase", "name": "Status", "type": "string" }, { "jsonPath": ".metadata.creationTimestamp", "name": "Age", "type": "date" } ], "name": "v1alpha1", "schema": { "openAPIV3Schema": { "description": "MultiClusterEngine is the Schema for the multiclusterengines\nAPI", "properties": { "apiVersion": { "description": "APIVersion defines the versioned schema of this representation\nof an object. Servers should convert recognized schemas to the latest\ninternal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", "type": "string" }, "kind": { "description": "Kind is a string value representing the REST resource this\nobject represents. Servers may infer this from the endpoint the client\nsubmits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", "type": "string" }, "metadata": { "type": "object" }, "spec": { "description": "MultiClusterEngineSpec defines the desired state of MultiClusterEngine", "properties": { "imagePullSecret": { "description": "Override pull secret for accessing MultiClusterEngine\noperand and endpoint images", "type": "string" }, "nodeSelector": { "additionalProperties": { "type": "string" }, "description": "Set the nodeselectors", "type": "object" }, "targetNamespace": { "description": "Location where MCE resources will be placed", "type": "string" }, "tolerations": { "description": "Tolerations causes all components to tolerate any taints.", "items": { "description": "The pod this Toleration is attached to tolerates any\ntaint that matches the triple <key,value,effect> using the matching\noperator <operator>.", "properties": { "effect": { "description": "Effect indicates the taint effect to match. Empty\nmeans match all taint effects. When specified, allowed values\nare NoSchedule, PreferNoSchedule and NoExecute.", "type": "string" }, "key": { "description": "Key is the taint key that the toleration applies\nto. Empty means match all taint keys. If the key is empty,\noperator must be Exists; this combination means to match all\nvalues and all keys.", "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the\nvalue. Valid operators are Exists and Equal. Defaults to Equal.\nExists is equivalent to wildcard for value, so that a pod\ncan tolerate all taints of a particular category.", "type": "string" }, "tolerationSeconds": { "description": "TolerationSeconds represents the period of time\nthe toleration (which must be of effect NoExecute, otherwise\nthis field is ignored) tolerates the taint. By default, it\nis not set, which means tolerate the taint forever (do not\nevict). Zero and negative values will be treated as 0 (evict\nimmediately) by the system.", "format": "int64", "type": "integer" }, "value": { "description": "Value is the taint value the toleration matches\nto. If the operator is Exists, the value should be empty,\notherwise just a regular string.", "type": "string" } }, "type": "object" }, "type": "array" } }, "type": "object" }, "status": { "description": "MultiClusterEngineStatus defines the observed state of MultiClusterEngine", "properties": { "components": { "items": { "description": "ComponentCondition contains condition information for\ntracked components", "properties": { "kind": { "description": "The resource kind this condition represents", "type": "string" }, "lastTransitionTime": { "description": "LastTransitionTime is the last time the condition\nchanged from one status to another.", "format": "date-time", "type": "string" }, "message": { "description": "Message is a human-readable message indicating\ndetails about the last status change.", "type": "string" }, "name": { "description": "The component name", "type": "string" }, "reason": { "description": "Reason is a (brief) reason for the condition's\nlast status change.", "type": "string" }, "status": { "description": "Status is the status of the condition. One of True,\nFalse, Unknown.", "type": "string" }, "type": { "description": "Type is the type of the cluster condition.", "type": "string" } }, "type": "object" }, "type": "array" }, "conditions": { "items": { "properties": { "lastTransitionTime": { "description": "LastTransitionTime is the last time the condition\nchanged from one status to another.", "format": "date-time", "type": "string" }, "lastUpdateTime": { "description": "The last time this condition was updated.", "format": "date-time", "type": "string" }, "message": { "description": "Message is a human-readable message indicating\ndetails about the last status change.", "type": "string" }, "reason": { "description": "Reason is a (brief) reason for the condition's\nlast status change.", "type": "string" }, "status": { "description": "Status is the status of the condition. One of True,\nFalse, Unknown.", "type": "string" }, "type": { "description": "Type is the type of the cluster condition.", "type": "string" } }, "type": "object" }, "type": "array" }, "phase": { "description": "Latest observed overall state", "type": "string" } }, "type": "object" } }, "type": "object" } }, "served": true, "storage": true, "subresources": { "status": {} } } ] }, "status": { "acceptedNames": { "kind": "", "plural": "" }, "conditions": [], "storedVersions": [] } }
GET /apis/multicluster.openshift.io/v1alpha1/multiclusterengines
DELETE /apis/multicluster.openshift.io/v1alpha1/multiclusterengines/{name}
GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements
POST /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements
{ "apiVersion" : "cluster.open-cluster-management.io/v1beta1", "kind" : "Placement", "metadata" : { "name" : "placement1", "namespace": "ns1" }, "spec": { "predicates": [ { "requiredClusterSelector": { "labelSelector": { "matchLabels": { "vendor": "OpenShift" } } } } ] }, "status" : { } }
GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements/{placement_name}
DELETE /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placements/{placement_name}
GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions
POST /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions
{ "apiVersion" : "cluster.open-cluster-management.io/v1beta1", "kind" : "PlacementDecision", "metadata" : { "labels" : { "cluster.open-cluster-management.io/placement" : "placement1" }, "name" : "placement1-decision1", "namespace": "ns1" }, "status" : { } }
GET /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions/{placementdecision_name}
DELETE /cluster.open-cluster-management.io/v1beta1/namespaces/{namespace}/placementdecisions/{placementdecision_name}
<your-directory>/cluster-scoped-resources/gather-managed.log>
adm must-gather --image=registry.redhat.io/multicluster-engine/must-gather-rhel9:v2.6 --dest-dir=<directory>
REGISTRY=registry.example.com:5000 IMAGE=USDREGISTRY/multicluster-engine/must-gather-rhel9@sha256:ff9f37eb400dc1f7d07a9b6f2da9064992934b69847d17f59e385783c071b9d8> adm must-gather --image=USDIMAGE --dest-dir=./data
adm must-gather --image=registry.redhat.io/multicluster-engine/must-gather-rhel9:<v2.x> /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME
adm must-gather --image=registry.redhat.io/multicluster-engine/must-gather-rhel9:<v2.x> /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME --dest-dir=NAME ; tar -cvzf NAME.tgz NAME
REGISTRY=registry.example.com:5000 IMAGE=USDREGISTRY/multicluster-engine/must-gather-rhel9@sha256:ff9f37eb400dc1f7d07a9b6f2da9064992934b69847d17f59e385783c071b9d8 adm must-gather --image=USDIMAGE /usr/bin/gather hosted-cluster-namespace=HOSTEDCLUSTERNAMESPACE hosted-cluster-name=HOSTEDCLUSTERNAME --dest-dir=./data
This host is pending user action. Host timed out when pulling ignition. Check the host console... Rebooting
info: networking config is defined in the real root info: will not attempt to propagate initramfs networking
"bmac.agent-install.openshift.io/installer-args": "[\"--append-karg\", \"coreos.force_persist_ip\"]"
2024-02-22T09:56:19-05:00 ERROR HostedCluster deletion failed {"namespace": "clusters", "name": "hosted-0", "error": "context deadline exceeded"} 2024-02-22T09:56:19-05:00 ERROR Failed to destroy cluster {"error": "context deadline exceeded"}
get machine -n <hosted_cluster_namespace>
NAMESPACE NAME CLUSTER NODENAME PROVIDERID PHASE AGE VERSION clusters-hosted-0 hosted-0-9gg8b hosted-0-nhdbp Deleting 10h 4.15.0-rc.8
edit machines -n <hosted_cluster_namespace>
get agentmachine -n <hosted_cluster_namespace>
hcp destroy cluster agent --name <cluster_name>
reason: Unschedulable message: '0/6 nodes are available: 3 Insufficient cpu, 3 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.'
#!/bin/bash MCE_NAMESPACE=<namespace> delete multiclusterengine --all delete apiservice v1.admission.cluster.open-cluster-management.io v1.admission.work.open-cluster-management.io delete crd discoveredclusters.discovery.open-cluster-management.io discoveryconfigs.discovery.open-cluster-management.io delete mutatingwebhookconfiguration ocm-mutating-webhook managedclustermutators.admission.cluster.open-cluster-management.io delete validatingwebhookconfiguration ocm-validating-webhook delete ns USDMCE_NAMESPACE
-n multicluster-engine get pods -l app=managedcluster-import-controller-v2
-n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1
-n <managed_cluster_name> get secrets <managed_cluster_name>-import
-n multicluster-engine logs -l app=managedcluster-import-controller-v2 --tail=-1 | grep importconfig-controller
get managedcluster <managed_cluster_name> -o=jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' | grep ManagedClusterImportSucceeded
-n multicluster-engine logs -l app=managedcluster-import-controller-v2 -f
cluster_name=<your-managed-cluster-name>
kubeconfig_secret_name=USD(oc -n USD{cluster_name} get clusterdeployments USD{cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}')
-n USD{cluster_name} get secret USD{kubeconfig_secret_name} -ojsonpath={.data.kubeconfig} | base64 -d > kubeconfig.old
export KUBECONFIG=kubeconfig.old
get ns
cluster_name=<managed_cluster_name> kubeconfig_file=<path_to_kubeconfig>
kubeconfig=USD(cat USD{kubeconfig_file} | base64 -w0)
kubeconfig=USD(cat USD{kubeconfig_file} | base64)
kubeconfig_patch="[\{\"op\":\"replace\", \"path\":\"/data/kubeconfig\", \"value\":\"USD{kubeconfig}\"}, \{\"op\":\"replace\", \"path\":\"/data/raw-kubeconfig\", \"value\":\"USD{kubeconfig}\"}]"
kubeconfig_secret_name=USD(oc -n USD{cluster_name} get clusterdeployments USD{cluster_name} -ojsonpath='{.spec.clusterMetadata.adminKubeconfigSecretRef.name}')
-n USD{cluster_name} patch secrets USD{kubeconfig_secret_name} --type='json' -p="USD{kubeconfig_patch}"
get pod -n open-cluster-management-agent | grep klusterlet-registration-agent
logs <registration_agent_pod> -n open-cluster-management-agent
get infrastructure cluster -o yaml | grep apiServerURL
E0917 03:04:05.874759 1 manifestwork_controller.go:179] Reconcile work test-1-klusterlet-addon-workmgr fails with err: Failed to update work status with err Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks/test-1-klusterlet-addon-workmgr": x509: certificate signed by unknown authority E0917 03:04:05.874887 1 base_controller.go:231] "ManifestWorkAgent" controller failed to sync "test-1-klusterlet-addon-workmgr", err: Failed to update work status with err Get "api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks/test-1-klusterlet-addon-workmgr": x509: certificate signed by unknown authority E0917 03:04:37.245859 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManifestWork: failed to list *v1.ManifestWork: Get "api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/namespaces/test-1/manifestworks?resourceVersion=607424": x509: certificate signed by unknown authority
I0917 02:27:41.525026 1 event.go:282] Event(v1.ObjectReference{Kind:"Namespace", Namespace:"open-cluster-management-agent", Name:"open-cluster-management-agent", UID:"", APIVersion:"v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'ManagedClusterAvailableConditionUpdated' update managed cluster "test-1" available condition to "True", due to "Managed cluster is available" E0917 02:58:26.315984 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1beta1.CertificateSigningRequest: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true"": x509: certificate signed by unknown authority E0917 02:58:26.598343 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManagedCluster: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true": x509: certificate signed by unknown authority E0917 02:58:27.613963 1 reflector.go:127] k8s.io/client-go@v0.19.0/tools/cache/reflector.go:156: Failed to watch *v1.ManagedCluster: failed to list *v1.ManagedCluster: Get "https://api.aaa-ocp.dev02.location.com:6443/apis/cluster.management.io/v1/managedclusters?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dtest-1&resourceVersion=607408&timeout=9m33s&timeoutSeconds=573&watch=true"": x509: certificate signed by unknown authority
delete secret -n <cluster_name> <cluster_name>-import
delete secret -n <cluster_name> <cluster_name>-import
get secret -n <cluster_name> <cluster_name>-import -ojsonpath='{.data.import\.yaml}' | base64 --decode > import.yaml
apply -f import.yaml
edit managedcluster <cluster-name>
time="2020-08-07T15:27:55Z" level=error msg="Error: error setting up new vSphere SOAP client: Post https://147.1.1.1/sdk: x509: cannot validate certificate for xx.xx.xx.xx because it doesn't contain any IP SANs" time="2020-08-07T15:27:55Z" level=error
Error: error setting up new vSphere SOAP client: Post https://vspherehost.com/sdk: x509: certificate signed by unknown authority"
x509: certificate has expired or is not yet valid
time="2020-08-07T19:41:58Z" level=debug msg="vsphere_tag_category.category: Creating..." time="2020-08-07T19:41:58Z" level=error time="2020-08-07T19:41:58Z" level=error msg="Error: could not create category: POST https://vspherehost.com/rest/com/vmware/cis/tagging/category: 403 Forbidden" time="2020-08-07T19:41:58Z" level=error time="2020-08-07T19:41:58Z" level=error msg=" on ../tmp/openshift-install-436877649/main.tf line 54, in resource \"vsphere_tag_category\" \"category\":" time="2020-08-07T19:41:58Z" level=error msg=" 54: resource \"vsphere_tag_category\" \"category\" {"
failed to fetch Master Machines: failed to load asset \\\"Install Config\\\": invalid \\\"install-config.yaml\\\" file: platform.vsphere.dnsVIP: Invalid value: \\\"\\\": \\\"\\\" is not a valid IP
time="2020-08-11T14:31:38-04:00" level=debug msg="vsphereprivate_import_ova.import: Creating..." time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=error msg="Error: rpc error: code = Unavailable desc = transport is closing" time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=error time="2020-08-11T14:31:39-04:00" level=fatal msg="failed to fetch Cluster: failed to generate asset \"Cluster\": failed to create cluster: failed to apply Terraform: failed to complete the change"
ERROR ERROR Error: error reconfiguring virtual machine: error processing disk changes post-clone: disk.0: ServerFaultCode: NoPermission: RESOURCE (vm-71:2000), ACTION (queryAssociatedProfile): RESOURCE (vm-71), ACTION (PolicyIDByVirtualDisk)
get pod -n <new_cluster_name>
logs <new_cluster_name_provision_pod_name> -n <new_cluster_name> -c hive
describe clusterdeployments -n <new_cluster_name>
No subnets provided for zones
customresourcedefinition.apiextensions.k8s.io/klusterlets.operator.open-cluster-management.io configured clusterrole.rbac.authorization.k8s.io/klusterlet configured clusterrole.rbac.authorization.k8s.io/open-cluster-management:klusterlet-admin-aggregate-clusterrole configured clusterrolebinding.rbac.authorization.k8s.io/klusterlet configured namespace/open-cluster-management-agent configured secret/open-cluster-management-image-pull-credentials unchanged serviceaccount/klusterlet configured deployment.apps/klusterlet unchanged klusterlet.operator.open-cluster-management.io/klusterlet configured Error from server (BadRequest): error when creating "STDIN": Secret in version "v1" cannot be handled as a Secret: v1.Secret.ObjectMeta: v1.ObjectMeta.TypeMeta: Kind: Data: decode base64: illegal base64 data at input byte 1313, error found in #10 byte of ...|dhruy45="},"kind":"|..., bigger context ...|tye56u56u568yuo7i67i67i67o556574i"},"kind":"Secret","metadata":{"annotations":{"kube|
version
apply -f import.yaml
get klusterlets klusterlet -oyaml
api-resources --verbs=list --namespaced -o name | grep -E '^secrets|^serviceaccounts|^managedclusteraddons|^roles|^rolebindings|^manifestworks|^leases|^managedclusterinfo|^appliedmanifestworks'|^clusteroauths' | xargs -n 1 oc get --show-kind --ignore-not-found -n <cluster_name>
edit <resource_kind> <resource_name> -n <namespace>
delete ns <cluster-name>
delete secret auto-import-secret -n <cluster-namespace>
describe placement <placement-name>
Name: demo-placement Namespace: default Labels: <none> Annotations: <none> API Version: cluster.open-cluster-management.io/v1beta1 Kind: Placement Status: Conditions: Last Transition Time: 2022-09-30T07:39:45Z Message: Placement configurations check pass Reason: Succeedconfigured Status: False Type: PlacementMisconfigured Last Transition Time: 2022-09-30T07:39:45Z Message: No valid ManagedClusterSetBindings found in placement namespace Reason: NoManagedClusterSetBindings Status: False Type: PlacementSatisfied Number Of Selected Clusters: 0
Name: demo-placement Namespace: default Labels: <none> Annotations: <none> API Version: cluster.open-cluster-management.io/v1beta1 Kind: Placement Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DecisionCreate 2m10s placementController Decision demo-placement-decision-1 is created with placement demo-placement in namespace default Normal DecisionUpdate 2m10s placementController Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default Normal ScoreUpdate 2m10s placementController cluster1:0 cluster2:100 cluster3:200 Normal DecisionUpdate 3s placementController Decision demo-placement-decision-1 is updated with placement demo-placement in namespace default Normal ScoreUpdate 3s placementController cluster1:200 cluster2:145 cluster3:189 cluster4:200
ProvisioningError 51s metal3-baremetal-controller Image provisioning failed: Deploy step deploy.deploy failed with BadRequestError: HTTP POST https://<bmc_address>/redfish/v1/Managers/iDRAC.Embedded.1/VirtualMedia/CD/Actions/VirtualMedia.InsertMedia returned code 400. Base.1.8.GeneralError: A general error has occurred. See ExtendedInfo for more information Extended information: [ {"Message": "Unable to mount remote share https://<ironic_address>/redfish/boot-<uuid>.iso.", 'MessageArgs': ["https://<ironic_address>/redfish/boot-<uuid>.iso"], "MessageArgs@odata.count": 1, "MessageId": "IDRAC.2.5.RAC0720", "RelatedProperties": ["#/Image"], "RelatedProperties@odata.count": 1, "Resolution": "Retry the operation.", "Severity": "Informational"} ]
get vm -n <namespace>
get pods -n <namespace> | grep "import"
get pods -n <namespace> -l kubevirt.io=virt-launcher
get nodes -o yaml
get pods -A --field-selector=status.phase!=Running,status,phase!=Succeeded
patch ingresscontroller -n openshift-ingress-operator default --type=json -p '[{ "op": "add", "path": "/spec/routeAdmission", "value": {wildcardPolicy: "WildcardsAllowed"}}]'
get pods -n <hosted-control-plane-namespace> -l app=cloud-controller-manager
get pods -n openshift-cluster-csi-drivers -o wide -l app=kubevirt-csi-driver
get pods -n <hcp namespace> -l app=kubevirt-csi-driver
get imagecontentsourcepolicy -o json | jq -r '.items[].spec.repositoryDigestMirrors[0].mirrors[0]'
get clusterversion version -ojsonpath='{.status.desired.image}'
image extract --file /release-manifests/0000_50_installer_coreos-bootimages.yaml <payload_image> --confirm
cat 0000_50_installer_coreos-bootimages.yaml | yq -r .data.stream | jq -r '.architectures.x86_64.images.kubevirt."digest-ref"'
image mirror <rhcos_image> <internal_registry>
apiVersion: config.openshift.io/v1 kind: ImageDigestMirrorSet metadata: name: rhcos-boot-kubevirt spec: repositoryDigestMirrors: - mirrors: - <rhcos_image_no_digest> 1 source: virthost.ostest.test.metalkube.org:5000/localimages/ocp-v4.0-art-dev 2
apply -f rhcos-boot-kubevirt.yaml
E0809 18:45:29.450874 1 reflector.go:147] k8s.io/client-go@v0.29.4/tools/cache/reflector.go:229: Failed to watch *v1.CertificateSigningRequest: failed to list *v1.CertificateSigningRequest: Get "https://api.xxx.openshiftapps.com:443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=500&resourceVersion=0": tls: failed to verify certificate: x509: certificate signed by unknown authority
curl -s https://letsencrypt.org/certs/isrgrootx1.pem | base64 | tr -d "\n"
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerCABundle: "<your_ca_certificate>"
apply -f <filename>
get secret local-cluster-import -n local-cluster -o jsonpath={.data.import/.yaml} | base64 --decode > import.yaml
apply -f import.yaml
E0809 18:45:29.450874 1 reflector.go:147] k8s.io/client-go@v0.29.4/tools/cache/reflector.go:229: Failed to watch *v1.CertificateSigningRequest: failed to list *v1.CertificateSigningRequest: Get "https://api.xxx.openshiftapps.com:443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=500&resourceVersion=0": tls: failed to verify certificate: x509: certificate signed by unknown authority
curl -s https://letsencrypt.org/certs/isrgrootx1.pem | base64 | tr -d "\n"
apiVersion: config.open-cluster-management.io/v1alpha1 kind: KlusterletConfig metadata: name: global spec: hubKubeAPIServerCABundle: "<your_ca_certificate>"
apply -f <filename>
get secret <cluster_name>-import -n <cluster_name> -o jsonpath={.data.import/.yaml} | base64 --decode > <cluster_name>-import.yaml
apply -f <cluster_name>-import.yaml
get clusterversion version -o jsonpath='{.status.availableUpdates[*].version}'
-n <cluster_name> get managedclusterinfo <cluster_name> -o jsonpath='{.status.distributionInfo.ocp.availableUpdates[*]}'
-n <cluster_name> get ClusterCurator <cluster_name> -o yaml
-n <cluster_name> delete ClusterCurator <cluster_name>
-n open-cluster-management-agent-addon logs klusterlet-addon-workmgr-<your_pod_name>
logs -n multicluster-engine <assisted_service_pod_name> 1
error="failed to get pull secret for update: invalid pull secret data in secret pull-secret"
pull secret must contain auth for \"registry.redhat.io\"