multicluster engine operator with Red Hat Advanced Cluster Management Red Hat Advanced Cluster Management for Kubernetes 2.12 multicluster engine operator with Red Hat Advanced Cluster Management integration

Commands:
curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: addonDeploymentConfig metadata: name: addon-ns-config namespace: multicluster-engine spec: agentInstallNamespace: open-cluster-management-agent-addon-discovery
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: work-manager spec: addonMeta: displayName: work-manager installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All type: Placements
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: work-manager spec: addonMeta: displayName: work-manager installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All configs: - group: addon.open-cluster-management.io name: addon-ns-config namespace: multicluster-engine resource: addondeploymentconfigs type: Placements
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: managed-serviceaccount spec: addonMeta: displayName: managed-serviceaccount installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All configs: - group: addon.open-cluster-management.io name: addon-ns-config namespace: multicluster-engine resource: addondeploymentconfigs type: Placements
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ClusterManagementAddOn metadata: name: cluster-proxy spec: addonMeta: displayName: cluster-proxy installStrategy: placements: - name: global namespace: open-cluster-management-global-set rolloutStrategy: type: All configs: - group: addon.open-cluster-management.io name: addon-ns-config namespace: multicluster-engine resource: addondeploymentconfigs type: Placements
get deployment -n open-cluster-management-agent-addon-discovery
NAME READY UP-TO-DATE AVAILABLE AGE cluster-proxy-proxy-agent 1/1 1 1 24h klusterlet-addon-workmgr 1/1 1 1 24h managed-serviceaccount-addon-agent 1/1 1 1 24h
kind: KlusterletConfig apiVersion: config.open-cluster-management.io/v1alpha1 metadata: name: mce-import-klusterlet-config spec: installMode: type: noOperator noOperator: postfix: mce-import
label addondeploymentconfig addon-ns-config -n multicluster-engine cluster.open-cluster-management.io/backup=true
label addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine cluster.open-cluster-management.io/backup=true
label clustermanagementaddon work-manager cluster.open-cluster-management.io/backup=true
label clustermanagementaddon cluster-proxy cluster.open-cluster-management.io/backup=true
label clustermanagementaddon managed-serviceaccount cluster.open-cluster-management.io/backup=true
label KlusterletConfig mce-import-klusterlet-config cluster.open-cluster-management.io/backup=true
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: annotations: agent.open-cluster-management.io/klusterlet-config: mce-import-klusterlet-config 1 labels: cloud: auto-detect vendor: auto-detect name: mce-a 2 spec: hubAcceptsClient: true leaseDurationSeconds: 60
get managedcluster
NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE local-cluster true https://<api.acm-hub.com:port> True True 44h mce-a true https://<api.mce-a.com:port> True True 27s
patch addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine --type=merge -p '{"spec":{"agentInstallNamespace":"open-cluster-management-agent-addon-discovery"}}'
patch addondeploymentconfig hypershift-addon-deploy-config -n multicluster-engine --type=merge -p '{"spec":{"customizedVariables":[{"name":"disableMetrics","value": "true"},{"name":"disableHOManagement","value": "true"}]}}'
clusteradm addon enable --names hypershift-addon --clusters <managed-cluster-names>
get managedcluster
get deployment -n open-cluster-management-agent-addon-discovery
NAME READY UP-TO-DATE AVAILABLE AGE cluster-proxy-proxy-agent 1/1 1 1 24h klusterlet-addon-workmgr 1/1 1 1 24h hypershift-addon-agent 1/1 1 1 24h managed-serviceaccount-addon-agent 1/1 1 1 24h
apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: creationTimestamp: "2024-05-30T23:05:39Z" generation: 1 labels: hypershift.open-cluster-management.io/hc-name: hosted-cluster-1 hypershift.open-cluster-management.io/hc-namespace: clusters name: hosted-cluster-1 namespace: mce-1 resourceVersion: "1740725" uid: b4c36dca-a0c4-49f9-9673-f561e601d837 spec: apiUrl: https://a43e6fe6dcef244f8b72c30426fb6ae3-ea3fec7b113c88da.elb.us-west-1.amazonaws.com:6443 cloudProvider: aws creationTimestamp: "2024-05-30T23:02:45Z" credential: {} displayName: mce-1-hosted-cluster-1 importAsManagedCluster: false isManagedCluster: false name: hosted-cluster-1 openshiftVersion: 0.0.0 status: Active type: MultiClusterEngineHCP
apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-mce-hcp-autoimport namespace: open-cluster-management-global-set annotations: policy.open-cluster-management.io/standards: NIST SP 800-53 policy.open-cluster-management.io/categories: CM Configuration Management policy.open-cluster-management.io/controls: CM-2 Baseline Configuration policy.open-cluster-management.io/description: Discovered clusters that are of type MultiClusterEngineHCP can be automatically imported into ACM as managed clusters. This policy configure those discovered clusters so they are automatically imported. Fine tuning MultiClusterEngineHCP clusters to be automatically imported can be done by configure filters at the configMap or add annotation to the discoverd cluster. spec: disabled: false policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: mce-hcp-autoimport-config spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: v1 kind: ConfigMap metadata: name: discovery-config namespace: open-cluster-management-global-set data: rosa-filter: "" remediationAction: enforce 1 severity: low - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-mce-hcp-autoimport spec: remediationAction: enforce severity: low object-templates-raw: | {{- /* find the MultiClusterEngineHCP DiscoveredClusters */ -}} {{- range USDdc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }} {{- /* Check for the flag that indicates the import should be skipped */ -}} {{- USDskip := "false" -}} {{- range USDkey, USDvalue := USDdc.metadata.annotations }} {{- if and (eq USDkey "discovery.open-cluster-management.io/previously-auto-imported") (eq USDvalue "true") }} {{- USDskip = "true" }} {{- end }} {{- end }} {{- /* if the type is MultiClusterEngineHCP and the status is Active */ -}} {{- if and (eq USDdc.spec.status "Active") (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "mce-hcp-filter") USDdc.spec.displayName) (eq USDdc.spec.type "MultiClusterEngineHCP") (eq USDskip "false") }} - complianceType: musthave objectDefinition: apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: {{ USDdc.metadata.name }} namespace: {{ USDdc.metadata.namespace }} spec: importAsManagedCluster: true 2 {{- end }} {{- end }}
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: policy-mce-hcp-autoimport-placement namespace: open-cluster-management-global-set spec: tolerations: - key: cluster.open-cluster-management.io/unreachable operator: Exists - key: cluster.open-cluster-management.io/unavailable operator: Exists clusterSets: - global predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: local-cluster operator: In values: - "true"
apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: policy-mce-hcp-autoimport-placement-binding namespace: open-cluster-management-global-set placementRef: name: policy-mce-hcp-autoimport-placement apiGroup: cluster.open-cluster-management.io kind: Placement subjects: - name: policy-mce-hcp-autoimport apiGroup: policy.open-cluster-management.io kind: Policy
get policies.policy.open-cluster-management.io policy-mce-hcp-autoimport -n <namespace>
annotations: discovery.open-cluster-management.io/previously-auto-imported: "true"
apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: name: policy-rosa-autoimport annotations: policy.open-cluster-management.io/standards: NIST SP 800-53 policy.open-cluster-management.io/categories: CM Configuration Management policy.open-cluster-management.io/controls: CM-2 Baseline Configuration policy.open-cluster-management.io/description: OpenShift Service on AWS discovered clusters can be automatically imported into Red Hat Advanced Cluster Management as managed clusters with this policy. You can select and configure those managed clusters so you can import. Configure filters or add an annotation if you do not want all of your OpenShift Service on AWS clusters to be automatically imported. spec: remediationAction: inform 1 disabled: false policy-templates: - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: rosa-autoimport-config spec: object-templates: - complianceType: musthave objectDefinition: apiVersion: v1 kind: ConfigMap metadata: name: discovery-config namespace: open-cluster-management-global-set data: rosa-filter: "" 2 remediationAction: enforce severity: low - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-rosa-autoimport spec: remediationAction: enforce severity: low object-templates-raw: | {{- /* find the ROSA DiscoveredClusters */ -}} {{- range USDdc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }} {{- /* Check for the flag that indicates the import should be skipped */ -}} {{- USDskip := "false" -}} {{- range USDkey, USDvalue := USDdc.metadata.annotations }} {{- if and (eq USDkey "discovery.open-cluster-management.io/previously-auto-imported") (eq USDvalue "true") }} {{- USDskip = "true" }} {{- end }} {{- end }} {{- /* if the type is ROSA and the status is Active */ -}} {{- if and (eq USDdc.spec.status "Active") (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "rosa-filter") USDdc.spec.displayName) (eq USDdc.spec.type "ROSA") (eq USDskip "false") }} - complianceType: musthave objectDefinition: apiVersion: discovery.open-cluster-management.io/v1 kind: DiscoveredCluster metadata: name: {{ USDdc.metadata.name }} namespace: {{ USDdc.metadata.namespace }} spec: importAsManagedCluster: true {{- end }} {{- end }} - objectDefinition: apiVersion: policy.open-cluster-management.io/v1 kind: ConfigurationPolicy metadata: name: policy-rosa-managedcluster-status spec: remediationAction: enforce severity: low object-templates-raw: | {{- /* Use the same DiscoveredCluster list to check ManagedCluster status */ -}} {{- range USDdc := (lookup "discovery.open-cluster-management.io/v1" "DiscoveredCluster" "" "").items }} {{- /* Check for the flag that indicates the import should be skipped */ -}} {{- USDskip := "false" -}} {{- range USDkey, USDvalue := USDdc.metadata.annotations }} {{- if and (eq USDkey "discovery.open-cluster-management.io/previously-auto-imported") (eq USDvalue "true") }} {{- USDskip = "true" }} {{- end }} {{- end }} {{- /* if the type is ROSA and the status is Active */ -}} {{- if and (eq USDdc.spec.status "Active") (contains (fromConfigMap "open-cluster-management-global-set" "discovery-config" "rosa-filter") USDdc.spec.displayName) (eq USDdc.spec.type "ROSA") (eq USDskip "false") }} - complianceType: musthave objectDefinition: apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: {{ USDdc.spec.displayName }} namespace: {{ USDdc.spec.displayName }} status: conditions: - type: ManagedClusterConditionAvailable status: "True" {{- end }} {{- end }}
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Placement metadata: name: placement-openshift-plus-hub spec: predicates: - requiredClusterSelector: labelSelector: matchExpressions: - key: name operator: In values: - local-cluster
apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: name: binding-policy-rosa-autoimport placementRef: apiGroup: cluster.open-cluster-management.io kind: Placement name: placement-policy-rosa-autoimport subjects: - apiGroup: policy.open-cluster-management.io kind: Policy name: policy-rosa-autoimport
get policies.policy.open-cluster-management.io policy-rosa-autoimport -n <namespace>
{{ if .Spec.Proxy }} proxy: {{ .Spec.Proxy | toYaml | indent 4 }} {{ end }}
name: "{{ .SpecialVars.CurrentNode.HostName }}" namespace: "{{ .Spec.ClusterName }}"
apiVersion: v1 data: AgentClusterInstall: |- siteconfig.open-cluster-management.io/sync-wave: "1" ClusterDeployment: |- siteconfig.open-cluster-management.io/sync-wave: "1" InfraEnv: |- siteconfig.open-cluster-management.io/sync-wave: "2" KlusterletAddonConfig: |- siteconfig.open-cluster-management.io/sync-wave: "3" ManagedCluster: |- siteconfig.open-cluster-management.io/sync-wave: "3" kind: ConfigMap metadata: name: assisted-installer-templates namespace: example-namespace
apiVersion: siteconfig.open-cluster-management.io/v1alpha1 kind: ClusterInstance metadata: name: "example-sno" namespace: "example-sno" spec: [...] clusterName: "example-sno" extraAnnotations: 1 ClusterDeployment: myClusterAnnotation: success extraLabels: 2 ManagedCluster: common: "true" group-du: "" nodes: - hostName: "example-sno.example.redhat.com" role: "master" extraAnnotations: 3 BareMetalHost: myNodeAnnotation: success extraLabels: 4 BareMetalHost: "testExtraLabel": "success"
get managedclusters example-sno -ojsonpath='{.metadata.labels}' | jq
common: true
get bmh example-sno.example.redhat.com -n example-sno -ojsonpath='{.metadata.annotations}' | jq
myNodeAnnotation: success
patch multiclusterhubs.operator.open-cluster-management.io multiclusterhub -n rhacm --type json --patch '[{"op": "add", "path":"/spec/overrides/components/-", "value": {"name":"siteconfig","enabled": true}}]'
-n rhacm get po | grep siteconfig
siteconfig-controller-manager-6fdd86cc64-sdg87 2/2 Running 0 43s
-n rhacm get cm
NAME DATA AGE ai-cluster-templates-v1 5 97s ai-node-templates-v1 2 97s ibi-cluster-templates-v1 3 97s ibi-node-templates-v1 3 97s
apiVersion: v1 kind: Namespace metadata: name: example-sno
apply -f clusterinstance-namespace.yaml
apiVersion: v1 kind: Secret metadata: name: pull-secret namespace: example-sno 1 data: .dockerconfigjson: <encoded_docker_configuration> 2 type: kubernetes.io/dockerconfigjson
apply -f pull-secret.yaml
apiVersion: v1 data: password: <password> username: <username> kind: Secret metadata: name: example-bmh-secret namespace: "example-sno" 1 type: Opaque
apply -f example-bmc-secret.yaml
apiVersion: v1 kind: ConfigMap metadata: name: enable-crun namespace: example-sno 1 data: enable-crun-master.yaml: | apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: enable-crun-master spec: machineConfigPoolSelector: matchLabels: pools.operator.machineconfiguration.openshift.io/master: "" containerRuntimeConfig: defaultRuntime: crun enable-crun-worker.yaml: | apiVersion: machineconfiguration.openshift.io/v1 kind: ContainerRuntimeConfig metadata: name: enable-crun-worker spec: machineConfigPoolSelector: matchLabels: pools.operator.machineconfiguration.openshift.io/worker: "" containerRuntimeConfig: defaultRuntime: crun
apply -f enable-crun.yaml
apiVersion: siteconfig.open-cluster-management.io/v1alpha1 kind: ClusterInstance metadata: name: "example-clusterinstance" namespace: "example-sno" 1 spec: holdInstallation: false extraManifestsRefs: 2 - name: extra-machine-configs - name: enable-crun pullSecretRef: name: "pull-secret" 3 [...] clusterName: "example-sno" 4 [...] clusterImageSetNameRef: "img4.17-x86-64" [...] templateRefs: 5 - name: ibi-cluster-templates-v1 namespace: rhacm [...] nodes: [...] bmcCredentialsName: 6 name: "example-bmh-secret" [...] templateRefs: 7 - name: ibi-node-templates-v1 namespace: rhacm [...]
apply -f clusterinstance-ibi.yaml
get clusterinstance <cluster_name> -n <target_namespace> -o yaml
message: Applied site config manifests reason: Completed status: "True" type: RenderedTemplatesApplied
get clusterinstance <cluster_name> -n <target_namespace> -o jsonpath='{.status.manifestsRendered}'
delete clusterinstance <cluster_name> -n <target_namespace>
get clusterinstance <cluster_name> -n <target_namespace>
Error from server (NotFound): clusterinstances.siteconfig.open-cluster-management.io "<cluster_name>" not found
apiVersion: v1 kind: ConfigMap metadata: name: my-custom-secret namespace: rhacm data: MySecret: |- apiVersion: v1 kind: Secret metadata: name: "{{ .Spec.ClusterName }}-my-custom-secret-key" namespace: "clusters" annotations: siteconfig.open-cluster-management.io/sync-wave: "1" 1 type: Opaque data: key: <key>
apply -f my-custom-secret.yaml
spec: templateRefs: - name: ai-cluster-templates-v1.yaml namespace: rhacm - name: my-custom-secret.yaml namespace: rhacm
apply -f clusterinstance-my-custom-secret.yaml
spec: nodes: - hostName: "worker-node2.example.com" role: "worker" ironicInspect: "" extraAnnotations: BareMetalHost: bmac.agent-install.openshift.io/remove-agent-and-node-on-delete: "true"
apply -f <clusterinstance>.yaml
get bmh -n <clusterinstance_namespace> worker-node2.example.com -ojsonpath='{.metadata.annotations}' | jq
{ "baremetalhost.metal3.io/detached": "assisted-service-controller", "bmac.agent-install.openshift.io/hostname": "worker-node2.example.com", "bmac.agent-install.openshift.io/remove-agent-and-node-on-delete": "true" "bmac.agent-install.openshift.io/role": "master", "inspect.metal3.io": "disabled", "siteconfig.open-cluster-management.io/sync-wave": "1", }
spec: nodes: - hostName: "worker-node2.example.com" pruneManifests: - apiVersion: metal3.io/v1alpha1 kind: BareMetalHost
apply -f <clusterinstance>.yaml
get bmh -n <clusterinstance_namespace> --watch --kubeconfig <hub_cluster_kubeconfig_filename>
NAME STATE CONSUMER ONLINE ERROR AGE master-node1.example.com provisioned true 81m worker-node2.example.com deprovisioning true 44m worker-node2.example.com powering off before delete true 20h worker-node2.example.com deleting true 50m
get agents -n <clusterinstance_namespace> --kubeconfig <hub_cluster_kubeconfig_filename>
NAME CLUSTER APPROVED ROLE STAGE master-node1.example.com <managed_cluster_name> true master Done master-node2.example.com <managed_cluster_name> true master Done master-node3.example.com <managed_cluster_name> true master Done worker-node1.example.com <managed_cluster_name> true worker Done
get nodes --kubeconfig <managed_cluster_kubeconfig_filename>
NAME STATUS ROLES AGE VERSION worker-node2.example.com NotReady,SchedulingDisabled worker 19h v1.30.5 worker-node1.example.com Ready worker 19h v1.30.5 master-node1.example.com Ready control-plane,master 19h v1.30.5 master-node2.example.com Ready control-plane,master 19h v1.30.5 master-node3.example.com Ready control-plane,master 19h v1.30.5
spec: nodes: - hostName: "<host_name>" role: "worker" templateRefs: - name: ai-node-templates-v1 namespace: rhacm bmcAddress: "<bmc_address>" bmcCredentialsName: name: "<bmc_credentials_name>" bootMACAddress: "<boot_mac_address>"
apply -f <clusterinstance>.yaml
get bmh -n <clusterinstance_namespace> --watch --kubeconfig <hub_cluster_kubeconfig_filename>
NAME STATE CONSUMER ONLINE ERROR AGE master-node1.example.com provisioned true 81m worker-node2.example.com provisioning true 44m
get agents -n <clusterinstance_namespace> --kubeconfig <hub_cluster_kubeconfig_filename>
NAME CLUSTER APPROVED ROLE STAGE master-node1.example.com <managed_cluster_name> true master Done master-node2.example.com <managed_cluster_name> true master Done master-node3.example.com <managed_cluster_name> true master Done worker-node1.example.com <managed_cluster_name> false worker worker-node2.example.com <managed_cluster_name> true worker Starting installation worker-node2.example.com <managed_cluster_name> true worker Installing worker-node2.example.com <managed_cluster_name> true worker Writing image to disk worker-node2.example.com <managed_cluster_name> true worker Waiting for control plane worker-node2.example.com <managed_cluster_name> true worker Rebooting worker-node2.example.com <managed_cluster_name> true worker Joined worker-node2.example.com <managed_cluster_name> true worker Done
get nodes --kubeconfig <managed_cluster_kubeconfig_filename>
NAME STATUS ROLES AGE VERSION worker-node2.example.com Ready worker 1h v1.30.5 worker-node1.example.com Ready worker 19h v1.30.5 master-node1.example.com Ready control-plane,master 19h v1.30.5 master-node2.example.com Ready control-plane,master 19h v1.30.5 master-node3.example.com Ready control-plane,master 19h v1.30.5