Installing an on-premise cluster with the Agent-based Installer OpenShift Container Platform 4.18 Installing an on-premise OpenShift Container Platform cluster with the Agent-based Installer Red Hat OpenShift Documentation Team

Commands:
apiVersion: v1 baseDomain: test.example.com metadata: name: sno-cluster fips: True
apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall metadata: annotations: agent-install.openshift.io/install-config-overrides: '{"fips": True}' name: sno-cluster namespace: sno-cluster-test
apiVersion: v1beta1 kind: AgentConfig metadata: name: example-cluster rendezvousIP: 192.168.111.80 hosts: - hostname: master-1 role: master interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a5 - hostname: master-2 role: master interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a6 - hostname: master-3 role: master interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a7 - hostname: worker-1 role: worker interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a8
- name: master-0 role: master rootDeviceHints: deviceName: "/dev/sda"
apiVersion: v1alpha1 kind: AgentConfig metadata: name: sno-cluster rendezvousIP: 192.168.111.80 1
cat > agent-config.yaml << EOF apiVersion: v1alpha1 kind: AgentConfig metadata: name: sno-cluster rendezvousIP: 192.168.111.80 1 hosts: - hostname: master-0 interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a5 2 networkConfig: interfaces: - name: eno1 type: ethernet state: up mac-address: 00:ef:44:21:e6:a5 ipv4: enabled: true address: - ip: 192.168.111.80 3 prefix-length: 23 4 dhcp: false dns-resolver: config: server: - 192.168.111.1 5 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.1 6 next-hop-interface: eno1 table-id: 254 EOF
apiVersion: agent-install.openshift.io/v1beta1 kind: NMStateConfig metadata: name: master-0 namespace: openshift-machine-api labels: cluster0-nmstate-label-name: cluster0-nmstate-label-value spec: config: interfaces: - name: eth0 type: ethernet state: up mac-address: 52:54:01:aa:aa:a1 ipv4: enabled: true address: - ip: 192.168.122.2 1 prefix-length: 23 2 dhcp: false dns-resolver: config: server: - 192.168.122.1 3 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.122.1 4 next-hop-interface: eth0 table-id: 254 interfaces: - name: eth0 macAddress: 52:54:01:aa:aa:a1 5
USDTTL 1W @ IN SOA ns1.example.com. root ( 2019070700 ; serial 3H ; refresh (3 hours) 30M ; retry (30 minutes) 2W ; expiry (2 weeks) 1W ) ; minimum (1 week) IN NS ns1.example.com. IN MX 10 smtp.example.com. ; ; ns1.example.com. IN A 192.168.1.5 smtp.example.com. IN A 192.168.1.5 ; helper.example.com. IN A 192.168.1.5 helper.ocp4.example.com. IN A 192.168.1.5 ; api.ocp4.example.com. IN A 192.168.1.5 1 api-int.ocp4.example.com. IN A 192.168.1.5 2 ; *.apps.ocp4.example.com. IN A 192.168.1.5 3 ; master0.ocp4.example.com. IN A 192.168.1.97 4 master1.ocp4.example.com. IN A 192.168.1.98 5 master2.ocp4.example.com. IN A 192.168.1.99 6 ; worker0.ocp4.example.com. IN A 192.168.1.11 7 worker1.ocp4.example.com. IN A 192.168.1.7 8 ; ;EOF
USDTTL 1W @ IN SOA ns1.example.com. root ( 2019070700 ; serial 3H ; refresh (3 hours) 30M ; retry (30 minutes) 2W ; expiry (2 weeks) 1W ) ; minimum (1 week) IN NS ns1.example.com. ; 5.1.168.192.in-addr.arpa. IN PTR api.ocp4.example.com. 1 5.1.168.192.in-addr.arpa. IN PTR api-int.ocp4.example.com. 2 ; 97.1.168.192.in-addr.arpa. IN PTR master0.ocp4.example.com. 3 98.1.168.192.in-addr.arpa. IN PTR master1.ocp4.example.com. 4 99.1.168.192.in-addr.arpa. IN PTR master2.ocp4.example.com. 5 ; 11.1.168.192.in-addr.arpa. IN PTR worker0.ocp4.example.com. 6 7.1.168.192.in-addr.arpa. IN PTR worker1.ocp4.example.com. 7 ; ;EOF
global log 127.0.0.1 local2 pidfile /var/run/haproxy.pid maxconn 4000 daemon defaults mode http log global option dontlognull option http-server-close option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 1m timeout server 1m timeout http-keep-alive 10s timeout check 10s maxconn 3000 listen api-server-6443 1 bind *:6443 mode tcp server master0 master0.ocp4.example.com:6443 check inter 1s server master1 master1.ocp4.example.com:6443 check inter 1s server master2 master2.ocp4.example.com:6443 check inter 1s listen machine-config-server-22623 2 bind *:22623 mode tcp server master0 master0.ocp4.example.com:22623 check inter 1s server master1 master1.ocp4.example.com:22623 check inter 1s server master2 master2.ocp4.example.com:22623 check inter 1s listen ingress-router-443 3 bind *:443 mode tcp balance source server worker0 worker0.ocp4.example.com:443 check inter 1s server worker1 worker1.ocp4.example.com:443 check inter 1s listen ingress-router-80 4 bind *:80 mode tcp balance source server worker0 worker0.ocp4.example.com:80 check inter 1s server worker1 worker1.ocp4.example.com:80 check inter 1s
apiVersion: v1alpha1 kind: AgentConfig rendezvousIP: 10.10.10.14 hosts: - hostname: master0 role: master interfaces: - name: enp0s4 macAddress: 00:21:50:90:c0:10 - name: enp0s5 macAddress: 00:21:50:90:c0:20 networkConfig: interfaces: - name: bond0.300 1 type: vlan 2 state: up vlan: base-iface: bond0 id: 300 ipv4: enabled: true address: - ip: 10.10.10.14 prefix-length: 24 dhcp: false - name: bond0 3 type: bond 4 state: up mac-address: 00:21:50:90:c0:10 5 ipv4: enabled: false ipv6: enabled: false link-aggregation: mode: active-backup 6 options: miimon: "150" 7 port: - enp0s4 - enp0s5 dns-resolver: 8 config: server: - 10.10.10.11 - 10.10.10.12 routes: config: - destination: 0.0.0.0/0 next-hop-address: 10.10.10.10 9 next-hop-interface: bond0.300 10 table-id: 254
apiVersion: v1alpha1 kind: AgentConfig rendezvousIP: 10.10.10.14 hosts: - hostname: worker-1 interfaces: - name: eno1 macAddress: 0c:42:a1:55:f3:06 - name: eno2 macAddress: 0c:42:a1:55:f3:07 networkConfig: 1 interfaces: 2 - name: eno1 3 type: ethernet 4 state: up mac-address: 0c:42:a1:55:f3:06 ipv4: enabled: true dhcp: false 5 ethernet: sr-iov: total-vfs: 2 6 ipv6: enabled: false - name: sriov:eno1:0 type: ethernet state: up 7 ipv4: enabled: false 8 ipv6: enabled: false dhcp: false - name: sriov:eno1:1 type: ethernet state: down - name: eno2 type: ethernet state: up mac-address: 0c:42:a1:55:f3:07 ipv4: enabled: true ethernet: sr-iov: total-vfs: 2 ipv6: enabled: false - name: sriov:eno2:0 type: ethernet state: up ipv4: enabled: false ipv6: enabled: false - name: sriov:eno2:1 type: ethernet state: down - name: bond0 type: bond state: up min-tx-rate: 100 9 max-tx-rate: 200 10 link-aggregation: mode: active-backup 11 options: primary: sriov:eno1:0 12 port: - sriov:eno1:0 - sriov:eno2:0 ipv4: address: - ip: 10.19.16.57 13 prefix-length: 23 dhcp: false enabled: true ipv6: enabled: false dns-resolver: config: server: - 10.11.5.160 - 10.2.70.215 routes: config: - destination: 0.0.0.0/0 next-hop-address: 10.19.17.254 next-hop-interface: bond0 14 table-id: 254
apiVersion: v1 baseDomain: example.com 1 compute: 2 - name: worker replicas: 0 3 architecture: amd64 controlPlane: 4 name: master replicas: 1 5 architecture: amd64 metadata: name: sno-cluster 6 networking: clusterNetwork: - cidr: 10.128.0.0/14 7 hostPrefix: 23 8 networkType: OVNKubernetes 9 serviceNetwork: 10 - 172.30.0.0/16 platform: none: {} 11 fips: false 12 pullSecret: '{"auths": ...}' 13 sshKey: 'ssh-ed25519 AAAA...' 14
networking: clusterNetwork: - cidr: 172.21.0.0/16 hostPrefix: 23 - cidr: fd02::/48 hostPrefix: 64 machineNetwork: - cidr: 192.168.11.0/16 - cidr: 2001:DB8::/32 serviceNetwork: - 172.22.0.0/16 - fd03::/112 networkType: OVNKubernetes platform: baremetal: apiVIPs: - 192.168.11.3 - 2001:DB8::4 ingressVIPs: - 192.168.11.4 - 2001:DB8::5
oc adm release mirror
To use the new mirrored repository to install, add the following section to the install-config.yaml: imageContentSources: mirrors: virthost.ostest.test.metalkube.org:5000/localimages/local-release-image source: quay.io/openshift-release-dev/ocp-v4.0-art-dev mirrors: virthost.ostest.test.metalkube.org:5000/localimages/local-release-image source: registry.ci.openshift.org/ocp/release
spec: repositoryDigestMirrors: - mirrors: - virthost.ostest.test.metalkube.org:5000/openshift/release source: quay.io/openshift-release-dev/ocp-v4.0-art-dev - mirrors: - virthost.ostest.test.metalkube.org:5000/openshift/release-images source: quay.io/openshift-release-dev/ocp-release
additionalTrustBundle: | -----BEGIN CERTIFICATE----- ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ -----END CERTIFICATE-----
[[registry]] location = "registry.ci.openshift.org/ocp/release" mirror-by-digest-only = true [[registry.mirror]] location = "virthost.ostest.test.metalkube.org:5000/localimages/local-release-image" [[registry]] location = "quay.io/openshift-release-dev/ocp-v4.0-art-dev" mirror-by-digest-only = true [[registry.mirror]] location = "virthost.ostest.test.metalkube.org:5000/localimages/local-release-image"
mkdir ~/<directory_name>
cat << EOF > ./my-cluster/install-config.yaml apiVersion: v1 baseDomain: test.example.com compute: - architecture: amd64 1 hyperthreading: Enabled name: worker replicas: 0 controlPlane: architecture: amd64 hyperthreading: Enabled name: master replicas: 1 metadata: name: sno-cluster 2 networking: clusterNetwork: - cidr: fd01::/48 hostPrefix: 64 machineNetwork: - cidr: fd2e:6f44:5dd8:c956::/120 networkType: OVNKubernetes 3 serviceNetwork: - fd02::/112 platform: 4 none: {} pullSecret: '<pull_secret>' 5 sshKey: '<ssh_pub_key>' 6 additionalTrustBundle: | 7 -----BEGIN CERTIFICATE----- ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ -----END CERTIFICATE----- imageContentSources: 8 - mirrors: - <local_registry>/<local_repository_name>/release source: quay.io/openshift-release-dev/ocp-release - mirrors: - <local_registry>/<local_repository_name>/release source: quay.io/openshift-release-dev/ocp-v4.0-art-dev EOF
cat > agent-config.yaml << EOF apiVersion: v1beta1 kind: AgentConfig metadata: name: sno-cluster rendezvousIP: fd2e:6f44:5dd8:c956::50 1 EOF
openshift-install --dir <install_directory> agent create image
./openshift-install --dir <install_directory> agent wait-for bootstrap-complete \ 1 --log-level=info 2
................................................................ ................................................................ INFO Bootstrap configMap status is complete INFO cluster bootstrap is complete
openshift-install --dir <install_directory> agent wait-for install-complete 1
................................................................ ................................................................ INFO Cluster is installed INFO Install complete! INFO To access the cluster as the system:admin user when using 'oc', run INFO export KUBECONFIG=/home/core/installer/auth/kubeconfig INFO Access the OpenShift web-console here: https://console-openshift-console.apps.sno-cluster.test.example.com
./openshift-install --dir <installation_directory> agent wait-for bootstrap-complete --log-level=debug
ERROR Bootstrap failed to complete: : bootstrap process timed out: context deadline exceeded
ssh core@<node-ip> agent-gather -O >agent-gather.tar.xz
./openshift-install --dir <install_directory> agent wait-for install-complete --log-level=debug
export KUBECONFIG=<install_directory>/auth/kubeconfig
oc adm must-gather
tar cvaf must-gather.tar.gz <must_gather_directory>
./openshift-install version
./openshift-install 4.18.0 built from commit abc123def456 release image quay.io/openshift-release-dev/ocp-release@sha256:123abc456def789ghi012jkl345mno678pqr901stu234vwx567yz0 release architecture amd64
oc adm release info <release_image> -o jsonpath="{ .metadata.metadata}" 1
release.openshift.io architecture: multi
sudo dnf install /usr/bin/nmstatectl -y
mkdir ~/<directory_name>
cat << EOF > ./<directory_name>/install-config.yaml apiVersion: v1 baseDomain: test.example.com compute: - architecture: amd64 1 hyperthreading: Enabled name: worker replicas: 0 controlPlane: architecture: amd64 hyperthreading: Enabled name: master replicas: 1 metadata: name: sno-cluster 2 networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 machineNetwork: - cidr: 192.168.0.0/16 networkType: OVNKubernetes 3 serviceNetwork: - 172.30.0.0/16 platform: 4 none: {} pullSecret: '<pull_secret>' 5 sshKey: '<ssh_pub_key>' 6 EOF
networking: clusterNetwork: - cidr: 172.21.0.0/16 hostPrefix: 23 - cidr: fd02::/48 hostPrefix: 64 machineNetwork: - cidr: 192.168.11.0/16 - cidr: 2001:DB8::/32 serviceNetwork: - 172.22.0.0/16 - fd03::/112 networkType: OVNKubernetes platform: baremetal: apiVIPs: - 192.168.11.3 - 2001:DB8::4 ingressVIPs: - 192.168.11.4 - 2001:DB8::5
cat > agent-config.yaml << EOF apiVersion: v1beta1 kind: AgentConfig metadata: name: sno-cluster rendezvousIP: 192.168.111.80 1 hosts: 2 - hostname: master-0 3 interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a5 rootDeviceHints: 4 deviceName: /dev/sdb networkConfig: 5 interfaces: - name: eno1 type: ethernet state: up mac-address: 00:ef:44:21:e6:a5 ipv4: enabled: true address: - ip: 192.168.111.80 prefix-length: 23 dhcp: false dns-resolver: config: server: - 192.168.111.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.2 next-hop-interface: eno1 table-id: 254 EOF
mkdir <installation_directory>/openshift
variant: openshift version: 4.18.0 metadata: labels: machineconfiguration.openshift.io/role: worker name: 98-var-partition storage: disks: - device: /dev/disk/by-id/<device_name> 1 partitions: - label: var start_mib: <partition_start_offset> 2 size_mib: <partition_size> 3 number: 5 filesystems: - device: /dev/disk/by-partlabel/var path: /var format: xfs mount_options: [defaults, prjquota] 4 with_mount_unit: true
butane USDHOME/clusterconfig/98-var-partition.bu -o USDHOME/clusterconfig/openshift/98-var-partition.yaml
openshift-install agent create cluster-manifests --dir <installation_directory>
cd <installation_directory>/cluster-manifests
cd ../mirror
openshift-install agent create cluster-manifests --dir <installation_directory>
cd <installation_directory>/cluster-manifests
diskEncryption: enableOn: all 1 mode: tang 2 tangServers: "server1": "http://tang-server-1.example.com:7500" 3
openshift-install --dir <install_directory> agent create image
virt-install --name <vm_name> --autostart --memory=<memory> --cpu host --vcpus=<vcpus> --cdrom <agent_iso_image> \ 1 --disk pool=default,size=<disk_pool_size> --network network:default,mac=<mac_address> --graphics none --noautoconsole --os-variant rhel9.0 --wait=-1
virt-install --name <vm_name> --autostart --ram=16384 --cpu host --vcpus=8 --location <path_to_kernel_initrd_image>,kernel=kernel.img,initrd=initrd.img \ 1 --disk <qcow_image_path> --network network:macvtap ,mac=<mac_address> --graphics none --noautoconsole --wait=-1 --extra-args "rd.neednet=1 nameserver=<nameserver>" --extra-args "ip=<IP>::<nameserver>::<hostname>:enc1:none" --extra-args "coreos.live.rootfs_url=http://<http_server>:8080/agent.s390x-rootfs.img" --extra-args "random.trust_cpu=on rd.luks.options=discard" --extra-args "ignition.firstboot ignition.platform.id=metal" --extra-args "console=tty1 console=ttyS1,115200n8" --extra-args "coreos.inst.persistent-kargs=console=tty1 console=ttyS1,115200n8" --extra-args "fips=1" \ 2 --osinfo detect=on,require=off
./openshift-install --dir <install_directory> agent wait-for bootstrap-complete \ 1 --log-level=info 2
................................................................ ................................................................ INFO Bootstrap configMap status is complete INFO cluster bootstrap is complete
openshift-install --dir <install_directory> agent wait-for install-complete 1
................................................................ ................................................................ INFO Cluster is installed INFO Install complete! INFO To access the cluster as the system:admin user when using 'oc', run INFO export KUBECONFIG=/home/core/installer/auth/kubeconfig INFO Access the OpenShift web-console here: https://console-openshift-console.apps.sno-cluster.test.example.com
apiVIP: 192.168.11.3 ingressVIP: 192.168.11.4 clusterDeploymentRef: name: mycluster imageSetRef: name: openshift-4.18 networking: clusterNetwork: - cidr: 172.21.0.0/16 hostPrefix: 23 - cidr: fd02::/48 hostPrefix: 64 machineNetwork: - cidr: 192.168.11.0/16 - cidr: 2001:DB8::/32 serviceNetwork: - 172.22.0.0/16 - fd03::/112 networkType: OVNKubernetes
apiVersion: extensions.hive.openshift.io/v1beta1 kind: AgentClusterInstall metadata: name: test-agent-cluster-install namespace: cluster0 spec: clusterDeploymentRef: name: ostest imageSetRef: name: openshift-4.18 networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 serviceNetwork: - 172.30.0.0/16 provisionRequirements: controlPlaneAgents: 1 workerAgents: 0 sshPublicKey: <ssh_public_key>
apiVersion: hive.openshift.io/v1 kind: ClusterDeployment metadata: name: ostest namespace: cluster0 spec: baseDomain: test.metalkube.org clusterInstallRef: group: extensions.hive.openshift.io kind: AgentClusterInstall name: test-agent-cluster-install version: v1beta1 clusterName: ostest controlPlaneConfig: servingCertificates: {} platform: agentBareMetal: agentSelector: matchLabels: bla: aaa pullSecretRef: name: pull-secret
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: name: openshift-4.18 spec: releaseImage: registry.ci.openshift.org/ocp/release:4.18.0-0.nightly-2022-06-06-025509
apiVersion: agent-install.openshift.io/v1beta1 kind: InfraEnv metadata: name: myinfraenv namespace: cluster0 spec: clusterRef: name: ostest namespace: cluster0 cpuArchitecture: aarch64 pullSecretRef: name: pull-secret sshAuthorizedKey: <ssh_public_key> nmStateConfigLabelSelector: matchLabels: cluster0-nmstate-label-name: cluster0-nmstate-label-value
apiVersion: agent-install.openshift.io/v1beta1 kind: NMStateConfig metadata: name: master-0 namespace: openshift-machine-api labels: cluster0-nmstate-label-name: cluster0-nmstate-label-value spec: config: interfaces: - name: eth0 type: ethernet state: up mac-address: 52:54:01:aa:aa:a1 ipv4: enabled: true address: - ip: 192.168.122.2 prefix-length: 23 dhcp: false dns-resolver: config: server: - 192.168.122.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.122.1 next-hop-interface: eth0 table-id: 254 interfaces: - name: "eth0" macAddress: 52:54:01:aa:aa:a1
apiVersion: v1 kind: Secret type: kubernetes.io/dockerconfigjson metadata: name: pull-secret namespace: cluster0 stringData: .dockerconfigjson: <pull_secret>
./openshift-install --dir <installation_directory> agent wait-for bootstrap-complete --log-level=debug
ERROR Bootstrap failed to complete: : bootstrap process timed out: context deadline exceeded
ssh core@<node-ip> agent-gather -O >agent-gather.tar.xz
./openshift-install --dir <install_directory> agent wait-for install-complete --log-level=debug
export KUBECONFIG=<install_directory>/auth/kubeconfig
oc adm must-gather
tar cvaf must-gather.tar.gz <must_gather_directory>
sudo dnf install /usr/bin/nmstatectl -y
mkdir ~/<directory_name>
cat << EOF > ./<directory_name>/install-config.yaml apiVersion: v1 baseDomain: test.example.com compute: - architecture: amd64 1 hyperthreading: Enabled name: worker replicas: 0 controlPlane: architecture: amd64 hyperthreading: Enabled name: master replicas: 1 metadata: name: sno-cluster 2 networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 machineNetwork: - cidr: 192.168.0.0/16 networkType: OVNKubernetes 3 serviceNetwork: - 172.30.0.0/16 platform: 4 none: {} pullSecret: '<pull_secret>' 5 sshKey: '<ssh_pub_key>' 6 EOF
networking: clusterNetwork: - cidr: 172.21.0.0/16 hostPrefix: 23 - cidr: fd02::/48 hostPrefix: 64 machineNetwork: - cidr: 192.168.11.0/16 - cidr: 2001:DB8::/32 serviceNetwork: - 172.22.0.0/16 - fd03::/112 networkType: OVNKubernetes platform: baremetal: apiVIPs: - 192.168.11.3 - 2001:DB8::4 ingressVIPs: - 192.168.11.4 - 2001:DB8::5
cat > agent-config.yaml << EOF apiVersion: v1beta1 kind: AgentConfig metadata: name: sno-cluster rendezvousIP: 192.168.111.80 1 hosts: 2 - hostname: master-0 3 interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a5 rootDeviceHints: 4 deviceName: /dev/sdb networkConfig: 5 interfaces: - name: eno1 type: ethernet state: up mac-address: 00:ef:44:21:e6:a5 ipv4: enabled: true address: - ip: 192.168.111.80 prefix-length: 23 dhcp: false dns-resolver: config: server: - 192.168.111.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.2 next-hop-interface: eno1 table-id: 254 EOF
apiVersion: v1beta1 kind: AgentConfig metadata: name: sno-cluster rendezvousIP: 192.168.111.80 bootArtifactsBaseURL: <asset_server_URL>
openshift-install agent create pxe-files
boot-artifacts ├─ agent.x86_64-initrd.img ├─ agent.x86_64.ipxe ├─ agent.x86_64-rootfs.img └─ agent.x86_64-vmlinuz
ai.ip_cfg_override=1
rd.neednet=1 cio_ignore=all,!condev console=ttysclp0 coreos.live.rootfs_url=<coreos_url> 1 ip=<ip>::<gateway>:<netmask>:<hostname>::none nameserver=<dns> rd.znet=qeth,<network_adaptor_range>,layer2=1 rd.<disk_type>=<adapter> 2 rd.zfcp=<adapter>,<wwpn>,<lun> random.trust_cpu=on 3 zfcp.allow_lun_scan=0 ai.ip_cfg_override=1 ignition.firstboot ignition.platform.id=metal random.trust_cpu=on
rd.neednet=1 console=ttysclp0 coreos.live.rootfs_url=<rootfs_url> \ 1 ip=172.18.78.2::172.18.78.1:255.255.255.0:::none nameserver=172.18.78.1 \ 2 zfcp.allow_lun_scan=0 \ 3 ai.ip_cfg_override=1 rd.znet=qeth,0.0.bdd0,0.0.bdd1,0.0.bdd2,layer2=1 rd.dasd=0.0.4411 \ 4 rd.zfcp=0.0.8001,0x50050763040051e3,0x4000406300000000 \ 5 fips=1 \ 6 random.trust_cpu=on rd.luks.options=discard ignition.firstboot ignition.platform.id=metal console=tty1 console=ttyS1,115200n8 coreos.inst.persistent-kargs="console=tty1 console=ttyS1,115200n8"
ipl c
virt-install --name <vm_name> --autostart --ram=16384 --cpu host --vcpus=8 --location <path_to_kernel_initrd_image>,kernel=kernel.img,initrd=initrd.img \ 1 --disk <qcow_image_path> --network network:macvtap ,mac=<mac_address> --graphics none --noautoconsole --wait=-1 --extra-args "rd.neednet=1 nameserver=<nameserver>" --extra-args "ip=<IP>::<nameserver>::<hostname>:enc1:none" --extra-args "coreos.live.rootfs_url=http://<http_server>:8080/agent.s390x-rootfs.img" --extra-args "random.trust_cpu=on rd.luks.options=discard" --extra-args "ignition.firstboot ignition.platform.id=metal" --extra-args "console=tty1 console=ttyS1,115200n8" --extra-args "coreos.inst.persistent-kargs=console=tty1 console=ttyS1,115200n8" --osinfo detect=on,require=off
virt-install --name <vm_name> --autostart --ram=16384 --cpu host --vcpus=8 --location <path_to_kernel_initrd_image>,kernel=kernel.img,initrd=initrd.img \ 1 --disk <qcow_image_path> --network network:macvtap ,mac=<mac_address> --graphics none --noautoconsole --wait=-1 --extra-args "rd.neednet=1 nameserver=<nameserver>" --extra-args "ip=<IP>::<nameserver>::<hostname>:enc1:none" --extra-args "coreos.live.rootfs_url=http://<http_server>:8080/agent.s390x-rootfs.img" --extra-args "random.trust_cpu=on rd.luks.options=discard" --extra-args "ignition.firstboot ignition.platform.id=metal" --extra-args "console=tty1 console=ttyS1,115200n8" --extra-args "coreos.inst.persistent-kargs=console=tty1 console=ttyS1,115200n8" --extra-args "fips=1" \ 2 --osinfo detect=on,require=off
rd.neednet=1 cio_ignore=all,!condev console=ttysclp0 ignition.firstboot ignition.platform.id=metal coreos.live.rootfs_url=http://<http_server>/rhcos-<version>-live-rootfs.<architecture>.img \ 1 coreos.inst.persistent-kargs=console=ttysclp0 ip=<ip>::<gateway>:<netmask>:<hostname>::none nameserver=<dns> \ 2 rd.znet=qeth,<network_adaptor_range>,layer2=1 rd.<disk_type>=<adapter> \ 3 fips=1 \ 4 zfcp.allow_lun_scan=0 ai.ip_cfg_override=1 random.trust_cpu=on rd.luks.options=discard
boot-artifacts ├─ agent.s390x-generic.ins ├─ agent.s390x-initrd.addrsize ├─ agent.s390x-rootfs.img └─ agent.s390x-kernel.img └─ agent.s390x-rootfs.img
KERNEL_IMG_PATH='./kernel.img'
INITRD_IMG_PATH='./initrd.img'
CMDLINE_PATH='./generic.prm'
kernel_size=USD(stat -c%s USDKERNEL_IMG_PATH)
initrd_size=USD(stat -c%s USDINITRD_IMG_PATH)
BYTE_PER_MIB=USD(( 1024 * 1024 )) offset=USD(( (kernel_size + BYTE_PER_MIB - 1) / BYTE_PER_MIB * BYTE_PER_MIB ))
INITRD_IMG_NAME=USD(echo USDINITRD_IMG_PATH | rev | cut -d '/' -f 1 | rev)
KERNEL_OFFSET=0x00000000
KERNEL_CMDLINE_OFFSET=0x00010480
INITRD_ADDR_SIZE_OFFSET=0x00010408
OFFSET_HEX=(printf '0x%08x\n' offset)
printf "USD(printf '%016x\n' USDinitrd_size)" | xxd -r -p > temp_size.bin
cat temp_address.bin temp_size.bin > "USDINITRD_IMG_NAME.addrsize"
rm -rf temp_address.bin temp_size.bin
cat > generic.ins <<EOF USDKERNEL_IMG_PATH USDKERNEL_OFFSET USDINITRD_IMG_PATH USDOFFSET_HEX USDINITRD_IMG_NAME.addrsize USDINITRD_ADDR_SIZE_OFFSET USDCMDLINE_PATH USDKERNEL_CMDLINE_OFFSET EOF
sudo dnf install /usr/bin/nmstatectl -y
mkdir ~/<directory_name>
cat << EOF > ./<directory_name>/install-config.yaml apiVersion: v1 baseDomain: test.example.com compute: - architecture: amd64 1 hyperthreading: Enabled name: worker replicas: 0 controlPlane: architecture: amd64 hyperthreading: Enabled name: master replicas: 1 metadata: name: sno-cluster 2 networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 machineNetwork: - cidr: 192.168.0.0/16 networkType: OVNKubernetes 3 serviceNetwork: - 172.30.0.0/16 platform: 4 none: {} pullSecret: '<pull_secret>' 5 sshKey: '<ssh_pub_key>' 6 EOF
networking: clusterNetwork: - cidr: 172.21.0.0/16 hostPrefix: 23 - cidr: fd02::/48 hostPrefix: 64 machineNetwork: - cidr: 192.168.11.0/16 - cidr: 2001:DB8::/32 serviceNetwork: - 172.22.0.0/16 - fd03::/112 networkType: OVNKubernetes platform: baremetal: apiVIPs: - 192.168.11.3 - 2001:DB8::4 ingressVIPs: - 192.168.11.4 - 2001:DB8::5
cat > agent-config.yaml << EOF apiVersion: v1beta1 kind: AgentConfig metadata: name: sno-cluster rendezvousIP: 192.168.111.80 1 hosts: 2 - hostname: master-0 3 interfaces: - name: eno1 macAddress: 00:ef:44:21:e6:a5 rootDeviceHints: 4 deviceName: /dev/sdb networkConfig: 5 interfaces: - name: eno1 type: ethernet state: up mac-address: 00:ef:44:21:e6:a5 ipv4: enabled: true address: - ip: 192.168.111.80 prefix-length: 23 dhcp: false dns-resolver: config: server: - 192.168.111.1 routes: config: - destination: 0.0.0.0/0 next-hop-address: 192.168.111.2 next-hop-interface: eno1 table-id: 254 minimalISO: true 6 EOF
openshift-install --dir <install_directory> agent create image
cat << EOF > agent.ipxe !ipxe set initiator-iqn <iscsi_initiator_base>:\USD{hostname} sanboot --keep iscsi:<iscsi_network_subnet>.1::::<iscsi_target_base>:\USD{hostname} EOF
cat << EOF > agent.ipxe !ipxe set initiator-iqn iqn.2023-01.com.example:\USD{hostname} sanboot --keep iscsi:192.168.45.1::::iqn.2023-01.com.example:\USD{hostname} EOF
kind: ImageSetConfiguration apiVersion: mirror.openshift.io/v1alpha2 archiveSize: 4 1 storageConfig: 2 imageURL: <your-local-registry-dns-name>:<your-local-registry-port>/mirror/oc-mirror-metadata 3 skipTLS: true mirror: platform: architectures: - "amd64" channels: - name: stable-4.18 4 type: ocp additionalImages: - name: registry.redhat.io/ubi9/ubi:latest operators: - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.18 5 packages: 6 - name: multicluster-engine 7 - name: local-storage-operator 8
oc mirror --dest-skip-tls --config ocp-mce-imageset.yaml docker://<your-local-registry-dns-name>:<your-local-registry-port>
imageContentSources: - source: "quay.io/openshift-release-dev/ocp-release" mirrors: - "<your-local-registry-dns-name>:<your-local-registry-port>/openshift/release-images" - source: "quay.io/openshift-release-dev/ocp-v4.0-art-dev" mirrors: - "<your-local-registry-dns-name>:<your-local-registry-port>/openshift/release" - source: "registry.redhat.io/ubi9" mirrors: - "<your-local-registry-dns-name>:<your-local-registry-port>/ubi9" - source: "registry.redhat.io/multicluster-engine" mirrors: - "<your-local-registry-dns-name>:<your-local-registry-port>/multicluster-engine" - source: "registry.redhat.io/rhel8" mirrors: - "<your-local-registry-dns-name>:<your-local-registry-port>/rhel8" - source: "registry.redhat.io/redhat" mirrors: - "<your-local-registry-dns-name>:<your-local-registry-port>/redhat"
additionalTrustBundle: | -----BEGIN CERTIFICATE----- zzzzzzzzzzz -----END CERTIFICATE-------
openshift-install agent create cluster-manifests
apiVersion: v1 kind: Namespace metadata: labels: openshift.io/cluster-monitoring: "true" name: multicluster-engine
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: multicluster-engine-operatorgroup namespace: multicluster-engine spec: targetNamespaces: - multicluster-engine
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: multicluster-engine namespace: multicluster-engine spec: channel: "stable-2.3" name: multicluster-engine source: redhat-operators sourceNamespace: openshift-marketplace
apiVersion: v1 kind: Namespace metadata: annotations: openshift.io/cluster-monitoring: "true" name: openshift-local-storage
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: local-operator-group namespace: openshift-local-storage spec: targetNamespaces: - openshift-local-storage
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: local-storage-operator namespace: openshift-local-storage spec: installPlanApproval: Automatic name: local-storage-operator source: redhat-operators sourceNamespace: openshift-marketplace
<assets_directory> ├─ install-config.yaml ├─ agent-config.yaml └─ /openshift ├─ mce_namespace.yaml ├─ mce_operatorgroup.yaml ├─ mce_subscription.yaml ├─ lso_namespace.yaml ├─ lso_operatorgroup.yaml └─ lso_subscription.yaml
openshift-install agent create image --dir <assets_directory>
openshift-install agent wait-for install-complete --dir <assets_directory>
apiVersion: local.storage.openshift.io/v1 kind: LocalVolume metadata: name: assisted-service namespace: openshift-local-storage spec: logLevel: Normal managementState: Managed storageClassDevices: - devicePaths: - /dev/vda - /dev/vdb storageClassName: assisted-service volumeMode: Filesystem
oc wait localvolume -n openshift-local-storage assisted-service --for condition=Available --timeout 10m
The `devicePath` is an example and may vary depending on the actual hardware configuration used.
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterengine spec: {}
apiVersion: agent-install.openshift.io/v1beta1 kind: AgentServiceConfig metadata: name: agent namespace: assisted-installer spec: databaseStorage: storageClassName: assisted-service accessModes: - ReadWriteOnce resources: requests: storage: 10Gi filesystemStorage: storageClassName: assisted-service accessModes: - ReadWriteOnce resources: requests: storage: 10Gi
apiVersion: hive.openshift.io/v1 kind: ClusterImageSet metadata: name: "4.18" spec: releaseImage: quay.io/openshift-release-dev/ocp-release:4.18.0-x86_64
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: labels: local-cluster: "true" cloud: auto-detect vendor: auto-detect name: local-cluster spec: hubAcceptsClient: true
oc wait -n multicluster-engine managedclusters local-cluster --for condition=ManagedClusterJoined=True --timeout 10m
oc get managedcluster NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE local-cluster true https://<your cluster url>:6443 True True 77m
apiVersion:
baseDomain:
metadata:
metadata: name:
platform:
pullSecret:
{ "auths":{ "cloud.openshift.com":{ "auth":"b3Blb=", "email":"you@example.com" }, "quay.io":{ "auth":"b3Blb=", "email":"you@example.com" } } }
networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 - cidr: fd00:10:128::/56 hostPrefix: 64 serviceNetwork: - 172.30.0.0/16 - fd00:172:16::/112
networking:
networking: networkType:
networking: clusterNetwork:
networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 - cidr: fd01::/48 hostPrefix: 64
networking: clusterNetwork: cidr:
networking: clusterNetwork: hostPrefix:
networking: serviceNetwork:
networking: serviceNetwork: - 172.30.0.0/16 - fd02::/112
networking: machineNetwork:
networking: machineNetwork: - cidr: 10.0.0.0/16
networking: machineNetwork: cidr:
networking: ovnKubernetesConfig: ipv4: internalJoinSubnet:
additionalTrustBundle:
capabilities:
capabilities: baselineCapabilitySet:
capabilities: additionalEnabledCapabilities:
cpuPartitioningMode:
compute:
compute: architecture:
compute: hyperthreading:
compute: name:
compute: platform:
compute: replicas:
featureSet:
controlPlane:
controlPlane: architecture:
controlPlane: hyperthreading:
controlPlane: name:
controlPlane: platform:
controlPlane: replicas:
credentialsMode:
fips:
imageContentSources:
imageContentSources: source:
imageContentSources: mirrors:
publish:
sshKey:
platform: baremetal: clusterProvisioningIP:
platform: baremetal: provisioningNetwork:
platform: baremetal: provisioningMACAddress:
platform: baremetal: provisioningNetworkCIDR:
platform: baremetal: provisioningNetworkInterface:
platform: baremetal: provisioningDHCPRange:
platform: baremetal: hosts:
platform: baremetal: hosts: name:
platform: baremetal: hosts: bootMACAddress:
platform: baremetal: hosts: bmc:
platform: baremetal: hosts: bmc: username:
platform: baremetal: hosts: bmc: password:
platform: baremetal: hosts: bmc: address:
platform: baremetal: hosts: bmc: disableCertificateVerification:
platform: vsphere:
platform: vsphere: failureDomains:
platform: vsphere: failureDomains: name:
platform: vsphere: failureDomains: region:
platform: vsphere: failureDomains: server:
platform: vsphere: failureDomains: zone:
platform: vsphere: failureDomains: topology: computeCluster:
platform: vsphere: failureDomains: topology: datacenter:
platform: vsphere: failureDomains: topology: datastore:
platform: vsphere: failureDomains: topology: folder:
platform: vsphere: failureDomains: topology: networks:
platform: vsphere: failureDomains: topology: resourcePool:
platform: vsphere: failureDomains: topology template:
platform: vsphere: vcenters:
platform: vsphere: vcenters: datacenters:
platform: vsphere: vcenters: password:
platform: vsphere: vcenters: port:
platform: vsphere: vcenters: server:
platform: vsphere: vcenters: user:
platform: vsphere: cluster:
platform: vsphere: datacenter:
platform: vsphere: defaultDatastore:
platform: vsphere: folder:
platform: vsphere: password:
platform: vsphere: resourcePool:
platform: vsphere: username:
platform: vsphere: vCenter:
apiVersion:
metadata:
metadata: name:
rendezvousIP:
bootArtifactsBaseURL:
additionalNTPSources:
hosts:
hosts: hostname:
hosts: interfaces:
hosts: interfaces: name:
hosts: interfaces: macAddress:
hosts: role:
hosts: rootDeviceHints:
hosts: rootDeviceHints: deviceName:
hosts: networkConfig:
minimalISO: