Configuring Red Hat build of MicroShift 4.18 Configuring MicroShift Red Hat OpenShift Documentation Team

Commands:
microshift show-config
apiServer: advertiseAddress: 10.44.0.0/32 1 auditLog: maxFileAge: 0 maxFileSize: 200 maxFiles: 10 profile: Default namedCertificates: - certPath: "" keyPath: "" names: - "" subjectAltNames: [] debugging: logLevel: "Normal" dns: baseDomain: microshift.example.com etcd: memoryLimitMB: 0 ingress: defaultHTTPVersion: 1 forwardedHeaderPolicy: "" httpCompression: mimeTypes: - "" httpEmptyRequestsPolicy: Respond listenAddress: - "" logEmptyRequests: Log ports: http: 80 https: 443 routeAdmissionPolicy: namespaceOwnership: InterNamespaceAllowed status: Managed tuningOptions: clientFinTimeout: "" clientTimeout: "" headerBufferBytes: 0 headerBufferMaxRewriteBytes: 0 healthCheckInterval: "" maxConnections: 0 serverFinTimeout: "" serverTimeout: "" threadCount: 0 tlsInspectDelay: "" tunnelTimeout: "" kubelet: manifests: kustomizePaths: - /usr/lib/microshift/manifests - /usr/lib/microshift/manifests.d/* - /etc/microshift/manifests - /etc/microshift/manifests.d/* network: clusterNetwork: - 10.42.0.0/16 serviceNetwork: - 10.43.0.0/16 serviceNodePortRange: 30000-32767 node: hostnameOverride: "" nodeIP: "" 2 nodeIPv6: "" storage: driver: "" 3 optionalCsiComponents: 4 - ""
container_memory_working_set_bytes{container=`router`,namespace=`openshift-ingress`}`
container_memory_working_set_bytes{container=`router`,namespace=`openshift-ingress`}/container_processes{container=`router`,namespace=`openshift-ingress`}
apiServer: subjectAltNames: - host1 - host2
apiServer: subjectAltNames: - hostZ
apiServer: subjectAltNames: - hostZ
apiServer: subjectAltNames: - host1 - host2 - hostZ
apiServer: subjectAltNames: - host1 - host2 - hostZ
apiServer: advertiseAddress: "microshift-example"
apiServer: auditLog: maxFileAge: 12
apiServer: advertiseAddress: "microshift-example" auditLog: maxFileAge: 12
apiServer: advertiseAddress: "microshift-example"
apiServer: auditLog: maxFileAge: 12
apiServer: subjectAltNames: - a.com - b.com
apiServer: subjectAltNames: - c.com - d.com
apiServer: advertiseAddress: "microshift-example" auditLog: maxFileAge: 12 subjectAltNames: - c.com - d.com
apiServer: network: clusterNetwork: - fd01::/48 1 serviceNetwork: - fd02::/112 2 node: nodeIP: 2600:1f14:1c48:ee00:2d76:3190:5bc2:5aef 3
sudo systemctl start microshift
oc get node -o jsonpath='{.items[].spec.podCIDRs[]}'
fd01::/48
oc get pod -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system csi-snapshot-controller-bb7cb654b-rqrt6 1/1 Running 0 65s fd01:0:0:1::5 microshift-9 <none> <none> openshift-dns dns-default-cjn66 2/2 Running 0 62s fd01:0:0:1::9 microshift-9 <none> <none> openshift-dns node-resolver-ppnjb 1/1 Running 0 63s 2001:db9:ca7:ff::1db8 microshift-9 <none> <none> openshift-ingress router-default-6d97d7b8b6-wdtmg 1/1 Running 0 61s fd01:0:0:1::8 microshift-9 <none> <none> openshift-ovn-kubernetes ovnkube-master-gfvp5 4/4 Running 0 63s 2001:db9:ca7:ff::1db8 microshift-9 <none> <none> openshift-ovn-kubernetes ovnkube-node-bnpjh 1/1 Running 0 63s 2001:db9:ca7:ff::1db8 microshift-9 <none> <none> openshift-service-ca service-ca-5d7bd9db6-j25bd 1/1 Running 0 60s fd01:0:0:1::4 microshift-9 <none> <none> openshift-storage lvms-operator-656cd9b59b-bwr47 1/1 Running 0 63s fd01:0:0:1::7 microshift-9 <none> <none> openshift-storage vg-manager-f7dmk 1/1 Running 0 27s fd01:0:0:1::a microshift-9 <none> <none>
oc get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE default kubernetes ClusterIP fd02::1 <none> 443/TCP 3m42s openshift-dns dns-default ClusterIP fd02::a <none> 53/UDP,53/TCP,9154/TCP 2m58s openshift-ingress router-default LoadBalancer fd02::f2e6 2001:db9:ca7:ff::1db8,fd01:0:0:1::2,fd02::1:0,fd69::2 80:31133/TCP,443:31996/TCP 2m58s openshift-ingress router-internal-default ClusterIP fd02::c55e <none> 80/TCP,443/TCP,1936/TCP 2m58s openshift-storage lvms-operator-metrics-service ClusterIP fd02::7afb <none> 443/TCP 2m58s openshift-storage lvms-webhook-service ClusterIP fd02::d8dd <none> 443/TCP 2m58s openshift-storage vg-manager-metrics-service ClusterIP fd02::fc1 <none> 443/TCP 2m58s
apiServer: apiServer: subjectAltNames: - 192.168.113.117 - 2001:db9:ca7:ff::1db8 network: clusterNetwork: - 10.42.0.0/16 - fd01::/48 1 serviceNetwork: - 10.43.0.0/16 - fd02::/112 2 node: nodeIP: 192.168.113.117 3 nodeIPv6: 2001:db9:ca7:ff::1db8 4
sudo systemctl start microshift
oc get pod -n openshift-ingress router-default-5b75594b4-w7w6s -o jsonpath='{.status.podIPs}'
10.42.0.4, fd01:0:0:1::4
oc get pod -n openshift-ovn-kubernetes ovnkube-master-2fm2k -o jsonpath='{.status.podIPs}'
192.168.113.117, 2001:db9:ca7:ff::1db8
apiServer: subjectAltNames: - 192.168.113.117 - 2001:db9:ca7:ff::1db8 1 network: clusterNetwork: - 10.42.0.0/16 2 - fd01::/48 3 serviceNetwork: - 10.43.0.0/16 - fd02::/112 4 node: nodeIP: 192.168.113.117 5 nodeIPv6: 2001:db9:ca7:ff::1db8 6
sudo systemctl restart microshift
oc get pod -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system csi-snapshot-controller-bb7cb654b-7s5ql 1/1 Running 0 46m 10.42.0.6 microshift-9 <none> <none> openshift-dns dns-default-zxkqn 2/2 Running 0 46m 10.42.0.5 microshift-9 <none> <none> openshift-dns node-resolver-r2h5z 1/1 Running 0 46m 192.168.113.117 microshift-9 <none> <none> openshift-ingress router-default-5b75594b4-228z7 1/1 Running 0 2m5s 10.42.0.3 microshift-9 <none> <none> openshift-ovn-kubernetes ovnkube-master-bltk7 4/4 Running 2 (2m32s ago) 2m36s 192.168.113.117 microshift-9 <none> <none> openshift-ovn-kubernetes ovnkube-node-9ghgs 1/1 Running 2 (2m32s ago) 46m 192.168.113.117 microshift-9 <none> <none> openshift-service-ca service-ca-5d7bd9db6-qgwgw 1/1 Running 0 46m 10.42.0.7 microshift-9 <none> <none> openshift-storage lvms-operator-656cd9b59b-8rpf4 1/1 Running 0 46m 10.42.0.8 microshift-9 <none> <none> openshift-storage vg-manager-wqmh4 1/1 Running 2 (2m39s ago) 46m 10.42.0.10 microshift-9 <none> <none>
oc get pod -n openshift-ovn-kubernetes ovnkube-master-bltk7 -o jsonpath='{.status.podIPs}'
192.168.113.117, 2001:db9:ca7:ff::1db8
oc get pod -n openshift-ingress router-default-5b75594b4-228z7 -o jsonpath='{.status.podIPs}'
10.42.0.3, fd01:0:0:1::3
kind: Service apiVersion: v1 metadata: name: microshift-new-service labels: app: microshift-application spec: type: NodePort ipFamilyPolicy: `PreferDualStack` 1
I1006 16:09:50.985852 60651 helper_linux.go:73] Found default gateway interface br-ex 192.168.127.1 I1006 16:09:50.985923 60651 helper_linux.go:73] Found default gateway interface ens4 fe80::5054:ff:febe:bcd4 F1006 16:09:50.985939 60651 ovnkube.go:130] multiple gateway interfaces detected: br-ex ens4
I0512 19:07:17.589083 108432 helper_linux.go:74] Found default gateway interface br-ex 192.168.123.1 F0512 19:07:17.589141 108432 ovnkube.go:133] failed to get default gateway interface
apiServer: ingress: defaultHTTPVersion: 1 forwardedHeaderPolicy: Append httpCompression: mimeTypes: - "" httpEmptyRequestsPolicy: Respond logEmptyRequests: Log tuningOptions: clientFinTimeout: 1s clientTimeout: 30s headerBufferBytes: 0 headerBufferMaxRewriteBytes: 0 healthCheckInterval: 5s maxConnections: 0 serverFinTimeout: 1s serverTimeout: 30s threadCount: 4 tlsInspectDelay: 5s tunnelTimeout: 1h
sudo systemctl start microshift
sudo systemctl restart microshift
oc get pods -n openshift-ingress
NAME READY STATUS RESTARTS AGE router-default-8649b5bf65-w29cn 1/1 Running 0 6m10s
/var/lib/microshift/resources/kubeadmin/ ├── kubeconfig 1 ├── alt-name-1 2 │ └── kubeconfig ├── 1.2.3.4 3 │ └── kubeconfig └── microshift-rhel9 4 └── kubeconfig
clusters: - cluster: certificate-authority-data: <base64 CA> server: https://localhost:6443
mkdir -p ~/.kube/
sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config
chmod go-r ~/.kube/config
oc get all -A
clusters: - cluster: certificate-authority-data: <base64 CA> server: https://microshift-rhel9:6443
cat /etc/microshift/config.yaml
cat /var/lib/microshift/resources/kubeadmin/<hostname>/kubeconfig
dns: baseDomain: example.com node: hostnameOverride: "microshift-rhel9" 1 nodeIP: 10.0.0.1 apiServer: subjectAltNames: - alt-name-1 2 - 1.2.3.4 3
sudo systemctl restart microshift
cat /var/lib/microshift/resources/kubeadmin/alt-name-1/kubeconfig
clusters: - cluster: certificate-authority-data: <base64 CA> server: https://alt-name-1:6443 1
[user@microshift]USD sudo firewall-cmd --permanent --zone=public --add-port=6443/tcp && sudo firewall-cmd --reload
[user@microshift]USD oc get all -A
[user@workstation]USD mkdir -p ~/.kube/
[user@workstation]USD MICROSHIFT_MACHINE=<name or IP address of MicroShift machine>
[user@workstation]USD ssh <user>@USDMICROSHIFT_MACHINE "sudo cat /var/lib/microshift/resources/kubeadmin/USDMICROSHIFT_MACHINE/kubeconfig" > ~/.kube/config
chmod go-r ~/.kube/config
[user@workstation]USD oc get all -A
apiServer: namedCertificates: - certPath: ~/certs/api_fqdn_1.crt 1 keyPath: ~/certs/api_fqdn_1.key 2 - certPath: ~/certs/api_fqdn_2.crt keyPath: ~/certs/api_fqdn_2.key names: 3 - api_fqdn_1 - *.apps.external.com
systemctl microshift restart
export KUBECONFIG=~/custom-kubeconfigs/kubeconfig 1
oc --certificate-authority ~/certs/ca.ca get node
get node NAME STATUS ROLES AGE VERSION dhcp-1-235-195.arm.example.com Ready control-plane,master,worker 76m v1.31.3
oc config set clusters.microshift.certificate-authority /tmp/certificate-authority-data-new.crt
oc config view --flatten
apiVersion: v1 clusters: - cluster: certificate-authority: /tmp/certificate-authority-data-new.crt 1 server: https://api.ci-ln-k0gim2b-76ef8.aws-2.ci.openshift.org:6443 name: ci-ln-k0gim2b-76ef8 contexts: - context: cluster: ci-ln-k0gim2b-76ef8 user: name: current-context: kind: Config preferences: {}
curl --cacert /tmp/caCert.pem https://USD{fqdn_name}:6443/healthz -v
Server certificate: subject: CN=kas-test-cert_server start date: Mar 12 11:39:46 2024 GMT expire date: Mar 12 11:39:46 2025 GMT subjectAltName: host "dhcp-1-235-3.arm.eng.rdu2.redhat.com" matched cert's "dhcp-1-235-3.arm.eng.rdu2.redhat.com" issuer: CN=kas-test-cert_ca SSL certificate verify ok.
journalctl -u microshift -b0 | grep tls-sni-cert-key
Jan 24 14:53:00 localhost.localdomain microshift[45313]: kube-apiserver I0124 14:53:00.649099 45313 flags.go:64] FLAG: --tls-sni-cert-key="[/home/eslutsky/dev/certs/server.crt,/home/eslutsky/dev/certs/server.key;/var/lib/microshift/certs/kube-apiserver-external-signer/kube-external-serving/server.crt,/var/lib/microshift/certs/kube-apiserver-external-signer/kube-external-serving/server.key;/var/lib/microshift/certs/kube-apiserver-localhost-signer/kube-apiserver-localhost-serving/server.crt,/var/lib/microshift/certs/kube-apiserver-localhost-signer/kube-apiserver-localhost-serving/server.key;/var/lib/microshift/certs/kube-apiserver-service-network-signer/kube-apiserver-service-network-serving/server.crt,/var/lib/microshift/certs/kube-apiserver-service-network-signer/kube-apiserver-service-network-serving/server.key
openssl s_client -connect <SNI_ADDRESS>:6443 -showcerts | openssl x509 -text -noout -in - | grep -C 1 "Alternative\|CN"
sudo microshift-cleanup-data --cert
Stopping MicroShift services Removing MicroShift certificates MicroShift service was stopped Cleanup succeeded
sudo systemctl start microshift
systemctl show --property=SubState --value greenboot-healthcheck.service
systemctl show --property=ExecMainStatus --value greenboot-healthcheck.service
cat /run/motd.d/boot-status
apiServer: . auditLog: maxFileAge: 7 1 maxFileSize: 200 2 maxFiles: 1 3 profile: Default 4 .
sudo systemctl stop microshift
sudo mv /var/log/kube-apiserver <~/kube-apiserver> 1
sudo ln -s <~/kube-apiserver> /var/log/kube-apiserver 1
sudo systemctl restart microshift
sudo microshift show-config --mode effective
auditLog: maxFileSize: 200 maxFiles: 1 maxFileAge: 7 profile: AllRequestBodies
sudo ls -ltrh /var/log/kube-apiserver/audit.log
-rw-------. 1 root root 46M Mar 12 09:52 /var/log/kube-apiserver/audit.log
sudo ls -ltrh /var/log/kube-apiserver/
total 6.0M -rw-------. 1 root root 2.0M Mar 12 10:56 audit-2024-03-12T14-56-16.267.log -rw-------. 1 root root 2.0M Mar 12 10:56 audit-2024-03-12T14-56-49.444.log -rw-------. 1 root root 962K Mar 12 10:57 audit.log
storage: {} 1
sudo systemctl start microshift
storage driver: - "none" 1
sudo systemctl enable --now microshift
sudo dnf install -y microshift-low-latency
apiServer: kubelet: 1 cpuManagerPolicy: static 2 cpuManagerPolicyOptions: full-pcpus-only: "true" 3 cpuManagerReconcilePeriod: 5s memoryManagerPolicy: Static 4 topologyManagerPolicy: single-numa-node reservedSystemCPUs: 0-1 5 reservedMemory: - limits: memory: 1100Mi 6 numaNode: 0 kubeReserved: memory: 500Mi systemReserved: memory: 500Mi evictionHard: 7 imagefs.available: "15%" 8 memory.available: "100Mi" 9 nodefs.available: "10%" 10 nodefs.inodesFree: "5%" 11 evictionPressureTransitionPeriod: 5m 12
Isolate cores 2-7 for running application workloads isolated_cores=2-7 1 Size of the hugepages hugepages_size=2M 2 Number of hugepages hugepages=0 Additional kernel arguments additional_args= 3 CPU set to be offlined offline_cpu_set= 4
sudo tuned-adm profile microshift-baseline
cat /proc/cmdline
BOOT_IMAGE=(hd0,msdos2)/ostree/rhel-7f82ccd9595c3c70af16525470e32c6a81c9138c4eae6c79ab86d5a2d108d7fc/vmlinuz-5.14.0-427.31.1.el9_4.x86_64+rt crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M rd.lvm.lv=rhel/root fips=0 console=ttyS0,115200n8 root=/dev/mapper/rhel-root rw ostree=/ostree/boot.1/rhel/7f82ccd9595c3c70af16525470e32c6a81c9138c4eae6c79ab86d5a2d108d7fc/0 skew_tick=1 tsc=reliable rcupdate.rcu_normal_after_boot=1 nohz=on nohz_full=2,4-5 rcu_nocbs=2,4-5 tuned.non_isolcpus=0000000b intel_pstate=disable nosoftlockup hugepagesz=2M hugepages=10
profile: microshift-baseline 1 reboot_after_apply: True 2
sudo systemctl enable microshift-tuned.service
sudo systemctl start microshift-tuned.service
sudo subscription-manager repos --enable rhel-9-for-x86_64-rt-rpms
sudo dnf install -y kernel-rt
RTVER=USD(rpm -q --queryformat '%{version}-%{release}.%{arch}' kernel-rt | sort | tail -1)
sudo grubby --set-default="/boot/vmlinuz-USD{RTVER}+rt"
[[packages]] name = "microshift-low-latency" version = "*" Kernel RT is supported only on the x86_64 architecture [customizations.kernel] name = "kernel-rt" [customizations.services] enabled = ["microshift", "microshift-tuned"] [[customizations.files]] path = "/etc/microshift/config.yaml" data = """ kubelet: cpuManagerPolicy: static cpuManagerPolicyOptions: full-pcpus-only: "true" cpuManagerReconcilePeriod: 5s memoryManagerPolicy: Static topologyManagerPolicy: single-numa-node reservedSystemCPUs: 0-1 reservedMemory: - limits: memory: 1100Mi numaNode: 0 kubeReserved: memory: 500Mi systemReserved: memory: 500Mi evictionHard: imagefs.available: 15% memory.available: 100Mi nodefs.available: 10% nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 5m """ [[customizations.files]] path = "/etc/tuned/microshift-baseline-variables.conf" data = """ Isolated cores should be complementary to the kubelet configuration reserved CPUs. Isolated and reserved CPUs must contain all online CPUs. Core #3 is for testing offlining, therefore it is skipped. isolated_cores=2,4-5 hugepages_size=2M hugepages=10 additional_args=test1=on test2=true dummy offline_cpu_set=3 """ [[customizations.files]] path = "/etc/microshift/tuned.yaml" data = """ profile: microshift-baseline reboot_after_apply: True """
cpu-load-balancing.crio.io: "disable" irq-load-balancing.crio.io: "disable" cpu-quota.crio.io: "disable" cpu-load-balancing.crio.io: "disable" cpu-freq-governor.crio.io: "<governor>"
apiVersion: v1 kind: Pod metadata: name: oslat annotations: cpu-load-balancing.crio.io: "disable" 1 irq-load-balancing.crio.io: "disable" 2 cpu-quota.crio.io: "disable" 3 cpu-c-states.crio.io: "disable" 4 cpu-freq-governor.crio.io: "<governor>" 5 spec: runtimeClassName: microshift-low-latency 6 containers: - name: oslat image: quay.io/container-perf-tools/oslat imagePullPolicy: Always resources: requests: memory: "400Mi" cpu: "2" limits: memory: "400Mi" cpu: "2" env: - name: tool value: "oslat" - name: manual value: "n" - name: PRIO value: "1" - name: delay value: "0" - name: RUNTIME_SECONDS value: "60" - name: TRACE_THRESHOLD value: "" - name: EXTRA_ARGS value: "" securityContext: privileged: true capabilities: add: - SYS_NICE - IPC_LOCK
name = "microshift-low-latency" description = "RHEL 9.4 and MicroShift configured for low latency" version = "0.0.1" modules = [] groups = [] distro = "rhel-94" [[packages]] name = "microshift" version = "*" [[packages]] name = "microshift-greenboot" version = "*" [[packages]] name = "microshift-networking" version = "*" [[packages]] name = "microshift-selinux" version = "*" [[packages]] name = "microshift-low-latency" version = "*" Kernel RT is only available for x86_64 [customizations.kernel] name = "kernel-rt" [customizations.services] enabled = ["microshift", "microshift-tuned"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] [customizations.firewall.services] enabled = ["mdns", "ssh", "http", "https"] [[customizations.firewall.zones]] name = "trusted" sources = ["10.42.0.0/16", "169.254.169.1"] [[customizations.files]] path = "/etc/microshift/config.yaml" data = """ kubelet: cpuManagerPolicy: static cpuManagerPolicyOptions: full-pcpus-only: "true" cpuManagerReconcilePeriod: 5s memoryManagerPolicy: Static topologyManagerPolicy: single-numa-node reservedSystemCPUs: 0-1 reservedMemory: - limits: memory: 1100Mi numaNode: 0 kubeReserved: memory: 500Mi systemReserved: memory: 500Mi evictionHard: imagefs.available: 15% memory.available: 100Mi nodefs.available: 10% nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 5m """ [[customizations.files]] path = "/etc/tuned/microshift-baseline-variables.conf" data = """ Isolated cores should be complementary to the kubelet configuration reserved CPUs. Isolated and reserved CPUs must contain all online CPUs. Core #3 is for testing offlining, therefore it is skipped. isolated_cores=2,4-5 hugepages_size=2M hugepages=10 additional_args=test1=on test2=true dummy offline_cpu_set=3 """ [[customizations.files]] path = "/etc/microshift/tuned.yaml" data = """ profile: microshift-baseline reboot_after_apply: True """
{ "management": { "cpuset": "0,6,7" 1 } }
kubelet: reservedSystemCPUs: 0,6,7 1 cpuManagerPolicy: static cpuManagerPolicyOptions: full-pcpus-only: "true" 2 cpuManagerReconcilePeriod: 5s
[crio.runtime] infra_ctr_cpuset = "0,6,7" [crio.runtime.workloads.management] activation_annotation = "target.workload.openshift.io/management" annotation_prefix = "resources.workload.openshift.io" resources = { "cpushares" = 0, "cpuset" = "0,6,7" }
[Service] CPUAffinity=0,6,7
[Service] CPUAffinity=0,6,7
[Service] CPUAffinity=0,6,7
[Service] CPUAffinity=0,6,7