Upgrading Red Hat Advanced Cluster Security for Kubernetes 4.6 Upgrading Red Hat Advanced Cluster Security for Kubernetes Red Hat OpenShift Documentation Team

Commands:
spec: central: db: isEnabled: Default 1 persistence: persistentVolumeClaim: 2 claimName: central-db size: 100Gi storageClassName: <storage-class-name>
oc create secret generic external-db-password \ 1 --from-file=password=<password.txt> 2
spec: tls: additionalCAs: - name: db-ca content: | <certificate> central: db: isEnabled: Default 1 connectionString: "host=<host> port=5432 user=<user> sslmode=verify-ca" passwordSecret: name: external-db-password
oc -n rhacs-operator \ 1 patch subscriptions.operators.coreos.com rhacs-operator --type=merge --patch='{ "spec": { "channel": "stable" }}'
oc -n rhacs-operator delete subscription rhacs-operator
kubectl -n rhacs-operator delete subscription rhacs-operator
oc -n rhacs-operator delete csv -l operators.coreos.com/rhacs-operator.rhacs-operator
kubectl -n rhacs-operator delete csv -l operators.coreos.com/rhacs-operator.rhacs-operator
curl -k -s -u <user>:<password> https://<central hostname>/v1/centralhealth/upgradestatus | jq -r .upgradeStatus.forceRollbackTo
oc scale -n <central namespace> -replicas=0 deploy/central
kubectl scale -n <central namespace> -replicas=0 deploy/central
apiVersion: v1 kind: Pod metadata: name: get-previous-db-version spec: containers: - name: get-previous-db-version image: registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:<rollback version> command: - sh args: - '-c' - "cat /var/lib/stackrox/.previous/migration_version.yaml | grep '^image:' | cut -f 2 -d : | tr -d ' '" volumeMounts: - name: stackrox-db mountPath: /var/lib/stackrox volumes: - name: stackrox-db persistentVolumeClaim: claimName: stackrox-db
oc create -n <central namespace> -f pod.yaml
kubectl create -n <central namespace> -f pod.yaml
oc logs -n <central namespace> get-previous-db-version
kubectl logs -n <central namespace> get-previous-db-version
oc get configmap -n <central namespace> central-config -o yaml | sed -e "s/forceRollbackVersion: none/forceRollbackVersion: <version>/" | oc -n <central namespace> apply -f -
kubectl get configmap -n <central namespace> central-config -o yaml | sed -e "s/forceRollbackVersion: none/forceRollbackVersion: <version>/" | kubectl -n <central namespace> apply -f -
oc set image -n <central namespace> deploy/central central=registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:<version>
kubectl set image -n <central namespace> deploy/central central=registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:<version>
Clone to Migrate ".previous", ""
curl -k -s -u <user>:<password> https://<central hostname>/v1/centralhealth/upgradestatus | jq -r .upgradeStatus.forceRollbackTo
apiVersion: v1 kind: Pod metadata: name: get-previous-db-version spec: containers: - name: get-previous-db-version image: registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:<rollback version> command: - sh args: - '-c' - "cat /var/lib/stackrox/.previous/migration_version.yaml | grep '^image:' | cut -f 2 -d : | tr -d ' '" volumeMounts: - name: stackrox-db mountPath: /var/lib/stackrox volumes: - name: stackrox-db persistentVolumeClaim: claimName: stackrox-db
Clone to Migrate ".previous", ""
oc -n <namespace> get pod -l app=central-db 1
oc -n <namespace> describe po/<central-db-pod-name> 1
Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling 54s default-scheduler 0/7 nodes are available: 1 Insufficient memory, 3 node(s) had untolerated taint {node-role.kubernetes.io/master: }, 4 Insufficient cpu. preemption: 0/7 nodes are available: 3 Preemption is not helpful for scheduling, 4 No preemption victims found for incoming pod.
oc -n rhacs-operator describe centrals.platform.stackrox.io 1
oc -n rhacs-operator describe securedclusters.platform.stackrox.io 1
Conditions: Last Transition Time: 2023-04-19T10:49:57Z Status: False Type: Deployed Last Transition Time: 2023-04-19T10:49:57Z Status: True Type: Initialized Last Transition Time: 2023-04-19T10:59:10Z Message: Deployment.apps "central" is invalid: spec.template.spec.containers[0].resources.requests: Invalid value: "50": must be less than or equal to cpu limit Reason: ReconcileError Status: True Type: Irreconcilable Last Transition Time: 2023-04-19T10:49:57Z Message: No proxy configuration is desired Reason: NoProxyConfig Status: False Type: ProxyConfigFailed Last Transition Time: 2023-04-19T10:49:57Z Message: Deployment.apps "central" is invalid: spec.template.spec.containers[0].resources.requests: Invalid value: "50": must be less than or equal to cpu limit Reason: InstallError Status: True Type: ReleaseFailed
-n rhacs-operator logs deploy/rhacs-operator-controller-manager manager 1
roxctl -e "USDROX_CENTRAL_ADDRESS" central backup
central: db: resources: requests: memory: 16Gi cpu: 8 limits: memory: 16Gi cpu: 8
chmod +x create_certificate_values_file.sh
create_certificate_values_file.sh values-private.yaml
helm repo update
helm search repo -l rhacs/
helm upgrade -n stackrox stackrox-central-services rhacs/central-services --version <current-rhacs-version> \ 1 -f values-private.yaml --set central.db.password.generate=true --set central.db.serviceTLS.generate=true --set central.db.persistence.persistentVolumeClaim.createClaim=true
helm upgrade -n stackrox stackrox-secured-cluster-services rhacs/secured-cluster-services --version <current-rhacs-version> \ 1 -f values-private.yaml
helm upgrade -n stackrox stackrox-central-services rhacs/central-services --version <current-rhacs-version> --reuse-values -f values-private.yaml --set central.db.password.generate=false --set central.db.serviceTLS.generate=false --set central.db.persistence.persistentVolumeClaim.createClaim=false
helm upgrade -n stackrox stackrox-central-services rhacs/central-services --version <previous_rhacs_74_version> \ 1 --set central.db.enabled=false
oc -n stackrox delete pvc central-db 1
roxctl -e "USDROX_CENTRAL_ADDRESS" central backup
ROXPATH=USD(which roxctl) && rm -f USDROXPATH 1
arch="USD(uname -m | sed "s/x86_64//")"; arch="USD{arch:+-USDarch}"
curl -L -f -o roxctl "https://mirror.openshift.com/pub/rhacs/assets/4.6.3/bin/Linux/roxctlUSD{arch}"
chmod +x roxctl
echo USDPATH
roxctl version
arch="USD(uname -m | sed "s/x86_64//")"; arch="USD{arch:+-USDarch}"
curl -L -f -o roxctl "https://mirror.openshift.com/pub/rhacs/assets/4.6.3/bin/Darwin/roxctlUSD{arch}"
xattr -c roxctl
chmod +x roxctl
echo USDPATH
roxctl version
curl -f -O https://mirror.openshift.com/pub/rhacs/assets/4.6.3/bin/Windows/roxctl.exe
roxctl version
oc -n stackrox set image deploy/central central=registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:4.6.3 1
oc get deploy -n stackrox -o wide
oc get pod -n stackrox --watch
oc -n stackrox edit deploy/central 1
oc -n stackrox set image deploy/scanner scanner=registry.redhat.io/advanced-cluster-security/rhacs-scanner-rhel8:4.6.3 1
oc get deploy -n stackrox -o wide
oc get pod -n stackrox --watch
oc -n stackrox edit deploy/scanner 1
oc logs -n stackrox deploy/central -c central 1
No database restore directory found (this is not an error). Migrator: 2023/04/19 17:58:54: starting DB compaction Migrator: 2023/04/19 17:58:54: Free fraction of 0.0391 (40960/1048576) is < 0.7500. Will not compact badger 2023/04/19 17:58:54 INFO: All 1 tables opened in 2ms badger 2023/04/19 17:58:55 INFO: Replaying file id: 0 at offset: 846357 badger 2023/04/19 17:58:55 INFO: Replay took: 50.324ms badger 2023/04/19 17:58:55 DEBUG: Value log discard stats empty Migrator: 2023/04/19 17:58:55: DB is up to date. Nothing to do here. badger 2023/04/19 17:58:55 INFO: Got compaction priority: {level:0 score:1.73 dropPrefix:[]} version: 2023/04/19 17:58:55.189866 ensure.go:49: Info: Version found in the DB was current. We're good to go!
oc -n stackrox set image deploy/sensor sensor=registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:4.6.3 1
oc -n stackrox set image ds/collector compliance=registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:4.6.3 1
oc -n stackrox set image ds/collector collector=registry.redhat.io/advanced-cluster-security/rhacs-collector-rhel8:4.6.3 1
oc -n stackrox set image ds/collector collector=registry.redhat.io/advanced-cluster-security/rhacs-collector-slim-rhel8:{rhacs-version}
oc -n stackrox set image deploy/admission-control admission-control=registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:4.6.3
[[ -z "USD(oc -n stackrox get deployment sensor -o yaml | grep POD_NAMESPACE)" ]] && oc -n stackrox patch deployment sensor --type=json -p '[{"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}}]'
[[ -z "USD(oc -n stackrox get deployment admission-control -o yaml | grep POD_NAMESPACE)" ]] && oc -n stackrox patch deployment admission-control --type=json -p '[{"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}}]'
oc -n stackrox describe pods | grep 'openshift.io/scc\|^Name:'
Name: admission-control-6f4dcc6b4c-2phwd openshift.io/scc: stackrox-admission-control # Name: central-575487bfcb-sjdx8 openshift.io/scc: stackrox-central Name: central-db-7c7885bb-6bgbd openshift.io/scc: stackrox-central-db Name: collector-56nkr openshift.io/scc: stackrox-collector # Name: scanner-68fc55b599-f2wm6 openshift.io/scc: stackrox-scanner Name: scanner-68fc55b599-fztlh # Name: sensor-84545f86b7-xgdwf openshift.io/scc: stackrox-sensor #
apiVersion: rbac.authorization.k8s.io/v1 kind: Role 1 metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: central app.kubernetes.io/instance: stackrox-central-services app.kubernetes.io/name: stackrox app.kubernetes.io/part-of: stackrox-central-services app.kubernetes.io/version: 4.4.0 name: use-central-db-scc 2 namespace: stackrox 3 Rules: 4 - apiGroups: - security.openshift.io resourceNames: - nonroot-v2 resources: - securitycontextconstraints verbs: - use - - - apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: central app.kubernetes.io/instance: stackrox-central-services app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: stackrox app.kubernetes.io/part-of: stackrox-central-services app.kubernetes.io/version: 4.4.0 name: use-central-scc namespace: stackrox rules: - apiGroups: - security.openshift.io resourceNames: - nonroot-v2 resources: - securitycontextconstraints verbs: - use - - - apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: scanner app.kubernetes.io/instance: stackrox-central-services app.kubernetes.io/name: stackrox app.kubernetes.io/part-of: stackrox-central-services app.kubernetes.io/version: 4.4.0 name: use-scanner-scc namespace: stackrox rules: - apiGroups: - security.openshift.io resourceNames: - nonroot-v2 resources: - securitycontextconstraints verbs: - use - - - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding 5 metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: central app.kubernetes.io/instance: stackrox-central-services app.kubernetes.io/name: stackrox app.k ubernetes.io/part-of: stackrox-central-services app.kubernetes.io/version: 4.4.0 name: central-db-use-scc 6 namespace: stackrox roleRef: 7 apiGroup: rbac.authorization.k8s.io kind: Role name: use-central-db-scc subjects: 8 - kind: ServiceAccount name: central-db namespace: stackrox - - - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: central app.kubernetes.io/instance: stackrox-central-services app.kubernetes.io/name: stackrox app.kubernetes.io/part-of: stackrox-central-services app.kubernetes.io/version: 4.4.0 name: central-use-scc namespace: stackrox roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: use-central-scc subjects: - kind: ServiceAccount name: central namespace: stackrox - - - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: scanner app.kubernetes.io/instance: stackrox-central-services app.kubernetes.io/name: stackrox app.kubernetes.io/part-of: stackrox-central-services app.kubernetes.io/version: 4.4.0 name: scanner-use-scc namespace: stackrox roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: use-scanner-scc subjects: - kind: ServiceAccount name: scanner namespace: stackrox - - -
oc -n stackrox create -f ./update-central.yaml
apiVersion: rbac.authorization.k8s.io/v1 kind: Role 1 metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: collector app.kubernetes.io/instance: stackrox-secured-cluster-services app.kubernetes.io/name: stackrox app.kubernetes.io/part-of: stackrox-secured-cluster-services app.kubernetes.io/version: 4.4.0 auto-upgrade.stackrox.io/component: sensor name: use-privileged-scc 2 namespace: stackrox 3 rules: 4 - apiGroups: - security.openshift.io resourceNames: - privileged resources: - securitycontextconstraints verbs: - use - - - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding 5 metadata: annotations: email: support@stackrox.com owner: stackrox labels: app.kubernetes.io/component: collector app.kubernetes.io/instance: stackrox-secured-cluster-services app.kubernetes.io/name: stackrox app.kubernetes.io/part-of: stackrox-secured-cluster-services app.kubernetes.io/version: 4.4.0 auto-upgrade.stackrox.io/component: sensor name: collector-use-scc 6 namespace: stackrox roleRef: 7 apiGroup: rbac.authorization.k8s.io kind: Role name: use-privileged-scc subjects: 8 - kind: ServiceAccount name: collector namespace: stackrox - - -
oc -n stackrox create -f ./update-scs.yaml
oc delete scc/stackrox-central scc/stackrox-central-db scc/stackrox-scanner
oc delete scc/stackrox-admission-control scc/stackrox-collector scc/stackrox-sensor
oc -n stackrox describe pods | grep 'openshift.io/scc\|^Name:'
oc -n stackrox edit deploy/sensor 1
oc -n stackrox edit deploy/collector 1
oc -n stackrox edit deploy/admission-control 1
oc get deploy,ds -n stackrox -o wide 1
oc get pod -n stackrox --watch 1
oc -n stackrox patch daemonset/collector -p '{"spec":{"template":{"spec":{"containers":[{"name":"compliance","env":[{"name":"ROX_METRICS_PORT","value":"disabled"},{"name":"ROX_NODE_SCANNING_ENDPOINT","value":"127.0.0.1:8444"},{"name":"ROX_NODE_SCANNING_INTERVAL","value":"4h"},{"name":"ROX_NODE_SCANNING_INTERVAL_DEVIATION","value":"24m"},{"name":"ROX_NODE_SCANNING_MAX_INITIAL_WAIT","value":"5m"},{"name":"ROX_RHCOS_NODE_SCANNING","value":"true"},{"name":"ROX_CALL_NODE_INVENTORY_ENABLED","value":"true"}]}]}}}}'
oc -n stackrox patch daemonset/collector -p '{"spec":{"template":{"spec":{"containers":[{"name":"compliance","env":[{"name":"ROX_METRICS_PORT","value":":9091"},{"name":"ROX_NODE_SCANNING_ENDPOINT","value":"127.0.0.1:8444"},{"name":"ROX_NODE_SCANNING_INTERVAL","value":"4h"},{"name":"ROX_NODE_SCANNING_INTERVAL_DEVIATION","value":"24m"},{"name":"ROX_NODE_SCANNING_MAX_INITIAL_WAIT","value":"5m"},{"name":"ROX_RHCOS_NODE_SCANNING","value":"true"},{"name":"ROX_CALL_NODE_INVENTORY_ENABLED","value":"true"}]}]}}}}'
oc -n stackrox patch daemonset/collector -p '{"spec":{"template":{"spec":{"volumes":[{"name":"tmp-volume","emptyDir":{}},{"name":"cache-volume","emptyDir":{"sizeLimit":"200Mi"}}]}}}}'
oc -n stackrox patch daemonset/collector -p '{"spec":{"template":{"spec":{"containers":[{"command":["/scanner","--nodeinventory","--config=",""],"env":[{"name":"ROX_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"ROX_CLAIR_V4_SCANNING","value":"true"},{"name":"ROX_COMPLIANCE_OPERATOR_INTEGRATION","value":"true"},{"name":"ROX_CSV_EXPORT","value":"false"},{"name":"ROX_DECLARATIVE_CONFIGURATION","value":"false"},{"name":"ROX_INTEGRATIONS_AS_CONFIG","value":"false"},{"name":"ROX_NETPOL_FIELDS","value":"true"},{"name":"ROX_NETWORK_DETECTION_BASELINE_SIMULATION","value":"true"},{"name":"ROX_NETWORK_GRAPH_PATTERNFLY","value":"true"},{"name":"ROX_NODE_SCANNING_CACHE_TIME","value":"3h36m"},{"name":"ROX_NODE_SCANNING_INITIAL_BACKOFF","value":"30s"},{"name":"ROX_NODE_SCANNING_MAX_BACKOFF","value":"5m"},{"name":"ROX_PROCESSES_LISTENING_ON_PORT","value":"false"},{"name":"ROX_QUAY_ROBOT_ACCOUNTS","value":"true"},{"name":"ROX_ROXCTL_NETPOL_GENERATE","value":"true"},{"name":"ROX_SOURCED_AUTOGENERATED_INTEGRATIONS","value":"false"},{"name":"ROX_SYSLOG_EXTRA_FIELDS","value":"true"},{"name":"ROX_SYSTEM_HEALTH_PF","value":"false"},{"name":"ROX_VULN_MGMT_WORKLOAD_CVES","value":"false"}],"image":"registry.redhat.io/advanced-cluster-security/rhacs-scanner-slim-rhel8:4.6.3","imagePullPolicy":"IfNotPresent","name":"node-inventory","ports":[{"containerPort":8444,"name":"grpc","protocol":"TCP"}],"volumeMounts":[{"mountPath":"/host","name":"host-root-ro","readOnly":true},{"mountPath":"/tmp/","name":"tmp-volume"},{"mountPath":"/cache","name":"cache-volume"}]}]}}}}'
oc -n stackrox rollout undo deploy/central 1
oc -n stackrox rollout undo deploy/central 1
oc -n stackrox edit configmap/central-config 1
data: central-config.yaml: | maintenance: safeMode: false compaction: enabled: true bucketFillFraction: .5 freeFractionThreshold: 0.75 forceRollbackVersion: <x.x.x.x> 1
oc -n stackrox \ 1 set image deploy/central central=registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:<x.x.x.x> 2
Upgrader failed to execute PreflightStage of the roll-forward workflow: executing stage "Run preflight checks": preflight check "Kubernetes authorization" reported errors. This usually means that access is denied. Have you configured this Secured Cluster for automatically receiving upgrades?"
"Upgrade initialization error: The upgrader pods have trouble pulling the new image: Error pulling image: (...) (<image_reference:tag>: not found)"
"Upgrade initialization error: Pod terminated: (Error)"
kubectl -n <namespace> logs deploy/sensor-upgrader 1