Business continuity Red Hat Advanced Cluster Management for Kubernetes 2.12 Business continuity

Commands:
apiVersion: my.group/v1alpha1 kind: MyResource metadata: labels: cluster.open-cluster-management.io/backup: ""
apiVersion: my.group/v1alpha1 kind: MyResource metadata: labels: velero.io/exclude-from-backup: "true"
apiVersion: my.group/v1alpha1 kind: MyResource metadata: labels: cluster.open-cluster-management.io/backup: cluster-activation
installer.open-cluster-management.io/oadp-subscription-spec: '{"channel": "stable-1.4"}'
apiVersion: operator.open-cluster-management.io/v1 kind: MultiClusterHub metadata: annotations: installer.open-cluster-management.io/oadp-subscription-spec: '{"channel": "stable-1.5","installPlanApproval": "Automatic","name": "redhat-oadp-operator","source": "redhat-operators","sourceNamespace": "openshift-marketplace"}' name: multiclusterhub spec: {}
apiVersion: oadp.openshift.io/v1alpha1 kind: DataProtectionApplication metadata: name: dpa-sample spec: configuration: velero: defaultPlugins: - openshift - aws restic: enable: true backupLocations: - name: default velero: provider: aws default: true objectStorage: bucket: my-bucket prefix: my-prefix config: region: us-east-1 profile: "default" credential: name: cloud-credentials key: cloud snapshotLocations: - name: default velero: provider: aws config: region: us-west-2 profile: "default"
apiVersion: operator.open-cluster-management.io/v1 kind: MultiClusterHub metadata: annotations: installer.open-cluster-management.io/oadp-subscription-spec: '{"source": "redhat-operator-index"}'
get catalogsource -A
NAMESPACE NAME DISPLAY TYPE PUBLISHER AGE openshift-marketplace acm-custom-registry Advanced Cluster Management grpc Red Hat 42h openshift-marketplace multiclusterengine-catalog MultiCluster Engine grpc Red Hat 42h openshift-marketplace redhat-operator-index grpc 42h
apiVersion: operator.open-cluster-management.io/v1 kind: MultiClusterHub metadata: name: multiclusterhub namespace: open-cluster-management spec: availabilityConfig: High enableClusterBackup: false imagePullSecret: multiclusterhub-operator-pull-secret ingress: sslCiphers: - ECDHE-ECDSA-AES256-GCM-SHA384 - ECDHE-RSA-AES256-GCM-SHA384 - ECDHE-ECDSA-AES128-GCM-SHA256 - ECDHE-RSA-AES128-GCM-SHA256 overrides: components: - enabled: true name: multiclusterhub-repo - enabled: true name: search - enabled: true name: management-ingress - enabled: true name: console - enabled: true name: insights - enabled: true name: grc - enabled: true name: cluster-lifecycle - enabled: true name: volsync - enabled: true name: multicluster-engine - enabled: true name: cluster-backup separateCertificateManagement: false
create -f cluster_v1beta1_backupschedule.yaml
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm namespace: open-cluster-management-backup spec: veleroSchedule: 0 */2 * * * 1 veleroTtl: 120h 2 useManagedServiceAccount: true 3 paused: false 4
get BackupSchedule -n open-cluster-management-backup
get schedules -A | grep acm
get backupschedule -A
NAMESPACE NAME PHASE MESSAGE openshift-adp schedule-hub-1 BackupCollision Backup acm-resources-schedule-20220301234625, from cluster with id [be97a9eb-60b8-4511-805c-298e7c0898b3] is using the same storage location. This is a backup collision with current cluster [1f30bfe5-0588-441c-889e-eaf0ae55f941] backup. Review and resolve the collision then create a new BackupSchedule resource to resume backups from this cluster.
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: veleroManagedClustersBackupName: latest veleroCredentialsBackupName: latest veleroResourcesBackupName: latest
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm-passive-sync namespace: open-cluster-management-backup spec: syncRestoreWithNewBackups: true # restore again when new backups are available restoreSyncInterval: 10m # check for new backups every 10 minutes cleanupBeforeRestore: CleanupRestored veleroManagedClustersBackupName: skip veleroCredentialsBackupName: latest veleroResourcesBackupName: latest
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm-passive namespace: open-cluster-management-backup spec: cleanupBeforeRestore: CleanupRestored veleroManagedClustersBackupName: skip veleroCredentialsBackupName: latest veleroResourcesBackupName: latest
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm-passive-activate namespace: open-cluster-management-backup spec: cleanupBeforeRestore: CleanupRestored veleroManagedClustersBackupName: latest veleroCredentialsBackupName: skip veleroResourcesBackupName: skip
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: cleanupBeforeRestore: None veleroManagedClustersBackupName: latest veleroCredentialsBackupName: latest veleroResourcesBackupName: latest
describe -n open-cluster-management-backup restore-acm
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: veleroManagedClustersBackupSchedule: latest veleroCredentialsBackupSchedule: latest veleroResourcesBackupSchedule: latest
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: veleroManagedClustersBackupName: latest veleroCredentialsBackupName: skip veleroResourcesBackupName: skip
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: veleroManagedClustersBackupName: acm-managed-clusters-schedule-20210902205438 veleroCredentialsBackupName: skip veleroResourcesBackupName: skip
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-filter-sample namespace: open-cluster-management-backup spec: cleanupBeforeRestore: None veleroCredentialsBackupName: latest veleroResourcesBackupName: latest veleroManagedClustersBackupName: latest excludedResources: - ManagedCluster orLabelSelectors: - matchExpressions: - values: - vb-managed-cls-2 key: name operator: In
spec: includedResources includedNamespaces excludedResources excludedNamespaces LabelSelector OrLabelSelector
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-filter-sample namespace: open-cluster-management-backup spec: cleanupBeforeRestore: None veleroCredentialsBackupName: latest veleroResourcesBackupName: latest veleroManagedClustersBackupName: latest excludedResources: - ManagedCluster includedNamespaces: - vb-managed-cls-2
spec: includedResources includedNamespaces excludedResources excludedNamespaces LabelSelector OrLabelSelector
describe -n open-cluster-management-backup <restore-name>
Spec: Cleanup Before Restore: CleanupRestored Restore Sync Interval: 4m Sync Restore With New Backups: true Velero Credentials Backup Name: latest Velero Managed Clusters Backup Name: skip Velero Resources Backup Name: latest Status: Last Message: Velero restores have run to completion, restore will continue to sync with new backups Phase: Enabled Velero Credentials Restore Name: example-acm-credentials-schedule-20220406171919 Velero Resources Restore Name: example-acm-resources-schedule-20220406171920 Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Prepare to restore: 76m Restore controller Cleaning up resources for backup acm-credentials-hive-schedule-20220406155817 Normal Prepare to restore: 76m Restore controller Cleaning up resources for backup acm-credentials-cluster-schedule-20220406155817 Normal Prepare to restore: 76m Restore controller Cleaning up resources for backup acm-credentials-schedule-20220406155817 Normal Prepare to restore: 76m Restore controller Cleaning up resources for backup acm-resources-generic-schedule-20220406155817 Normal Prepare to restore: 76m Restore controller Cleaning up resources for backup acm-resources-schedule-20220406155817 Normal Velero restore created: 74m Restore controller example-acm-credentials-schedule-20220406155817 Normal Velero restore created: 74m Restore controller example-acm-resources-generic-schedule-20220406155817 Normal Velero restore created: 74m Restore controller example-acm-resources-schedule-20220406155817 Normal Velero restore created: 74m Restore controller example-acm-credentials-cluster-schedule-20220406155817 Normal Velero restore created: 74m Restore controller example-acm-credentials-hive-schedule-20220406155817 Normal Prepare to restore: 64m Restore controller Cleaning up resources for backup acm-resources-schedule-20220406165328 Normal Prepare to restore: 62m Restore controller Cleaning up resources for backup acm-credentials-hive-schedule-20220406165328 Normal Prepare to restore: 62m Restore controller Cleaning up resources for backup acm-credentials-cluster-schedule-20220406165328 Normal Prepare to restore: 62m Restore controller Cleaning up resources for backup acm-credentials-schedule-20220406165328 Normal Prepare to restore: 62m Restore controller Cleaning up resources for backup acm-resources-generic-schedule-20220406165328 Normal Velero restore created: 61m Restore controller example-acm-credentials-cluster-schedule-20220406165328 Normal Velero restore created: 61m Restore controller example-acm-credentials-schedule-20220406165328 Normal Velero restore created: 61m Restore controller example-acm-resources-generic-schedule-20220406165328 Normal Velero restore created: 61m Restore controller example-acm-resources-schedule-20220406165328 Normal Velero restore created: 61m Restore controller example-acm-credentials-hive-schedule-20220406165328 Normal Prepare to restore: 38m Restore controller Cleaning up resources for backup acm-resources-generic-schedule-20220406171920 Normal Prepare to restore: 38m Restore controller Cleaning up resources for backup acm-resources-schedule-20220406171920 Normal Prepare to restore: 36m Restore controller Cleaning up resources for backup acm-credentials-hive-schedule-20220406171919 Normal Prepare to restore: 36m Restore controller Cleaning up resources for backup acm-credentials-cluster-schedule-20220406171919 Normal Prepare to restore: 36m Restore controller Cleaning up resources for backup acm-credentials-schedule-20220406171919 Normal Velero restore created: 36m Restore controller example-acm-credentials-cluster-schedule-20220406171919 Normal Velero restore created: 36m Restore controller example-acm-credentials-schedule-20220406171919 Normal Velero restore created: 36m Restore controller example-acm-resources-generic-schedule-20220406171920 Normal Velero restore created: 36m Restore controller example-acm-resources-schedule-20220406171920 Normal Velero restore created: 36m Restore controller example-acm-credentials-hive-schedule-20220406171919
apiVersion: multicluster.openshift.io/v1 kind: MultiClusterEngine metadata: name: multiclusterhub spec: overrides: components: - enabled: true name: managedserviceaccount
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm-msa namespace: open-cluster-management-backup spec: veleroSchedule: veleroTtl: 120h useManagedServiceAccount: true
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm-msa namespace: open-cluster-management-backup spec: veleroSchedule: veleroTtl: 120h useManagedServiceAccount: true managedServiceAccountTTL: 300h
apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: name: managed-cluster-name spec: hubAcceptsClient: true leaseDurationSeconds: 60 managedClusterClientConfigs: url: <apiserver>
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm-msa namespace: open-cluster-management-backup spec: veleroSchedule: veleroTtl: 120h useManagedServiceAccount: false
spec: backupLocations: - velero: config: kmsKeyId: 502b409c-4da1-419f-a16e-eif453b3i49f profile: default region: us-east-1
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm-msa namespace: open-cluster-management-backup spec: veleroSchedule: 0 */1 * * * veleroTtl: 120h useManagedServiceAccount: true paused: true
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: cleanupBeforeRestore: None veleroManagedClustersBackupName: latest veleroCredentialsBackupName: latest veleroResourcesBackupName: latest
-f apply restore-acm.yaml
annotations: import.open-cluster-management.io/disable-auto-import: ''
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: cleanupBeforeRestore: None veleroManagedClustersBackupName: latest veleroCredentialsBackupName: latest veleroResourcesBackupName: latest
-f apply restore-acm.yaml
resources: limits: cpu: "1" memory: 256Mi requests: cpu: 500m memory: 128Mi
limits: cpu: "2" memory: 1Gi requests: cpu: 500m memory: 256Mi
apiVersion: oadp.openshift.io/v1alpha1 kind: DataProtectionApplication metadata: name: velero namespace: open-cluster-management-backup spec: configuration: velero: podConfig: resourceAllocations: limits: cpu: "2" memory: 1Gi requests: cpu: 500m memory: 256Mi
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm-msa namespace: open-cluster-management-backup spec: veleroSchedule: 0 */1 * * * veleroTtl: 120h useManagedServiceAccount: true paused: false
kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: cleanupBeforeRestore: None veleroManagedClustersBackupName: latest veleroCredentialsBackupName: latest veleroResourcesBackupName: latest excludedResources: 1 - ManagedCluster excludedNamespaces: 2 - managed-cls-1 - managed-cls-2 - managed-cls-n
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm-msa namespace: open-cluster-management-backup spec: veleroSchedule: "0 */1 * * *" veleroTtl: 120h useManagedServiceAccount: true paused: false
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: Restore metadata: name: restore-acm namespace: open-cluster-management-backup spec: cleanupBeforeRestore: CleanupRestored veleroManagedClustersBackupName: latest veleroCredentialsBackupName: latest veleroResourcesBackupName: latest
apiVersion: cluster.open-cluster-management.io/v1beta1 kind: BackupSchedule metadata: name: schedule-acm-msa namespace: open-cluster-management-backup spec: veleroSchedule: "0 */1 * * *" veleroTtl: 120h useManagedServiceAccount: true paused: false
addons.open-cluster-management.io/volsync=true
get csv -n openshift-operators
label managedcluster <managed-cluster-1> "addons.open-cluster-management.io/volsync"="true"
label managedcluster <managed-cluster-2> "addons.open-cluster-management.io/volsync"="true"
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: volsync namespace: <managed-cluster-1-namespace> spec: {}
apply -f volsync-mcao.yaml
annotations: operator-subscription-channel: stable-0.9
create ns <destination-ns>
apiVersion: volsync.backube/v1alpha1 kind: ReplicationDestination metadata: name: <destination> namespace: <destination-ns> spec: rsyncTLS: serviceType: LoadBalancer 1 copyMethod: Snapshot capacity: 2Gi 2 accessModes: [ReadWriteOnce] storageClassName: gp2-csi volumeSnapshotClassName: csi-aws-vsc
create -n <destination-ns> -f replication_destination.yaml
ADDRESS=`oc get replicationdestination <destination> -n <destination-ns> --template={{.status.rsyncTLS.address}}` echo USDADDRESS
a831264645yhrjrjyer6f9e4a02eb2-5592c0b3d94dd376.elb.us-east-1.amazonaws.com
KEYSECRET=`oc get replicationdestination <destination> -n <destination-ns> --template={{.status.rsyncTLS.keySecret}}` echo USDKEYSECRET
volsync-rsync-tls-destination-name
get secret -n <destination-ns> USDKEYSECRET -o yaml > /tmp/secret.yaml
vi /tmp/secret.yaml
create -f /tmp/secret.yaml
apiVersion: volsync.backube/v1alpha1 kind: ReplicationSource metadata: name: <source> 1 namespace: <source-ns> 2 spec: sourcePVC: <persistent_volume_claim> 3 trigger: schedule: "*/3 * * * *" #/* rsyncTLS: keySecret: <mykeysecret> 4 address: <my.host.com> 5 copyMethod: Snapshot storageClassName: gp2-csi volumeSnapshotClassName: csi-aws-vsc
sed -i "s/<my.host.com>/USDADDRESS/g" replication_source.yaml sed -i "s/<mykeysecret>/USDKEYSECRET/g" replication_source.yaml create -n <source> -f replication_source.yaml
describe ReplicationSource -n <source-ns> <source>
Status: Conditions: Last Transition Time: 2021-10-14T20:48:00Z Message: Synchronization in-progress Reason: SyncInProgress Status: True Type: Synchronizing Last Transition Time: 2021-10-14T20:41:41Z Message: Reconcile complete Reason: ReconcileComplete Status: True Type: Reconciled Last Sync Duration: 5m20.764642395s Last Sync Time: 2021-10-14T20:47:01Z Next Sync Time: 2021-10-14T20:48:00Z
create ns <destination-ns>
apiVersion: volsync.backube/v1alpha1 kind: ReplicationDestination metadata: name: <destination> namespace: <destination-ns> spec: rsync: serviceType: LoadBalancer copyMethod: Snapshot capacity: 2Gi accessModes: [ReadWriteOnce] storageClassName: gp2-csi volumeSnapshotClassName: csi-aws-vsc
create -n <destination-ns> -f replication_destination.yaml
ADDRESS=`oc get replicationdestination <destination> -n <destination-ns> --template={{.status.rsync.address}}` echo USDADDRESS
a831264645yhrjrjyer6f9e4a02eb2-5592c0b3d94dd376.elb.us-east-1.amazonaws.com
SSHKEYS=`oc get replicationdestination <destination> -n <destination-ns> --template={{.status.rsync.sshKeys}}` echo USDSSHKEYS
volsync-rsync-dst-src-destination-name
get secret -n <destination-ns> USDSSHKEYS -o yaml > /tmp/secret.yaml
vi /tmp/secret.yaml
create -f /tmp/secret.yaml
apiVersion: volsync.backube/v1alpha1 kind: ReplicationSource metadata: name: <source> namespace: <source-ns> spec: sourcePVC: <persistent_volume_claim> trigger: schedule: "*/3 * * * *" #/* rsync: sshKeys: <mysshkeys> address: <my.host.com> copyMethod: Snapshot storageClassName: gp2-csi volumeSnapshotClassName: csi-aws-vsc
sed -i "s/<my.host.com>/USDADDRESS/g" replication_source.yaml sed -i "s/<mysshkeys>/USDSSHKEYS/g" replication_source.yaml create -n <source> -f replication_source.yaml
describe ReplicationSource -n <source-ns> <source>
Status: Conditions: Last Transition Time: 2021-10-14T20:48:00Z Message: Synchronization in-progress Reason: SyncInProgress Status: True Type: Synchronizing Last Transition Time: 2021-10-14T20:41:41Z Message: Reconcile complete Reason: ReconcileComplete Status: True Type: Reconciled Last Sync Duration: 5m20.764642395s Last Sync Time: 2021-10-14T20:47:01Z Next Sync Time: 2021-10-14T20:48:00Z
apiVersion: v1 kind: Secret metadata: name: restic-config type: Opaque stringData: RESTIC_REPOSITORY: <my-restic-repository> RESTIC_PASSWORD: <my-restic-password> AWS_ACCESS_KEY_ID: access AWS_SECRET_ACCESS_KEY: password
apiVersion: volsync.backube/v1alpha1 kind: ReplicationSource metadata: name: mydata-backup spec: sourcePVC: <source> trigger: schedule: "*/30 * * * *" #\* restic: pruneIntervalDays: 14 repository: <restic-config> retain: hourly: 6 daily: 5 weekly: 4 monthly: 2 yearly: 1 copyMethod: Clone # The StorageClass to use when creating the PiT copy (same as source PVC if omitted) #storageClassName: my-sc-name # The VSC to use if the copy method is Snapshot (default if omitted) #volumeSnapshotClassName: my-vsc-name
annotate namespace <namespace> volsync.backube/privileged-movers=true
kind: PersistentVolumeClaim apiVersion: v1 metadata: name: <pvc-name> spec: accessModes: - ReadWriteOnce resources: requests: storage: 3Gi
apiVersion: volsync.backube/v1alpha1 kind: ReplicationDestination metadata: name: <destination> spec: trigger: manual: restore-once restic: repository: <restic-repo> destinationPVC: <pvc-name> copyMethod: Direct
apiVersion: volsync.backube/v1alpha1 kind: ReplicationSource metadata: name: <source> namespace: <source-ns> spec: sourcePVC: <source-pvc> trigger: schedule: "*/6 * * * *" #\* rclone: rcloneConfigSection: <intermediate-s3-bucket> rcloneDestPath: <destination-bucket> rcloneConfig: <rclone-secret> copyMethod: Snapshot storageClassName: <my-sc-name> volumeSnapshotClassName: <my-vsc>
apiVersion: volsync.backube/v1alpha1 kind: ReplicationDestination metadata: name: database-destination namespace: dest spec: trigger: schedule: "3,9,15,21,27,33,39,45,51,57 * * * *" #/* rclone: rcloneConfigSection: <intermediate-s3-bucket> rcloneDestPath: <destination-bucket> rcloneConfig: <rclone-secret> copyMethod: Snapshot accessModes: [ReadWriteOnce] capacity: 10Gi storageClassName: <my-sc> volumeSnapshotClassName: <my-vsc>
annotate namespace <namespace> volsync.backube/privileged-movers=true
apiVersion: v1 kind: PersistentVolumeClaim metadata: name: <pvc-name> namespace: <destination-ns> spec: accessModes: - ReadWriteOnce dataSourceRef: kind: ReplicationDestination apiGroup: volsync.backube name: <replicationdestination_to_replace> resources: requests: storage: 2Gi
kubectl get pvc -n <destination-ns>
spec: trigger: schedule: "*/6 * * * *"
apiVersion: v1 data: # echo -n 1:23b7395fafc3e842bd8ac0fe142e6ad1 | base64 psk.txt: MToyM2I3Mzk1ZmFmYzNlODQyYmQ4YWMwZmUxNDJlNmFkMQ== kind: Secret metadata: name: tls-key-secret type: Opaque