Backup and restore Red Hat build of MicroShift 4.18 Backup and restore the Red Hat build of MicroShift database Red Hat OpenShift Documentation Team

Commands:
sudo systemctl stop microshift
sudo crictl ps -a
sudo systemctl stop kubepods.slice
sudo microshift backup /var/lib/microshift-backups/<my_manual_backup>
??? I1017 07:38:16.770506 5900 data_manager.go:92] "Copying data to backup directory" storage="/var/lib/microshift-backups" name="test" data="/var/lib/microshift" ??? I1017 07:38:16.770713 5900 data_manager.go:227] "Starting copy" cmd="/bin/cp --verbose --recursive --preserve --reflink=auto /var/lib/microshift /var/lib/microshift-backups/test" ??? I1017 07:38:16.776162 5900 data_manager.go:241] "Finished copy" cmd="/bin/cp --verbose --recursive --preserve --reflink=auto /var/lib/microshift /var/lib/microshift-backups/test" ??? I1017 07:38:16.776256 5900 data_manager.go:125] "Copied data to backup directory" backup="/var/lib/microshift-backups/test" data="/var/lib/microshift"
sudo microshift backup /mnt/<other_backups_location>/<another_manual_backup>
sudo microshift restore /var/lib/microshift-backups/<my_manual_backup>
??? I1017 07:39:52.055165 6007 data_manager.go:131] "Copying backup to data directory" storage="/var/lib/microshift-backups" name="test" data="/var/lib/microshift" ??? I1017 07:39:52.055243 6007 data_manager.go:154] "Renaming existing data dir" data="/var/lib/microshift" renamedTo="/var/lib/microshift.saved" ??? I1017 07:39:52.055326 6007 data_manager.go:227] "Starting copy" cmd="/bin/cp --verbose --recursive --preserve --reflink=auto /var/lib/microshift-backups/test /var/lib/microshift" ??? I1017 07:39:52.061363 6007 data_manager.go:241] "Finished copy" cmd="/bin/cp --verbose --recursive --preserve --reflink=auto /var/lib/microshift-backups/test /var/lib/microshift" ??? I1017 07:39:52.061404 6007 data_manager.go:175] "Removing temporary data directory" path="/var/lib/microshift.saved" ??? I1017 07:39:52.063745 6007 data_manager.go:180] "Copied backup to data directory" name="test" data="/var/lib/microshift"
sudo microshift restore /<mnt>/<other_backups_location>/<another_manual_backup>
sudo microshift backup --auto-recovery <path_of_directory> 1
??? I1104 09:18:52.100725 8906 system.go:58] "OSTree deployments" deployments=[{"id":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1","booted":true,"staged":false,"pinned":false},{"id":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0","booted":false,"staged":false,"pinned":false}] ??? I1104 09:18:52.100895 8906 data_manager.go:83] "Copying data to backup directory" storage="/var/lib/microshift-auto-recovery" name="20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1" data="/var/lib/microshift" ??? I1104 09:18:52.102296 8906 disk_space.go:33] Calculated size of "/var/lib/microshift": 261M - increasing by 10% for safety: 287M ??? I1104 09:18:52.102321 8906 disk_space.go:44] Calculated available disk space for "/var/lib/microshift-auto-recovery": 1658M ??? I1104 09:18:52.105700 8906 atomic_dir_copy.go:66] "Made an intermediate copy" cmd="/bin/cp --verbose --recursive --preserve --reflink=auto /var/lib/microshift /var/lib/microshift-auto-recovery/20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1.tmp.99142" ??? I1104 09:18:52.105732 8906 atomic_dir_copy.go:115] "Renamed to final destination" src="/var/lib/microshift-auto-recovery/20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1.tmp.99142" dest="/var/lib/microshift-auto-recovery/20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1" ??? I1104 09:18:52.105749 8906 data_manager.go:120] "Copied data to backup directory" backup="/var/lib/microshift-auto-recovery/20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1" data="/var/lib/microshift" /var/lib/microshift-auto-recovery/20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1
sudo ls -la <path_of_directory> 1
sudo microshift restore --auto-recovery <path_of_directory> 1
??? I1104 09:19:28.617225 8950 state.go:80] "Read state from the disk" state={"LastBackup":"20241022101528_default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0"} ??? I1104 09:19:28.617323 8950 storage.go:78] "Auto-recovery backup storage read and parsed" dirs=["20241022101255_default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0","20241022101520_default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0","20241022101528_default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0","20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1","restored"] backups=[{"CreationTime":"2024-10-22T10:12:55Z","Version":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0"},{"CreationTime":"2024-10-22T10:15:20Z","Version":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0"},{"CreationTime":"2024-10-22T10:15:28Z","Version":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0"},{"CreationTime":"2024-11-04T09:18:52Z","Version":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1"}] ??? I1104 09:19:28.617350 8950 storage.go:40] "Filtered list of backups - removed previously restored backup" removed="20241022101528_default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0" newList=[{"CreationTime":"2024-10-22T10:12:55Z","Version":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0"},{"CreationTime":"2024-10-22T10:15:20Z","Version":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0"},{"CreationTime":"2024-11-04T09:18:52Z","Version":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1"}] ??? I1104 09:19:28.633237 8950 system.go:58] "OSTree deployments" deployments=[{"id":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1","booted":true,"staged":false,"pinned":false},{"id":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0","booted":false,"staged":false,"pinned":false}] ??? I1104 09:19:28.633258 8950 storage.go:49] "Filtered list of backups by version" version="default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1" newList=[{"CreationTime":"2024-11-04T09:18:52Z","Version":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1"}] ??? I1104 09:19:28.633268 8950 restore.go:170] "Potential backups" bz=[{"CreationTime":"2024-11-04T09:18:52Z","Version":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1"}] ??? I1104 09:19:28.633277 8950 restore.go:173] "Candidate backup for restore" b={"CreationTime":"2024-11-04T09:18:52Z","Version":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1"} ??? I1104 09:19:28.634007 8950 disk_space.go:33] Calculated size of "/var/lib/microshift-auto-recovery/20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1": 261M - increasing by 10% for safety: 287M ??? I1104 09:19:28.634096 8950 disk_space.go:44] Calculated available disk space for "/var/lib": 1658M ??? I1104 09:19:28.634507 8950 disk_space.go:33] Calculated size of "/var/lib/microshift": 261M - increasing by 10% for safety: 287M ??? I1104 09:19:28.634522 8950 disk_space.go:44] Calculated available disk space for "/var/lib/microshift-auto-recovery": 1658M ??? I1104 09:19:28.649719 8950 system.go:58] "OSTree deployments" deployments=[{"id":"default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1","booted":true,"staged":false,"pinned":false},{"id":"default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0","booted":false,"staged":false,"pinned":false}] ??? I1104 09:19:28.653880 8950 atomic_dir_copy.go:66] "Made an intermediate copy" cmd="/bin/cp --verbose --recursive --preserve --reflink=auto /var/lib/microshift /var/lib/microshift-auto-recovery/failed/20241104091928_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1.tmp.22742" ??? I1104 09:19:28.657362 8950 atomic_dir_copy.go:66] "Made an intermediate copy" cmd="/bin/cp --verbose --recursive --preserve --reflink=auto /var/lib/microshift-auto-recovery/20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1 /var/lib/microshift.tmp.482" ??? I1104 09:19:28.657385 8950 state.go:40] "Saving intermediate state" state="{\"LastBackup\":\"20241104091852_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1\"}" path="/var/lib/microshift-auto-recovery/state.json.tmp.41544" ??? I1104 09:19:28.662438 8950 atomic_dir_copy.go:115] "Renamed to final destination" src="/var/lib/microshift.tmp.482" dest="/var/lib/microshift" ??? I1104 09:19:28.662451 8950 state.go:46] "Moving state file to final path" intermediatePath="/var/lib/microshift-auto-recovery/state.json.tmp.41544" finalPath="/var/lib/microshift-auto-recovery/state.json" ??? I1104 09:19:28.662521 8950 atomic_dir_copy.go:115] "Renamed to final destination" src="/var/lib/microshift-auto-recovery/failed/20241104091928_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1.tmp.22742" dest="/var/lib/microshift-auto-recovery/failed/20241104091928_default-b3442053c9ce69310cd54140d8d592234c5306e4c5132de6efe615f79c84300a.1" ??? I1104 09:19:28.662969 8950 atomic_dir_copy.go:115] "Renamed to final destination" src="/var/lib/microshift-auto-recovery/20241022101528_default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0" dest="/var/lib/microshift-auto-recovery/restored/20241022101528_default-a129624b9233fa54fe3574f1aa211bc2d85e1052b52245fe7d83f10c2f6d28e3.0" ??? I1104 09:19:28.662983 8950 restore.go:141] "Auto-recovery restore completed".
sudo systemctl restart microshift
oc get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE default i-06166fbb376f14a8bus-west-2computeinternal-debug-qtwcr 1/1 Running 0 46m kube-system csi-snapshot-controller-5c6586d546-lprv4 1/1 Running 0 51m openshift-dns dns-default-45jl7 2/2 Running 0 50m openshift-dns node-resolver-7wmzf 1/1 Running 0 51m openshift-ingress router-default-78b86fbf9d-qvj9s 1/1 Running 0 51m openshift-ovn-kubernetes ovnkube-master-5rfhh 4/4 Running 0 51m openshift-ovn-kubernetes ovnkube-node-gcnt6 1/1 Running 0 51m openshift-service-ca service-ca-bf5b7c9f8-pn6rk 1/1 Running 0 51m openshift-storage topolvm-controller-549f7fbdd5-7vrmv 5/5 Running 0 51m openshift-storage topolvm-node-rht2m 3/3 Running 0 50m
sudo mkdir -p /usr/lib/systemd/system/microshift.service.d
sudo tee /usr/lib/systemd/system/microshift.service.d/10-auto-recovery.conf > /dev/null <<'EOF' [Unit] OnFailure=microshift-auto-recovery.service StartLimitIntervalSec=25s 1 EOF
sudo tee /usr/lib/systemd/system/microshift-auto-recovery.service > /dev/null <<'EOF' [Unit] Description=MicroShift auto-recovery [Service] Type=oneshot ExecStart=/usr/bin/microshift-auto-recovery [Install] WantedBy=multi-user.target EOF
sudo tee /usr/bin/microshift-auto-recovery > /dev/null <<'EOF' #!/usr/bin/env bash set -xeuo pipefail If greenboot uses a non-default file for clearing boot_counter, use boot_success instead. if grep -q "/boot/grubenv" /usr/libexec/greenboot/greenboot-grub2-set-success; then if grub2-editenv - list | grep -q ^boot_success=0; then echo "Greenboot didn't decide the system is healthy after staging new deployment." echo "Quitting to not interfere with the process" exit 0 fi else if grub2-editenv - list | grep -q ^boot_counter=; then echo "Greenboot didn't decide the system is healthy after staging a new deployment." echo "Quitting to not interfere with the process" exit 0 fi fi /usr/bin/microshift restore --auto-recovery /var/lib/microshift-auto-recovery /usr/bin/systemctl reset-failed microshift /usr/bin/systemctl start microshift echo "DONE" EOF
sudo chmod +x /usr/bin/microshift-auto-recovery
sudo systemctl daemon-reload
[[customizations.directories]] path = "/etc/systemd/system/microshift.service.d" [[customizations.directories]] path = "/etc/bin" [[customizations.files]] path = "/etc/systemd/system/microshift.service.d/10-auto-recovery.conf" data = """ [Unit] OnFailure=microshift-auto-recovery.service """ [[customizations.files]] path = "/etc/systemd/system/microshift-auto-recovery.service" data = """ [Unit] Description=MicroShift auto-recovery [Service] Type=oneshot ExecStart=/etc/bin/microshift-auto-recovery [Install] WantedBy=multi-user.target """ [[customizations.files]] path = "/etc/bin/microshift-auto-recovery" mode = "0755" data = """ #!/usr/bin/env bash set -xeuo pipefail If greenboot uses a non-default file for clearing boot_counter, use boot_success instead. if grep -q "/boot/grubenv" /usr/libexec/greenboot/greenboot-grub2-set-success; then if grub2-editenv - list | grep -q ^boot_success=0; then echo "Greenboot didn't decide the system is healthy after staging a new deployment." echo "Quitting to not interfere with the process" exit 0 fi else if grub2-editenv - list | grep -q ^boot_counter=; then echo "Greenboot didn't decide the system is healthy after staging a new deployment." echo "Quitting to not interfere with the process" exit 0 fi fi /usr/bin/microshift restore --auto-recovery /var/lib/microshift-auto-recovery /usr/bin/systemctl reset-failed microshift /usr/bin/systemctl start microshift echo "DONE" """
RUN mkdir -p /usr/lib/systemd/system/microshift.service.d COPY ./auto-rec/10-auto-recovery.conf /usr/lib/systemd/system/microshift.service.d/10-auto-recovery.conf COPY ./auto-rec/microshift-auto-recovery.service /usr/lib/systemd/system/ COPY ./auto-rec/microshift-auto-recovery /usr/bin/ RUN chmod +x /usr/bin/microshift-auto-recovery
PULL_SECRET=~/.pull-secret.json USER_PASSWD=<your_redhat_user_password> IMAGE_NAME=microshift-4.18-bootc sudo podman build --authfile "USD{PULL_SECRET}" -t "USD{IMAGE_NAME}" --build-arg USER_PASSWD="USD{USER_PASSWD}" -f Containerfile
sudo podman images "USD{IMAGE_NAME}"
REPOSITORY TAG IMAGE ID CREATED SIZE localhost/microshift-4.18-bootc latest 193425283c00 2 minutes ago 2.31 GB