Repository: rook/rook
Pull request #15761 (2025-04-22 06:33:35)

osd: fix osd disk cleanup for mpath setups
ceph-volume raw list will return the main disk instead of the mpath setup. `ceph-volume lvm zap` on the main disks fails. This PR uses `lsblk /mnt/<diskreference> ` to fetch the actual mapath disk and use it for cleaning

Testing: 

Before the fix: Notice that `ceph-volume raw list` returns OSD info with device name as `/dev/sdb`. And running `ceph-volume lvm zap` on the parent disk causes failure. 
```
[santosh@hetzner-dominguez ~]$ oc logs rook-ceph-osd-prepare-a349f13b45cd5b2d3f756a6f216d59c2-vfc6q
Defaulted container "provision" out of: provision, copy-bins (init), blkdevmapper (init)
2025/04/21 14:14:18 maxprocs: Leaving GOMAXPROCS=16: CPU quota undefined
2025-04-21 14:14:18.766927 I | cephcmd: desired devices to configure osds: [{Name:/mnt/ocs-deviceset-odf-disks-0-data-0jtnxv OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-04-21 14:14:18.768013 I | rookcmd: starting Rook v4.18.1-0.555802e077c136b73462af33dd7a60ee2ce3ec7f with arguments '/rook/rook ceph osd provision'
2025-04-21 14:14:18.768024 I | rookcmd: flag values: --cluster-id=6ebe7230-b4da-47b9-8eb6-30356c548fe7, --cluster-name=ocs-storagecluster-cephcluster, --data-device-filter=, --data-device-path-filter=, --data-devices=[{"id":"/mnt/ocs-deviceset-odf-disks-0-data-0jtnxv","storeConfig":{"osdsPerDevice":1}}], --encrypted-device=false, --force-format=false, --help=false, --location=, --log-level=DEBUG, --metadata-device=, --node-name=ocs-deviceset-odf-disks-0-data-0jtnxv, --osd-crush-device-class=ssd, --osd-crush-initial-weight=, --osd-database-size=0, --osd-store-type=bluestore, --osd-wal-size=576, --osds-per-device=1, --pvc-backed-osd=true, --replace-osd=0
2025-04-21 14:14:18.768033 I | ceph-spec: parsing mon endpoints: a=172.30.212.132:3300,b=172.30.151.172:3300,c=172.30.165.77:3300
2025-04-21 14:14:18.772936 I | op-osd: CRUSH location=root=default host=santander-worker-1-ddomingu-com
2025-04-21 14:14:18.772970 I | cephcmd: crush location of osd: root=default host=santander-worker-1-ddomingu-com
2025-04-21 14:14:18.774169 I | cephclient: writing config file /var/lib/rook/openshift-storage/openshift-storage.config
2025-04-21 14:14:18.774298 I | cephclient: generated admin config in /var/lib/rook/openshift-storage
2025-04-21 14:14:18.774399 D | cephclient: config file @ /etc/ceph/ceph.conf:
[global]
fsid                               = 044f8070-5ce0-4edf-b0fc-7b25d1a35c1b
mon initial members                = a b c
mon host                           = [v2:172.30.212.132:3300],[v2:172.30.151.172:3300],[v2:172.30.165.77:3300]
bdev_flock_retry                   = 20
mon_osd_full_ratio                 = .85
mon_osd_backfillfull_ratio         = .8
mon_osd_nearfull_ratio             = .75
mon_max_pg_per_osd                 = 600
mon_pg_warn_max_object_skew        = 0
mon_data_avail_warn                = 15
mon_warn_on_pool_no_redundancy     = false
bluestore_prefer_deferred_size_hdd = 0

[osd]
osd_memory_target_cgroup_limit_ratio = 0.8

[client.admin]
keyring = /var/lib/rook/openshift-storage/client.admin.keyring
2025-04-21 14:14:18.774407 I | cephcmd: destroying osd.0 and cleaning its backing device
2025-04-21 14:14:18.774539 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list  --format json
2025-04-21 14:14:18.938827 D | cephosd: {}
2025-04-21 14:14:18.938853 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-04-21 14:14:18.938876 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list --format json
2025-04-21 14:14:22.189418 D | cephosd: {
    "fa7638ac-1031-444e-adf6-fcf0fe476410": {
        "ceph_fsid": "044f8070-5ce0-4edf-b0fc-7b25d1a35c1b",
        **"device": "/dev/sdb",**
        "osd_id": 0,
        "osd_uuid": "fa7638ac-1031-444e-adf6-fcf0fe476410",
        "type": "bluestore-rdr"
    }
}
2025-04-21 14:14:22.189480 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-04-21 14:14:22.189484 I | cephosd: destroying osd.0
2025-04-21 14:14:22.189494 D | exec: Running command: ceph osd destroy osd.0 --yes-i-really-mean-it --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-04-21 14:14:22.357408 I | cephosd: successfully destroyed osd.0
2025-04-21 14:14:22.357425 I | cephosd: zap OSD.0 path "/dev/sdb"
**2025-04-21 14:14:22.357432 D | exec: Running command: stdbuf -oL ceph-volume lvm zap /dev/sdb --destroy**
2025-04-21 14:14:22.723280 C | rookcmd: failed to destroy OSD 0.: failed to zap osd.0 path "/dev/sdb". --> Zapping: /dev/sdb
--> Removing all BlueStore signature on /dev/sdb if any...
Running command: /usr/bin/ceph-bluestore-tool zap-device --dev /dev/sdb --yes-i-really-really-mean-it
 stderr: error from zap: (16) Device or resource busy
2025-04-21T14:14:22.707+0000 7f0ba088ba40 -1 bdev(0x562f80e7a000 /dev/sdb) open open got: (16) Device or resource busy
Traceback (most recent call last):
  File "/usr/sbin/ceph-volume", line 33, in <module>
    sys.exit(load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')())
  File "/usr/lib/python3.9/site-packages/ceph_volume/main.py", line 54, in __init__
    self.main(self.argv)
  File "/usr/lib/python3.9/site-packages/ceph_volume/decorators.py", line 59, in newfunc
    return f(*a, **kw)
  File "/usr/lib/python3.9/site-packages/ceph_volume/main.py", line 166, in main
    terminal.dispatch(self.mapper, subcommand_args)
  File "/usr/lib/python3.9/site-packages/ceph_volume/terminal.py", line 194, in dispatch
    instance.main()
  File "/usr/lib/python3.9/site-packages/ceph_volume/devices/lvm/main.py", line 46, in main
    terminal.dispatch(self.mapper, self.argv)
  File "/usr/lib/python3.9/site-packages/ceph_volume/terminal.py", line 194, in dispatch
    instance.main()
  File "/usr/lib/python3.9/site-packages/ceph_volume/devices/lvm/zap.py", line 431, in main
    self.zap()
  File "/usr/lib/python3.9/site-packages/ceph_volume/decorators.py", line 16, in is_root
    return func(*a, **kw)
  File "/usr/lib/python3.9/site-packages/ceph_volume/devices/lvm/zap.py", line 307, in zap
    self.zap_raw_device(device)
  File "/usr/lib/python3.9/site-packages/ceph_volume/devices/lvm/zap.py", line 289, in zap_raw_device
    zap_device(device.path)
  File "/usr/lib/python3.9/site-packages/ceph_volume/devices/lvm/zap.py", line 25, in zap_device
    zap_bluestore(path)
  File "/usr/lib/python3.9/site-packages/ceph_volume/devices/lvm/zap.py", line 36, in zap_bluestore
    process.run([
  File "/usr/lib/python3.9/site-packages/ceph_volume/process.py", line 147, in run
    raise RuntimeError(msg)
RuntimeError: command returned non-zero exit status: 1.: exit status 1
[santosh@hetzner-dominguez ~]$
```


After the fix: Notice that `ceph-volume raw list` still returns OSD info with device name as `/dev/sdb`. But don't rely on this output in case of OSDs on PVC.  Rook executes `lsblk /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz` to get the actual mpath and then runs `ceph-volume lvm zap` on it. 

```
[santosh@hetzner-dominguez ~]$ oc logs rook-ceph-osd-prepare-4b076a89c44ed494aeb1dceba52e2618-4x6d4
Defaulted container "provision" out of: provision, copy-bins (init), blkdevmapper (init)
2025/04/22 06:20:14 maxprocs: Leaving GOMAXPROCS=16: CPU quota undefined
2025-04-22 06:20:14.865610 I | cephcmd: desired devices to configure osds: [{Name:/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-04-22 06:20:14.866180 I | rookcmd: starting Rook v1.15.0-194.g4138b6100 with arguments '/rook/rook ceph osd provision'
2025-04-22 06:20:14.866198 I | rookcmd: flag values: --cluster-id=6ebe7230-b4da-47b9-8eb6-30356c548fe7, --cluster-name=ocs-storagecluster-cephcluster, --data-device-filter=, --data-device-path-filter=, --data-devices=[{"id":"/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz","storeConfig":{"osdsPerDevice":1}}], --encrypted-device=false, --force-format=false, --help=false, --location=, --log-level=DEBUG, --metadata-device=, --node-name=ocs-deviceset-odf-disks-0-data-1mx2vz, --osd-crush-device-class=ssd, --osd-crush-initial-weight=, --osd-database-size=0, --osd-store-type=bluestore, --osd-wal-size=576, --osds-per-device=1, --pvc-backed-osd=true, --replace-osd=2
2025-04-22 06:20:14.866201 I | ceph-spec: parsing mon endpoints: a=172.30.212.132:3300,b=172.30.151.172:3300,c=172.30.165.77:3300
2025-04-22 06:20:14.871120 I | op-osd: CRUSH location=root=default host=santander-worker-0-ddomingu-com
2025-04-22 06:20:14.871130 I | cephcmd: crush location of osd: root=default host=santander-worker-0-ddomingu-com
2025-04-22 06:20:14.872374 I | cephclient: writing config file /var/lib/rook/openshift-storage/openshift-storage.config
2025-04-22 06:20:14.872492 I | cephclient: generated admin config in /var/lib/rook/openshift-storage
2025-04-22 06:20:14.872576 D | cephclient: config file @ /etc/ceph/ceph.conf:
[global]
fsid                               = 044f8070-5ce0-4edf-b0fc-7b25d1a35c1b
mon initial members                = a b c
mon host                           = [v2:172.30.212.132:3300],[v2:172.30.151.172:3300],[v2:172.30.165.77:3300]
bdev_flock_retry                   = 20
mon_osd_full_ratio                 = .85
mon_osd_backfillfull_ratio         = .8
mon_osd_nearfull_ratio             = .75
mon_max_pg_per_osd                 = 600
mon_pg_warn_max_object_skew        = 0
mon_data_avail_warn                = 15
mon_warn_on_pool_no_redundancy     = false
bluestore_prefer_deferred_size_hdd = 0

[osd]
osd_memory_target_cgroup_limit_ratio = 0.8

[client.admin]
keyring = /var/lib/rook/openshift-storage/client.admin.keyring
2025-04-22 06:20:14.872583 I | cephcmd: destroying osd.2 and cleaning its backing device
2025-04-22 06:20:14.872684 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list  --format json
2025-04-22 06:20:15.032513 D | cephosd: {}
2025-04-22 06:20:15.032542 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-04-22 06:20:15.032568 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list --format json
2025-04-22 06:20:18.169947 D | cephosd: {
    "273032f8-75c2-480f-b28a-cda610924382": {
        "ceph_fsid": "044f8070-5ce0-4edf-b0fc-7b25d1a35c1b",
        **"device": "/dev/sdb",**
        "osd_id": 2,
        "osd_uuid": "273032f8-75c2-480f-b28a-cda610924382",
        "type": "bluestore-rdr"
    }
}
2025-04-22 06:20:18.170004 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-04-22 06:20:18.170008 I | cephosd: destroying osd.2
2025-04-22 06:20:18.170017 D | exec: Running command: ceph osd destroy osd.2 --yes-i-really-mean-it --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-04-22 06:20:18.394552 I | cephosd: successfully destroyed osd.2
2025-04-22 06:20:18.394583 D | exec: Running command: lsblk /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-22 06:20:18.396371 D | sys: lsblk output: "SIZE=\"10737418240\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpatha\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"ceph_bluestore\""
**2025-04-22 06:20:18.396387 I | cephosd: zap OSD.2 path "/dev/mapper/mpatha"
2025-04-22 06:20:18.396391 D | exec: Running command: stdbuf -oL ceph-volume lvm zap /dev/mapper/mpatha --destroy
2025-04-22 06:20:19.034227 I | cephosd: --> Zapping: /dev/mapper/mpatha
--> Removing all BlueStore signature on /dev/mapper/mpatha if any...
Running command: /usr/bin/ceph-bluestore-tool zap-device --dev /dev/mapper/mpatha --yes-i-really-really-mean-it
Running command: /usr/bin/dd if=/dev/zero of=/dev/mapper/mpatha bs=1M count=10 conv=fsync
--> Zapping successful for: <Raw Device: /dev/mapper/mpatha>**
2025-04-22 06:20:19.034248 I | cephosd: successfully zapped osd.2 path "/dev/mapper/mpatha"
2025-04-22 06:20:19.034251 I | cephosd: sleeping after destroying the OSD.....
2025-04-22 06:25:19.036098 D | exec: Running command: dmsetup version
2025-04-22 06:25:19.037474 I | cephosd: Library version:   1.02.198-RHEL9 (2024-05-16)
Driver version:    4.48.0
2025-04-22 06:25:19.044202 I | cephosd: discovering hardware
2025-04-22 06:25:19.044215 D | exec: Running command: lsblk /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-22 06:25:19.046264 D | sys: lsblk output: "SIZE=\"10737418240\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpatha\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-04-22 06:25:19.046289 D | exec: Running command: udevadm info --query=property /dev/dm-0
2025-04-22 06:25:19.049740 D | sys: udevadm info output: "DEVPATH=/devices/virtual/block/dm-0\nDEVNAME=/dev/dm-0\nDEVTYPE=disk\nDISKSEQ=6\nMAJOR=253\nMINOR=0\nSUBSYSTEM=block\nUSEC_INITIALIZED=13954134\nDM_UDEV_DISABLE_LIBRARY_FALLBACK_FLAG=1\nDM_UDEV_PRIMARY_SOURCE_FLAG=1\nDM_UDEV_RULES_VSN=2\nDM_NAME=mpatha\nDM_UUID=mpath-360014058a405ccb67f84b5bafee7e158\nDM_SUSPENDED=0\nMPATH_DEVICE_READY=1\nMPATH_SBIN_PATH=/sbin\nDM_TYPE=scsi\nDM_WWN=0x60014058a405ccb67f84b5bafee7e158\nDM_SERIAL=360014058a405ccb67f84b5bafee7e158\nNVME_HOST_IFACE=none\nSYSTEMD_READY=1\nDEVLINKS=/dev/disk/by-id/scsi-360014058a405ccb67f84b5bafee7e158 /dev/disk/by-id/wwn-0x60014058a405ccb67f84b5bafee7e158 /dev/disk/by-id/dm-name-mpatha /dev/mapper/mpatha /dev/disk/by-id/dm-uuid-mpath-360014058a405ccb67f84b5bafee7e158\nTAGS=:systemd:\nCURRENT_TAGS=:systemd:"
2025-04-22 06:25:19.049753 I | cephosd: creating and starting the osds
2025-04-22 06:25:19.049762 D | cephosd: desiredDevices are [{Name:/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-04-22 06:25:19.049764 D | cephosd: context.Devices are:
2025-04-22 06:25:19.049775 D | cephosd: &{Name:/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz Parent: HasChildren:false DevLinks:/dev/disk/by-id/scsi-360014058a405ccb67f84b5bafee7e158 /dev/disk/by-id/wwn-0x60014058a405ccb67f84b5bafee7e158 /dev/disk/by-id/dm-name-mpatha /dev/mapper/mpatha /dev/disk/by-id/dm-uuid-mpath-360014058a405ccb67f84b5bafee7e158 Size:10737418240 UUID: Serial: Type:data Rotational:true Readonly:false Partitions:[] Filesystem: Mountpoint: Vendor: Model: WWN: WWNVendorExtension: Empty:false CephVolumeData: RealPath:/dev/mapper/mpatha KernelName:dm-0 Encrypted:false}
2025-04-22 06:25:19.049779 I | cephosd: old lsblk can't detect bluestore signature, so try to detect here
2025-04-22 06:25:19.049800 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz
2025-04-22 06:25:19.054893 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz". Device /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz is not a valid LUKS device.: exit status 1
2025-04-22 06:25:19.054908 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --format json
2025-04-22 06:25:19.441253 D | cephosd: {}
2025-04-22 06:25:19.441274 I | cephosd: 0 ceph-volume raw osd devices configured on this node
2025-04-22 06:25:19.441279 I | cephosd: device "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz" is available.
2025-04-22 06:25:19.441283 I | cephosd: "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz" found in the desired devices
2025-04-22 06:25:19.441289 I | cephosd: device "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz" is selected by the device filter/name "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz"
2025-04-22 06:25:19.445981 I | cephosd: configuring osd devices: {"Entries":{"data":{"Data":-1,"Metadata":null,"Config":{"Name":"/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz","OSDsPerDevice":1,"MetadataDevice":"","DatabaseSizeMB":0,"DeviceClass":"ssd","InitialWeight":"","IsFilter":false,"IsDevicePathFilter":false},"PersistentDevicePaths":["/dev/disk/by-id/scsi-360014058a405ccb67f84b5bafee7e158","/dev/disk/by-id/wwn-0x60014058a405ccb67f84b5bafee7e158","/dev/disk/by-id/dm-name-mpatha","/dev/mapper/mpatha","/dev/disk/by-id/dm-uuid-mpath-360014058a405ccb67f84b5bafee7e158"],"DeviceInfo":{"name":"/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-360014058a405ccb67f84b5bafee7e158 /dev/disk/by-id/wwn-0x60014058a405ccb67f84b5bafee7e158 /dev/disk/by-id/dm-name-mpatha /dev/mapper/mpatha /dev/disk/by-id/dm-uuid-mpath-360014058a405ccb67f84b5bafee7e158","size":10737418240,"uuid":"","serial":"","type":"data","rotational":true,"readOnly":false,"Partitions":null,"filesystem":"","mountpoint":"","vendor":"","model":"","wwn":"","wwnVendorExtension":"","empty":false,"real-path":"/dev/mapper/mpatha","kernel-name":"dm-0"},"RestoreOSD":false}}}
2025-04-22 06:25:19.446016 I | cephclient: getting or creating ceph auth key "client.bootstrap-osd"
2025-04-22 06:25:19.446030 D | exec: Running command: ceph auth get-or-create-key client.bootstrap-osd mon allow profile bootstrap-osd --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-04-22 06:25:19.647197 D | exec: Running command: lsblk /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-22 06:25:19.649064 D | sys: lsblk output: "SIZE=\"10737418240\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpatha\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-04-22 06:25:19.649084 I | cephosd: configuring new device "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz"
2025-04-22 06:25:19.649091 D | exec: Running command: stdbuf -oL ceph-volume --log-path /var/log/ceph/ocs-deviceset-odf-disks-0-data-1mx2vz raw prepare --bluestore --data /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --osd-id 2 --crush-device-class ssd
2025-04-22 06:25:23.304333 I | cephosd: stderr: Unknown device "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz": No such device
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd tree -f json
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new c68f5ab4-6bea-4b19-b6d7-a26ca28a0cfa 2
Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-2
Running command: /usr/bin/chown -R ceph:ceph /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz
Running command: /usr/bin/ln -s /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz /var/lib/ceph/osd/ceph-2/block
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-2/activate.monmap
 stderr: got monmap epoch 4
--> Creating keyring file for osd.2
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2/keyring
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2/
Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 2 --monmap /var/lib/ceph/osd/ceph-2/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-2/ --osd-uuid c68f5ab4-6bea-4b19-b6d7-a26ca28a0cfa --setuser ceph --setgroup ceph
 stderr: 2025-04-22T06:25:20.878+0000 7f7783058740 -1 bluestore(/var/lib/ceph/osd/ceph-2//block) _read_bdev_label /var/lib/ceph/osd/ceph-2//block data at 0, unable to decode label
 stderr: 2025-04-22T06:25:20.878+0000 7f7783058740 -1 bluestore(/var/lib/ceph/osd/ceph-2//block) _read_bdev_label /var/lib/ceph/osd/ceph-2//block data at 40000000, unable to decode label
 stderr: 2025-04-22T06:25:20.878+0000 7f7783058740 -1 bluestore(/var/lib/ceph/osd/ceph-2//block) No valid bdev label found
 stderr: 2025-04-22T06:25:21.144+0000 7f7783058740 -1 bluestore(/var/lib/ceph/osd/ceph-2/) _read_fsid unparsable uuid
--> ceph-volume raw dmcrypt prepare successful for: /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz
2025-04-22 06:25:23.304369 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --format json
2025-04-22 06:25:23.594257 D | cephosd: {}
2025-04-22 06:25:23.594283 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-04-22 06:25:23.594296 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz
2025-04-22 06:25:23.599887 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz". Device /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz is not a valid LUKS device.: exit status 1
2025-04-22 06:25:23.599911 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --format json
2025-04-22 06:25:23.967754 D | cephosd: {
    "c68f5ab4-6bea-4b19-b6d7-a26ca28a0cfa": {
        "ceph_fsid": "044f8070-5ce0-4edf-b0fc-7b25d1a35c1b",
        **"device": "/dev/mapper/mpatha",**
        "osd_id": 2,
        "osd_uuid": "c68f5ab4-6bea-4b19-b6d7-a26ca28a0cfa",
        "type": "bluestore"
    }
}
2025-04-22 06:25:23.967814 D | exec: Running command: lsblk /mnt/ocs-deviceset-odf-disks-0-data-1mx2vz --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-22 06:25:23.969561 D | sys: lsblk output: "SIZE=\"10737418240\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpatha\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"ceph_bluestore\""
2025-04-22 06:25:23.969576 I | cephosd: setting device class "hdd" for device "/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz"
2025-04-22 06:25:23.969581 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-04-22 06:25:23.969602 I | cephosd: devices = [{ID:2 Cluster:ceph UUID:c68f5ab4-6bea-4b19-b6d7-a26ca28a0cfa DevicePartUUID: DeviceClass:hdd BlockPath:/mnt/ocs-deviceset-odf-disks-0-data-1mx2vz MetadataPath: WalPath: SkipLVRelease:true Location:root=default host=santander-worker-0-ddomingu-com LVBackedPV:false CVMode:raw Store:bluestore TopologyAffinity: Encrypted:false ExportService:false NodeName: PVCName:}]
[santosh@hetzner-dominguez ~]$
```

All the OSDs were successfully migrated from bluestore-rdr to bluestore.  CephCluster spec status: 

```
 storage:
      deviceClasses:
      - name: ssd
      osd:
        storeType:
          bluestore: 3
```


Testing on s390x

```
[root@m4202001 ~]# oc logs rook-ceph-osd-prepare-427fb81218e33f194c38bbedae1c3d4e-gtrj7
Defaulted container "provision" out of: provision, copy-bins (init), blkdevmapper (init)
2025/04/24 16:00:38 maxprocs: Leaving GOMAXPROCS=16: CPU quota undefined
2025-04-24 16:00:38.761320 I | cephcmd: desired devices to configure osds: [{Name:/mnt/ocs-deviceset-localblock-0-data-2qrvwp OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-04-24 16:00:38.762881 I | rookcmd: starting Rook v4.18.2-0.d5f0836d2fdc127dbe052aefe5416b724943cf72 with arguments '/rook/rook ceph osd provision'
2025-04-24 16:00:38.762906 I | rookcmd: flag values: --cluster-id=ae5adf8e-c2c9-4560-860a-d56288ef101d, --cluster-name=ocs-storagecluster-cephcluster, --data-device-filter=, --data-device-path-filter=, --data-devices=[{"id":"/mnt/ocs-deviceset-localblock-0-data-2qrvwp","storeConfig":{"osdsPerDevice":1}}], --encrypted-device=false, --force-format=false, --help=false, --location=, --log-level=DEBUG, --metadata-device=, --node-name=ocs-deviceset-localblock-0-data-2qrvwp, --osd-crush-device-class=ssd, --osd-crush-initial-weight=, --osd-database-size=0, --osd-store-type=bluestore, --osd-wal-size=576, --osds-per-device=1, --pvc-backed-osd=true, --replace-osd=0
2025-04-24 16:00:38.762921 I | ceph-spec: parsing mon endpoints: c=172.31.87.114:3300,a=172.31.155.225:3300,b=172.31.190.253:3300
2025-04-24 16:00:38.785941 I | op-osd: CRUSH location=root=default host=worker-2-m4202001-lnxero1-boe
2025-04-24 16:00:38.785964 I | cephcmd: crush location of osd: root=default host=worker-2-m4202001-lnxero1-boe
2025-04-24 16:00:38.790323 I | cephclient: writing config file /var/lib/rook/openshift-storage/openshift-storage.config
2025-04-24 16:00:38.790473 I | cephclient: generated admin config in /var/lib/rook/openshift-storage
2025-04-24 16:00:38.790589 D | cephclient: config file @ /etc/ceph/ceph.conf:
[global]
fsid                               = 7fa6d45c-5ad7-490d-8d15-905a842b3df8
mon initial members                = c a b
mon host                           = [v2:172.31.87.114:3300],[v2:172.31.155.225:3300],[v2:172.31.190.253:3300]
bdev_flock_retry                   = 20
mon_osd_full_ratio                 = .85
mon_osd_backfillfull_ratio         = .8
mon_osd_nearfull_ratio             = .75
mon_max_pg_per_osd                 = 600
mon_pg_warn_max_object_skew        = 0
mon_data_avail_warn                = 15
mon_warn_on_pool_no_redundancy     = false
bluestore_prefer_deferred_size_hdd = 0

[osd]
osd_memory_target_cgroup_limit_ratio = 0.8

[client.admin]
keyring = /var/lib/rook/openshift-storage/client.admin.keyring
2025-04-24 16:00:38.790625 I | cephcmd: destroying osd.0 and cleaning its backing device
2025-04-24 16:00:38.790842 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list  --format json
2025-04-24 16:00:39.630965 D | cephosd: {}
2025-04-24 16:00:39.631048 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-04-24 16:00:39.631112 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list --format json
2025-04-24 16:00:49.492278 D | cephosd: {
    "e5bba83f-62c7-420e-99f3-9509e841690b": {
        "ceph_fsid": "7fa6d45c-5ad7-490d-8d15-905a842b3df8",
        "device": "/dev/sdg",
        "osd_id": 0,
        "osd_uuid": "e5bba83f-62c7-420e-99f3-9509e841690b",
        "type": "bluestore-rdr"
    }
}
2025-04-24 16:00:49.492432 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-04-24 16:00:49.492443 I | cephosd: destroying osd.0
2025-04-24 16:00:49.492465 D | exec: Running command: ceph osd destroy osd.0 --yes-i-really-mean-it --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-04-24 16:00:49.969693 I | cephosd: successfully destroyed osd.0
2025-04-24 16:00:49.969749 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-2qrvwp --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-24 16:00:49.973481 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"0\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"ceph_bluestore\""
2025-04-24 16:00:49.973510 I | cephosd: zap OSD.0 path "/dev/mapper/mpathb"
2025-04-24 16:00:49.973529 D | exec: Running command: stdbuf -oL ceph-volume lvm zap /dev/mapper/mpathb --destroy
2025-04-24 16:00:51.358828 I | cephosd: --> Zapping: /dev/mapper/mpathb
--> Removing all BlueStore signature on /dev/mapper/mpathb if any...
Running command: /usr/bin/ceph-bluestore-tool zap-device --dev /dev/mapper/mpathb --yes-i-really-really-mean-it
Running command: /usr/bin/dd if=/dev/zero of=/dev/mapper/mpathb bs=1M count=10 conv=fsync
 stderr: 10+0 records in
10+0 records out
 stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0181228 s, 579 MB/s
--> Zapping successful for: <Raw Device: /dev/mapper/mpathb>
2025-04-24 16:00:51.358890 I | cephosd: successfully zapped osd.0 path "/dev/mapper/mpathb"
2025-04-24 16:00:51.358912 D | exec: Running command: dmsetup version
2025-04-24 16:00:51.361134 I | cephosd: Library version:   1.02.198-RHEL9 (2024-05-16)
Driver version:    4.48.0
2025-04-24 16:00:51.391540 I | cephosd: discovering hardware
2025-04-24 16:00:51.391572 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-2qrvwp --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-24 16:00:51.396412 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"0\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-04-24 16:00:51.396454 D | exec: Running command: udevadm info --query=property /dev/dm-0
2025-04-24 16:00:51.403269 D | sys: udevadm info output: "DEVPATH=/devices/virtual/block/dm-0\nDEVNAME=/dev/dm-0\nDEVTYPE=disk\nDISKSEQ=9\nMAJOR=253\nMINOR=0\nSUBSYSTEM=block\nUSEC_INITIALIZED=13017468\nDM_UDEV_DISABLE_LIBRARY_FALLBACK_FLAG=1\nDM_UDEV_PRIMARY_SOURCE_FLAG=1\nDM_UDEV_RULES_VSN=2\nDM_NAME=mpathb\nDM_UUID=mpath-3600507630affd0a40000000000000337\nDM_SUSPENDED=0\nMPATH_DEVICE_READY=1\nMPATH_SBIN_PATH=/sbin\nDM_TYPE=scsi\nDM_WWN=0x600507630affd0a40000000000000337\nDM_SERIAL=3600507630affd0a40000000000000337\nNVME_HOST_IFACE=none\nSYSTEMD_READY=1\nDEVLINKS=/dev/disk/by-id/wwn-0x600507630affd0a40000000000000337 /dev/disk/by-id/dm-uuid-mpath-3600507630affd0a40000000000000337 /dev/disk/by-id/dm-name-mpathb /dev/disk/by-id/scsi-3600507630affd0a40000000000000337 /dev/mapper/mpathb\nTAGS=:systemd:\nCURRENT_TAGS=:systemd:"
2025-04-24 16:00:51.403293 I | cephosd: creating and starting the osds
2025-04-24 16:00:51.403316 D | cephosd: desiredDevices are [{Name:/mnt/ocs-deviceset-localblock-0-data-2qrvwp OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-04-24 16:00:51.403325 D | cephosd: context.Devices are:
2025-04-24 16:00:51.403349 D | cephosd: &{Name:/mnt/ocs-deviceset-localblock-0-data-2qrvwp Parent: HasChildren:false DevLinks:/dev/disk/by-id/wwn-0x600507630affd0a40000000000000337 /dev/disk/by-id/dm-uuid-mpath-3600507630affd0a40000000000000337 /dev/disk/by-id/dm-name-mpathb /dev/disk/by-id/scsi-3600507630affd0a40000000000000337 /dev/mapper/mpathb Size:536870912000 UUID: Serial: Type:data Rotational:false Readonly:false Partitions:[] Filesystem: Mountpoint: Vendor: Model: WWN: WWNVendorExtension: Empty:false CephVolumeData: RealPath:/dev/mapper/mpathb KernelName:dm-0 Encrypted:false}
2025-04-24 16:00:51.403355 I | cephosd: old lsblk can't detect bluestore signature, so try to detect here
2025-04-24 16:00:51.403418 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-localblock-0-data-2qrvwp
2025-04-24 16:00:51.425072 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-localblock-0-data-2qrvwp" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-localblock-0-data-2qrvwp". Device /mnt/ocs-deviceset-localblock-0-data-2qrvwp is not a valid LUKS device.: exit status 1
2025-04-24 16:00:51.425144 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-localblock-0-data-2qrvwp --format json
2025-04-24 16:00:52.011082 D | cephosd: {}
2025-04-24 16:00:52.011213 I | cephosd: 0 ceph-volume raw osd devices configured on this node
2025-04-24 16:00:52.011235 I | cephosd: device "/mnt/ocs-deviceset-localblock-0-data-2qrvwp" is available.
2025-04-24 16:00:52.011259 I | cephosd: "/mnt/ocs-deviceset-localblock-0-data-2qrvwp" found in the desired devices
2025-04-24 16:00:52.011280 I | cephosd: device "/mnt/ocs-deviceset-localblock-0-data-2qrvwp" is selected by the device filter/name "/mnt/ocs-deviceset-localblock-0-data-2qrvwp"
2025-04-24 16:00:52.020595 I | cephosd: configuring osd devices: {"Entries":{"data":{"Data":-1,"Metadata":null,"Config":{"Name":"/mnt/ocs-deviceset-localblock-0-data-2qrvwp","OSDsPerDevice":1,"MetadataDevice":"","DatabaseSizeMB":0,"DeviceClass":"ssd","InitialWeight":"","IsFilter":false,"IsDevicePathFilter":false},"PersistentDevicePaths":["/dev/disk/by-id/wwn-0x600507630affd0a40000000000000337","/dev/disk/by-id/dm-uuid-mpath-3600507630affd0a40000000000000337","/dev/disk/by-id/dm-name-mpathb","/dev/disk/by-id/scsi-3600507630affd0a40000000000000337","/dev/mapper/mpathb"],"DeviceInfo":{"name":"/mnt/ocs-deviceset-localblock-0-data-2qrvwp","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/wwn-0x600507630affd0a40000000000000337 /dev/disk/by-id/dm-uuid-mpath-3600507630affd0a40000000000000337 /dev/disk/by-id/dm-name-mpathb /dev/disk/by-id/scsi-3600507630affd0a40000000000000337 /dev/mapper/mpathb","size":536870912000,"uuid":"","serial":"","type":"data","rotational":false,"readOnly":false,"Partitions":null,"filesystem":"","mountpoint":"","vendor":"","model":"","wwn":"","wwnVendorExtension":"","empty":false,"real-path":"/dev/mapper/mpathb","kernel-name":"dm-0"},"RestoreOSD":false}}}
2025-04-24 16:00:52.020686 I | cephclient: getting or creating ceph auth key "client.bootstrap-osd"
2025-04-24 16:00:52.020698 D | exec: Running command: ceph auth get-or-create-key client.bootstrap-osd mon allow profile bootstrap-osd --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-04-24 16:00:52.553741 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-2qrvwp --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-24 16:00:52.558317 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"0\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-04-24 16:00:52.558366 I | cephosd: configuring new device "/mnt/ocs-deviceset-localblock-0-data-2qrvwp"
2025-04-24 16:00:52.558384 D | exec: Running command: stdbuf -oL ceph-volume --log-path /var/log/ceph/ocs-deviceset-localblock-0-data-2qrvwp raw prepare --bluestore --data /mnt/ocs-deviceset-localblock-0-data-2qrvwp --osd-id 0 --crush-device-class ssd
2025-04-24 16:00:59.475379 I | cephosd: stderr: Unknown device "/mnt/ocs-deviceset-localblock-0-data-2qrvwp": No such device
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd tree -f json
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 9f5b9cba-1159-46dd-9891-4b6214b2c72b 0
Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0
Running command: /usr/bin/chown -R ceph:ceph /mnt/ocs-deviceset-localblock-0-data-2qrvwp
Running command: /usr/bin/ln -s /mnt/ocs-deviceset-localblock-0-data-2qrvwp /var/lib/ceph/osd/ceph-0/block
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap
 stderr: got monmap epoch 4
--> Creating keyring file for osd.0
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/
Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 9f5b9cba-1159-46dd-9891-4b6214b2c72b --setuser ceph --setgroup ceph
 stderr: 2025-04-24T16:00:55.547+0000 3ff8e465900 -1 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label /var/lib/ceph/osd/ceph-0//block data at 0, unable to decode label
 stderr: 2025-04-24T16:00:55.547+0000 3ff8e465900 -1 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label /var/lib/ceph/osd/ceph-0//block data at 40000000, unable to decode label
 stderr: 2025-04-24T16:00:55.547+0000 3ff8e465900 -1 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label /var/lib/ceph/osd/ceph-0//block data at 280000000, unable to decode label
 stderr: 2025-04-24T16:00:55.547+0000 3ff8e465900 -1 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label /var/lib/ceph/osd/ceph-0//block data at 1900000000, unable to decode label
 stderr: 2025-04-24T16:00:55.547+0000 3ff8e465900 -1 bluestore(/var/lib/ceph/osd/ceph-0//block) No valid bdev label found
 stderr: 2025-04-24T16:00:55.957+0000 3ff8e465900 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _read_fsid unparsable uuid
--> ceph-volume raw dmcrypt prepare successful for: /mnt/ocs-deviceset-localblock-0-data-2qrvwp
2025-04-24 16:00:59.475450 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list /mnt/ocs-deviceset-localblock-0-data-2qrvwp --format json
2025-04-24 16:01:00.860965 D | cephosd: {}
2025-04-24 16:01:00.861022 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-04-24 16:01:00.861037 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-localblock-0-data-2qrvwp
2025-04-24 16:01:00.870026 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-localblock-0-data-2qrvwp" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-localblock-0-data-2qrvwp". Device /mnt/ocs-deviceset-localblock-0-data-2qrvwp is not a valid LUKS device.: exit status 1
2025-04-24 16:01:00.870058 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-localblock-0-data-2qrvwp --format json
2025-04-24 16:01:01.485912 D | cephosd: {
    "9f5b9cba-1159-46dd-9891-4b6214b2c72b": {
        "ceph_fsid": "7fa6d45c-5ad7-490d-8d15-905a842b3df8",
        "device": "/dev/mapper/mpathb",
        "osd_id": 0,
        "osd_uuid": "9f5b9cba-1159-46dd-9891-4b6214b2c72b",
        "type": "bluestore"
    }
}
2025-04-24 16:01:01.486051 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-2qrvwp --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-04-24 16:01:01.493874 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"0\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-0\" MOUNTPOINT=\"\" FSTYPE=\"ceph_bluestore\""
2025-04-24 16:01:01.493910 I | cephosd: setting device class "ssd" for device "/mnt/ocs-deviceset-localblock-0-data-2qrvwp"
2025-04-24 16:01:01.493920 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-04-24 16:01:01.493959 I | cephosd: devices = [{ID:0 Cluster:ceph UUID:9f5b9cba-1159-46dd-9891-4b6214b2c72b DevicePartUUID: DeviceClass:ssd BlockPath:/mnt/ocs-deviceset-localblock-0-data-2qrvwp MetadataPath: WalPath: SkipLVRelease:true Location:root=default host=worker-2-m4202001-lnxero1-boe LVBackedPV:false CVMode:raw Store:bluestore TopologyAffinity: Encrypted:false ExportService:false NodeName: PVCName:}]
[root@m4202001 ~]#
```


<!-- Thank you for contributing to Rook! -->

<!-- STEPS TO FOLLOW:
  1. Add a description of the changes (frequently the same as the commit description)
  2. Enter the issue number next to "Resolves #" below (if there is no tracking issue resolved, **remove that section**)
  3. Review our Contributing documentation at https://rook.io/docs/rook/latest/Contributing/development-flow/
  4. Follow the steps in the checklist below, starting with the **Commit Message Formatting**.
-->

<!-- Uncomment this section with the issue number if an issue is being resolved
**Issue resolved by this Pull Request:**
Resolves #
--->

**Checklist:**

- [ ] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/latest/Contributing/development-flow/#commit-structure).
- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/latest/Contributing/development-flow/#submitting-a-pull-request)
- [ ] [Pending release notes](https://github.com/rook/rook/blob/master/PendingReleaseNotes.md) updated with breaking and/or notable changes for the next minor release.
- [ ] Documentation has been updated, if necessary.
- [ ] Unit tests have been added, if necessary.
- [ ] Integration tests have been added, if necessary.



Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 820398c0d..bddc9b255 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -36,7 +36,7 @@ jobs:
 
       - name: build rook
         run: |
-          GOPATH=$(go env GOPATH) make clean && make -j$nproc BUILD_CONTAINER_IMAGE=false build
+          GOPATH=$(go env GOPATH) make clean && make -j$nproc IMAGES='ceph' BUILD_CONTAINER_IMAGE=false build
 
       - name: validate build
         run: tests/scripts/validate_modified_files.sh build
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 1726ba7c4..4d2cfd168 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -64,6 +64,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15
+        uses: github/codeql-action/upload-sarif@fc7e4a0fa01c3cca5fd6a1fddec5c0740c977aa2 # v3.28.14
         with:
           sarif_file: results.sarif
diff --git a/.mergify.yml b/.mergify.yml
index d00d2913a..d913cf835 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -235,17 +235,17 @@ pull_request_rules:
       - "check-success=canary-tests / raw-disk-with-object (quay.io/ceph/ceph:v19)"
       - "check-success=canary-tests / two-object-one-zone (quay.io/ceph/ceph:v19)"
       - "check-success=canary-tests / two-osds-in-device (quay.io/ceph/ceph:v19)"
-      - "check-success=TestCephHelmSuite (v1.28.15)"
-      - "check-success=TestCephHelmSuite (v1.32.3)"
-      - "check-success=TestCephMultiClusterDeploySuite (v1.32.3)"
-      - "check-success=TestCephObjectSuite (v1.28.15)"
-      - "check-success=TestCephObjectSuite (v1.32.3)"
-      - "check-success=TestCephSmokeSuite (v1.28.15)"
-      - "check-success=TestCephSmokeSuite (v1.32.3)"
-      - "check-success=TestCephUpgradeSuite (v1.28.15)"
-      - "check-success=TestCephUpgradeSuite (v1.32.3)"
-      - "check-success=TestHelmUpgradeSuite (v1.28.15)"
-      - "check-success=TestHelmUpgradeSuite (v1.32.3)"
+      - "check-success=TestCephHelmSuite (v1.27.16)"
+      - "check-success=TestCephHelmSuite (v1.32.0)"
+      - "check-success=TestCephMultiClusterDeploySuite (v1.32.0)"
+      - "check-success=TestCephObjectSuite (v1.27.16)"
+      - "check-success=TestCephObjectSuite (v1.32.0)"
+      - "check-success=TestCephSmokeSuite (v1.27.16)"
+      - "check-success=TestCephSmokeSuite (v1.32.0)"
+      - "check-success=TestCephUpgradeSuite (v1.27.16)"
+      - "check-success=TestCephUpgradeSuite (v1.32.0)"
+      - "check-success=TestHelmUpgradeSuite (v1.27.16)"
+      - "check-success=TestHelmUpgradeSuite (v1.32.0)"
     actions:
       merge:
         method: merge
diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
index 130f58986..0f776f523 100755
--- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md
+++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
@@ -26,7 +26,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
 * `external`:
     * `enable`: if `true`, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled **all** the other options will be ignored except `cephVersion.image` and `dataDirHostPath`. See [external cluster configuration](external-cluster/external-cluster.md). If `cephVersion.image` is left blank, Rook will refuse the creation of extra CRs like object, file and nfs.
 * `cephVersion`: The version information for launching the ceph daemons.
-    * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v19.2.2`. For more details read the [container images section](#ceph-container-images).
+    * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v19.2.1`. For more details read the [container images section](#ceph-container-images).
         For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/).
         To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version.
         Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v19` will be updated each time a new Squid build is released.
@@ -121,8 +121,8 @@ These are general purpose Ceph container with all necessary daemons and dependen
 | -------------------- | --------------------------------------------------------- |
 | vRELNUM              | Latest release in this series (e.g., **v19** = Squid)     |
 | vRELNUM.Y            | Latest stable release in this stable series (e.g., v19.2) |
-| vRELNUM.Y.Z          | A specific release (e.g., v19.2.2)                        |
-| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v19.2.2-20250409)                 |
+| vRELNUM.Y.Z          | A specific release (e.g., v19.2.1)                        |
+| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v19.2.1-20250202)                 |
 
 A specific will contain a specific release of Ceph as well as security fixes from the Operating System.
 
@@ -144,18 +144,9 @@ A specific will contain a specific release of Ceph as well as security fixes fro
     This setting only applies to new monitors that are created when the requested
     number of monitors increases, or when a monitor fails and is recreated. An
     [example CRD configuration is provided below](./pvc-cluster.md).
-
-    **Note:** This field should not be used if you are defining a specific `volumeClaimTemplate`
-    for each zone in the `zones` section, as it will be overridden by the zone-specific configurations.
-
 * `failureDomainLabel`: The label that is expected on each node where the mons
     are expected to be deployed. The labels must be found in the list of
     well-known [topology labels](#osd-topology).
-
-    **Note:** This setting is an alternative to specifying a `topologySpreadConstraints` rule with the
-    chosen label (e.g., `topology.kubernetes.io/zone`) for the mons under the `placement` section, as
-    described in the [Placement Configuration Settings](#placement-configuration-settings).
-
 * `externalMonIDs`: ID list of external mons deployed outside of Rook cluster
     and not managed by Rook. If set, Rook will not remove external mons from quorum
     and populate external mons addresses to mon endpoints for CSI.
@@ -177,11 +168,6 @@ A specific will contain a specific release of Ceph as well as security fixes fro
         number of monitors increases, or when a monitor fails and is recreated. An
         [example CRD configuration is provided below](./pvc-cluster.md).
 
-    **Note:** Configuring this section applies specific affinity rules to assign each mon to one
-    of the specified zones. When this section is used, there is no need to define mon placement
-    under the[Placement Configuration Settings](#placement-configuration-settings), as the zone
-    settings will take precedence.
-
 * `stretchCluster`: The stretch cluster settings that define the zones (or other failure domain labels) across which to configure the cluster.
     * `failureDomainLabel`: The label that is expected on each node where the cluster is expected to be deployed. The labels must be found
     in the list of well-known [topology labels](#osd-topology).
@@ -419,11 +405,8 @@ In stretch clusters, if the `arbiter` placement is specified, that placement wil
 Neither will the `arbiter` placement be merged with the `all` placement to allow the arbiter to be fully independent of other daemon placement.
 The remaining mons will still use the `mon` and/or `all` sections.
 
-Other considerations of the placement settings include:
-    - If OSDs are defined using `storageClassDeviceSets`, placement settings must be specified within each device set and the `osd` key here will be ignored.
-    - If OSDs are created via other settings (e.g., `useAllDevices` or `storage.nodes`), the `osd` key in this section applies.
-    - When `zones` are configured in the `mon` settings, specific affinity rules are automatically generated to assign each mon to one of the defined zones, without needing to configure this section.
-    - Other components from separate CRDs, such as metadata servers in the `CephFilesystem` CRD, have their own placement configurations and are not affected by this section, even if the `all` key is used.
+!!! note
+    Placement of OSD pods is controlled using the [Storage Class Device Set](#storage-class-device-sets), not the general `placement` configuration.
 
 A Placement configuration is specified (according to the kubernetes PodSpec) as:
 
@@ -452,7 +435,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -559,7 +542,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -689,7 +672,7 @@ kubectl -n rook-ceph get CephCluster -o yaml
       deviceClasses:
       - name: hdd
     version:
-      image: quay.io/ceph/ceph:v19.2.2
+      image: quay.io/ceph/ceph:v19.2.1
       version: 16.2.6-0
     conditions:
     - lastHeartbeatTime: "2021-03-02T21:22:11Z"
diff --git a/Documentation/CRDs/Cluster/host-cluster.md b/Documentation/CRDs/Cluster/host-cluster.md
index ec86c08a2..a9cd511a2 100644
--- a/Documentation/CRDs/Cluster/host-cluster.md
+++ b/Documentation/CRDs/Cluster/host-cluster.md
@@ -22,7 +22,7 @@ metadata:
 spec:
   cephVersion:
     # see the "Cluster Settings" section below for more details on which image of ceph to run
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -49,7 +49,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -101,7 +101,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
diff --git a/Documentation/CRDs/Cluster/pvc-cluster.md b/Documentation/CRDs/Cluster/pvc-cluster.md
index 59439f6ea..6485e1e8d 100644
--- a/Documentation/CRDs/Cluster/pvc-cluster.md
+++ b/Documentation/CRDs/Cluster/pvc-cluster.md
@@ -18,7 +18,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -72,7 +72,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     allowUnsupported: false
   dashboard:
     enabled: true
@@ -128,7 +128,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
diff --git a/Documentation/CRDs/Cluster/stretch-cluster.md b/Documentation/CRDs/Cluster/stretch-cluster.md
index f46494a68..bc45459cd 100644
--- a/Documentation/CRDs/Cluster/stretch-cluster.md
+++ b/Documentation/CRDs/Cluster/stretch-cluster.md
@@ -34,7 +34,7 @@ spec:
       - name: b
       - name: c
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     allowUnsupported: true
   # Either storageClassDeviceSets or the storage section can be specified for creating OSDs.
   # This example uses all devices for simplicity.
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index dcd304780..6cf3bcde8 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -2948,19 +2948,6 @@ int64
 <p>ObservedGeneration is the latest generation observed by the controller.</p>
 </td>
 </tr>
-<tr>
-<td>
-<code>secrets</code><br/>
-<em>
-<a href="#ceph.rook.io/v1.SecretReference">
-[]SecretReference
-</a>
-</em>
-</td>
-<td>
-<em>(Optional)</em>
-</td>
-</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.CIDR">CIDR
@@ -7546,34 +7533,6 @@ string
 <p>The authentication mechanism for this topic (PLAIN/SCRAM-SHA-512/SCRAM-SHA-256/GSSAPI/OAUTHBEARER)</p>
 </td>
 </tr>
-<tr>
-<td>
-<code>UserSecretRef</code><br/>
-<em>
-<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core">
-Kubernetes core/v1.SecretKeySelector
-</a>
-</em>
-</td>
-<td>
-<em>(Optional)</em>
-<p>The kafka user name to use for authentication</p>
-</td>
-</tr>
-<tr>
-<td>
-<code>PasswordSecretRef</code><br/>
-<em>
-<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core">
-Kubernetes core/v1.SecretKeySelector
-</a>
-</em>
-</td>
-<td>
-<em>(Optional)</em>
-<p>The kafka password to use for authentication</p>
-</td>
-</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.KerberosConfigFiles">KerberosConfigFiles
@@ -12528,7 +12487,7 @@ int32
 <h3 id="ceph.rook.io/v1.SecretReference">SecretReference
 </h3>
 <p>
-(<em>Appears on:</em><a href="#ceph.rook.io/v1.BucketTopicStatus">BucketTopicStatus</a>, <a href="#ceph.rook.io/v1.ObjectStoreUserStatus">ObjectStoreUserStatus</a>)
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.ObjectStoreUserStatus">ObjectStoreUserStatus</a>)
 </p>
 <div>
 </div>
diff --git a/Documentation/Getting-Started/Prerequisites/prerequisites.md b/Documentation/Getting-Started/Prerequisites/prerequisites.md
index fa040450a..182f6e5f5 100644
--- a/Documentation/Getting-Started/Prerequisites/prerequisites.md
+++ b/Documentation/Getting-Started/Prerequisites/prerequisites.md
@@ -7,7 +7,7 @@ and Rook is granted the required privileges (see below for more information).
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.28** through **v1.32** are supported.
+Kubernetes versions **v1.27** through **v1.32** are supported.
 
 ## CPU Architecture
 
diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md
index 4a03835b1..913868a71 100644
--- a/Documentation/Getting-Started/quickstart.md
+++ b/Documentation/Getting-Started/quickstart.md
@@ -12,7 +12,7 @@ This guide will walk through the basic setup of a Ceph cluster and enable K8s ap
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.28** through **v1.32** are supported.
+Kubernetes versions **v1.27** through **v1.32** are supported.
 
 ## CPU Architecture
 
diff --git a/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md b/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md
index 3c0a1d20e..fe7e2e7e8 100644
--- a/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md
+++ b/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md
@@ -13,6 +13,7 @@ This chart is a simple packaging of templates that will optionally create Rook r
 
 ## Prerequisites
 
+* Kubernetes 1.22+
 * Helm 3.x
 * Install the [Rook Operator chart](operator-chart.md)
 
@@ -23,6 +24,8 @@ The [configuration](#configuration) section lists the parameters that can be con
 recommended that the rook operator be installed into the `rook-ceph` namespace. The clusters can be installed
 into the same namespace as the operator or a separate namespace.
 
+Rook currently publishes builds of this chart to the `release` and `master` channels.
+
 **Before installing, review the values.yaml to confirm if the default settings need to be updated.**
 
 * If the operator was installed in a namespace other than `rook-ceph`, the namespace
@@ -35,7 +38,7 @@ into the same namespace as the operator or a separate namespace.
 
 ### **Release**
 
-The `release` channel is the most recent release of Rook that is considered stable for the community.
+The release channel is the most recent release of Rook that is considered stable for the community.
 
 The example install assumes you have first installed the [Rook Operator Helm Chart](operator-chart.md)
 and created your customized values.yaml.
diff --git a/Documentation/Helm-Charts/ceph-cluster-chart.md b/Documentation/Helm-Charts/ceph-cluster-chart.md
index cf49dbdee..d8fd88f91 100644
--- a/Documentation/Helm-Charts/ceph-cluster-chart.md
+++ b/Documentation/Helm-Charts/ceph-cluster-chart.md
@@ -16,6 +16,7 @@ This chart is a simple packaging of templates that will optionally create Rook r
 
 ## Prerequisites
 
+* Kubernetes 1.22+
 * Helm 3.x
 * Install the [Rook Operator chart](operator-chart.md)
 
@@ -26,6 +27,8 @@ The [configuration](#configuration) section lists the parameters that can be con
 recommended that the rook operator be installed into the `rook-ceph` namespace. The clusters can be installed
 into the same namespace as the operator or a separate namespace.
 
+Rook currently publishes builds of this chart to the `release` and `master` channels.
+
 **Before installing, review the values.yaml to confirm if the default settings need to be updated.**
 
 * If the operator was installed in a namespace other than `rook-ceph`, the namespace
@@ -38,7 +41,7 @@ into the same namespace as the operator or a separate namespace.
 
 ### **Release**
 
-The `release` channel is the most recent release of Rook that is considered stable for the community.
+The release channel is the most recent release of Rook that is considered stable for the community.
 
 The example install assumes you have first installed the [Rook Operator Helm Chart](operator-chart.md)
 and created your customized values.yaml.
diff --git a/Documentation/Helm-Charts/operator-chart.gotmpl.md b/Documentation/Helm-Charts/operator-chart.gotmpl.md
index a8f4a8943..3a045f409 100644
--- a/Documentation/Helm-Charts/operator-chart.gotmpl.md
+++ b/Documentation/Helm-Charts/operator-chart.gotmpl.md
@@ -11,6 +11,7 @@ This chart bootstraps a [rook-ceph-operator](https://github.com/rook/rook) deplo
 
 ## Prerequisites
 
+* Kubernetes 1.22+
 * Helm 3.x
 
 See the [Helm support matrix](https://helm.sh/docs/topics/version_skew/) for more details.
@@ -24,9 +25,11 @@ The Ceph Operator helm chart will install the basic components necessary to crea
 
 The `helm install` command deploys rook on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the `rook-ceph` namespace (you will install your clusters into separate namespaces).
 
+Rook currently publishes builds of the Ceph operator to the `release` and `master` channels.
+
 ### **Release**
 
-The `release` channel is the most recent release of Rook that is considered stable for the community.
+The release channel is the most recent release of Rook that is considered stable for the community.
 
 ```console
 helm repo add rook-release https://charts.rook.io/release
diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md
index 64afdc653..ace2f972d 100644
--- a/Documentation/Helm-Charts/operator-chart.md
+++ b/Documentation/Helm-Charts/operator-chart.md
@@ -14,6 +14,7 @@ This chart bootstraps a [rook-ceph-operator](https://github.com/rook/rook) deplo
 
 ## Prerequisites
 
+* Kubernetes 1.22+
 * Helm 3.x
 
 See the [Helm support matrix](https://helm.sh/docs/topics/version_skew/) for more details.
@@ -27,9 +28,11 @@ The Ceph Operator helm chart will install the basic components necessary to crea
 
 The `helm install` command deploys rook on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the `rook-ceph` namespace (you will install your clusters into separate namespaces).
 
+Rook currently publishes builds of the Ceph operator to the `release` and `master` channels.
+
 ### **Release**
 
-The `release` channel is the most recent release of Rook that is considered stable for the community.
+The release channel is the most recent release of Rook that is considered stable for the community.
 
 ```console
 helm repo add rook-release https://charts.rook.io/release
@@ -57,7 +60,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` |
 | `csi.cephFSPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. | `1` |
 | `csi.cephcsi.repository` | Ceph CSI image repository | `"quay.io/cephcsi/cephcsi"` |
-| `csi.cephcsi.tag` | Ceph CSI image tag | `"v3.14.0"` |
+| `csi.cephcsi.tag` | Ceph CSI image tag | `"v3.13.1"` |
 | `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port | `9081` |
 | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Deployments and DaemonSets Pods | `nil` |
 | `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster | `nil` |
diff --git a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md
index f9c9ee9e8..3b439e27e 100644
--- a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md
+++ b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md
@@ -249,22 +249,6 @@ The simplest way to trigger the update is to add [annotations or labels](../../C
 to the CephCluster CR for the daemons you want to restart. The operator will then proceed with a rolling
 update, similar to any other update to the cluster.
 
-### Node-specific OSD settings
-
-The OSD ceph config settings can also be customized per-node. This may be helpful for some ceph.conf settings that need to be unique per node depending on the hardware. This can be configured by creating a node-specific configmap that will be loaded for all OSDs and OSD prepare jobs on that node, instead of the default settings that are loaded from the rook-config-override configmap.
-
-The node-specific configmaps must have the label:
-
-```yaml
-node.config.rook.io/osd
-```
-
-The configmaps must follow the naming convention:
-
-```yaml
-rook-config-override-<hostname>
-```
-
 ### Example
 
 In this example we will set the default pool `size` to two, and tell OSD
diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md
index 5b59f41e3..013a4885b 100644
--- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md
+++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md
@@ -217,10 +217,10 @@ CSI-Addons supports the following operations:
 
 Ceph-CSI supports encrypting PersistentVolumeClaims (PVCs) for both RBD and CephFS.
 This can be achieved using LUKS for RBD and fscrypt for CephFS. More details on encrypting RBD PVCs can be found
-[here](https://github.com/ceph/ceph-csi/blob/v3.14.0/docs/deploy-rbd.md#encryption-for-rbd-volumes),
+[here](https://github.com/ceph/ceph-csi/blob/v3.13.1/docs/deploy-rbd.md#encryption-for-rbd-volumes),
 which includes a full list of supported encryption configurations.
-More details on encrypting CephFS PVCs can be found [here](https://github.com/ceph/ceph-csi/blob/v3.14.0/docs/deploy-cephfs.md#cephfs-volume-encryption).
-A sample KMS configmap can be found [here](https://github.com/ceph/ceph-csi/blob/v3.14.0/examples/kms/vault/kms-config.yaml).
+More details on encrypting CephFS PVCs can be found [here](https://github.com/ceph/ceph-csi/blob/v3.13.1/docs/deploy-cephfs.md#cephfs-volume-encryption).
+A sample KMS configmap can be found [here](https://github.com/ceph/ceph-csi/blob/v3.13.1/examples/kms/vault/kms-config.yaml).
 
 !!! note
     Not all KMS are compatible with fscrypt. Generally, KMS that either store secrets to use directly (like Vault)
diff --git a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
index 955f22aeb..1acb2f816 100644
--- a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
+++ b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
@@ -18,7 +18,7 @@ kubectl -n $ROOK_OPERATOR_NAMESPACE edit configmap rook-ceph-operator-config
 The default upstream images are included below, which you can change to your desired images.
 
 ```yaml
-ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.14.0"
+ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.13.1"
 ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0"
 ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"
 ROOK_CSI_ATTACHER_IMAGE: "registry.k8s.io/sig-storage/csi-attacher:v4.8.0"
diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md
index 8aa76d68d..7653b0947 100644
--- a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md
+++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md
@@ -66,67 +66,50 @@ spec:
 #     ackLevel: broker [17]
 #     useSSL: false [18]
 #     mechanism: SCRAM-SHA-512 [19]
-#     userSecretRef: [20]
-#       name: foo-kafka
-#       key: user
-#     passwordSecretRef: [21]
-#       name: foo-kafka
-#       key: pass
-```
-
-```yaml
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: foo-kafka
-  namespace: rook-ceph
-stringData:
-  user: bar
-  pass: baz
 ```
 
 1. `name` of the `CephBucketTopic`
-    + In case of AMQP endpoint, the name is used for the AMQP topic ("routing key" for a topic exchange)
+    + In case of AMQP endpoint, the name is used for the AMQP topic (“routing key” for a topic exchange)
     + In case of Kafka endpoint, the name is used as the Kafka topic
 2. `namespace`(optional) of the `CephBucketTopic`. Should match the namespace of the CephBucketNotification associated with this CephBucketTopic, and the OBC with the label referencing the CephBucketNotification
 3. `objectStoreName` is the name of the object store in which the topic should be created. This must be the same object store used for the buckets associated with the notifications referencing this topic.
 4. `objectStoreNamespace` is the namespace of the object store in which the topic should be created
 5. `opaqueData` (optional) is added to all notifications triggered by a notifications associated with the topic
-6. `persistent` (optional) indicates whether notifications to this endpoint are persistent (=asynchronous) or sent synchronously (`false` by default)
+6. `persistent` (optional) indicates whether notifications to this endpoint are persistent (=asynchronous) or sent synchronously (“false” by default)
 7. `endpoint` to which to send the notifications to. Exactly one of the endpoints must be defined: `http`, `amqp`, `kafka`
 8. `http` (optional) hold the spec for an HTTP endpoint. The format of the URI would be: `http[s]://<fqdn>[:<port>][/<resource>]`
     + port defaults to: 80/443 for HTTP/S accordingly
-9. `disableVerifySSL` indicates whether the RGW is going to verify the SSL certificate of the HTTP server in case HTTPS is used (`false` by default)
-10. `sendCloudEvents`: (optional) send the notifications with the [CloudEvents header](https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md). (`false` by default)
+9. `disableVerifySSL` indicates whether the RGW is going to verify the SSL certificate of the HTTP server in case HTTPS is used ("false" by default)
+10. `sendCloudEvents`: (optional) send the notifications with the [CloudEvents header](https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md). ("false" by default)
 11. `amqp` (optional) hold the spec for an AMQP endpoint. The format of the URI would be: `amqp[s]://[<user>:<password>@]<fqdn>[:<port>][/<vhost>]`
     + port defaults to: 5672/5671 for AMQP/S accordingly
     + user/password defaults to: guest/guest
     + user/password may only be provided if HTTPS is used with the RGW. If not, topic creation request will be rejected
     + vhost defaults to: “/”
-12. `disableVerifySSL` (optional) indicates whether the RGW is going to verify the SSL certificate of the AMQP server in case AMQPS is used (`false` by default)
+12. `disableVerifySSL` (optional) indicates whether the RGW is going to verify the SSL certificate of the AMQP server in case AMQPS is used ("false" by default)
 13. `ackLevel` (optional) indicates what kind of ack the RGW is waiting for after sending the notifications:
-    + `none`: message is considered "delivered" if sent to broker
-    + `broker`: message is considered "delivered" if acked by broker (default)
-    + `routable`: message is considered "delivered" if broker can route to a consumer
+    + “none”: message is considered “delivered” if sent to broker
+    + “broker”: message is considered “delivered” if acked by broker (default)
+    + “routable”: message is considered “delivered” if broker can route to a consumer
 14. `exchange` in the AMQP broker that would route the notifications. Different topics pointing to the same endpoint must use the same exchange
 15. `kafka` (optional) hold the spec for a Kafka endpoint. The format of the URI would be: `kafka://[<user>:<password>@]<fqdn>[:<port]`
     + port defaults to: 9092
     + user/password may only be provided if HTTPS is used with the RGW. If not, topic creation request will be rejected
     + user/password may only be provided together with `useSSL`, if not, the connection to the broker would fail
-16. `disableVerifySSL` (optional) indicates whether the RGW is going to verify the SSL certificate of the Kafka server in case `useSSL` flag is used (`false` by default)
+16. `disableVerifySSL` (optional) indicates whether the RGW is going to verify the SSL certificate of the Kafka server in case `useSSL` flag is used ("false" by default)
 17. `ackLevel` (optional) indicates what kind of ack the RGW is waiting for after sending the notifications:
-    + `none`: message is considered "delivered" if sent to broker
-    + `broker`: message is considered "delivered" if acked by broker (default)
-18. `useSSL` (optional) indicates that secure connection will be used for connecting with the broker (`false` by default)
+    + “none”: message is considered “delivered” if sent to broker
+    + “broker”: message is considered “delivered” if acked by broker (default)
+18. `useSSL` (optional) indicates that secure connection will be used for connecting with the broker (“false” by default)
 19. `mechanism` (optional) select which authentication mechanism to use, one of:
-    + `PLAIN` (default)
-    + `SCRAM-SHA-512`
-    + `SCRAM-SHA-256`
-    + `GSSAPI`
-    + `OAUTHBEARER`
-20. `userSecretRef` (optional) A [SecretKeySelector](https://pkg.go.dev/k8s.io/api/core/v1#SecretKeySelector) to set the Kafka endpoint username
-21. `passwordSecretRef` (optional) A [SecretKeySelector](https://pkg.go.dev/k8s.io/api/core/v1#SecretKeySelector) to set the Kafka endpoint password
+    + “PLAIN” (default)
+    + “SCRAM-SHA-512”
+    + “SCRAM-SHA-256”
+    + “GSSAPI”
+    + “OAUTHBEARER”
+
+!!! note
+    In case of Kafka and AMQP, the consumer of the notifications is not required to ack the notifications, since the broker persists the messages before delivering them to their final destinations.
 
 ### CephBucketNotification Custom Resource
 
diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
index 138fd61cc..5eb89365b 100644
--- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
+++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
@@ -614,16 +614,16 @@ spec:
   store: my-store
   clusterNamespace: rook-ceph
   keys:
-    - accessKeyRef:
+    - acccessKeyRef
         name: foo-s3
         key: AWS_ACCESS_KEY_ID
-      secretKeyRef:
+      secretKeyRef
         name: foo-s3
         key: AWS_SECRET_ACCESS_KEY
-    - accessKeyRef:
+    - acccessKeyRef
         name: bar-s3
         key: AWS_ACCESS_KEY_ID
-      secretKeyRef:
+      secretKeyRef
         name: bar-s3
         key: AWS_SECRET_ACCESS_KEY
 ```
diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md
index c3ba61186..efc1c77d4 100644
--- a/Documentation/Upgrade/ceph-upgrade.md
+++ b/Documentation/Upgrade/ceph-upgrade.md
@@ -39,7 +39,7 @@ Official Ceph container images can be found on [Quay](https://quay.io/repository
 
 These images are tagged in a few ways:
 
-* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v19.2.2-20250409`).
+* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v19.2.1-20250202`).
     These tags are recommended for production clusters, as there is no possibility for the cluster to
     be heterogeneous with respect to the version of Ceph running in containers.
 * Ceph major version tags (e.g., `v19`) are useful for development and test clusters so that the
@@ -56,7 +56,7 @@ CephCluster CRD (`spec.cephVersion.image`).
 
 ```console
 ROOK_CLUSTER_NAMESPACE=rook-ceph
-NEW_CEPH_IMAGE='quay.io/ceph/ceph:v19.2.2-20250409'
+NEW_CEPH_IMAGE='quay.io/ceph/ceph:v19.2.1-20250202'
 kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $ROOK_CLUSTER_NAMESPACE --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}"
 ```
 
@@ -68,7 +68,7 @@ employed by the new Rook operator release. Employing an outdated Ceph version wi
 in unexpected behaviour.
 
 ```console
-kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v19.2.2-20250409
+kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v19.2.1-20250202
 ```
 
 #### **3. Wait for the pod updates**
@@ -86,9 +86,9 @@ Confirm the upgrade is completed when the versions are all on the desired Ceph v
 kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"ceph-version="}{.metadata.labels.ceph-version}{"\n"}{end}' | sort | uniq
 This cluster is not yet finished:
     ceph-version=v18.2.4-0
-    ceph-version=v19.2.2-0
+    ceph-version=v19.2.1-0
 This cluster is finished:
-    ceph-version=v19.2.2-0
+    ceph-version=v19.2.1-0
 ```
 
 #### **4. Verify cluster health**
diff --git a/Documentation/Upgrade/rook-upgrade.md b/Documentation/Upgrade/rook-upgrade.md
index ad8de677d..700eac7cc 100644
--- a/Documentation/Upgrade/rook-upgrade.md
+++ b/Documentation/Upgrade/rook-upgrade.md
@@ -14,7 +14,7 @@ We welcome feedback and opening issues!
 
 ## Supported Versions
 
-This guide is for upgrading from **Rook v1.16.x to Rook v1.17.x**.
+This guide is for upgrading from **Rook v1.15.x to Rook v1.16.x**.
 
 Please refer to the upgrade guides from previous releases for supported upgrade paths.
 Rook upgrades are only supported between official releases.
@@ -22,7 +22,6 @@ Rook upgrades are only supported between official releases.
 For a guide to upgrade previous versions of Rook, please refer to the version of documentation for
 those releases.
 
-* [Upgrade 1.15 to 1.16](https://rook.io/docs/rook/v1.16/Upgrade/rook-upgrade/)
 * [Upgrade 1.14 to 1.15](https://rook.io/docs/rook/v1.15/Upgrade/rook-upgrade/)
 * [Upgrade 1.13 to 1.14](https://rook.io/docs/rook/v1.14/Upgrade/rook-upgrade/)
 * [Upgrade 1.12 to 1.13](https://rook.io/docs/rook/v1.13/Upgrade/rook-upgrade/)
@@ -51,37 +50,18 @@ those releases.
     official releases. Builds from the master branch can have functionality changed or removed at any
     time without compatibility support and without prior notice.
 
-## Breaking changes in v1.17
-
-* The minimum supported Kubernetes version is v1.28.
-
-* S3 users provisioned via CephObjectStoreUser resources no longer allow multiple credentials to
-    exist on underlying S3 users, unless explicitly managed by Rook. Rook will purge all but one of
-    the undeclared credentials. This could be a user observable regression for administrators who
-    manually edited/rotated S3 user credentials for CephObjectStoreUsers. For affected users, Rook
-    now provides first-class
-    [user credential management support](../Storage-Configuration/Object-Storage-RGW/object-storage.md#managing-user-s3-credentials).
-    For more details, see [#15359](https://github.com/rook/rook/issues/15359).
-
-* Some ObjectBucketClaim options were added in Rook v1.16 that allowed more control over buckets.
-    These controls allow users to self-serve their own S3 policies, which many administrators might
-    consider a risk, depending on their environment. Rook has taken steps to ensure potentially risky
-    configurations are disabled by default to ensure the safest off-the-shelf configurations.
-    Administrators who wish to allow users to use the full range of OBC configurations must use the
-    new `ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS` to enable users to set potentially risky options.
-    See [#15376](https://github.com/rook/rook/pull/15376) for more information.
-
-* Kafka notifications configured via CephBucketTopic resources will now default
-    to setting the Kafka authentication mechanism to `PLAIN`. Previously, no auth
-    mechanism was specified by default.  It was possible to set the auth mechanism
-    via `CephBucketTopic.spec.endpoint.kafka.opaqueData`.  However, setting
-    `&mechanism=<auth type>` via `opaqueData` is no longer possible. If any auth
-    mechanism other than `PLAIN` is in use, modification to `CephBucketTopic`
-    resources is required.
-
-    See:
-    - [#15711](https://github.com/rook/rook/issues/15711)
-    - [#15554](https://github.com/rook/rook/pull/15554)
+## Breaking changes in v1.16
+
+* The minimum supported Kubernetes version is v1.27.
+
+* Rook no longer supports Ceph v17 (Quincy) as it is end of life. Rook v1.15 clusters should
+    [upgrade Ceph](./ceph-upgrade.md) to at least Ceph v18 (Reef) before upgrading Rook to v1.16.
+    Be aware of known [Ceph v18.2.4 ARM issues](https://github.com/rook/rook/issues/14502).
+
+* Rook fully deprecated CSI "holder" pods in Rook v1.16. Any Rook v1.15 CephCluster with
+    `csi-*plugin-holder-*` pods present in the Rook operator namespace must follow holder pod
+    removal migration steps outlined in
+    [v1.15 documentation](https://rook.io/docs/rook/v1.15/CRDs/Cluster/network-providers/?h=depre#holder-pod-deprecation).
 
 ## Considerations
 
@@ -97,11 +77,11 @@ With this upgrade guide, there are a few notes to consider:
 
 Unless otherwise noted due to extenuating requirements, upgrades from one patch release of Rook to
 another are as simple as updating the common resources and the image of the Rook operator. For
-example, when Rook v1.17.1 is released, the process of updating from v1.17.0 is as simple as running
+example, when Rook v1.16.1 is released, the process of updating from v1.16.0 is as simple as running
 the following:
 
 ```console
-git clone --single-branch --depth=1 --branch v1.17.1 https://github.com/rook/rook.git
+git clone --single-branch --depth=1 --branch v1.16.1 https://github.com/rook/rook.git
 cd rook/deploy/examples
 ```
 
@@ -109,11 +89,11 @@ If the Rook Operator or CephCluster are deployed into a different namespace than
 `rook-ceph`, see the [Update common resources and CRDs](#1-update-common-resources-and-crds)
 section for instructions on how to change the default namespaces in `common.yaml`.
 
-Then, apply the latest changes from v1.17, and update the Rook Operator image.
+Then, apply the latest changes from v1.16, and update the Rook Operator image.
 
 ```console
 kubectl apply -f common.yaml -f crds.yaml
-kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.17.1
+kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.16.1
 ```
 
 As exemplified above, it is a good practice to update Rook common resources from the example
@@ -147,9 +127,9 @@ In order to successfully upgrade a Rook cluster, the following prerequisites mus
 
 ## Rook Operator Upgrade
 
-The examples given in this guide upgrade a live Rook cluster running `v1.16.6` to
-the version `v1.17.0`. This upgrade should work from any official patch release of Rook v1.16 to any
-official patch release of v1.17.
+The examples given in this guide upgrade a live Rook cluster running `v1.15.6` to
+the version `v1.16.0`. This upgrade should work from any official patch release of Rook v1.15 to any
+official patch release of v1.16.
 
 Let's get started!
 
@@ -210,7 +190,7 @@ kubectl apply -f deploy/examples/monitoring/rbac.yaml
 !!! hint
     The operator is automatically updated when using Helm charts.
 
-The largest portion of the upgrade is triggered when the operator's image is updated to `v1.17.x`.
+The largest portion of the upgrade is triggered when the operator's image is updated to `v1.16.x`.
 When the operator is updated, it will proceed to update all of the Ceph daemons.
 
 ```console
@@ -241,18 +221,18 @@ watch --exec kubectl -n $ROOK_CLUSTER_NAMESPACE get deployments -l rook_cluster=
 ```
 
 As an example, this cluster is midway through updating the OSDs. When all deployments report `1/1/1`
-availability and `rook-version=v1.17.0`, the Ceph cluster's core components are fully updated.
+availability and `rook-version=v1.16.0`, the Ceph cluster's core components are fully updated.
 
 ```console
 Every 2.0s: kubectl -n rook-ceph get deployment -o j...
 
-rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.17.0
-rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.17.0
-rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.17.0
-rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.17.0
-rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.17.0
-rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.16.6
-rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.16.6
+rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.16.0
+rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.15.6
+rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.15.6
 ```
 
 An easy check to see if the upgrade is totally finished is to check that there is only one
@@ -261,14 +241,14 @@ An easy check to see if the upgrade is totally finished is to check that there i
 ```console
 # kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"rook-version="}{.metadata.labels.rook-version}{"\n"}{end}' | sort | uniq
 This cluster is not yet finished:
-  rook-version=v1.16.6
-  rook-version=v1.17.0
+  rook-version=v1.15.6
+  rook-version=v1.16.0
 This cluster is finished:
-  rook-version=v1.17.0
+  rook-version=v1.16.0
 ```
 
 ### **5. Verify the updated cluster**
 
-At this point, the Rook operator should be running version `rook/ceph:v1.17.0`.
+At this point, the Rook operator should be running version `rook/ceph:v1.16.0`.
 
 Verify the CephCluster health using the [health verification doc](health-verification.md).
diff --git a/Makefile b/Makefile
index 488ade028..f1b6ab12c 100644
--- a/Makefile
+++ b/Makefile
@@ -143,11 +143,7 @@ endif
 install: build.common
 	@$(MAKE) go.install
 
-.PHONY: check
-check: test ## Runs checks (unit tests)
-
-.PHONY: test
-test: ## Runs unit tests.
+check test: ## Runs unit tests.
 	@$(MAKE) go.test.unit
 
 test-integration: ## Runs integration tests.
@@ -162,7 +158,7 @@ fmt: $(YQ) ## Check formatting of go sources.
 
 fmt-fix: $(YQ) ## Reformatting of go sources.
 	@$(MAKE) go.fmt-fix
-
+ 
 golangci-lint: $(YQ)
 	@$(MAKE) go.golangci-lint
 
@@ -175,15 +171,12 @@ yamllint:
 	yamllint -c .yamllint deploy/examples/ --no-warnings
 
 .PHONY: lint
-lint: yamllint pylint shellcheck checkmake vet markdownlint golangci-lint ## Run various linters
+lint: yamllint pylint shellcheck vet markdownlint golangci-lint ## Run various linters
 
 .PHONY: pylint
 pylint:
 	pylint $(shell find $(ROOT_DIR) -name '*.py') -E
 
-.PHONY: checkmake
-checkmake:
-	checkmake Makefile
 .PHONY: shellcheck
 shellcheck:
 	shellcheck --severity=warning --format=gcc --shell=bash $(shell find $(ROOT_DIR) -type f -name '*.sh') build/reset build/sed-in-place
@@ -218,9 +211,8 @@ gen-rbac: $(HELM) $(YQ) ## Generate RBAC from Helm charts
 	HELM=$(HELM) ./build/rbac/gen-common.sh
 	HELM=$(HELM) ./build/rbac/gen-nfs-rbac.sh
 
-gen.docs: docs ## generate docs
-.PHONY: docs
-docs: helm-docs ## generate documentation
+gen.docs: docs
+docs: helm-docs
 gen.helm-docs: helm-docs
 helm-docs: $(HELM_DOCS) ## Use helm-docs to generate documentation from helm charts
 	$(HELM_DOCS) -c deploy/charts/rook-ceph \
@@ -251,8 +243,9 @@ generate: gen.codegen gen.crds gen.rbac gen.docs gen.crd-docs ## Update all gene
 # ====================================================================================
 # Help
 define HELPTEXT
-available options:
+Options:
     DEBUG        Whether to generate debug symbols. Default is 0.
+    IMAGES       Backend images to make. All by default. See: /rook/images/ dir
     PLATFORM     The platform to build.
     SUITE        The test suite to run.
     TESTFILTER   Tests to run in a suite.
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index 039978b87..11608157c 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -1,6 +1,24 @@
-# v1.18 Pending Release Notes
+# v1.17 Pending Release Notes
 
 ## Breaking Changes
 
+Object:
+
+- Some ObjectBucketClaim options were added in Rook v1.16 that allowed more control over buckets.
+    These controls allow users to self-serve their own S3 policies, which many administrators might
+    consider a risk, depending on their environment. Rook has taken steps to ensure potentially risky
+    configurations are disabled by default to ensure the safest off-the-shelf configurations.
+    Administrators who wish to allow users to use the full range of OBC configurations must use the
+    new `ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS` to enable users to set potentially risky options.
+    See https://github.com/rook/rook/pull/15376 for more information.
+
+- Add credentials/keys management to CephObjectStoreUser. Note that this will
+    cause manually created credentials/keys on an rgw user to be purged. This
+    behavior is necessary to ensure that keys that were explicitly declared and
+    then undeclared are purged. This could be a user observable regression.
+    (see [#15359](https://github.com/rook/rook/issues/15359)
 
 ## Features
+
+- Support external mons for local Rook cluster (see [#14733](https://github.com/rook/rook/issues/14733)).
+- Manage EndpointSlice resources containing monitor IPs to support DNS-based resolution for Ceph clients (see [#14986](https://github.com/rook/rook/issues/14986)).
diff --git a/build/makelib/common.mk b/build/makelib/common.mk
index d87a9cdce..d61242293 100644
--- a/build/makelib/common.mk
+++ b/build/makelib/common.mk
@@ -100,9 +100,12 @@ SED_IN_PLACE = $(ROOT_DIR)/build/sed-in-place
 export SED_IN_PLACE
 
 # This is a neat little target that prints any variable value from the Makefile
-# Usage: make echo.PLATFORM
+# Usage: make echo.IMAGES echo.PLATFORM
 echo.%: ; @echo $* = $($*)
 
+# Select which images (backends) to make; default to all possible images
+IMAGES ?= ceph
+
 COMMA := ,
 SPACE :=
 SPACE +=
diff --git a/build/release/Makefile b/build/release/Makefile
index 99928ef02..83f4d3c2d 100644
--- a/build/release/Makefile
+++ b/build/release/Makefile
@@ -17,6 +17,14 @@ all: build
 include ../makelib/common.mk
 include ../makelib/helm.mk
 
+# ====================================================================================
+# Options
+
+CHANNEL ?= master
+ifeq ($(filter master release,$(CHANNEL)),)
+$(error invalid channel $(CHANNEL))
+endif
+
 # When running the tag pipeline or release build we always want to add the alpha, beta, or rc suffix
 # if provided. Otherwise, it's a master build where we don't want to apply the suffix.
 ifneq ($(TAG_WITH_SUFFIX),true)
@@ -47,27 +55,29 @@ ifeq ($(COMMIT_HASH),)
 override COMMIT_HASH := $(shell git rev-parse HEAD)
 endif
 
+REMOTE_NAME ?= origin
+
 PLATFORMS ?= $(ALL_PLATFORMS)
 
 # "helm" flavor is for pushing to s3/https endpoint
 # "charts" flavor is for pushing to oci endpoint
+ifneq ($(filter master release-%,$(BRANCH_NAME)),)
 FLAVORS ?= output images docs helm charts
+else
+FLAVORS ?= output
+override BRANCH_NAME := pr/$(BRANCH_NAME)
+endif
 
 DOCKER_REGISTRY ?= docker.io/rook
 QUAY_REGISTRY ?= quay.io/rook
 GHCR_REGISTRY ?= ghcr.io/rook
-ifeq ($(TAGGED_RELEASE),false)
-# Publish always to dockerhub, including master and interim builds
-REGISTRIES = $(DOCKER_REGISTRY)
-else
-# Publish the images to all registries for tagged releases
-REGISTRIES = $(DOCKER_REGISTRY) $(QUAY_REGISTRY) $(GHCR_REGISTRY)
-endif
+REGISTRIES ?= $(DOCKER_REGISTRY) $(QUAY_REGISTRY) $(GHCR_REGISTRY)
 IMAGE_ARCHS := $(subst linux_,,$(filter linux_%,$(PLATFORMS)))
 IMAGE_PLATFORMS := $(subst _,/,$(subst $(SPACE),$(COMMA),$(filter linux_%,$(PLATFORMS))))
 IMAGE_PLATFORMS_COMMA := $(shell echo "$(IMAGE_PLATFORMS)" | sed 's/ /,/g' | sed 's/.$$//')
 
 S3_BUCKET ?= rook.releases
+S3_CP := aws s3 cp --only-show-errors
 S3_SYNC := aws s3 sync --only-show-errors
 S3_SYNC_DEL := aws s3 sync --only-show-errors --delete
 
@@ -89,10 +99,33 @@ $(MANIFEST_TOOL):
 build: $(addprefix build.,$(FLAVORS)) ;
 publish: $(addprefix publish.,$(FLAVORS)) ;
 promote: $(addprefix promote.,$(FLAVORS)) ;
+clean: $(addprefix clean.,$(FLAVORS)) ;
 
 # catch all for unimplemented targets / flavors
 %: ; @:
 
+# ====================================================================================
+# tagging a release
+
+VERSION_REGEX := ^v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)[-]*\([alpha|beta|rc]\)*[\.]*\([0-9]*\)$$
+VERSION_VALID := $(shell echo "$(VERSION)" | grep -q '$(VERSION_REGEX)' && echo 1 || echo 0)
+VERSION_MAJOR := $(shell echo "$(VERSION)" | sed -e 's/$(VERSION_REGEX)/\1/')
+VERSION_MINOR := $(shell echo "$(VERSION)" | sed -e 's/$(VERSION_REGEX)/\2/')
+VERSION_PATCH := $(shell echo "$(VERSION)" | sed -e 's/$(VERSION_REGEX)/\3/')
+
+tag:
+ifneq ($(VERSION_VALID),1)
+	$(error invalid version $(VERSION), must be a semantic version with v[Major].[Minor].[Patch] and an optional suffix such as -alpha.0, -beta.1, or -rc.2)
+endif
+	@echo === tagging commit hash $(COMMIT_HASH) with $(VERSION) and parsed v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_PATCH)
+	git tag -f -m "release $(VERSION)" $(VERSION) $(COMMIT_HASH)
+	git push $(REMOTE_NAME) $(VERSION)
+	set -e; if ! git ls-remote --heads $(REMOTE_NAME) | grep -q refs/heads/release-$(VERSION_MAJOR).$(VERSION_MINOR); then \
+		echo === creating new release branch release-$(VERSION_MAJOR).$(VERSION_MINOR) ;\
+		git branch -f release-$(VERSION_MAJOR).$(VERSION_MINOR) $(COMMIT_HASH) ;\
+		git push $(REMOTE_NAME) release-$(VERSION_MAJOR).$(VERSION_MINOR) ;\
+	fi
+
 # ====================================================================================
 # docs
 
@@ -113,16 +146,16 @@ publish.docs:
 
 # ====================================================================================
 # helm
-HELM_CHANNEL = release
+
 HELM_TEMP := $(shell mktemp -d)
-HELM_URL := $(HELM_BASE_URL)/$(HELM_CHANNEL)
+HELM_URL := $(HELM_BASE_URL)/$(CHANNEL)
 
 promote.helm: $(HELM)
 #	copy existing charts to a temp dir, then combine with new charts, reindex, and upload
-	$(S3_SYNC) s3://$(HELM_S3_BUCKET)/$(HELM_CHANNEL) $(HELM_TEMP)
+	$(S3_SYNC) s3://$(HELM_S3_BUCKET)/$(CHANNEL) $(HELM_TEMP)
 	$(S3_SYNC) s3://$(S3_BUCKET)/build/$(BRANCH_NAME)/$(VERSION)/charts $(HELM_TEMP)
 	$(HELM) repo index --url $(HELM_URL) $(HELM_TEMP)
-	$(S3_SYNC_DEL) $(HELM_TEMP) s3://$(HELM_S3_BUCKET)/$(HELM_CHANNEL)
+	$(S3_SYNC_DEL) $(HELM_TEMP) s3://$(HELM_S3_BUCKET)/$(CHANNEL)
 	rm -fr $(HELM_TEMP)
 
 # set the helm cart version number.
@@ -144,6 +177,9 @@ build.chart.$(1):
 	mkdir -p $(HELM_OUTPUT_DIR)
 	$(HELM) package --version $(HELM_CHART_VERSION) --app-version $(VERSION) $(HELM_CHARTS_DIR)/$(1) --destination $(HELM_OUTPUT_DIR)
 build.all.charts: build.chart.$(1)
+clean.chart.$(1):
+	@rm -fr $(HELM_OUTPUT_DIR)/$(1)-$(HELM_CHART_VERSION).tgz
+clean.all.charts: clean.chart.$(1)
 endef
 $(foreach c,$(HELM_CHARTS),$(eval $(call chart.build,$(c))))
 
@@ -161,42 +197,54 @@ $(foreach r,$(REGISTRIES), $(foreach c,$(HELM_CHARTS),$(eval $(call chart.target
 publish.output:
 	$(S3_SYNC_DEL) $(OUTPUT_DIR) s3://$(S3_BUCKET)/build/$(BRANCH_NAME)/$(VERSION)
 promote.output:
-	$(S3_SYNC_DEL) s3://$(S3_BUCKET)/build/$(BRANCH_NAME)/$(VERSION) s3://$(S3_BUCKET)/$(HELM_CHANNEL)/$(VERSION)
-	$(S3_SYNC_DEL) s3://$(S3_BUCKET)/build/$(BRANCH_NAME)/$(VERSION) s3://$(S3_BUCKET)/$(HELM_CHANNEL)/current
+	$(S3_SYNC_DEL) s3://$(S3_BUCKET)/build/$(BRANCH_NAME)/$(VERSION) s3://$(S3_BUCKET)/$(CHANNEL)/$(VERSION)
+	$(S3_SYNC_DEL) s3://$(S3_BUCKET)/build/$(BRANCH_NAME)/$(VERSION) s3://$(S3_BUCKET)/$(CHANNEL)/current
 
 # ====================================================================================
 # images
 
-# 1: registry 2: arch
+# 1: registry 2: image, 3: arch
 define repo.targets
-build.image.$(1).$(2):
-	$(DOCKERCMD) tag $(BUILD_REGISTRY)/ceph-$(2) $(1)/ceph-$(2):$(VERSION)
-	$(DOCKERCMD) tag $(BUILD_REGISTRY)/ceph-$(2) $(1)/ceph-$(2):$(BRANCH_NAME)
-	@# Save image as _output/images/linux_<arch>/ceph.tar.gz (no builds for darwin or windows)
-	mkdir -p $(OUTPUT_DIR)/images/linux_$(2)
-	$(DOCKERCMD) save $(BUILD_REGISTRY)/ceph-$(2) | gzip -c > $(OUTPUT_DIR)/images/linux_$(2)/ceph.tar.gz
-build.all.images: build.image.$(1).$(2)
-publish.image.$(1).$(2):
-	@$(DOCKERCMD) push $(1)/ceph-$(2):$(VERSION)
-	@$(DOCKERCMD) push $(1)/ceph-$(2):$(BRANCH_NAME)
-
-publish.all.images.$(1): publish.image.$(1).$(2)
+build.image.$(1).$(2).$(3):
+	$(DOCKERCMD) tag $(BUILD_REGISTRY)/$(2)-$(3) $(1)/$(2)-$(3):$(VERSION)
+	@# Save image as _output/images/linux_<arch>/<image>.tar.gz (no builds for darwin or windows)
+	mkdir -p $(OUTPUT_DIR)/images/linux_$(3)
+	$(DOCKERCMD) save $(BUILD_REGISTRY)/$(2)-$(3) | gzip -c > $(OUTPUT_DIR)/images/linux_$(3)/$(2).tar.gz
+build.all.images: build.image.$(1).$(2).$(3)
+publish.image.$(1).$(2).$(3): ; @$(DOCKERCMD) push $(1)/$(2)-$(3):$(VERSION)
+publish.all.images: publish.image.$(1).$(2).$(3)
+# tag the master image, but do not tag the release image with a generic channel tag
+promote.image.$(1).$(2).$(3):
+	$(DOCKERCMD) pull $(1)/$(2)-$(3):$(VERSION)
+	$(DOCKERCMD) tag $(1)/$(2)-$(3):$(VERSION) $(1)/$(2)-$(3):$(CHANNEL)
+	[ "$(CHANNEL)" = "release" ] || $(DOCKERCMD) push $(1)/$(2)-$(3):$(CHANNEL)
+promote.all.images: promote.image.$(1).$(2).$(3)
+clean.image.$(1).$(2).$(3):
+	[ -z "$$$$($(DOCKERCMD) images -q $(1)/$(2)-$(3):$(VERSION))" ] || $(DOCKERCMD) rmi $(1)/$(2)-$(3):$(VERSION)
+	[ -z "$$$$($(DOCKERCMD) images -q $(1)/$(2)-$(3):$(CHANNEL))" ] || $(DOCKERCMD) rmi $(1)/$(2)-$(3):$(CHANNEL)
+clean.all.images: clean.image.$(1).$(2).$(3)
 endef
-$(foreach r,$(REGISTRIES), $(foreach a,$(IMAGE_ARCHS),$(eval $(call repo.targets,$(r),$(a)))))
-
-define repo.manifest.targets
-publish.manifest.image.$(1): publish.all.images.$(1) $(MANIFEST_TOOL)
-	$(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(1)/ceph-ARCH:$(VERSION) --target $(1)/ceph:$(VERSION)
-	$(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(1)/ceph-ARCH:$(BRANCH_NAME) --target $(1)/ceph:$(BRANCH_NAME)
+$(foreach r,$(REGISTRIES), $(foreach i,$(IMAGES), $(foreach a,$(IMAGE_ARCHS),$(eval $(call repo.targets,$(r),$(i),$(a))))))
 
-publish.images: publish.manifest.image.$(1)
-endef
-$(foreach r,$(REGISTRIES),$(eval $(call repo.manifest.targets,$(r))))
+publish.manifest.image.%: publish.all.images $(MANIFEST_TOOL)
+	$(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(DOCKER_REGISTRY)/$*-ARCH:$(VERSION) --target $(DOCKER_REGISTRY)/$*:$(VERSION)
+	$(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(QUAY_REGISTRY)/$*-ARCH:$(VERSION) --target $(QUAY_REGISTRY)/$*:$(VERSION)
+	$(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(GHCR_REGISTRY)/$*-ARCH:$(VERSION) --target $(GHCR_REGISTRY)/$*:$(VERSION)
 
+# add the "master" tag to the master image, but do not add the "release" tag for the release channel
+promote.manifest.image.%: promote.all.images $(MANIFEST_TOOL)
+	[ "$(CHANNEL)" = "release" ] || $(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(DOCKER_REGISTRY)/$*-ARCH:$(VERSION) --target $(DOCKER_REGISTRY)/$*:$(CHANNEL)
+	[ "$(CHANNEL)" = "release" ] || $(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(QUAY_REGISTRY)/$*-ARCH:$(VERSION) --target $(QUAY_REGISTRY)/$*:$(CHANNEL)
+	[ "$(CHANNEL)" = "release" ] || $(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS_COMMA) --template $(GHCR_REGISTRY)/$*-ARCH:$(VERSION) --target $(GHCR_REGISTRY)/$*:$(CHANNEL)
 
 build.images: build.all.images
+publish.images: $(addprefix publish.manifest.image.,$(IMAGES))
+promote.images: $(addprefix promote.manifest.image.,$(IMAGES))
+clean.images: clean.all.images
+
 build.charts: build.all.charts
 promote.charts: promote.all.charts
+clean.charts: clean.all.charts
 
 # ====================================================================================
 # Help
@@ -207,12 +255,16 @@ help:
 	@echo ''
 	@echo 'Targets:'
 	@echo '    build        Build all release artifacts.'
+	@echo '    clean        Remove all release artifacts.'
 	@echo '    publish      Publish all release artifacts.'
-	@echo '    promote      Publish the helm charts.'
+	@echo '    promote      Promote a build to a channel.'
+	@echo '    tag          Tag a build for release.'
 	@echo ''
 	@echo 'Options:'
 	@echo '    VERSION      Sets the release version.'
 	@echo '    BRANCH_NAME  Name of the branch we are releasing from.'
+	@echo '    CHANNEL      Sets the release channel. Can be set to master,'
+	@echo '                 or release. Default is not set.'
 	@echo '    PLATFORMS    The supported platforms to build when running.'
 	@echo '                 the build.all target. The default is'
 	@echo '                 all supported platforms'
diff --git a/deploy/charts/rook-ceph-cluster/templates/_helpers.tpl b/deploy/charts/rook-ceph-cluster/templates/_helpers.tpl
index 6f0a0f9a2..dbb065ffa 100644
--- a/deploy/charts/rook-ceph-cluster/templates/_helpers.tpl
+++ b/deploy/charts/rook-ceph-cluster/templates/_helpers.tpl
@@ -11,3 +11,14 @@ Return the target Kubernetes version.
 {{- define "capabilities.kubeVersion" -}}
 {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}}
 {{- end }}
+
+{{/*
+Return the appropriate apiVersion for ingress.
+*/}}
+{{- define "capabilities.ingress.apiVersion" -}}
+{{- if semverCompare "<1.19-0" (include "capabilities.kubeVersion" .) -}}
+{{- print "networking.k8s.io/v1beta1" -}}
+{{- else -}}
+{{- print "networking.k8s.io/v1" -}}
+{{- end }}
+{{- end }}
diff --git a/deploy/charts/rook-ceph-cluster/templates/cephobjectstore-ingress.yaml b/deploy/charts/rook-ceph-cluster/templates/cephobjectstore-ingress.yaml
index 693c2ce16..bb0934124 100644
--- a/deploy/charts/rook-ceph-cluster/templates/cephobjectstore-ingress.yaml
+++ b/deploy/charts/rook-ceph-cluster/templates/cephobjectstore-ingress.yaml
@@ -1,7 +1,7 @@
 {{- range .Values.cephObjectStores }}
 {{- if dig "ingress" "enabled" false . }}
 ---
-apiVersion: networking.k8s.io/v1
+apiVersion: {{ include "capabilities.ingress.apiVersion" $ }}
 kind: Ingress
 metadata:
   name: {{ .name }}
@@ -16,11 +16,16 @@ spec:
         paths:
           - path: {{ .ingress.host.path | default "/" }}
             backend:
+{{- if (eq "networking.k8s.io/v1" (include "capabilities.ingress.apiVersion" $)) }}
               service:
                 name: rook-ceph-rgw-{{ .name }}
                 port:
-                  number: {{ .ingress.port | default .spec.gateway.securePort | default .spec.gateway.port }}
+                  number: {{ .spec.gateway.securePort | default .spec.gateway.port }}
             pathType: {{ .ingress.host.pathType | default "Prefix" }}
+{{- else }}
+              serviceName: rook-ceph-rgw-{{ .name }}
+              servicePort: {{ .spec.gateway.securePort | default .spec.gateway.port }}
+{{- end }}
   {{- with .ingress.ingressClassName }}
   ingressClassName: {{ . }}
   {{- end }}
diff --git a/deploy/charts/rook-ceph-cluster/templates/ingress.yaml b/deploy/charts/rook-ceph-cluster/templates/ingress.yaml
index 47746b462..f915575f4 100644
--- a/deploy/charts/rook-ceph-cluster/templates/ingress.yaml
+++ b/deploy/charts/rook-ceph-cluster/templates/ingress.yaml
@@ -1,13 +1,10 @@
 {{- if .Values.ingress.dashboard.host }}
 ---
-apiVersion: networking.k8s.io/v1
+apiVersion: {{ include "capabilities.ingress.apiVersion" . }}
 kind: Ingress
 metadata:
   name: {{ template "clusterName" . }}-dashboard
   namespace: {{ .Release.Namespace }} # namespace:cluster
-  {{- if .Values.ingress.dashboard.labels }}
-  labels: {{- toYaml .Values.ingress.dashboard.labels | nindent 4 }}
-  {{- end}}
   {{- if .Values.ingress.dashboard.annotations }}
   annotations: {{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }}
   {{- end }}
@@ -18,6 +15,7 @@ spec:
         paths:
           - path: {{ .Values.ingress.dashboard.host.path | default "/" }}
             backend:
+{{- if (eq "networking.k8s.io/v1" (include "capabilities.ingress.apiVersion" .)) }}
               service:
                 name: rook-ceph-mgr-dashboard
                 port:
@@ -27,6 +25,14 @@ spec:
                   name: http-dashboard
                   {{- end }}
             pathType: {{ .Values.ingress.dashboard.host.pathType | default "Prefix" }}
+{{- else }}
+              serviceName: rook-ceph-mgr-dashboard
+              {{- if .Values.cephClusterSpec.dashboard.ssl }}
+              servicePort: https-dashboard
+              {{- else }}
+              servicePort: http-dashboard
+              {{- end }}
+{{- end }}
   {{- if .Values.ingress.dashboard.ingressClassName }}
   ingressClassName: {{ .Values.ingress.dashboard.ingressClassName }}
   {{- end }}
diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml
index 3f336c5cc..865dea147 100644
--- a/deploy/charts/rook-ceph-cluster/values.yaml
+++ b/deploy/charts/rook-ceph-cluster/values.yaml
@@ -25,7 +25,7 @@ toolbox:
   # -- Enable Ceph debugging pod deployment. See [toolbox](../Troubleshooting/ceph-toolbox.md)
   enabled: false
   # -- Toolbox image, defaults to the image used by the Ceph cluster
-  image: #quay.io/ceph/ceph:v19.2.2
+  image: #quay.io/ceph/ceph:v19.2.1
   # -- Toolbox tolerations
   tolerations: []
   # -- Toolbox affinity
@@ -94,9 +94,9 @@ cephClusterSpec:
     # v18 is Reef, v19 is Squid
     # RECOMMENDATION: In production, use a specific version tag instead of the general v18 flag, which pulls the latest release and could result in different
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
-    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v19.2.2-20250409
+    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v19.2.1-20250202
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     # Whether to allow unsupported versions of Ceph. Currently Reef and Squid are supported.
     # Future versions such as Tentacle (v20) would require this to be set to `true`.
     # Do not set to true in production.
@@ -417,8 +417,6 @@ ingress:
   # -- Enable an ingress for the ceph-dashboard
   dashboard:
     {}
-    # labels:
-    #   external-dns/private: "true"
     # annotations:
     #   external-dns.alpha.kubernetes.io/hostname: dashboard.example.com
     #   nginx.ingress.kubernetes.io/rewrite-target: /ceph-dashboard/$2
@@ -629,9 +627,6 @@ cephObjectStores:
     ingress:
       # Enable an ingress for the ceph-objectstore
       enabled: false
-      # The ingress port by default will be the object store's "securePort" (if set), or the gateway "port".
-      # To override those defaults, set this ingress port to the desired port.
-      # port: 80
       # annotations: {}
       # host:
       #   name: objectstore.example.com
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index 3048672a6..a6ae5e771 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -954,50 +954,6 @@ spec:
                     kafka:
                       description: Spec of Kafka endpoint
                       properties:
-                        PasswordSecretRef:
-                          description: The kafka password to use for authentication
-                          properties:
-                            key:
-                              description: The key of the secret to select from.  Must be a valid secret key.
-                              type: string
-                            name:
-                              default: ""
-                              description: |-
-                                Name of the referent.
-                                This field is effectively required, but due to backwards compatibility is
-                                allowed to be empty. Instances of this type with an empty value here are
-                                almost certainly wrong.
-                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                              type: string
-                            optional:
-                              description: Specify whether the Secret or its key must be defined
-                              type: boolean
-                          required:
-                            - key
-                          type: object
-                          x-kubernetes-map-type: atomic
-                        UserSecretRef:
-                          description: The kafka user name to use for authentication
-                          properties:
-                            key:
-                              description: The key of the secret to select from.  Must be a valid secret key.
-                              type: string
-                            name:
-                              default: ""
-                              description: |-
-                                Name of the referent.
-                                This field is effectively required, but due to backwards compatibility is
-                                allowed to be empty. Instances of this type with an empty value here are
-                                almost certainly wrong.
-                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                              type: string
-                            optional:
-                              description: Specify whether the Secret or its key must be defined
-                              type: boolean
-                          required:
-                            - key
-                          type: object
-                          x-kubernetes-map-type: atomic
                         ackLevel:
                           default: broker
                           description: The ack level required for this topic (none/broker)
@@ -1061,26 +1017,6 @@ spec:
                   type: integer
                 phase:
                   type: string
-                secrets:
-                  items:
-                    properties:
-                      name:
-                        description: name is unique within a namespace to reference a secret resource.
-                        type: string
-                      namespace:
-                        description: namespace defines the space within which the secret name must be unique.
-                        type: string
-                      resourceVersion:
-                        type: string
-                      uid:
-                        description: |-
-                          UID is a type that holds unique ID values, including UUIDs.  Because we
-                          don't ONLY use UUIDs, this is an alias to string.  Being a type captures
-                          intent and helps make sure that UIDs and names do not get conflated.
-                        type: string
-                    type: object
-                    x-kubernetes-map-type: atomic
-                  type: array
               type: object
               x-kubernetes-preserve-unknown-fields: true
           required:
diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml
index a6b304e32..b16cf0061 100644
--- a/deploy/charts/rook-ceph/values.yaml
+++ b/deploy/charts/rook-ceph/values.yaml
@@ -489,7 +489,7 @@ csi:
     # -- Ceph CSI image repository
     repository: quay.io/cephcsi/cephcsi
     # -- Ceph CSI image tag
-    tag: v3.14.0
+    tag: v3.13.1
 
   registrar:
     # -- Kubernetes CSI registrar image repository
diff --git a/deploy/examples/cluster-external-management.yaml b/deploy/examples/cluster-external-management.yaml
index 03d20e9ae..290e58456 100644
--- a/deploy/examples/cluster-external-management.yaml
+++ b/deploy/examples/cluster-external-management.yaml
@@ -20,4 +20,4 @@ spec:
   dataDirHostPath: /var/lib/rook
   # providing an image is required, if you want to create other CRs (rgw, mds, nfs)
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2 # Should match external cluster version
+    image: quay.io/ceph/ceph:v19.2.1 # Should match external cluster version
diff --git a/deploy/examples/cluster-on-local-pvc.yaml b/deploy/examples/cluster-on-local-pvc.yaml
index 9c79125e1..73ca094de 100644
--- a/deploy/examples/cluster-on-local-pvc.yaml
+++ b/deploy/examples/cluster-on-local-pvc.yaml
@@ -174,7 +174,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-on-pvc.yaml b/deploy/examples/cluster-on-pvc.yaml
index d53ac3bd6..c07fd8224 100644
--- a/deploy/examples/cluster-on-pvc.yaml
+++ b/deploy/examples/cluster-on-pvc.yaml
@@ -34,7 +34,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-stretched-aws.yaml b/deploy/examples/cluster-stretched-aws.yaml
index 62b06de3f..eb742ea00 100644
--- a/deploy/examples/cluster-stretched-aws.yaml
+++ b/deploy/examples/cluster-stretched-aws.yaml
@@ -45,7 +45,7 @@ spec:
   mgr:
     count: 2
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     allowUnsupported: true
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-stretched.yaml b/deploy/examples/cluster-stretched.yaml
index 24a4c1b4e..677a8e53d 100644
--- a/deploy/examples/cluster-stretched.yaml
+++ b/deploy/examples/cluster-stretched.yaml
@@ -39,7 +39,7 @@ spec:
   mgr:
     count: 2
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     allowUnsupported: true
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml
index 79382e6d6..9a6ae45ee 100644
--- a/deploy/examples/cluster.yaml
+++ b/deploy/examples/cluster.yaml
@@ -19,9 +19,9 @@ spec:
     # v18 is Reef, v19 is Squid
     # RECOMMENDATION: In production, use a specific version tag instead of the general v19 flag, which pulls the latest release and could result in different
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
-    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v19.2.2-20250409
+    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v19.2.1-20250202
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
     # Whether to allow unsupported versions of Ceph. Currently Reef and Squid are supported.
     # Future versions such as Tentacle (v20) would require this to be set to `true`.
     # Do not set to true in production.
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 7b317bb66..01aa2fb90 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -954,50 +954,6 @@ spec:
                     kafka:
                       description: Spec of Kafka endpoint
                       properties:
-                        PasswordSecretRef:
-                          description: The kafka password to use for authentication
-                          properties:
-                            key:
-                              description: The key of the secret to select from.  Must be a valid secret key.
-                              type: string
-                            name:
-                              default: ""
-                              description: |-
-                                Name of the referent.
-                                This field is effectively required, but due to backwards compatibility is
-                                allowed to be empty. Instances of this type with an empty value here are
-                                almost certainly wrong.
-                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                              type: string
-                            optional:
-                              description: Specify whether the Secret or its key must be defined
-                              type: boolean
-                          required:
-                            - key
-                          type: object
-                          x-kubernetes-map-type: atomic
-                        UserSecretRef:
-                          description: The kafka user name to use for authentication
-                          properties:
-                            key:
-                              description: The key of the secret to select from.  Must be a valid secret key.
-                              type: string
-                            name:
-                              default: ""
-                              description: |-
-                                Name of the referent.
-                                This field is effectively required, but due to backwards compatibility is
-                                allowed to be empty. Instances of this type with an empty value here are
-                                almost certainly wrong.
-                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                              type: string
-                            optional:
-                              description: Specify whether the Secret or its key must be defined
-                              type: boolean
-                          required:
-                            - key
-                          type: object
-                          x-kubernetes-map-type: atomic
                         ackLevel:
                           default: broker
                           description: The ack level required for this topic (none/broker)
@@ -1061,26 +1017,6 @@ spec:
                   type: integer
                 phase:
                   type: string
-                secrets:
-                  items:
-                    properties:
-                      name:
-                        description: name is unique within a namespace to reference a secret resource.
-                        type: string
-                      namespace:
-                        description: namespace defines the space within which the secret name must be unique.
-                        type: string
-                      resourceVersion:
-                        type: string
-                      uid:
-                        description: |-
-                          UID is a type that holds unique ID values, including UUIDs.  Because we
-                          don't ONLY use UUIDs, this is an alias to string.  Being a type captures
-                          intent and helps make sure that UIDs and names do not get conflated.
-                        type: string
-                    type: object
-                    x-kubernetes-map-type: atomic
-                  type: array
               type: object
               x-kubernetes-preserve-unknown-fields: true
           required:
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
index 0de066e86..5e90e4a7f 100644
--- a/deploy/examples/images.txt
+++ b/deploy/examples/images.txt
@@ -1,8 +1,8 @@
  docker.io/rook/ceph:master
  gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:v20240513-v0.1.0-35-gefb3255
- quay.io/ceph/ceph:v19.2.2
+ quay.io/ceph/ceph:v19.2.1
  quay.io/ceph/cosi:v0.1.2
- quay.io/cephcsi/cephcsi:v3.14.0
+ quay.io/cephcsi/cephcsi:v3.13.1
  quay.io/csiaddons/k8s-sidecar:v0.12.0
  registry.k8s.io/sig-storage/csi-attacher:v4.8.0
  registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
index 095a0ef06..f866d0413 100644
--- a/deploy/examples/operator-openshift.yaml
+++ b/deploy/examples/operator-openshift.yaml
@@ -189,7 +189,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.14.0"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.13.1"
   # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0"
   # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.13.1"
   # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"
diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
index 256549683..300f59aa0 100644
--- a/deploy/examples/operator.yaml
+++ b/deploy/examples/operator.yaml
@@ -119,7 +119,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.14.0"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.13.1"
   # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0"
   # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.13.1"
   # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"
diff --git a/design/ceph/ceph-cluster-cleanup.md b/design/ceph/ceph-cluster-cleanup.md
index a227acb8f..c50b8d6d0 100644
--- a/design/ceph/ceph-cluster-cleanup.md
+++ b/design/ceph/ceph-cluster-cleanup.md
@@ -34,7 +34,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
diff --git a/go.mod b/go.mod
index a6cde1178..7bb46def5 100644
--- a/go.mod
+++ b/go.mod
@@ -17,10 +17,6 @@ replace (
 require (
 	github.com/IBM/keyprotect-go-client v0.15.1
 	github.com/aws/aws-sdk-go v1.55.6
-	github.com/aws/aws-sdk-go-v2/config v1.29.4
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.57
-	github.com/aws/aws-sdk-go-v2/service/sns v1.34.3
-	github.com/aws/smithy-go v1.22.2
 	github.com/banzaicloud/k8s-objectmatcher v1.8.0
 	github.com/ceph/ceph-csi-operator/api v0.0.0-20241211130321-1c8ad98a7d6e
 	github.com/ceph/ceph-csi/api v0.0.0-20241216133622-88b7e0d6684f
@@ -72,16 +68,9 @@ require (
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
 	github.com/ansel1/merry v1.8.0 // indirect
 	github.com/ansel1/merry/v2 v2.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.36.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.57 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -159,7 +148,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.36.0 // indirect
 	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/oauth2 v0.27.0 // indirect
+	golang.org/x/oauth2 v0.25.0 // indirect
 	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/term v0.30.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
diff --git a/go.sum b/go.sum
index 7513e72cf..0cc106c71 100644
--- a/go.sum
+++ b/go.sum
@@ -148,8 +148,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
 github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
 github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
-github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
+github.com/aws/aws-sdk-go-v2 v1.36.0 h1:b1wM5CcE65Ujwn565qcwgtOTT1aT4ADOHHgglKjG7fk=
+github.com/aws/aws-sdk-go-v2 v1.36.0/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
 github.com/aws/aws-sdk-go-v2/config v1.29.4 h1:ObNqKsDYFGr2WxnoXKOhCvTlf3HhwtoGgc+KmZ4H5yg=
@@ -158,10 +158,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.57 h1:kFQDsbdBAR3GZsB8xA+51ptEnq9
 github.com/aws/aws-sdk-go-v2/credentials v1.17.57/go.mod h1:2kerxPUUbTagAr/kkaHiqvj/bcYHzi2qiJS/ZinllU0=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 h1:7lOW8NUwE9UZekS1DYoiPdVAqZ6A+LheHWb+mHbNOq8=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27/go.mod h1:w1BASFIPOPUae7AgaH4SbjNbfdkxuggLyGfNFTn8ITY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 h1:lWm9ucLSRFiI4dQQafLrEOmEDGry3Swrz0BIRdiHJqQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31/go.mod h1:Huu6GG0YTfbPphQkDSo4dEGmQRTKb9k9G7RdtyQWxuI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 h1:ACxDklUKKXb48+eg5ROZXi1vDgfMyfIA/WyvqHcHI0o=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31/go.mod h1:yadnfsDwqXeVaohbGc/RaD287PuyRw2wugkh5ZL2J6k=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME=
@@ -176,8 +176,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEH
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 h1:SAfh4pNx5LuTafKKWR02Y+hL3A+3TX8cTKG1OIAJaBk=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q=
-github.com/aws/aws-sdk-go-v2/service/sns v1.34.3 h1:iJtp/KnPsgMO4TSGfjqi3oGr+R73W7xWqDXHCbqdnv8=
-github.com/aws/aws-sdk-go-v2/service/sns v1.34.3/go.mod h1:PJtxxMdj747j8DeZENRTTYAz/lx/pADn/U0k7YNNiUY=
 github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 h1:c5WJ3iHz7rLIgArznb3JCSQT3uUMiz9DLZhIX+1G8ok=
 github.com/aws/aws-sdk-go-v2/service/sso v1.24.14/go.mod h1:+JJQTxB6N4niArC14YNtxcQtwEqzS3o9Z32n7q33Rfs=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 h1:f1L/JtUkVODD+k1+IiSJUUv8A++2qVr+Xvb3xWXETMU=
@@ -1157,8 +1155,8 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j
 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
-golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
+golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70=
+golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
diff --git a/images/ceph/Makefile b/images/ceph/Makefile
index d8fd65daf..962da3c0f 100755
--- a/images/ceph/Makefile
+++ b/images/ceph/Makefile
@@ -16,7 +16,7 @@ include ../image.mk
 
 # ====================================================================================
 # Image Build Options
-CEPH_VERSION ?= v19.2.2-20250409
+CEPH_VERSION ?= v19.2.1-20250202
 
 REGISTRY_NAME = quay.io
 BASEIMAGE = $(REGISTRY_NAME)/ceph/ceph:$(CEPH_VERSION)
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 0c80afaaa..72162221d 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -19,7 +19,6 @@ package v1
 import (
 	"time"
 
-	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -2301,8 +2300,6 @@ type BucketTopicStatus struct {
 	// ObservedGeneration is the latest generation observed by the controller.
 	// +optional
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
-	// +optional
-	Secrets []SecretReference `json:"secrets,omitempty"`
 }
 
 // CephBucketTopicList represents a list Ceph Object Store Bucket Notification Topics
@@ -2396,12 +2393,6 @@ type KafkaEndpointSpec struct {
 	// +kubebuilder:default=PLAIN
 	// +optional
 	Mechanism string `json:"mechanism,omitempty"`
-	// The kafka user name to use for authentication
-	// +optional
-	UserSecretRef *corev1.SecretKeySelector `json:"UserSecretRef,omitempty"`
-	// The kafka password to use for authentication
-	// +optional
-	PasswordSecretRef *corev1.SecretKeySelector `json:"PasswordSecretRef,omitempty"`
 }
 
 // +genclient
diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
index 36c747f75..556739e25 100644
--- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
@@ -238,11 +238,6 @@ func (in *BucketTopicStatus) DeepCopyInto(out *BucketTopicStatus) {
 		*out = new(string)
 		**out = **in
 	}
-	if in.Secrets != nil {
-		in, out := &in.Secrets, &out.Secrets
-		*out = make([]SecretReference, len(*in))
-		copy(*out, *in)
-	}
 	return
 }
 
@@ -2836,16 +2831,6 @@ func (in *HybridStorageSpec) DeepCopy() *HybridStorageSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KafkaEndpointSpec) DeepCopyInto(out *KafkaEndpointSpec) {
 	*out = *in
-	if in.UserSecretRef != nil {
-		in, out := &in.UserSecretRef, &out.UserSecretRef
-		*out = new(corev1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.PasswordSecretRef != nil {
-		in, out := &in.PasswordSecretRef, &out.PasswordSecretRef
-		*out = new(corev1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
 	return
 }
 
@@ -5076,7 +5061,7 @@ func (in *TopicEndpointSpec) DeepCopyInto(out *TopicEndpointSpec) {
 	if in.Kafka != nil {
 		in, out := &in.Kafka, &out.Kafka
 		*out = new(KafkaEndpointSpec)
-		(*in).DeepCopyInto(*out)
+		**out = **in
 	}
 	return
 }
diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod
index 926a94edd..97441dafc 100644
--- a/pkg/apis/go.mod
+++ b/pkg/apis/go.mod
@@ -72,7 +72,7 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/crypto v0.36.0 // indirect
 	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/oauth2 v0.27.0 // indirect
+	golang.org/x/oauth2 v0.25.0 // indirect
 	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/term v0.30.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
diff --git a/pkg/apis/go.sum b/pkg/apis/go.sum
index 1afcdfc6e..66d66bc0a 100644
--- a/pkg/apis/go.sum
+++ b/pkg/apis/go.sum
@@ -939,8 +939,8 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j
 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
-golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
+golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70=
+golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
diff --git a/pkg/daemon/ceph/client/mirror_health.go b/pkg/daemon/ceph/client/mirror_health.go
index 89ac49920..332a49b9c 100644
--- a/pkg/daemon/ceph/client/mirror_health.go
+++ b/pkg/daemon/ceph/client/mirror_health.go
@@ -112,9 +112,8 @@ func (c *mirrorChecker) CheckMirroringHealth() error {
 	}
 
 	// On success
-	if mirrorStatus.Summary != nil {
-		c.UpdateStatusMirroring(mirrorStatus.Summary, mirrorInfo, snapSchedStatus, "")
-	}
+	c.UpdateStatusMirroring(mirrorStatus.Summary, mirrorInfo, snapSchedStatus, "")
+
 	return nil
 }
 
diff --git a/pkg/daemon/ceph/client/status.go b/pkg/daemon/ceph/client/status.go
index 71a4b51dd..327ede035 100644
--- a/pkg/daemon/ceph/client/status.go
+++ b/pkg/daemon/ceph/client/status.go
@@ -149,7 +149,7 @@ type Fsmap struct {
 		Rank         int    `json:"rank"`
 		Name         string `json:"name"`
 		Status       string `json:"status"`
-		Gid          uint64 `json:"gid"`
+		Gid          int    `json:"gid"`
 	} `json:"by_rank"`
 	UpStandby int `json:"up:standby"`
 }
diff --git a/pkg/daemon/ceph/osd/remove.go b/pkg/daemon/ceph/osd/remove.go
index 676b76ee7..8cea71dcd 100644
--- a/pkg/daemon/ceph/osd/remove.go
+++ b/pkg/daemon/ceph/osd/remove.go
@@ -262,16 +262,19 @@ func DestroyOSD(context *clusterd.Context, clusterInfo *client.ClusterInfo, id i
 	}
 	logger.Infof("successfully destroyed osd.%d", osdInfo.ID)
 
-	if isPVC && osdInfo.Encrypted {
-		// remove the dm device
+	// in case of OSD on PVs, fetch the actual device name for the mounted /mnt/<pvc-name>
+	if isPVC {
 		pvcName := os.Getenv(oposd.PVCNameEnvVarName)
-		target := oposd.EncryptionDMName(pvcName, oposd.DmcryptBlockType)
-		err = removeEncryptedDevice(context, target)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to remove dm device %q", target)
+
+		// remove the dm device
+		if osdInfo.Encrypted {
+			target := oposd.EncryptionDMName(pvcName, oposd.DmcryptBlockType)
+			err = removeEncryptedDevice(context, target)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to remove dm device %q", target)
+			}
 		}
-		// ceph-volume uses `/dev/mapper/*` for encrypted disks. This is not a block device. So we need to fetch the corresponding
-		// block device for cleanup using `ceph-volume lvm zap`
+		// fetch the actual device for cleanup
 		blockPath := fmt.Sprintf("/mnt/%s", pvcName)
 		diskInfo, err := clusterd.PopulateDeviceInfo(blockPath, context.Executor)
 		if err != nil {
diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go
index 8f3e64bbd..fcf54bf3a 100644
--- a/pkg/operator/ceph/client/controller.go
+++ b/pkg/operator/ceph/client/controller.go
@@ -158,14 +158,10 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
 	observedGeneration := cephClient.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephClient)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephClient)
 	if err != nil {
 		return reconcile.Result{}, *cephClient, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the cephclient %q after adding finalizer", cephClient.Name)
-		return reconcile.Result{}, *cephClient, nil
-	}
 
 	// The CR was just created, initializing status fields
 	if cephClient.Status == nil {
diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go
index 7ed598cfa..75b310a58 100644
--- a/pkg/operator/ceph/cluster/controller.go
+++ b/pkg/operator/ceph/cluster/controller.go
@@ -274,14 +274,10 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
 	}
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephCluster)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephCluster)
 	if err != nil {
 		return reconcile.Result{}, *cephCluster, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the ceph cluster %q after adding finalizer", cephCluster.Name)
-		return reconcile.Result{}, *cephCluster, nil
-	}
 
 	// DELETE: the CR was deleted
 	if !cephCluster.GetDeletionTimestamp().IsZero() {
diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go
index dccde5d2b..01d59d92e 100644
--- a/pkg/operator/ceph/cluster/mon/service.go
+++ b/pkg/operator/ceph/cluster/mon/service.go
@@ -74,10 +74,6 @@ func (c *Cluster) createService(mon *monConfig) (*v1.Service, error) {
 }
 
 func (c *Cluster) exportService(service *v1.Service, monDaemon string) (string, error) {
-	// defer removing the mon canary deployment to after the service is exported because DNS
-	// query on <service>.<ns>.svc.clusterset.local requires the mon canary pod to be running
-	defer c.removeCanaryDeployments(monCanaryLabelSelector + fmt.Sprintf(",mon=%s", monDaemon))
-
 	logger.Infof("exporting service %q", service.Name)
 	exportedIP, err := k8sutil.ExportService(c.ClusterInfo.Context, c.context, service, c.spec.Network.MultiClusterService.ClusterID)
 	if err != nil {
@@ -85,5 +81,9 @@ func (c *Cluster) exportService(service *v1.Service, monDaemon string) (string,
 	}
 	logger.Infof("mon %q exported IP is %s", service.Name, exportedIP)
 
+	// remove mon canary deployment only after the service is exported because DNS
+	// query on <service>.<ns>.svc.clusterset.local requires the mon pod be running
+	c.removeCanaryDeployments(monCanaryLabelSelector + fmt.Sprintf(",mon=%s", monDaemon))
+
 	return exportedIP, nil
 }
diff --git a/pkg/operator/ceph/cluster/osd/key_rotation.go b/pkg/operator/ceph/cluster/osd/key_rotation.go
index 07d84d7f1..5ad7afdcb 100644
--- a/pkg/operator/ceph/cluster/osd/key_rotation.go
+++ b/pkg/operator/ceph/cluster/osd/key_rotation.go
@@ -279,7 +279,7 @@ func (c *Cluster) reconcileKeyRotationCronJob() error {
 			continue
 		}
 
-		logger.Infof("starting OSD key rotation cron job for osd %d", osd.ID)
+		logger.Infof("starting OSD key rotation cron job for osd %q", osd.ID)
 		cj, err := c.makeKeyRotationCronJob(pvcName, osd, osdProps)
 		if err != nil {
 			return errors.Wrap(err, "failed to make key rotation cron job")
diff --git a/pkg/operator/ceph/cluster/predicate.go b/pkg/operator/ceph/cluster/predicate.go
index bf9578de1..2da2439df 100644
--- a/pkg/operator/ceph/cluster/predicate.go
+++ b/pkg/operator/ceph/cluster/predicate.go
@@ -183,8 +183,7 @@ func watchControllerPredicate[T *cephv1.CephCluster](ctx context.Context, c clie
 				return false
 
 			} else if objOld.GetGeneration() != objNew.GetGeneration() {
-				logger.Debugf("reconciling CephCluster %q with changed generation", objNew.Name)
-				return true
+				logger.Debugf("skipping resource %q update with unchanged spec", objNew.Name)
 			}
 
 			return false
diff --git a/pkg/operator/ceph/controller/finalizer.go b/pkg/operator/ceph/controller/finalizer.go
index b64837a78..e214e88dd 100644
--- a/pkg/operator/ceph/controller/finalizer.go
+++ b/pkg/operator/ceph/controller/finalizer.go
@@ -52,28 +52,25 @@ func remove(list []string, s string) []string {
 
 // AddFinalizerIfNotPresent adds a finalizer an object to avoid instant deletion
 // of the object without finalizing it.
-func AddFinalizerIfNotPresent(ctx context.Context, client client.Client, obj client.Object) (bool, error) {
+func AddFinalizerIfNotPresent(ctx context.Context, client client.Client, obj client.Object) error {
 	objectFinalizer := buildFinalizerName(obj.GetObjectKind().GroupVersionKind().Kind)
+
 	accessor, err := meta.Accessor(obj)
 	if err != nil {
-		return false, errors.Wrap(err, "failed to get meta information of object")
+		return errors.Wrap(err, "failed to get meta information of object")
 	}
 
 	if !contains(accessor.GetFinalizers(), objectFinalizer) {
 		logger.Infof("adding finalizer %q on %q", objectFinalizer, accessor.GetName())
 		accessor.SetFinalizers(append(accessor.GetFinalizers(), objectFinalizer))
-		originalGeneration := obj.GetGeneration()
 
 		// Update CR with finalizer
 		if err := client.Update(ctx, obj); err != nil {
-			return false, errors.Wrapf(err, "failed to add finalizer %q on %q", objectFinalizer, accessor.GetName())
+			return errors.Wrapf(err, "failed to add finalizer %q on %q", objectFinalizer, accessor.GetName())
 		}
-		newGeneration := obj.GetGeneration()
-		logger.Debugf("when adding finalizer on %q, original generation %d, new generation %d", accessor.GetName(), originalGeneration, newGeneration)
-		return originalGeneration != newGeneration, nil
 	}
 
-	return false, nil
+	return nil
 }
 
 // RemoveFinalizer removes a finalizer from an object
diff --git a/pkg/operator/ceph/controller/predicate.go b/pkg/operator/ceph/controller/predicate.go
index 25971fc40..cf51f2332 100644
--- a/pkg/operator/ceph/controller/predicate.go
+++ b/pkg/operator/ceph/controller/predicate.go
@@ -104,8 +104,7 @@ func WatchControllerPredicate[T client.Object](scheme *runtime.Scheme) predicate
 				logger.Debugf("resource %q: %q is going to be deleted", kind, nsName)
 				return true
 			} else if objOld.GetGeneration() != objNew.GetGeneration() {
-				logger.Debugf("reconciling %s %q with changed generation", kind, nsName.String())
-				return true
+				logger.Debugf("skipping resource %q: %q, update event with unchanged spec", kind, nsName)
 			}
 			return false
 		},
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index e7d4c979f..c889d858d 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -132,7 +132,7 @@ var (
 // manually challenging.
 var (
 	// image names
-	DefaultCSIPluginImage   = "quay.io/cephcsi/cephcsi:v3.14.0"
+	DefaultCSIPluginImage   = "quay.io/cephcsi/cephcsi:v3.13.1"
 	DefaultRegistrarImage   = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0"
 	DefaultProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"
 	DefaultAttacherImage    = "registry.k8s.io/sig-storage/csi-attacher:v4.8.0"
diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go
index 76e7c2764..51ffecee5 100644
--- a/pkg/operator/ceph/file/controller.go
+++ b/pkg/operator/ceph/file/controller.go
@@ -214,14 +214,10 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil
 	observedGeneration := cephFilesystem.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephFilesystem)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephFilesystem)
 	if err != nil {
 		return reconcile.Result{}, *cephFilesystem, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the filesystem %q after adding finalizer", cephFilesystem.Name)
-		return reconcile.Result{}, *cephFilesystem, nil
-	}
 
 	// The CR was just created, initialize status as 'Progressing'
 	if cephFilesystem.Status == nil {
diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go
index 78cf5e198..ae387dc31 100644
--- a/pkg/operator/ceph/file/subvolumegroup/controller.go
+++ b/pkg/operator/ceph/file/subvolumegroup/controller.go
@@ -155,14 +155,10 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
 	observedGeneration := cephFilesystemSubVolumeGroup.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephFilesystemSubVolumeGroup)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephFilesystemSubVolumeGroup)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the subvolume group %q after adding finalizer", cephFilesystemSubVolumeGroup.Name)
-		return reconcile.Result{}, nil
-	}
 
 	// The CR was just created, initializing status fields
 	if cephFilesystemSubVolumeGroup.Status == nil {
diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go
index ad121d4a1..27c4be1e6 100644
--- a/pkg/operator/ceph/nfs/controller.go
+++ b/pkg/operator/ceph/nfs/controller.go
@@ -176,14 +176,10 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul
 	observedGeneration := cephNFS.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephNFS)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephNFS)
 	if err != nil {
 		return reconcile.Result{}, *cephNFS, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the nfs %q after adding finalizer", cephNFS.Name)
-		return reconcile.Result{}, *cephNFS, nil
-	}
 
 	// The CR was just created, initializing status fields
 	if cephNFS.Status == nil {
diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go
index e81899926..e1b010dc4 100644
--- a/pkg/operator/ceph/object/controller.go
+++ b/pkg/operator/ceph/object/controller.go
@@ -293,14 +293,10 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
 	observedGeneration := cephObjectStore.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephObjectStore)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephObjectStore)
 	if err != nil {
 		return reconcile.Result{}, *cephObjectStore, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the object store %q after adding finalizer", cephObjectStore.Name)
-		return reconcile.Result{}, *cephObjectStore, nil
-	}
 
 	// The CR was just created, initializing status fields
 	if cephObjectStore.Status == nil {
@@ -441,9 +437,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
 	// CREATE/UPDATE
 	_, err = r.reconcileCreateObjectStore(cephObjectStore, request.NamespacedName, cephCluster.Spec)
 	if err != nil && kerrors.IsNotFound(err) {
-		// A not found error may mean ceph is still initializing, but there might be some other error
-		// so we log the error and requeue
-		logger.Warningf("object store %q reconcile failed, ceph may still be initializing. %v", request.NamespacedName.String(), err)
+		logger.Info(opcontroller.OperatorNotInitializedMessage)
 		return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephObjectStore, nil
 	} else if err != nil {
 		result, err := r.setFailedStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, "failed to create object store deployments", err)
diff --git a/pkg/operator/ceph/object/topic/controller.go b/pkg/operator/ceph/object/topic/controller.go
index 84d4d6a71..0250baf15 100644
--- a/pkg/operator/ceph/object/topic/controller.go
+++ b/pkg/operator/ceph/object/topic/controller.go
@@ -19,8 +19,6 @@ package topic
 
 import (
 	"context"
-	"slices"
-	"sort"
 
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
@@ -30,16 +28,12 @@ import (
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/ceph/reporting"
 	"github.com/rook/rook/pkg/operator/k8sutil"
-	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
-	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
@@ -91,95 +85,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 		return err
 	}
 
-	// watch for kafka secrets
-	// watch secrets referenced by CephBucketTopic.spec.endpoint.kafka.{userSecretRef,passwordSecretRef}
-	const (
-		// disable warning: G101: Potential hardcoded credentials (gosec)
-		// nolint:gosec
-		secretNameField = "spec.endpoint.kafka.secretNames"
-	)
-
-	err = mgr.GetFieldIndexer().IndexField(
-		context.TODO(),
-		&cephv1.CephBucketTopic{},
-		secretNameField,
-		func(obj client.Object) []string {
-			var secretNames []string
-			topic, ok := obj.(*cephv1.CephBucketTopic)
-			if !ok {
-				return nil
-			}
-
-			if topic.Spec.Endpoint.Kafka == nil {
-				return nil
-			}
-
-			kafka := topic.Spec.Endpoint.Kafka
-
-			if kafka.UserSecretRef != nil {
-				secretNames = append(secretNames, kafka.UserSecretRef.Name)
-			}
-
-			if kafka.PasswordSecretRef != nil {
-				secretNames = append(secretNames, kafka.PasswordSecretRef.Name)
-			}
-
-			slices.Sort(secretNames)
-			return slices.Compact(secretNames)
-		},
-	)
-	if err != nil {
-		return errors.Wrapf(err, "failed to setup IndexField for CephBucketTopic.Spec.Endpoint.Kafka.{UserSecretRef,PasswordSecretRef}")
-	}
-
-	// Always trigger a reconcile when a secret is deleted. This will cause a
-	// reconciliation failure to happen immediately in hopes of alerting the end
-	// user to the configuration problem.
-	changedOrDeleted := predicate.Or(
-		predicate.TypedResourceVersionChangedPredicate[*corev1.Secret]{},
-		predicate.TypedFuncs[*corev1.Secret]{
-			DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Secret]) bool {
-				return true
-			},
-		},
-	)
-
-	err = c.Watch(
-		source.Kind(
-			mgr.GetCache(),
-			&corev1.Secret{},
-			handler.TypedEnqueueRequestsFromMapFunc(
-				func(ctx context.Context, secret *corev1.Secret) []reconcile.Request {
-					referencingTopics := &cephv1.CephBucketTopicList{}
-					err := r.(*ReconcileBucketTopic).client.List(ctx, referencingTopics, &client.ListOptions{
-						FieldSelector: fields.OneTermEqualSelector(secretNameField, secret.GetName()),
-						Namespace:     secret.GetNamespace(),
-					})
-					if err != nil {
-						logger.Errorf("failed to list CephBucketTopic(s) while handling event for secret %q in namespace %q. %v", secret.GetName(), secret.GetNamespace(), err)
-						return []reconcile.Request{}
-					}
-
-					requests := make([]reconcile.Request, len(referencingTopics.Items))
-					for i, item := range referencingTopics.Items {
-						requests[i] = reconcile.Request{
-							NamespacedName: types.NamespacedName{
-								Name:      item.GetName(),
-								Namespace: item.GetNamespace(),
-							},
-						}
-					}
-					logger.Tracef("CephBucketTopic(s) referencing Secret %q in namespace %q: %v", secret.GetName(), secret.GetNamespace(), requests)
-					return requests
-				},
-			),
-			changedOrDeleted,
-		),
-	)
-	if err != nil {
-		return errors.Wrapf(err, "failed to configure watch for Secret(s)")
-	}
-
 	return nil
 }
 
@@ -191,7 +96,7 @@ func (r *ReconcileBucketTopic) Reconcile(context context.Context, request reconc
 	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
 	reconcileResponse, err := r.reconcile(request)
 	if err != nil {
-		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcileFailedStatus, nil, nil)
+		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcileFailedStatus, nil)
 		logger.Errorf("failed to reconcile %v", err)
 	}
 
@@ -216,13 +121,14 @@ func (r *ReconcileBucketTopic) reconcile(request reconcile.Request) (reconcile.R
 	observedGeneration := cephBucketTopic.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephBucketTopic)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephBucketTopic)
 	if err != nil {
 		return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to add finalizer to CephBucketTopic %q", request.NamespacedName)
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the object bucket topic %q after adding finalizer", cephBucketTopic.Name)
-		return reconcile.Result{}, nil
+
+	// The CR was just created, initializing status fields
+	if cephBucketTopic.Status == nil {
+		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.EmptyStatus, nil)
 	}
 
 	// Make sure a CephCluster is present otherwise do nothing
@@ -278,24 +184,24 @@ func (r *ReconcileBucketTopic) reconcile(request reconcile.Request) (reconcile.R
 	}
 
 	// Start object reconciliation, updating status for this
-	r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcilingStatus, nil, nil)
+	r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcilingStatus, nil)
 
 	// create topic
-	topicARN, referencedSecrets, err := r.createCephBucketTopic(cephBucketTopic)
+	topicARN, err := r.createCephBucketTopic(cephBucketTopic)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrapf(err, "topic creation failed for CephBucketTopic %q", request.NamespacedName)
 	}
 
 	// update ObservedGeneration in status a the end of reconcile
 	// Set Ready status, we are done reconciling
-	r.updateStatus(observedGeneration, request.NamespacedName, k8sutil.ReadyStatus, topicARN, referencedSecrets)
+	r.updateStatus(observedGeneration, request.NamespacedName, k8sutil.ReadyStatus, topicARN)
 
 	// Return and do not requeue
 	return reconcile.Result{}, nil
 }
 
-func (r *ReconcileBucketTopic) createCephBucketTopic(topic *cephv1.CephBucketTopic) (topicARN *string, referencedSecrets *map[types.UID]*corev1.Secret, err error) {
-	topicARN, referencedSecrets, err = createTopicFunc(
+func (r *ReconcileBucketTopic) createCephBucketTopic(topic *cephv1.CephBucketTopic) (topicARN *string, err error) {
+	topicARN, err = createTopicFunc(
 		provisioner{
 			client:           r.client,
 			context:          r.context,
@@ -322,7 +228,7 @@ func (r *ReconcileBucketTopic) deleteCephBucketTopic(topic *cephv1.CephBucketTop
 }
 
 // updateStatus updates the topic with a given status
-func (r *ReconcileBucketTopic) updateStatus(observedGeneration int64, nsName types.NamespacedName, status string, topicARN *string, referencedSecrets *map[types.UID]*corev1.Secret) {
+func (r *ReconcileBucketTopic) updateStatus(observedGeneration int64, nsName types.NamespacedName, status string, topicARN *string) {
 	topic := &cephv1.CephBucketTopic{}
 	if err := r.client.Get(r.opManagerContext, nsName, topic); err != nil {
 		if kerrors.IsNotFound(err) {
@@ -341,32 +247,6 @@ func (r *ReconcileBucketTopic) updateStatus(observedGeneration int64, nsName typ
 	if observedGeneration != k8sutil.ObservedGenerationNotAvailable {
 		topic.Status.ObservedGeneration = observedGeneration
 	}
-
-	logger.Debugf("updating CephBucketTopic %q .status.secrets to %+v.", nsName, referencedSecrets)
-
-	if referencedSecrets != nil {
-		secretsStatus := []cephv1.SecretReference{}
-
-		for _, secret := range *referencedSecrets {
-			secretsStatus = append(secretsStatus, cephv1.SecretReference{
-				SecretReference: corev1.SecretReference{
-					Name:      secret.Name,
-					Namespace: secret.Namespace,
-				},
-				UID:             secret.UID,
-				ResourceVersion: secret.ResourceVersion,
-			})
-		}
-
-		// assume map key ordering is unstable between reconciles and sort the slice
-		// by secret name
-		sort.Slice(secretsStatus, func(i, j int) bool {
-			return secretsStatus[i].Name < secretsStatus[j].Name
-		})
-
-		topic.Status.Secrets = secretsStatus
-	}
-
 	if err := reporting.UpdateStatus(r.client, topic); err != nil {
 		logger.Errorf("failed to set CephBucketTopic %q status to %q. error %v", nsName, status, err)
 		return
diff --git a/pkg/operator/ceph/object/topic/provisioner.go b/pkg/operator/ceph/object/topic/provisioner.go
index 510ba489a..1f6c844f9 100644
--- a/pkg/operator/ceph/object/topic/provisioner.go
+++ b/pkg/operator/ceph/object/topic/provisioner.go
@@ -20,11 +20,11 @@ package topic
 import (
 	"context"
 	"crypto/hmac"
+
 	//nolint:gosec // sha1 is needed for v2 signatures
 	"crypto/sha1"
 	"encoding/base64"
 	"net/http"
-	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -43,7 +43,6 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/ceph/object"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -144,12 +143,9 @@ func createSNSClient(p provisioner, objectStoreName types.NamespacedName) (*sns.
 	return snsClient, nil
 }
 
-func createTopicAttributes(p provisioner, topic *cephv1.CephBucketTopic) (map[string]*string, *map[types.UID]*corev1.Secret, error) {
+func createTopicAttributes(topic *cephv1.CephBucketTopic) map[string]*string {
 	attr := make(map[string]*string)
 	nsName := types.NamespacedName{Name: topic.Name, Namespace: topic.Namespace}
-	// Currently, referencedSecrets is only used by the kafka endpoint but it
-	// could be used by other endpoints in the future.
-	referencedSecrets := make(map[types.UID]*corev1.Secret)
 
 	attr["OpaqueData"] = &topic.Spec.OpaqueData
 	persistent := strconv.FormatBool(topic.Spec.Persistent)
@@ -173,48 +169,8 @@ func createTopicAttributes(p provisioner, topic *cephv1.CephBucketTopic) (map[st
 		attr["cloudevents"] = &cloudEvents
 	}
 	if topic.Spec.Endpoint.Kafka != nil {
-		kafka := topic.Spec.Endpoint.Kafka
-
-		uri, err := url.Parse(kafka.URI)
-		if err != nil {
-			// URI could contain a passphrase...
-			return nil, nil, errors.Wrapf(err, "failed to parse CephBucketTopic %q .spec.endpoint.kafka.URI %q", nsName, kafka.URI)
-		}
-
-		// If UserSecretRef or PasswordRef is set, we need to parse the URI and insert the
-		// credentials as http basic auth. If basic auth was already set as part of
-		// the URI string, it will be overridden.
-		if kafka.UserSecretRef != nil || kafka.PasswordSecretRef != nil {
-			var user, pass string
-
-			if kafka.UserSecretRef != nil {
-				var secret *corev1.Secret
-				user, secret, err = p.getSecretValue(kafka.UserSecretRef, topic.Namespace)
-				if err != nil {
-					return nil, nil, errors.Wrapf(err, "failed to get secret value from CephBucketTopic %q .spec.endpoint.kafka.userSecretRef %q", nsName, kafka.UserSecretRef)
-				}
-
-				logger.Debugf("CephBucketTopic %q references secret %q", nsName, client.ObjectKeyFromObject(secret))
-				referencedSecrets[secret.UID] = secret
-			}
-			if kafka.PasswordSecretRef != nil {
-				var secret *corev1.Secret
-				pass, secret, err = p.getSecretValue(kafka.PasswordSecretRef, topic.Namespace)
-				if err != nil {
-					return nil, nil, errors.Wrapf(err, "failed to get secret value from CephBucketTopic %q .spec.endpoint.kafka.passwordSecretRef %q", nsName, kafka.PasswordSecretRef)
-				}
-				logger.Debugf("CephBucketTopic %q references secret %q", nsName, client.ObjectKeyFromObject(secret))
-				referencedSecrets[secret.UID] = secret
-			}
-
-			uri.User = url.UserPassword(user, pass)
-		}
-
-		// do not log passphrases, if set
-		logger.Infof("creating CephBucketTopic %q with endpoint %q", nsName, uri.Redacted())
-
-		kafkaUri := uri.String()
-		attr["push-endpoint"] = &kafkaUri
+		logger.Infof("creating CephBucketTopic %q with endpoint %q", nsName, topic.Spec.Endpoint.Kafka.URI)
+		attr["push-endpoint"] = &topic.Spec.Endpoint.Kafka.URI
 		useSSL = strconv.FormatBool(topic.Spec.Endpoint.Kafka.UseSSL)
 		attr["use-ssl"] = &useSSL
 		attr["kafka-ack-level"] = &topic.Spec.Endpoint.Kafka.AckLevel
@@ -223,34 +179,30 @@ func createTopicAttributes(p provisioner, topic *cephv1.CephBucketTopic) (map[st
 		attr["mechanism"] = &topic.Spec.Endpoint.Kafka.Mechanism
 	}
 
-	return attr, &referencedSecrets, nil
+	return attr
 }
 
 // Allow overriding this function for unit tests
 var createTopicFunc = createTopic
 
-func createTopic(p provisioner, topic *cephv1.CephBucketTopic) (*string, *map[types.UID]*corev1.Secret, error) {
+func createTopic(p provisioner, topic *cephv1.CephBucketTopic) (*string, error) {
 	nsName := types.NamespacedName{Name: topic.Name, Namespace: topic.Namespace}
 
 	snsClient, err := createSNSClient(p, types.NamespacedName{Name: topic.Spec.ObjectStoreName, Namespace: topic.Spec.ObjectStoreNamespace})
 	if err != nil {
-		return nil, nil, errors.Wrapf(err, "failed to create SNS client for CephBucketTopic %q provisioning", nsName)
-	}
-	attr, referencedSecrets, err := createTopicAttributes(p, topic)
-	if err != nil {
-		return nil, nil, errors.Wrapf(err, "failed to generate topic attributes for CephBucketTopic %q", nsName)
+		return nil, errors.Wrapf(err, "failed to create SNS client for CephBucketTopic %q provisioning", nsName)
 	}
 	topicOutput, err := snsClient.CreateTopic(&sns.CreateTopicInput{
 		Name:       &topic.Name,
-		Attributes: attr,
+		Attributes: createTopicAttributes(topic),
 	})
 	if err != nil {
-		return nil, nil, errors.Wrapf(err, "failed to provision CephBucketTopic %q", nsName)
+		return nil, errors.Wrapf(err, "failed to provision CephBucketTopic %q", nsName)
 	}
 
 	logger.Infof("CephBucketTopic %q created with ARN %q", nsName, *topicOutput.TopicArn)
 
-	return topicOutput.TopicArn, referencedSecrets, nil
+	return topicOutput.TopicArn, nil
 }
 
 // Allow overriding this function for unit tests
@@ -306,21 +258,3 @@ func GetProvisioned(cl client.Client, ctx context.Context, topicName types.Names
 
 	return bucketTopic, nil
 }
-
-// getSecretValue returns the value of key in a kubernetes secret
-func (p *provisioner) getSecretValue(selector *corev1.SecretKeySelector, namespace string) (string, *corev1.Secret, error) {
-	secret := &corev1.Secret{}
-	namespacedName := types.NamespacedName{
-		Name:      selector.Name,
-		Namespace: namespace,
-	}
-	if err := p.client.Get(p.opManagerContext, namespacedName, secret); err != nil {
-		return "", nil, errors.Wrapf(err, "failed to get secret %q", namespacedName)
-	}
-	value, ok := secret.Data[selector.Key]
-	if !ok {
-		return "", nil, errors.Errorf("failed to find key %q in secret %q", selector.Key, namespacedName)
-	}
-
-	return string(value), secret, nil
-}
diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go
index f26fe7d9e..8488d47f2 100644
--- a/pkg/operator/ceph/object/user/controller.go
+++ b/pkg/operator/ceph/object/user/controller.go
@@ -245,14 +245,10 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci
 	observedGeneration := cephObjectStoreUser.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephObjectStoreUser)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephObjectStoreUser)
 	if err != nil {
 		return reconcile.Result{}, *cephObjectStoreUser, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the object user %q after adding finalizer", cephObjectStoreUser.Name)
-		return reconcile.Result{}, *cephObjectStoreUser, nil
-	}
 
 	clusterNamespace := request.NamespacedName
 	clusterNamespace.Namespace = clusterStoreNamespace(cephObjectStoreUser)
diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go
index df70cb466..105265130 100644
--- a/pkg/operator/ceph/object/zone/controller.go
+++ b/pkg/operator/ceph/object/zone/controller.go
@@ -158,15 +158,10 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
 	// CR status will be updated at end of reconcile, so to reflect the reconcile has finished
 	observedGeneration := cephObjectZone.ObjectMeta.Generation
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephObjectZone)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephObjectZone)
 	if err != nil {
 		return reconcile.Result{}, *cephObjectZone, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the object zone %q after adding finalizer", cephObjectZone.Name)
-		return reconcile.Result{}, *cephObjectZone, nil
-	}
-
 	// The CR was just created, initializing status fields
 	if cephObjectZone.Status == nil {
 		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.EmptyStatus)
diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go
index be3eaba23..b04af10dc 100644
--- a/pkg/operator/ceph/pool/controller.go
+++ b/pkg/operator/ceph/pool/controller.go
@@ -189,22 +189,14 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
 	observedGeneration := cephBlockPool.ObjectMeta.Generation
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephBlockPool)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephBlockPool)
 	if err != nil {
 		return opcontroller.ImmediateRetryResult, *cephBlockPool, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the ceph block pool %q after adding finalizer", cephBlockPool.Name)
-		return reconcile.Result{}, *cephBlockPool, nil
-	}
 
-	var statusErr error
 	// The CR was just created, initializing status fields
 	if cephBlockPool.Status == nil {
-		err = r.updateStatus(request.NamespacedName, cephv1.ConditionProgressing, k8sutil.ObservedGenerationNotAvailable)
-		if err != nil {
-			logger.Errorf("failed to update %q status to %q: %v", request.NamespacedName, cephv1.ConditionProgressing, err)
-		}
+		r.updateStatus(request.NamespacedName, cephv1.ConditionProgressing, k8sutil.ObservedGenerationNotAvailable)
 	}
 
 	// Make sure a CephCluster is present otherwise do nothing
@@ -322,10 +314,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
 			logger.Info(opcontroller.OperatorNotInitializedMessage)
 			return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephBlockPool, nil
 		}
-		statusErr = r.updateStatus(request.NamespacedName, cephv1.ConditionFailure, k8sutil.ObservedGenerationNotAvailable)
-		if statusErr != nil {
-			logger.Errorf("failed to update %q status to %q: %v", request.NamespacedName, cephv1.ConditionFailure, statusErr)
-		}
+		r.updateStatus(request.NamespacedName, cephv1.ConditionFailure, k8sutil.ObservedGenerationNotAvailable)
 		return reconcileResponse, *cephBlockPool, errors.Wrapf(err, "failed to create pool %q.", cephBlockPool.GetName())
 	}
 
@@ -340,10 +329,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
 		// Always create a bootstrap peer token in case another cluster wants to add us as a peer
 		reconcileResponse, err = opcontroller.CreateBootstrapPeerSecret(r.context, clusterInfo, cephBlockPool, k8sutil.NewOwnerInfo(cephBlockPool, r.scheme))
 		if err != nil {
-			statusErr = r.updateStatus(request.NamespacedName, cephv1.ConditionFailure, k8sutil.ObservedGenerationNotAvailable)
-			if statusErr != nil {
-				logger.Errorf("failed to update %q status to %q: %v", request.NamespacedName, cephv1.ConditionFailure, statusErr)
-			}
+			r.updateStatus(request.NamespacedName, cephv1.ConditionFailure, k8sutil.ObservedGenerationNotAvailable)
 			return reconcileResponse, *cephBlockPool, errors.Wrapf(err, "failed to create rbd-mirror bootstrap peer for pool %q.", cephBlockPool.GetName())
 		}
 
@@ -365,7 +351,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
 
 		// update ObservedGeneration in status at the end of reconcile
 		// Set Ready status, we are done reconciling
-		statusErr = r.updateStatus(request.NamespacedName, cephv1.ConditionReady, observedGeneration)
+		r.updateStatus(request.NamespacedName, cephv1.ConditionReady, observedGeneration)
 
 		if cephBlockPool.Spec.StatusCheck.Mirror.Disabled {
 			// Stop monitoring the mirroring status of this pool
@@ -398,7 +384,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
 		}
 		// update ObservedGeneration in status at the end of reconcile
 		// Set Ready status, we are done reconciling
-		statusErr = r.updateStatus(request.NamespacedName, cephv1.ConditionReady, observedGeneration)
+		r.updateStatus(request.NamespacedName, cephv1.ConditionReady, observedGeneration)
 
 		// Stop monitoring the mirroring status of this pool
 		if blockPoolContextsExists && r.blockPoolContexts[blockPoolChannelKey].started {
@@ -408,10 +394,6 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
 		}
 	}
 
-	if statusErr != nil {
-		return opcontroller.ImmediateRetryResult, *cephBlockPool, errors.Wrapf(statusErr, "failed to update status of pool %q to %q.", cephBlockPool.Name, cephv1.ConditionReady)
-	}
-
 	// Return and do not requeue
 	logger.Debug("done reconciling")
 	return reconcile.Result{}, *cephBlockPool, nil
diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go
index 0d3c34edd..18a4d5de7 100644
--- a/pkg/operator/ceph/pool/radosnamespace/controller.go
+++ b/pkg/operator/ceph/pool/radosnamespace/controller.go
@@ -158,14 +158,10 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque
 	}
 
 	// Set a finalizer so we can do cleanup before the object goes away
-	generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephBlockPoolRadosNamespace)
+	err = opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephBlockPoolRadosNamespace)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "failed to add finalizer")
 	}
-	if generationUpdated {
-		logger.Infof("reconciling the rados namespace %q after adding finalizer", cephBlockPoolRadosNamespace.Name)
-		return reconcile.Result{}, nil
-	}
 
 	// The CR was just created, initializing status fields
 	if cephBlockPoolRadosNamespace.Status == nil {
diff --git a/pkg/operator/ceph/pool/status.go b/pkg/operator/ceph/pool/status.go
index d66274fc9..6d19a5ec4 100644
--- a/pkg/operator/ceph/pool/status.go
+++ b/pkg/operator/ceph/pool/status.go
@@ -18,7 +18,6 @@ limitations under the License.
 package pool
 
 import (
-	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
@@ -29,16 +28,16 @@ import (
 )
 
 // updateStatus updates a pool CR with the given status
-func (r *ReconcileCephBlockPool) updateStatus(poolName types.NamespacedName, status cephv1.ConditionType, observedGeneration int64) error {
+func (r *ReconcileCephBlockPool) updateStatus(poolName types.NamespacedName, status cephv1.ConditionType, observedGeneration int64) {
 	pool := &cephv1.CephBlockPool{}
 	err := r.client.Get(r.opManagerContext, poolName, pool)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("CephBlockPool resource not found. Ignoring since object must be deleted.")
-			return nil
+			return
 		}
 		logger.Warningf("failed to retrieve pool %q to update status to %q. %v", poolName, status, err)
-		return errors.Wrapf(err, "failed to retrieve pool %q to update status to %q", poolName, status)
+		return
 	}
 
 	if pool.Status == nil {
@@ -57,10 +56,9 @@ func (r *ReconcileCephBlockPool) updateStatus(poolName types.NamespacedName, sta
 	}
 	if err := reporting.UpdateStatus(r.client, pool); err != nil {
 		logger.Warningf("failed to set pool %q status to %q. %v", pool.Name, status, err)
-		return errors.Wrapf(err, "failed to set pool %q status to %q", pool.Name, status)
+		return
 	}
 	logger.Debugf("pool %q status updated to %q", poolName, status)
-	return nil
 }
 
 func updateStatusInfo(cephBlockPool *cephv1.CephBlockPool) {


Test patch:
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index 42b60d136..4bc77b1d8 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -1591,7 +1591,7 @@ jobs:
           tests/scripts/github-action-helper.sh deploy_first_rook_cluster
           kubectl create -f deploy/examples/object-multisite-test.yaml
           # wait for multisite-store to be created
-          tests/scripts/github-action-helper.sh wait_for cephobjectstore multisite-store rook-ceph 600
+          tests/scripts/github-action-helper.sh wait_for cephobjectstore multisite-store rook-ceph 480
 
       - name: prep second cluster pull realm config
         shell: bash --noprofile --norc -eo pipefail -x {0}
@@ -1610,7 +1610,7 @@ jobs:
           tests/scripts/github-action-helper.sh deploy_second_rook_cluster
           kubectl create -f deploy/examples/object-multisite-pull-realm-test.yaml
           # wait for realms to be pulled and zone-b-multisite-store to be created
-          tests/scripts/github-action-helper.sh wait_for cephobjectstore zone-b-multisite-store rook-ceph-secondary 600
+          tests/scripts/github-action-helper.sh wait_for cephobjectstore zone-b-multisite-store rook-ceph-secondary 480
 
       - name: wait for both ceph clusters to be ready
         shell: bash --noprofile --norc -eo pipefail -x {0}
diff --git a/.github/workflows/integration-test-config-latest-k8s/action.yaml b/.github/workflows/integration-test-config-latest-k8s/action.yaml
index 34a956a19..479973412 100644
--- a/.github/workflows/integration-test-config-latest-k8s/action.yaml
+++ b/.github/workflows/integration-test-config-latest-k8s/action.yaml
@@ -8,15 +8,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Free Disk Space (Ubuntu)
-      uses: jlumbroso/free-disk-space@main
-      with:
-        tool-cache: true # removes old golang version but saves ~6GB
-        large-packages: false # takes a long time to run b/c package mgr
-        android: false # takes a long time to run b/c package mgr
-        dotnet: false # takes a long time to run b/c package mgr
-        haskell: false # takes a long time to run b/c package mgr
-
     - name: setup golang
       uses: actions/setup-go@v5
       with:
diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
index f10a3cfda..d6e5700cb 100644
--- a/.github/workflows/integration-test-helm-suite.yaml
+++ b/.github/workflows/integration-test-helm-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-keystone-auth-suite.yaml b/.github/workflows/integration-test-keystone-auth-suite.yaml
index 0e253610e..74a06ac5d 100644
--- a/.github/workflows/integration-test-keystone-auth-suite.yaml
+++ b/.github/workflows/integration-test-keystone-auth-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
index 3c889669d..d369daa40 100644
--- a/.github/workflows/integration-test-mgr-suite.yaml
+++ b/.github/workflows/integration-test-mgr-suite.yaml
@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.32.3"]
+        kubernetes-versions: ["v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
index fe1088c24..8104deb3a 100644
--- a/.github/workflows/integration-test-multi-cluster-suite.yaml
+++ b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.32.3"]
+        kubernetes-versions: ["v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml
index 7b6d85a45..e00aac487 100644
--- a/.github/workflows/integration-test-object-suite.yaml
+++ b/.github/workflows/integration-test-object-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
index 67e1d7c7f..57c4d5d7e 100644
--- a/.github/workflows/integration-test-smoke-suite.yaml
+++ b/.github/workflows/integration-test-smoke-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
index cd7538f5b..acebf0a75 100644
--- a/.github/workflows/integration-test-upgrade-suite.yaml
+++ b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -73,7 +73,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml
index 37bf7900b..849ad4954 100644
--- a/.github/workflows/integration-tests-on-release.yaml
+++ b/.github/workflows/integration-tests-on-release.yaml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -61,7 +61,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -102,7 +102,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -140,7 +140,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -178,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.29.15", "v1.30.11", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -219,7 +219,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.28.15", "v1.32.3"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/pkg/daemon/ceph/client/status_test.go b/pkg/daemon/ceph/client/status_test.go
index a191fe39f..a3b791a41 100644
--- a/pkg/daemon/ceph/client/status_test.go
+++ b/pkg/daemon/ceph/client/status_test.go
@@ -43,7 +43,7 @@ var statusFakeRaw = []byte(`{
 			  "rank": 0,
 			  "name": "myfs-b",
 			  "status": "up:active",
-			  "gid": 13999249051405999408
+			  "gid": 14716
 			}
 		  ],
 		  "up:standby": 1
diff --git a/pkg/operator/ceph/client/controller_test.go b/pkg/operator/ceph/client/controller_test.go
index 0e61ab872..7420c1463 100644
--- a/pkg/operator/ceph/client/controller_test.go
+++ b/pkg/operator/ceph/client/controller_test.go
@@ -127,13 +127,9 @@ func TestCephClientController(t *testing.T) {
 	// A Pool resource with metadata and spec.
 	cephClient := &cephv1.CephClient{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       name,
-			Namespace:  namespace,
-			UID:        types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
-			Finalizers: []string{"cephclient.ceph.rook.io"},
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind: "CephClient",
+			Name:      name,
+			Namespace: namespace,
+			UID:       types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
 		},
 		Spec: cephv1.ClientSpec{
 			Caps: map[string]string{
diff --git a/pkg/operator/ceph/controller/finalizer_test.go b/pkg/operator/ceph/controller/finalizer_test.go
index e76f4662b..8c06ebbe1 100644
--- a/pkg/operator/ceph/controller/finalizer_test.go
+++ b/pkg/operator/ceph/controller/finalizer_test.go
@@ -46,10 +46,9 @@ func TestAddFinalizerIfNotPresent(t *testing.T) {
 	cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
 
 	assert.Empty(t, fakeObject.Finalizers)
-	generationUpdated, err := AddFinalizerIfNotPresent(context.TODO(), cl, fakeObject)
+	err := AddFinalizerIfNotPresent(context.TODO(), cl, fakeObject)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, fakeObject.Finalizers)
-	assert.False(t, generationUpdated)
 }
 
 func TestRemoveFinalizer(t *testing.T) {
diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go
index 4fb20fe02..94e71c8d5 100644
--- a/pkg/operator/ceph/csi/util_test.go
+++ b/pkg/operator/ceph/csi/util_test.go
@@ -280,7 +280,7 @@ func Test_getImage(t *testing.T) {
 		{
 			name: "test with default image",
 			args: args{
-				defaultImage: "quay.io/cephcsi/cephcsi:v3.14.0",
+				defaultImage: "quay.io/cephcsi/cephcsi:v3.13.1",
 			},
 			want: DefaultCSIPluginImage,
 		},
diff --git a/pkg/operator/ceph/file/controller_test.go b/pkg/operator/ceph/file/controller_test.go
index 9e6a7fa7e..8f1700b35 100644
--- a/pkg/operator/ceph/file/controller_test.go
+++ b/pkg/operator/ceph/file/controller_test.go
@@ -165,9 +165,8 @@ func TestCephFilesystemController(t *testing.T) {
 	// A Pool resource with metadata and spec.
 	fs := &cephv1.CephFilesystem{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       name,
-			Namespace:  namespace,
-			Finalizers: []string{"cephfilesystem.ceph.rook.io"},
+			Name:      name,
+			Namespace: namespace,
 		},
 		Spec: cephv1.FilesystemSpec{
 			MetadataServer: cephv1.MetadataServerSpec{
diff --git a/pkg/operator/ceph/file/subvolumegroup/controller_test.go b/pkg/operator/ceph/file/subvolumegroup/controller_test.go
index 198667f2d..0984a5770 100644
--- a/pkg/operator/ceph/file/subvolumegroup/controller_test.go
+++ b/pkg/operator/ceph/file/subvolumegroup/controller_test.go
@@ -42,7 +42,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
-func TestFilesystemSubvolumeGroupController(t *testing.T) {
+func TestCephClientController(t *testing.T) {
 	ctx := context.TODO()
 	// Set DEBUG logging
 	capnslog.SetGlobalLogLevel(capnslog.DEBUG)
@@ -56,13 +56,9 @@ func TestFilesystemSubvolumeGroupController(t *testing.T) {
 	// A cephFilesystemSubVolumeGroup resource with metadata and spec.
 	cephFilesystemSubVolumeGroup := &cephv1.CephFilesystemSubVolumeGroup{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       name,
-			Namespace:  namespace,
-			UID:        types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
-			Finalizers: []string{"cephfilesystemsubvolumegroup.ceph.rook.io"},
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind: "CephFilesystemSubvolumeGroup",
+			Name:      name,
+			Namespace: namespace,
+			UID:       types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
 		},
 		Spec: cephv1.CephFilesystemSubVolumeGroupSpec{
 			FilesystemName: namespace,
diff --git a/pkg/operator/ceph/nfs/controller_test.go b/pkg/operator/ceph/nfs/controller_test.go
index e631b3566..34bd23b76 100644
--- a/pkg/operator/ceph/nfs/controller_test.go
+++ b/pkg/operator/ceph/nfs/controller_test.go
@@ -141,9 +141,8 @@ func TestCephNFSController(t *testing.T) {
 	baseCephNFS := func() *cephv1.CephNFS {
 		return &cephv1.CephNFS{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:       name,
-				Namespace:  namespace,
-				Finalizers: []string{"cephnfs.ceph.rook.io"},
+				Name:      name,
+				Namespace: namespace,
 			},
 			Spec: cephv1.NFSGaneshaSpec{
 				RADOS: cephv1.GaneshaRADOSSpec{
diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go
index fe67e64c7..51a9818a2 100644
--- a/pkg/operator/ceph/object/controller_test.go
+++ b/pkg/operator/ceph/object/controller_test.go
@@ -343,9 +343,8 @@ func TestCephObjectStoreController(t *testing.T) {
 		// A Pool resource with metadata and spec.
 		objectStore := &cephv1.CephObjectStore{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:       store,
-				Namespace:  namespace,
-				Finalizers: []string{"cephobjectstore.ceph.rook.io"},
+				Name:      store,
+				Namespace: namespace,
 			},
 			Spec:     cephv1.ObjectStoreSpec{MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1}}, DataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1}}},
 			TypeMeta: controllerTypeMeta,
@@ -633,9 +632,8 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
 
 	objectStore := &cephv1.CephObjectStore{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       store,
-			Namespace:  namespace,
-			Finalizers: []string{"cephobjectstore.ceph.rook.io"},
+			Name:      store,
+			Namespace: namespace,
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind: "CephObjectStore",
diff --git a/pkg/operator/ceph/object/topic/controller_test.go b/pkg/operator/ceph/object/topic/controller_test.go
index 20c384785..70fee32e9 100644
--- a/pkg/operator/ceph/object/topic/controller_test.go
+++ b/pkg/operator/ceph/object/topic/controller_test.go
@@ -33,7 +33,6 @@ import (
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
-	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -72,9 +71,8 @@ func TestCephBucketTopicController(t *testing.T) {
 
 	bucketTopic := &cephv1.CephBucketTopic{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       name,
-			Namespace:  namespace,
-			Finalizers: []string{"cephbuckettopic.ceph.rook.io"},
+			Name:      name,
+			Namespace: namespace,
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind: "CephBucketTopic",
@@ -234,8 +232,8 @@ func TestCephBucketTopicController(t *testing.T) {
 
 		// mock the provisioner
 		expectedARN := "arn:aws:sns:" + store + "::" + bucketTopic.Name
-		createTopicFunc = func(p provisioner, topic *cephv1.CephBucketTopic) (*string, *map[types.UID]*corev1.Secret, error) {
-			return &expectedARN, nil, nil
+		createTopicFunc = func(p provisioner, topic *cephv1.CephBucketTopic) (*string, error) {
+			return &expectedARN, nil
 		}
 		defer func() { createTopicFunc = createTopic }()
 		res, err := r.Reconcile(ctx, req)
diff --git a/pkg/operator/ceph/object/topic/provisioner_test.go b/pkg/operator/ceph/object/topic/provisioner_test.go
index fd0920180..798a7b6c9 100644
--- a/pkg/operator/ceph/object/topic/provisioner_test.go
+++ b/pkg/operator/ceph/object/topic/provisioner_test.go
@@ -25,7 +25,6 @@ import (
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -63,12 +62,8 @@ func TestTopicAttributesCreation(t *testing.T) {
 				},
 			},
 		}
-
-		attrs, _, err := createTopicAttributes(provisioner{}, bucketTopic)
-		require.NoError(t, err)
-		assert.Equal(t, expectedAttrs, attrs)
+		assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic))
 	})
-
 	t.Run("test AMQP attributes", func(t *testing.T) {
 		uri := "amqp://my-rabbitmq-service:5672/vhost1"
 		ackLevel := "broker"
@@ -101,12 +96,8 @@ func TestTopicAttributesCreation(t *testing.T) {
 				},
 			},
 		}
-
-		attrs, _, err := createTopicAttributes(provisioner{}, bucketTopic)
-		require.NoError(t, err)
-		assert.Equal(t, expectedAttrs, attrs)
+		assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic))
 	})
-
 	t.Run("test Kafka attributes", func(t *testing.T) {
 		uri := "kafka://my-kafka-service:9092"
 		ackLevel := "broker"
@@ -141,9 +132,6 @@ func TestTopicAttributesCreation(t *testing.T) {
 				},
 			},
 		}
-
-		attrs, _, err := createTopicAttributes(provisioner{}, bucketTopic)
-		require.NoError(t, err)
-		assert.Equal(t, expectedAttrs, attrs)
+		assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic))
 	})
 }
diff --git a/pkg/operator/ceph/object/user/controller_test.go b/pkg/operator/ceph/object/user/controller_test.go
index 009c36cd9..4ae6702a4 100644
--- a/pkg/operator/ceph/object/user/controller_test.go
+++ b/pkg/operator/ceph/object/user/controller_test.go
@@ -113,9 +113,8 @@ func TestCephObjectStoreUserController(t *testing.T) {
 	cephObjectStore := &cephv1.CephObjectStore{}
 	objectUser := &cephv1.CephObjectStoreUser{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       name,
-			Namespace:  namespace,
-			Finalizers: []string{"cephobjectstoreuser.ceph.rook.io"},
+			Name:      name,
+			Namespace: namespace,
 		},
 		Spec: cephv1.ObjectStoreUserSpec{
 			Store: store,
diff --git a/pkg/operator/ceph/object/zone/controller_test.go b/pkg/operator/ceph/object/zone/controller_test.go
index 87966987b..e28e53bc5 100644
--- a/pkg/operator/ceph/object/zone/controller_test.go
+++ b/pkg/operator/ceph/object/zone/controller_test.go
@@ -163,7 +163,6 @@ func TestCephObjectZoneController(t *testing.T) {
 			Name:       name,
 			Namespace:  namespace,
 			Generation: 0,
-			Finalizers: []string{"cephobjectzone.ceph.rook.io"},
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind: "CephObjectZone",
diff --git a/pkg/operator/ceph/pool/controller_test.go b/pkg/operator/ceph/pool/controller_test.go
index 7891337c0..1e77b1807 100644
--- a/pkg/operator/ceph/pool/controller_test.go
+++ b/pkg/operator/ceph/pool/controller_test.go
@@ -218,13 +218,9 @@ func TestCephBlockPoolController(t *testing.T) {
 	// A Pool resource with metadata and spec.
 	pool := &cephv1.CephBlockPool{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       name,
-			Namespace:  namespace,
-			UID:        types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
-			Finalizers: []string{"cephblockpool.ceph.rook.io"},
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind: "CephBlockPool",
+			Name:      name,
+			Namespace: namespace,
+			UID:       types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
 		},
 		Spec: cephv1.NamedBlockPoolSpec{
 			PoolSpec: cephv1.PoolSpec{
diff --git a/pkg/operator/ceph/pool/radosnamespace/controller_test.go b/pkg/operator/ceph/pool/radosnamespace/controller_test.go
index dad2e6489..859128619 100644
--- a/pkg/operator/ceph/pool/radosnamespace/controller_test.go
+++ b/pkg/operator/ceph/pool/radosnamespace/controller_test.go
@@ -56,13 +56,9 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) {
 	// A cephBlockPoolRadosNamespace resource with metadata and spec.
 	cephBlockPoolRadosNamespace := &cephv1.CephBlockPoolRadosNamespace{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:       name,
-			Namespace:  namespace,
-			UID:        types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
-			Finalizers: []string{"cephblockpoolradosnamespace.ceph.rook.io"},
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind: "CephBlockPoolRadosNamespace",
+			Name:      name,
+			Namespace: namespace,
+			UID:       types.UID("c47cac40-9bee-4d52-823b-ccd803ba5bfe"),
 		},
 		Spec: cephv1.CephBlockPoolRadosNamespaceSpec{
 			BlockPoolName: namespace,
diff --git a/pkg/util/exec/exec_test.go b/pkg/util/exec/exec_test.go
index 06ba82335..aed52d7a5 100644
--- a/pkg/util/exec/exec_test.go
+++ b/pkg/util/exec/exec_test.go
@@ -147,7 +147,7 @@ func TestExecuteCommandWithTimeout(t *testing.T) {
 		{
 			name: "test stdin",
 			args: args{
-				timeout: 2 * time.Second,
+				timeout: 30 * time.Second,
 				command: "cat",
 				stdin:   &testString,
 				arg:     []string{},
@@ -158,7 +158,7 @@ func TestExecuteCommandWithTimeout(t *testing.T) {
 		{
 			name: "test nil stdin",
 			args: args{
-				timeout: 2 * time.Second,
+				timeout: 30 * time.Second,
 				command: "echo",
 				stdin:   nil,
 				arg:     []string{testString},
@@ -166,24 +166,13 @@ func TestExecuteCommandWithTimeout(t *testing.T) {
 			want:    testString,
 			wantErr: false,
 		},
-		{
-			name: "test err return",
-			args: args{
-				timeout: 2 * time.Second,
-				command: "false",
-				stdin:   nil,
-				arg:     []string{},
-			},
-			want:    "",
-			wantErr: true,
-		},
 		{
 			name: "test timeout",
 			args: args{
-				timeout: 5 * time.Millisecond,
-				command: "sleep",
+				timeout: 0 * time.Second,
+				command: "cat",
 				stdin:   &testString,
-				arg:     []string{"2"},
+				arg:     []string{},
 			},
 			want:    "",
 			wantErr: true,
diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go
index bb2e71cfb..ccecc7060 100644
--- a/tests/framework/installer/ceph_manifests.go
+++ b/tests/framework/installer/ceph_manifests.go
@@ -189,10 +189,6 @@ spec:
   disruptionManagement:
     managePodBudgets: true
     osdMaintenanceTimeout: 30
-  cephConfig:
-    global:
-      mon_data_avail_warn: "10"
-      rgw_allow_notification_secrets_in_cleartext: "true"
   healthCheck:
     daemonHealth:
       mon:
diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go
index f55677112..3d18c86a3 100644
--- a/tests/integration/ceph_object_test.go
+++ b/tests/integration/ceph_object_test.go
@@ -41,7 +41,6 @@ import (
 	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
 	"github.com/rook/rook/tests/integration/object/bucketowner"
-	topickafka "github.com/rook/rook/tests/integration/object/topic/kafka"
 	"github.com/rook/rook/tests/integration/object/user/userkeys"
 )
 
@@ -157,7 +156,6 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
 
 	bucketowner.TestObjectBucketClaimBucketOwner(s.T(), k8sh, installer, logger, tlsEnable)
 	userkeys.TestObjectStoreUserKeys(s.T(), k8sh, installer, logger, tlsEnable)
-	topickafka.TestBucketTopicKafka(s.T(), k8sh, installer, logger, tlsEnable)
 
 	bucketNotificationTestStoreName := "bucket-notification-" + storeName
 	createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable, swiftAndKeystone)
diff --git a/tests/integration/object/topic/kafka/kafka.go b/tests/integration/object/topic/kafka/kafka.go
deleted file mode 100644
index 3cb23bd24..000000000
--- a/tests/integration/object/topic/kafka/kafka.go
+++ /dev/null
@@ -1,701 +0,0 @@
-/*
-Copyright 2025 The Rook Authors. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kafka
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/url"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/service/sns"
-	"github.com/coreos/pkg/capnslog"
-	bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	corev1 "k8s.io/api/core/v1"
-	storagev1 "k8s.io/api/storage/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
-
-	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/installer"
-	"github.com/rook/rook/tests/framework/utils"
-	utilsns "github.com/rook/rook/tests/integration/object/util/sns"
-)
-
-var (
-	defaultName = "test-topickafka"
-
-	ns = &corev1.Namespace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: defaultName,
-		},
-	}
-
-	objectStore = &cephv1.CephObjectStore{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: defaultName,
-			// the CephObjectstore must be in the same ns as the CephCluster
-			Namespace: "object-ns",
-		},
-		Spec: cephv1.ObjectStoreSpec{
-			MetadataPool: cephv1.PoolSpec{
-				Replicated: cephv1.ReplicatedSpec{
-					Size:                   1,
-					RequireSafeReplicaSize: false,
-				},
-			},
-			DataPool: cephv1.PoolSpec{
-				Replicated: cephv1.ReplicatedSpec{
-					Size:                   1,
-					RequireSafeReplicaSize: false,
-				},
-			},
-			Gateway: cephv1.GatewaySpec{
-				Port:      80,
-				Instances: 1,
-			},
-			AllowUsersInNamespaces: []string{ns.Name},
-		},
-	}
-
-	objectStoreSvc = &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      objectStore.Name,
-			Namespace: objectStore.Namespace,
-		},
-		Spec: corev1.ServiceSpec{
-			Selector: map[string]string{
-				"app":               "rook-ceph-rgw",
-				"rook_cluster":      objectStore.Namespace,
-				"rook_object_store": objectStore.Name,
-			},
-			Ports: []corev1.ServicePort{
-				{
-					Name:       "http",
-					Port:       80,
-					Protocol:   corev1.ProtocolTCP,
-					TargetPort: intstr.FromInt(8080),
-				},
-			},
-			SessionAffinity: corev1.ServiceAffinityNone,
-			Type:            corev1.ServiceTypeNodePort,
-		},
-	}
-
-	secret1 = &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      defaultName + "-secret1",
-			Namespace: ns.Name,
-		},
-		Data: map[string][]byte{
-			"user-name": []byte("kafka-user1"),
-		},
-	}
-
-	secret2 = &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      defaultName + "-secret2",
-			Namespace: ns.Name,
-		},
-		Data: map[string][]byte{
-			"password": []byte("kafka-pass2"),
-		},
-	}
-
-	secret3 = &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      defaultName + "-secret3",
-			Namespace: ns.Name,
-		},
-		Data: map[string][]byte{
-			"user-name": []byte("kafka-user3"),
-		},
-	}
-
-	secret4 = &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      defaultName + "-secret4",
-			Namespace: ns.Name,
-		},
-		Data: map[string][]byte{
-			"password": []byte("kafka-pass4"),
-		},
-	}
-
-	storageClass = &storagev1.StorageClass{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: defaultName,
-		},
-		Provisioner: objectStore.Namespace + ".ceph.rook.io/bucket",
-		Parameters: map[string]string{
-			"objectStoreName":      objectStore.Name,
-			"objectStoreNamespace": objectStore.Namespace,
-		},
-	}
-
-	obc1 = &bktv1alpha1.ObjectBucketClaim{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      defaultName + "-obc1",
-			Namespace: ns.Name,
-			Labels: map[string]string{
-				"bucket-notification-" + bn1.Name: bn1.Name,
-			},
-		},
-		Spec: bktv1alpha1.ObjectBucketClaimSpec{
-			BucketName:       defaultName + "-obc1",
-			StorageClassName: storageClass.Name,
-		},
-	}
-
-	bt1 = &cephv1.CephBucketTopic{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      defaultName + "-topic1",
-			Namespace: ns.Name,
-		},
-		Spec: cephv1.BucketTopicSpec{
-			ObjectStoreName:      objectStore.Name,
-			ObjectStoreNamespace: objectStore.Namespace,
-			Persistent:           false,
-			Endpoint: cephv1.TopicEndpointSpec{
-				Kafka: &cephv1.KafkaEndpointSpec{
-					URI:      "kafka://kafka.example.com:9094",
-					AckLevel: "broker",
-					UseSSL:   false,
-					UserSecretRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret1.Name,
-						},
-						Key: "user-name",
-					},
-					PasswordSecretRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: secret2.Name,
-						},
-						Key: "password",
-					},
-				},
-			},
-		},
-	}
-
-	bn1 = &cephv1.CephBucketNotification{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      defaultName + "-notification1",
-			Namespace: ns.Name,
-		},
-		Spec: cephv1.BucketNotificationSpec{
-			Topic: bt1.Name,
-			Events: []cephv1.BucketNotificationEvent{
-				"s3:ObjectCreated:*",
-			},
-		},
-	}
-)
-
-func checkStatusSecrets(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBucketTopic, expectedSecrets []*corev1.Secret) {
-	t.Run(fmt.Sprintf("cephBucketTopic %q has .status.secrets set", bt.Name), func(t *testing.T) {
-		ctx := context.TODO()
-
-		liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt.Namespace).Get(ctx, bt.Name, metav1.GetOptions{})
-		require.NoError(t, err)
-
-		require.NotNil(t, liveBt.Status)
-		assert.Len(t, liveBt.Status.Secrets, len(expectedSecrets))
-
-		for _, secret := range expectedSecrets {
-			secretRef, err := func(secretName string, keys []cephv1.SecretReference) (cephv1.SecretReference, error) {
-				for _, secretRef := range keys {
-					if secretRef.Name == secretName {
-						return secretRef, nil
-					}
-				}
-				return cephv1.SecretReference{}, fmt.Errorf("secretReference for secret %q not found in CephObjectStoreUser.status.keys", secret.Name)
-			}(secret.Name, liveBt.Status.Secrets)
-			require.NoError(t, err)
-
-			// fetch the live secret for UID and ResourceVersion
-			liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
-			require.NoError(t, err)
-
-			assert.Equal(t, liveSecret.Name, secretRef.Name)
-			assert.Equal(t, liveSecret.Namespace, secretRef.Namespace)
-			assert.Equal(t, liveSecret.UID, secretRef.UID)
-			assert.Equal(t, liveSecret.ResourceVersion, secretRef.ResourceVersion)
-		}
-	})
-}
-
-func checkRgwTopicEndpoint(t *testing.T, snsClient *sns.Client, arn, user, pass string) {
-	t.Run(fmt.Sprintf("rgw topic arn %q has basic auth set", arn), func(t *testing.T) {
-		ctx := context.TODO()
-
-		var uri *url.URL
-
-		inSync := utils.Retry(40, time.Second, "rgw topic basic auth in sync", func() bool {
-			topicAttrs, err := snsClient.GetTopicAttributes(ctx, &sns.GetTopicAttributesInput{
-				TopicArn: &arn,
-			})
-			require.NoError(t, err)
-
-			// the sns endpoint attributes are returned as JSON
-			var endpointJSON map[string]interface{}
-			err = json.Unmarshal([]byte(topicAttrs.Attributes["EndPoint"]), &endpointJSON)
-			require.NoError(t, err)
-
-			uri, err = url.Parse(string(endpointJSON["EndpointAddress"].(string)))
-			require.NoError(t, err)
-
-			uriPassword, _ := uri.User.Password()
-
-			return user == uri.User.Username() && pass == uriPassword
-		})
-		require.True(t, inSync)
-
-		assert.Equal(t, "kafka", uri.Scheme)
-		assert.Equal(t, "kafka.example.com:9094", uri.Host)
-		assert.Equal(t, user, uri.User.Username())
-		uriPassword, _ := uri.User.Password()
-		assert.Equal(t, pass, uriPassword)
-	})
-}
-
-// Note that .status.ARN is nil while .status.phase == Reconciling
-func cephBucketTopicReady(t *testing.T, k8sh *utils.K8sHelper, bt *cephv1.CephBucketTopic) *cephv1.CephBucketTopic {
-	ctx := context.TODO()
-
-	var liveBt *cephv1.CephBucketTopic
-	btReady := utils.Retry(60, time.Second, fmt.Sprintf("CephBucketTopic %q is Ready", bt.Name), func() bool {
-		var err error
-		liveBt, err = k8sh.RookClientset.CephV1().CephBucketTopics(bt.Namespace).Get(ctx, bt.Name, metav1.GetOptions{})
-		if err != nil {
-			return false
-		}
-
-		if liveBt.Status == nil {
-			return false
-		}
-
-		return liveBt.Status.Phase == string(cephv1.ConditionReady)
-	})
-	require.True(t, btReady)
-	require.NotNil(t, liveBt.Status.ARN)
-
-	return liveBt
-}
-
-func TestBucketTopicKafka(t *testing.T, k8sh *utils.K8sHelper, installer *installer.CephInstaller, logger *capnslog.PackageLogger, tlsEnable bool) {
-	ctx := context.TODO()
-	var snsClient *sns.Client
-
-	t.Run("CephBucketTopic kafka", func(t *testing.T) {
-		if tlsEnable {
-			// Skip testing with and without TLS to reduce test time
-			t.Skip("skipping test for TLS enabled clusters")
-		}
-
-		t.Run(fmt.Sprintf("create ns %q", ns.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create CephObjectStore %q", objectStore.Name), func(t *testing.T) {
-			objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Create(ctx, objectStore, metav1.CreateOptions{})
-			require.NoError(t, err)
-
-			osReady := utils.Retry(180, time.Second, "CephObjectStore is Ready", func() bool {
-				liveOs, err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
-				if err != nil {
-					return false
-				}
-
-				if liveOs.Status == nil {
-					return false
-				}
-
-				return liveOs.Status.Phase == cephv1.ConditionReady
-			})
-			require.True(t, osReady)
-		})
-
-		t.Run(fmt.Sprintf("create svc %q", objectStoreSvc.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Create(ctx, objectStoreSvc, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret1.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Create(ctx, secret1, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret2.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret2.Namespace).Create(ctx, secret2, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret3.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Create(ctx, secret3, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create secret %q", secret4.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Create(ctx, secret4, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		// the sc, obc, and CephBucketNotification are essentially unused for testing but are created for completeness
-		t.Run(fmt.Sprintf("create sc %q", storageClass.Name), func(t *testing.T) {
-			_, err := k8sh.Clientset.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create obc %q", obc1.Name), func(t *testing.T) {
-			_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Create(ctx, obc1, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("create CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Create(ctx, bt1, metav1.CreateOptions{})
-			require.NoError(t, err)
-
-			// user creation may be slow right after rgw start up
-			cephBucketTopicReady(t, k8sh, bt1)
-		})
-
-		t.Run(fmt.Sprintf("create CephBucketNotification %q", bn1.Name), func(t *testing.T) {
-			_, err := k8sh.RookClientset.CephV1().CephBucketNotifications(bn1.Namespace).Create(ctx, bn1, metav1.CreateOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run("setup sns client", func(t *testing.T) {
-			var err error
-			snsClient, err = utilsns.NewClient(objectStore, objectStoreSvc, k8sh, installer, tlsEnable)
-			require.NoError(t, err)
-		})
-
-		{
-			liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-			checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret1.Data["user-name"]), string(secret2.Data["password"]))
-
-			secrets := []*corev1.Secret{secret1, secret2}
-
-			checkStatusSecrets(t, k8sh, bt1, secrets)
-		}
-
-		t.Run("updating referenced secrets reconciles rgw topic", func(t *testing.T) {
-			t.Run(fmt.Sprintf("update kafka auth on CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-				liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-
-				liveBt.Spec.Endpoint.Kafka.UserSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret3.Name,
-					},
-					Key: "user-name",
-				}
-
-				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret4.Name,
-					},
-					Key: "password",
-				}
-
-				_, err = k8sh.RookClientset.CephV1().CephBucketTopics(liveBt.Namespace).Update(ctx, liveBt, metav1.UpdateOptions{})
-				require.NoError(t, err)
-			})
-
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret3.Data["user-name"]), string(secret4.Data["password"]))
-
-				secrets := []*corev1.Secret{secret3, secret4}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
-		})
-
-		t.Run("removing referenced secrets reconciles rgw topic", func(t *testing.T) {
-			t.Run(fmt.Sprintf("delete kafka auth on CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-				liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-
-				liveBt.Spec.Endpoint.Kafka.UserSecretRef = nil
-				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = nil
-
-				_, err = k8sh.RookClientset.CephV1().CephBucketTopics(liveBt.Namespace).Update(ctx, liveBt, metav1.UpdateOptions{})
-				require.NoError(t, err)
-			})
-
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, "", "")
-
-				secrets := []*corev1.Secret{}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
-		})
-
-		t.Run("adding new referenced secrets reconciles rgw topic", func(t *testing.T) {
-			t.Run(fmt.Sprintf("add kafka auth on CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-				liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-
-				liveBt.Spec.Endpoint.Kafka.UserSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret3.Name,
-					},
-					Key: "user-name",
-				}
-
-				liveBt.Spec.Endpoint.Kafka.PasswordSecretRef = &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: secret4.Name,
-					},
-					Key: "password",
-				}
-
-				_, err = k8sh.RookClientset.CephV1().CephBucketTopics(liveBt.Namespace).Update(ctx, liveBt, metav1.UpdateOptions{})
-				require.NoError(t, err)
-			})
-
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(secret3.Data["user-name"]), string(secret4.Data["password"]))
-
-				secrets := []*corev1.Secret{secret3, secret4}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
-		})
-
-		t.Run("updating secrets reconciles rgw topic", func(t *testing.T) {
-			t.Run(fmt.Sprintf("update secret %q data", secret3.Name), func(t *testing.T) {
-				liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret3.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-
-				liveSecret.Data = map[string][]byte{
-					"user-name": []byte("kafka-user3-updated"),
-				}
-
-				_, err = k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Update(ctx, liveSecret, metav1.UpdateOptions{})
-				require.NoError(t, err)
-			})
-
-			t.Run(fmt.Sprintf("update secret %q data", secret4.Name), func(t *testing.T) {
-				liveSecret, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret4.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-
-				liveSecret.Data = map[string][]byte{
-					"password": []byte("kafka-pass4-updated"),
-				}
-
-				_, err = k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Update(ctx, liveSecret, metav1.UpdateOptions{})
-				require.NoError(t, err)
-			})
-
-			{
-				liveBt := cephBucketTopicReady(t, k8sh, bt1)
-
-				liveSecret3, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret3.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-
-				liveSecret4, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Get(ctx, secret4.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-
-				checkRgwTopicEndpoint(t, snsClient, *liveBt.Status.ARN, string(liveSecret3.Data["user-name"]), string(liveSecret4.Data["password"]))
-
-				secrets := []*corev1.Secret{liveSecret3, liveSecret4}
-
-				checkStatusSecrets(t, k8sh, bt1, secrets)
-			}
-		})
-
-		t.Run("deleting a referenced secret triggers a reconcile, which fails", func(t *testing.T) {
-			t.Run(fmt.Sprintf("delete secret %q", secret4.Name), func(t *testing.T) {
-				err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Delete(ctx, secret4.Name, metav1.DeleteOptions{})
-				require.NoError(t, err)
-			})
-
-			t.Run(fmt.Sprintf("CephBucketTopic %q has phase ReconcileFailed", bt1.Name), func(t *testing.T) {
-				btReady := utils.Retry(40, time.Second, "CephBucketTopic is ReconcileFailed", func() bool {
-					liveBt, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-					if err != nil {
-						return false
-					}
-
-					if liveBt.Status == nil {
-						return false
-					}
-
-					return liveBt.Status.Phase == string(cephv1.ReconcileFailed)
-				})
-				require.True(t, btReady)
-			})
-
-			t.Run(fmt.Sprintf("create secret %q", secret4.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Create(ctx, secret4, metav1.CreateOptions{})
-				require.NoError(t, err)
-			})
-
-			cephBucketTopicReady(t, k8sh, bt1)
-		})
-
-		t.Run(fmt.Sprintf("delete CephBucketNotification %q", bn1.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephBucketNotifications(bn1.Namespace).Delete(ctx, bn1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephBucketNotification is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephBucketNotifications(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-		})
-
-		t.Run(fmt.Sprintf("no CephBucketNotification(s) in ns %q", ns.Name), func(t *testing.T) {
-			list, err := k8sh.RookClientset.CephV1().CephBucketNotifications(ns.Name).List(ctx, metav1.ListOptions{})
-			require.NoError(t, err)
-
-			assert.Len(t, list.Items, 0)
-		})
-
-		t.Run(fmt.Sprintf("delete CephBucketTopic %q", bt1.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Delete(ctx, bt1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "CephBucketTopic is absent", func() bool {
-				_, err := k8sh.RookClientset.CephV1().CephBucketTopics(bt1.Namespace).Get(ctx, bt1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-		})
-
-		t.Run(fmt.Sprintf("no CephBucketTopic(s) in ns %q", ns.Name), func(t *testing.T) {
-			list, err := k8sh.RookClientset.CephV1().CephBucketTopics(ns.Name).List(ctx, metav1.ListOptions{})
-			require.NoError(t, err)
-
-			assert.Len(t, list.Items, 0)
-		})
-
-		t.Run(fmt.Sprintf("delete obc %q", obc1.Name), func(t *testing.T) {
-			// lookup ob name
-			liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Get(ctx, obc1.Name, metav1.GetOptions{})
-			require.NoError(t, err)
-			obName := liveObc.Spec.ObjectBucketName
-
-			// delete obc
-			err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Delete(ctx, obc1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-
-			absent := utils.Retry(40, time.Second, "OBC is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(obc1.Namespace).Get(ctx, obc1.Name, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-
-			absent = utils.Retry(40, time.Second, "OB is absent", func() bool {
-				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
-				return err != nil
-			})
-			assert.True(t, absent)
-		})
-
-		t.Run(fmt.Sprintf("no obc(s) in ns %q", ns.Name), func(t *testing.T) {
-			list, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(ns.Name).List(ctx, metav1.ListOptions{})
-			require.NoError(t, err)
-
-			assert.Len(t, list.Items, 0)
-		})
-
-		t.Run("CephBucketTopic deletion did not remove any secrets", func(t *testing.T) {
-			t.Run(fmt.Sprintf("secret %q still exists", secret1.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Get(ctx, secret1.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-			})
-
-			t.Run(fmt.Sprintf("secret %q still exists", secret2.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret2.Namespace).Get(ctx, secret2.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-			})
-
-			t.Run(fmt.Sprintf("secret %q still exists", secret3.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Get(ctx, secret3.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-			})
-
-			t.Run(fmt.Sprintf("secret %q still exists", secret4.Name), func(t *testing.T) {
-				_, err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Get(ctx, secret4.Name, metav1.GetOptions{})
-				require.NoError(t, err)
-			})
-		})
-
-		t.Run(fmt.Sprintf("delete sc %q", storageClass.Name), func(t *testing.T) {
-			err := k8sh.Clientset.StorageV1().StorageClasses().Delete(ctx, storageClass.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("delete secret %q", secret4.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret4.Namespace).Delete(ctx, secret4.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("delete secret %q", secret3.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Delete(ctx, secret3.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("delete secret %q", secret2.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret3.Namespace).Delete(ctx, secret2.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("delete secret %q", secret1.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Secrets(secret1.Namespace).Delete(ctx, secret1.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("no secrets in ns %q", ns.Name), func(t *testing.T) {
-			secrets, err := k8sh.Clientset.CoreV1().Secrets(ns.Name).List(ctx, metav1.ListOptions{})
-			require.NoError(t, err)
-
-			assert.Len(t, secrets.Items, 0)
-		})
-
-		t.Run(fmt.Sprintf("delete svc %q", objectStoreSvc.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Delete(ctx, objectStoreSvc.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("delete CephObjectStore %q", objectStore.Name), func(t *testing.T) {
-			err := k8sh.RookClientset.CephV1().CephObjectStores(objectStore.Namespace).Delete(ctx, objectStore.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-
-		t.Run(fmt.Sprintf("delete ns %q", ns.Name), func(t *testing.T) {
-			err := k8sh.Clientset.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{})
-			require.NoError(t, err)
-		})
-	})
-}
diff --git a/tests/integration/object/util/sns/sns.go b/tests/integration/object/util/sns/sns.go
deleted file mode 100644
index 3ea9445bb..000000000
--- a/tests/integration/object/util/sns/sns.go
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
-Copyright 2025 The Rook Authors. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package sns
-
-import (
-	"context"
-	"crypto/tls"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/url"
-
-	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/service/sns"
-	smithyendpoints "github.com/aws/smithy-go/endpoints"
-	"github.com/pkg/errors"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/tests/framework/installer"
-	"github.com/rook/rook/tests/framework/utils"
-)
-
-// based on the s3 endpoint example in the official docs:
-// https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-endpoints.html
-type snsResolverV2 struct {
-	endpoint string
-}
-
-func (r *snsResolverV2) ResolveEndpoint(ctx context.Context, params sns.EndpointParameters) (smithyendpoints.Endpoint, error) {
-	u, err := url.Parse(r.endpoint)
-	if err != nil {
-		return smithyendpoints.Endpoint{}, err
-	}
-	return smithyendpoints.Endpoint{
-		URI: *u,
-	}, nil
-}
-
-func GetS3Credentials(objectStore *cephv1.CephObjectStore, installer *installer.CephInstaller) (string, string, error) {
-	err, output := installer.Execute("radosgw-admin", []string{"user", "info", "--uid=dashboard-admin", fmt.Sprintf("--rgw-realm=%s", objectStore.Name)}, objectStore.Namespace)
-	if err != nil {
-		return "", "", errors.Wrap(err, "failed to get user info")
-	}
-
-	// extract api creds from json output
-	var userInfo map[string]interface{}
-	err = json.Unmarshal([]byte(output), &userInfo)
-	if err != nil {
-		return "", "", errors.Wrap(err, "failed to unmarshal user info")
-	}
-
-	accessKey, ok := userInfo["keys"].([]interface{})[0].(map[string]interface{})["access_key"].(string)
-	if !ok {
-		return "", "", errors.New("failed to get access key")
-	}
-
-	secretKey, ok := userInfo["keys"].([]interface{})[0].(map[string]interface{})["secret_key"].(string)
-	if !ok {
-		return "", "", errors.New("failed to get secret key")
-	}
-
-	return accessKey, secretKey, nil
-}
-
-func GetS3Endpoint(objectStore *cephv1.CephObjectStore, k8sh *utils.K8sHelper, tlsEnable bool) (string, error) {
-	ctx := context.TODO()
-
-	// extract rgw endpoint from k8s svc
-	svc, err := k8sh.Clientset.CoreV1().Services(objectStore.Namespace).Get(ctx, objectStore.Name, metav1.GetOptions{})
-	if err != nil {
-		return "", errors.Wrap(err, "failed to get objectstore svc")
-	}
-
-	schema := "http://"
-	httpClient := &http.Client{}
-
-	if tlsEnable {
-		schema = "https://"
-		httpClient.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				// nolint:gosec // skip TLS verification as this is a test
-				InsecureSkipVerify: true,
-			},
-		}
-	}
-	endpoint := schema + svc.Spec.ClusterIP + ":80"
-
-	return endpoint, nil
-}
-
-func NewClient(objectStore *cephv1.CephObjectStore, svc *corev1.Service, k8sh *utils.K8sHelper, installer *installer.CephInstaller, tlsEnable bool) (*sns.Client, error) {
-	ctx := context.TODO()
-
-	accessKey, secretKey, err := GetS3Credentials(objectStore, installer)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get s3 credentials")
-	}
-
-	cfg, err := config.LoadDefaultConfig(
-		ctx,
-		config.WithRegion("us-east-1"),
-		config.WithCredentialsProvider(
-			credentials.NewStaticCredentialsProvider(
-				accessKey,
-				secretKey,
-				"",
-			),
-		),
-	)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to load default aws config")
-	}
-
-	endpoint, err := GetS3Endpoint(objectStore, k8sh, tlsEnable)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get sns endpoint")
-	}
-
-	snsClient := sns.NewFromConfig(cfg, func(o *sns.Options) {
-		o.EndpointResolverV2 = &snsResolverV2{endpoint: endpoint}
-	})
-
-	// sanity check that client is working
-	_, err = snsClient.ListTopics(ctx, &sns.ListTopicsInput{})
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to sns client sanity check")
-	}
-
-	return snsClient, nil
-}
diff --git a/tests/manifests/test-cluster-on-pvc-encrypted.yaml b/tests/manifests/test-cluster-on-pvc-encrypted.yaml
index 48f7e6458..941d9f2fa 100644
--- a/tests/manifests/test-cluster-on-pvc-encrypted.yaml
+++ b/tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -14,7 +14,7 @@ spec:
           requests:
             storage: 5Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.2
+    image: quay.io/ceph/ceph:v19.2.1
   dashboard:
     enabled: false
   network:
diff --git a/tests/scripts/build-release.sh b/tests/scripts/build-release.sh
index a193c287e..cabbfcf80 100755
--- a/tests/scripts/build-release.sh
+++ b/tests/scripts/build-release.sh
@@ -13,17 +13,18 @@ function  build() {
     $MAKE mod.check
 }
 
-function publish_images_and_docs() {
+function publish() {
     build
     $MAKE -C build/release build BRANCH_NAME=${BRANCH_NAME} TAG_WITH_SUFFIX=${TAG_WITH_SUFFIX} GIT_API_TOKEN=${GIT_API_TOKEN}
     git status &
     git diff &
-    $MAKE -C build/release publish BRANCH_NAME=${BRANCH_NAME} TAG_WITH_SUFFIX=${TAG_WITH_SUFFIX} AWS_ACCESS_KEY_ID=${AWS_USR} AWS_SECRET_ACCESS_KEY=${AWS_PSW} GIT_API_TOKEN=${GIT_API_TOKEN} TAGGED_RELEASE=${TAGGED_RELEASE}
+    $MAKE -C build/release publish BRANCH_NAME=${BRANCH_NAME} TAG_WITH_SUFFIX=${TAG_WITH_SUFFIX} AWS_ACCESS_KEY_ID=${AWS_USR} AWS_SECRET_ACCESS_KEY=${AWS_PSW} GIT_API_TOKEN=${GIT_API_TOKEN}
 }
 
-function publish_charts() {
-    echo "Publishing helm charts for release build"
-    $MAKE -C build/release promote BRANCH_NAME=${BRANCH_NAME} TAG_WITH_SUFFIX=${TAG_WITH_SUFFIX} AWS_ACCESS_KEY_ID=${AWS_USR} AWS_SECRET_ACCESS_KEY=${AWS_PSW}
+function promote() {
+    # automatically promote the master builds
+    echo "Promoting from branch ${BRANCH_NAME}"
+    $MAKE -C build/release promote BRANCH_NAME=${BRANCH_NAME} TAG_WITH_SUFFIX=${TAG_WITH_SUFFIX} CHANNEL=${CHANNEL} AWS_ACCESS_KEY_ID=${AWS_USR} AWS_SECRET_ACCESS_KEY=${AWS_PSW}
 }
 
 #############
@@ -45,12 +46,14 @@ if [ -n "${GIT_API_TOKEN}" ]; then
     export DOCS_GIT_REPO="https://${GIT_API_TOKEN}@${DOCS_GIT_REPO}"
 fi
 
-TAGGED_RELEASE=false
+SHOULD_PROMOTE=true
 if [[ ${GITHUB_REF} =~ master ]]; then
     echo "Publishing from master"
+    CHANNEL=master
 else
     echo "Tagging with suffix for release and tagged builds"
     TAG_WITH_SUFFIX=true
+    CHANNEL=release
 
     # If a tag, find the source release branch
     if [[ $BRANCH_NAME = v* ]]; then
@@ -61,15 +64,15 @@ else
             exit 1
         fi
         echo "Publishing tag ${TAG_NAME} in branch ${BRANCH_NAME}"
-        TAGGED_RELEASE=true
     else
         echo "Publishing from release branch ${BRANCH_NAME}"
+        SHOULD_PROMOTE=false
     fi
 fi
 
 
-publish_images_and_docs
+publish
 
-if [[ "$TAGGED_RELEASE" = true ]]; then
-  publish_charts
+if [[ "$SHOULD_PROMOTE" = true ]]; then
+  promote
 fi
diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index 22d3c091e..a9f2f4759 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -181,7 +181,7 @@ function build_rook() {
   fi
   GOPATH=$(go env GOPATH) make clean
   for _ in $(seq 1 3); do
-    if ! o=$(make -j"$(nproc)" "$build_type"); then
+    if ! o=$(make -j"$(nproc)" IMAGES='ceph' "$build_type"); then
       case "$o" in
       *"$NETWORK_ERROR"*)
         echo "network failure occurred, retrying..."
@@ -562,7 +562,7 @@ function get_secret_key() {
 }
 
 function s3cmd() {
-  command timeout 200 s3cmd -v --config=s3cfg --access_key="${S3CMD_ACCESS_KEY}" --secret_key="${S3CMD_SECRET_KEY}" "$@"
+  command timeout 150 s3cmd -v --config=s3cfg --access_key="${S3CMD_ACCESS_KEY}" --secret_key="${S3CMD_SECRET_KEY}" "$@"
 }
 
 function write_object_read_from_replica_cluster() {