Troubleshooting Guide Red Hat Ceph Storage 8 Troubleshooting Red Hat Ceph Storage Red Hat Ceph Storage Documentation Team

Commands:
cephadm shell
ceph health detail
ceph -W cephadm
ceph config set global log_to_file true ceph config set global mon_cluster_log_to_file true
cephadm shell
ceph health detail HEALTH_WARN 1 osds down; 1 OSDs or CRUSH {nodes, device-classes} have {NOUP,NODOWN,NOIN,NOOUT} flags set [WRN] OSD_DOWN: 1 osds down osd.1 (root=default,host=host01) is down [WRN] OSD_FLAGS: 1 OSDs or CRUSH {nodes, device-classes} have {NOUP,NODOWN,NOIN,NOOUT} flags set osd.1 has flags noup
ceph health mute HEALTH_MESSAGE
ceph health mute OSD_DOWN
ceph health mute HEALTH_MESSAGE DURATION
ceph health mute OSD_DOWN 10m
ceph -s cluster: id: 81a4597a-b711-11eb-8cb8-001a4a000740 health: HEALTH_OK (muted: OSD_DOWN(9m) OSD_FLAGS(9m)) services: mon: 3 daemons, quorum host01,host02,host03 (age 33h) mgr: host01.pzhfuh(active, since 33h), standbys: host02.wsnngf, host03.xwzphg osd: 11 osds: 10 up (since 4m), 11 in (since 5d) data: pools: 1 pools, 1 pgs objects: 13 objects, 0 B usage: 85 MiB used, 165 GiB / 165 GiB avail pgs: 1 active+clean
ceph health mute HEALTH_MESSAGE DURATION --sticky
ceph health mute OSD_DOWN 1h --sticky
ceph health unmute HEALTH_MESSAGE
ceph health unmute OSD_DOWN
dnf install sos
sos report -a --all-logs
sos report --all-logs -e ceph_mgr,ceph_common,ceph_mon,ceph_osd,ceph_ansible,ceph_mds,ceph_rgw
debug_ms = 5 debug_mon = 20 debug_paxos = 20 debug_auth = 20
2022-05-12 12:37:04.278761 7f45a9afc700 10 mon.cephn2@0(leader).osd e322 e322: 2 osds: 2 up, 2 in 2022-05-12 12:37:04.278792 7f45a9afc700 10 mon.cephn2@0(leader).osd e322 min_last_epoch_clean 322 2022-05-12 12:37:04.278795 7f45a9afc700 10 mon.cephn2@0(leader).log v1010106 log 2022-05-12 12:37:04.278799 7f45a9afc700 10 mon.cephn2@0(leader).auth v2877 auth 2022-05-12 12:37:04.278811 7f45a9afc700 20 mon.cephn2@0(leader) e1 sync_trim_providers 2022-05-12 12:37:09.278914 7f45a9afc700 11 mon.cephn2@0(leader) e1 tick 2022-05-12 12:37:09.278949 7f45a9afc700 10 mon.cephn2@0(leader).pg v8126 v8126: 64 pgs: 64 active+clean; 60168 kB data, 172 MB used, 20285 MB / 20457 MB avail 2022-05-12 12:37:09.278975 7f45a9afc700 10 mon.cephn2@0(leader).paxosservice(pgmap 7511..8126) maybe_trim trim_to 7626 would only trim 115 < paxos_service_trim_min 250 2022-05-12 12:37:09.278982 7f45a9afc700 10 mon.cephn2@0(leader).osd e322 e322: 2 osds: 2 up, 2 in 2022-05-12 12:37:09.278989 7f45a9afc700 5 mon.cephn2@0(leader).paxos(paxos active c 1028850..1029466) is_readable = 1 - now=2021-08-12 12:37:09.278990 lease_expire=0.000000 has v0 lc 1029466 . 2022-05-12 12:59:18.769963 7f45a92fb700 1 -- 192.168.0.112:6789/0 <== osd.1 192.168.0.114:6800/2801 5724 ==== pg_stats(0 pgs tid 3045 v 0) v1 ==== 124+0+0 (2380105412 0 0) 0x5d96300 con 0x4d5bf40 2022-05-12 12:59:18.770053 7f45a92fb700 1 -- 192.168.0.112:6789/0 --> 192.168.0.114:6800/2801 -- pg_stats_ack(0 pgs tid 3045) v1 -- ?+0 0x550ae00 con 0x4d5bf40 2022-05-12 12:59:32.916397 7f45a9afc700 0 mon.cephn2@0(leader).data_health(1) update_stats avail 53% total 1951 MB, used 780 MB, avail 1053 MB . 2022-05-12 13:01:05.256263 7f45a92fb700 1 -- 192.168.0.112:6789/0 --> 192.168.0.113:6800/2410 -- mon_subscribe_ack(300s) v1 -- ?+0 0x4f283c0 con 0x4d5b440
debug_ms = 5 debug_osd = 20
2022-05-12 11:27:53.869151 7f5d55d84700 1 -- 192.168.17.3:0/2410 --> 192.168.17.4:6801/2801 -- osd_ping(ping e322 stamp 2021-08-12 11:27:53.869147) v2 -- ?+0 0x63baa00 con 0x578dee0 2022-05-12 11:27:53.869214 7f5d55d84700 1 -- 192.168.17.3:0/2410 --> 192.168.0.114:6801/2801 -- osd_ping(ping e322 stamp 2021-08-12 11:27:53.869147) v2 -- ?+0 0x638f200 con 0x578e040 2022-05-12 11:27:53.870215 7f5d6359f700 1 -- 192.168.17.3:0/2410 <== osd.1 192.168.0.114:6801/2801 109210 ==== osd_ping(ping_reply e322 stamp 2021-08-12 11:27:53.869147) v2 ==== 47+0+0 (261193640 0 0) 0x63c1a00 con 0x578e040 2022-05-12 11:27:53.870698 7f5d6359f700 1 -- 192.168.17.3:0/2410 <== osd.1 192.168.17.4:6801/2801 109210 ==== osd_ping(ping_reply e322 stamp 2021-08-12 11:27:53.869147) v2 ==== 47+0+0 (261193640 0 0) 0x6313200 con 0x578dee0 . 2022-05-12 11:28:10.432313 7f5d6e71f700 5 osd.0 322 tick 2022-05-12 11:28:10.432375 7f5d6e71f700 20 osd.0 322 scrub_random_backoff lost coin flip, randomly backing off 2022-05-12 11:28:10.432381 7f5d6e71f700 10 osd.0 322 do_waiters -- start 2022-05-12 11:28:10.432383 7f5d6e71f700 10 osd.0 322 do_waiters -- finish
ceph tell TYPE . ID injectargs --debug- SUBSYSTEM VALUE [-- NAME VALUE ]
ceph tell osd.0 injectargs --debug-osd 0/5
ceph daemon NAME config show | less
ceph daemon osd.0 config show | less
[global] debug_ms = 1/5 [mon] debug_mon = 20 debug_paxos = 1/5 debug_auth = 2 [osd] debug_osd = 1/5 debug_monc = 5/20 [mds] debug_mds = 1
rotate 7 weekly size SIZE compress sharedscripts
rotate 7 weekly size 500 MB compress sharedscripts size 500M
crontab -e
30 * * * * /usr/sbin/logrotate /etc/logrotate.d/ceph-d3bb5396-c404-11ee-9e65-002590fc2a2e >/dev/null 2>&1
logrotate -f
logrotate -f /etc/logrotate.d/ceph-12ab345c-1a2b-11ed-b736-fa163e4f6220
ll LOG_LOCATION
ll /var/log/ceph/12ab345c-1a2b-11ed-b736-fa163e4f6220 -rw-r--r--. 1 ceph ceph 412 Sep 28 09:26 opslog.log.1.gz
/usr/local/bin/s3cmd ls
/usr/local/bin/s3cmd mb s3:// NEW_BUCKET_NAME
/usr/local/bin/s3cmd mb s3://bucket1 Bucket `s3://bucket1` created
ll LOG_LOCATION
ll /var/log/ceph/12ab345c-1a2b-11ed-b736-fa163e4f6220 total 852 -rw-r--r--. 1 ceph ceph 920 Jun 29 02:17 opslog.log -rw-r--r--. 1 ceph ceph 412 Jun 28 09:26 opslog.log.1.gz
tail -f LOG_LOCATION /opslog.log
tail -f /var/log/ceph/12ab345c-1a2b-11ed-b736-fa163e4f6220/opslog.log {"bucket":"","time":"2022-09-29T06:17:03.133488Z","time_local":"2022-09- 29T06:17:03.133488+0000","remote_addr":"10.0.211.66","user":"test1", "operation":"list_buckets","uri":"GET / HTTP/1.1","http_status":"200","error_code":"","bytes_sent":232, "bytes_received":0,"object_size":0,"total_time":9,"user_agent":"","referrer": "","trans_id":"tx00000c80881a9acd2952a-006335385f-175e5-primary", "authentication_type":"Local","access_key_id":"1234","temp_url":false} {"bucket":"cn1","time":"2022-09-29T06:17:10.521156Z","time_local":"2022-09- 29T06:17:10.521156+0000","remote_addr":"10.0.211.66","user":"test1", "operation":"create_bucket","uri":"PUT /cn1/ HTTP/1.1","http_status":"200","error_code":"","bytes_sent":0, "bytes_received":0,"object_size":0,"total_time":106,"user_agent":"", "referrer":"","trans_id":"tx0000058d60c593632c017-0063353866-175e5-primary", "authentication_type":"Local","access_key_id":"1234","temp_url":false}
dnf install net-tools dnf install telnet
cat /etc/ceph/ceph.conf minimal ceph.conf for 57bddb48-ee04-11eb-9962-001a4a000672 [global] fsid = 57bddb48-ee04-11eb-9962-001a4a000672 mon_host = [v2:10.74.249.26:3300/0,v1:10.74.249.26:6789/0] [v2:10.74.249.163:3300/0,v1:10.74.249.163:6789/0] [v2:10.74.254.129:3300/0,v1:10.74.254.129:6789/0] [mon.host01] public network = 10.74.248.0/21
ip link list 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: ens3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000 link/ether 00:1a:4a:00:06:72 brd ff:ff:ff:ff:ff:ff
ping SHORT_HOST_NAME
ping host02
firewall-cmd --info-zone= ZONE telnet IP_ADDRESS PORT
firewall-cmd --info-zone=public public (active) target: default icmp-block-inversion: no interfaces: ens3 sources: services: ceph ceph-mon cockpit dhcpv6-client ssh ports: 9283/tcp 8443/tcp 9093/tcp 9094/tcp 3000/tcp 9100/tcp 9095/tcp protocols: masquerade: no forward-ports: source-ports: icmp-blocks: rich rules: telnet 192.168.0.22 9100
ethtool -S INTERFACE
ethtool -S ens3 | grep errors NIC statistics: rx_fcs_errors: 0 rx_align_errors: 0 rx_frame_too_long_errors: 0 rx_in_length_errors: 0 rx_out_length_errors: 0 tx_mac_errors: 0 tx_carrier_sense_errors: 0 tx_errors: 0 rx_errors: 0
ifconfig ens3: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 10.74.249.26 netmask 255.255.248.0 broadcast 10.74.255.255 inet6 fe80::21a:4aff:fe00:672 prefixlen 64 scopeid 0x20<link> inet6 2620:52:0:4af8:21a:4aff:fe00:672 prefixlen 64 scopeid 0x0<global> ether 00:1a:4a:00:06:72 txqueuelen 1000 (Ethernet) RX packets 150549316 bytes 56759897541 (52.8 GiB) RX errors 0 dropped 176924 overruns 0 frame 0 TX packets 55584046 bytes 62111365424 (57.8 GiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 inet6 ::1 prefixlen 128 scopeid 0x10<host> loop txqueuelen 1000 (Local Loopback) RX packets 9373290 bytes 16044697815 (14.9 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 9373290 bytes 16044697815 (14.9 GiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
netstat -ai Kernel Interface table Iface MTU RX-OK RX-ERR RX-DRP RX-OVR TX-OK TX-ERR TX-DRP TX-OVR Flg ens3 1500 311847720 0 364903 0 114341918 0 0 0 BMRU lo 65536 19577001 0 0 0 19577001 0 0 0 LRU
dnf install iperf3
iperf3 -s ----------------------------------------------------------- Server listening on 5201 -----------------------------------------------------------
iperf3 -c mon Connecting to host mon, port 5201 [ 4] local xx.x.xxx.xx port 52270 connected to xx.x.xxx.xx port 5201 [ ID] Interval Transfer Bandwidth Retr Cwnd [ 4] 0.00-1.00 sec 114 MBytes 954 Mbits/sec 0 409 KBytes [ 4] 1.00-2.00 sec 113 MBytes 945 Mbits/sec 0 409 KBytes [ 4] 2.00-3.00 sec 112 MBytes 943 Mbits/sec 0 454 KBytes [ 4] 3.00-4.00 sec 112 MBytes 941 Mbits/sec 0 471 KBytes [ 4] 4.00-5.00 sec 112 MBytes 940 Mbits/sec 0 471 KBytes [ 4] 5.00-6.00 sec 113 MBytes 945 Mbits/sec 0 471 KBytes [ 4] 6.00-7.00 sec 112 MBytes 937 Mbits/sec 0 488 KBytes [ 4] 7.00-8.00 sec 113 MBytes 947 Mbits/sec 0 520 KBytes [ 4] 8.00-9.00 sec 112 MBytes 939 Mbits/sec 0 520 KBytes [ 4] 9.00-10.00 sec 112 MBytes 939 Mbits/sec 0 520 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bandwidth Retr [ 4] 0.00-10.00 sec 1.10 GBytes 943 Mbits/sec 0 sender [ 4] 0.00-10.00 sec 1.10 GBytes 941 Mbits/sec receiver iperf Done.
ethtool INTERFACE
ethtool ens3 Settings for ens3: Supported ports: [ TP ] Supported link modes: 10baseT/Half 10baseT/Full 100baseT/Half 100baseT/Full 1000baseT/Half 1000baseT/Full Supported pause frame use: No Supports auto-negotiation: Yes Supported FEC modes: Not reported Advertised link modes: 10baseT/Half 10baseT/Full 100baseT/Half 100baseT/Full 1000baseT/Half 1000baseT/Full Advertised pause frame use: Symmetric Advertised auto-negotiation: Yes Advertised FEC modes: Not reported Link partner advertised link modes: 10baseT/Half 10baseT/Full 100baseT/Half 100baseT/Full 1000baseT/Full Link partner advertised pause frame use: Symmetric Link partner advertised auto-negotiation: Yes Link partner advertised FEC modes: Not reported Speed: 1000Mb/s 1 Duplex: Full 2 Port: Twisted Pair PHYAD: 1 Transceiver: internal Auto-negotiation: on MDI-X: off Supports Wake-on: g Wake-on: d Current message level: 0x000000ff (255) drv probe link timer ifdown ifup rx_err tx_err Link detected: yes 3
systemctl status chronyd
systemctl enable chronyd systemctl start chronyd
chronyc sources chronyc sourcestats chronyc tracking
HEALTH_WARN 1 mons down, quorum 1,2 mon.b,mon.c mon.a (rank 0) addr 127.0.0.1:6789/0 is down (out of quorum)
systemctl status ceph- FSID @ DAEMON_NAME systemctl start ceph- FSID @ DAEMON_NAME
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service systemctl start ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
Corruption: error in middle of record Corruption: 1 missing files; example: /var/lib/ceph/mon/mon.0/store.db/1234567.ldb
Caught signal (Bus error)
ceph daemon ID mon_status
ceph daemon mon.host01 mon_status
mon.a (rank 0) addr 127.0.0.1:6789/0 is down (out of quorum) mon.a addr 127.0.0.1:6789/0 clock skew 0.08235s > max 0.05s (latency 0.0045s)
2022-05-04 07:28:32.035795 7f806062e700 0 log [WRN] : mon.a 127.0.0.1:6789/0 clock skew 0.14s > max 0.05s 2022-05-04 04:31:25.773235 7f4997663700 0 log [WRN] : message from mon.1 was stamped 0.186257s in the future, clocks not synchronized
mon.ceph1 store is getting too big! 48031 MB >= 15360 MB -- 62% avail
du -sch /var/lib/ceph/ CLUSTER_FSID /mon. HOST_NAME /store.db/
du -sh /var/lib/ceph/b341e254-b165-11ed-a564-ac1f6bb26e8c/mon.host01/ 109M /var/lib/ceph/b341e254-b165-11ed-a564-ac1f6bb26e8c/mon.host01/ 47G /var/lib/ceph/mon/ceph-ceph1/store.db/ 47G total
{ "name": "mon.3", "rank": 2, "state": "peon", "election_epoch": 96, "quorum": [ 1, 2 ], "outside_quorum": [], "extra_probe_peers": [], "sync_provider": [], "monmap": { "epoch": 1, "fsid": "d5552d32-9d1d-436c-8db1-ab5fc2c63cd0", "modified": "0.000000", "created": "0.000000", "mons": [ { "rank": 0, "name": "mon.1", "addr": "172.25.1.10:6789\/0" }, { "rank": 1, "name": "mon.2", "addr": "172.25.1.12:6789\/0" }, { "rank": 2, "name": "mon.3", "addr": "172.25.1.13:6789\/0" } ] } }
ceph mon getmap -o /tmp/monmap
systemctl stop ceph- FSID @ DAEMON_NAME
systemctl stop ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
ceph-mon -i ID --extract-monmap /tmp/monmap
ceph-mon -i mon.a --extract-monmap /tmp/monmap
systemctl stop ceph- FSID @ DAEMON_NAME
systemctl stop ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
ceph-mon -i ID --inject-monmap /tmp/monmap
ceph-mon -i mon.host01 --inject-monmap /tmp/monmap
systemctl start ceph- FSID @ DAEMON_NAME
systemctl start ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
systemctl start ceph- FSID @ DAEMON_NAME
systemctl start ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
rm -rf /var/lib/ceph/mon/ CLUSTER_NAME - SHORT_HOST_NAME
rm -rf /var/lib/ceph/mon/remote-host1
ceph mon remove SHORT_HOST_NAME --cluster CLUSTER_NAME
ceph mon remove host01 --cluster remote
ceph tell mon. HOST_NAME compact
ceph tell mon.host01 compact
[mon] mon_compact_on_start = true
systemctl restart ceph- FSID @ DAEMON_NAME
systemctl restart ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
ceph mon stat
systemctl status ceph- FSID @ DAEMON_NAME systemctl stop ceph- FSID @ DAEMON_NAME
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service systemctl stop ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
ceph-monstore-tool /var/lib/ceph/ CLUSTER_FSID /mon. HOST_NAME compact
ceph-monstore-tool /var/lib/ceph/b404c440-9e4c-11ec-a28a-001a4a0001df/mon.host01 compact
systemctl start ceph- FSID @ DAEMON_NAME
systemctl start ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
firewall-cmd --add-port 6800-7300/tcp firewall-cmd --add-port 6800-7300/tcp --permanent
Corruption: error in middle of record Corruption: 1 missing files; e.g.: /var/lib/ceph/mon/mon.0/store.db/1234567.ldb
ceph-volume lvm list
mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-USDi
for i in { OSD_ID }; do restorecon /var/lib/ceph/osd/ceph-USDi; done
for i in { OSD_ID }; do chown -R ceph:ceph /var/lib/ceph/osd/ceph-USDi; done
ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev OSD-DATA --path /var/lib/ceph/osd/ceph- OSD-ID
ln -snf BLUESTORE DATABASE /var/lib/ceph/osd/ceph- OSD-ID /block.db
cd /root/ ms=/tmp/monstore/ db=/root/db/ db_slow=/root/db.slow/ mkdir USDms for host in USDosd_nodes; do echo "USDhost" rsync -avz USDms USDhost:USDms rsync -avz USDdb USDhost:USDdb rsync -avz USDdb_slow USDhost:USDdb_slow rm -rf USDms rm -rf USDdb rm -rf USDdb_slow sh -t USDhost <<EOF for osd in /var/lib/ceph/osd/ceph-*; do ceph-objectstore-tool --type bluestore --data-path \USDosd --op update-mon-db --mon-store-path USDms done EOF rsync -avz USDhost:USDms USDms rsync -avz USDhost:USDdb USDdb rsync -avz USDhost:USDdb_slow USDdb_slow done
ceph-authtool /etc/ceph/ceph.client.admin.keyring -n mon. --cap mon 'allow *' --gen-key cat /etc/ceph/ceph.client.admin.keyring [mon.] key = AQCleqldWqm5IhAAgZQbEzoShkZV42RiQVffnA== caps mon = "allow *" [client.admin] key = AQCmAKld8J05KxAArOWeRAw63gAwwZO5o75ZNQ== auid = 0 caps mds = "allow *" caps mgr = "allow *" caps mon = "allow *" caps osd = "allow *"
mv /root/db/*.sst /root/db.slow/*.sst /tmp/monstore/store.db
ceph-monstore-tool /tmp/monstore rebuild -- --keyring /etc/ceph/ceph.client.admin
mv /var/lib/ceph/mon/ceph- HOSTNAME /store.db /var/lib/ceph/mon/ceph- HOSTNAME /store.db.corrupted
scp -r /tmp/monstore/store.db HOSTNAME :/var/lib/ceph/mon/ceph- HOSTNAME /
chown -R ceph:ceph /var/lib/ceph/mon/ceph- HOSTNAME /store.db
umount /var/lib/ceph/osd/ceph-*
systemctl start ceph- FSID @ DAEMON_NAME
systemctl start ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@mon.host01.service
ceph -s
ceph auth import -i /etc/ceph/ceph.mgr. HOSTNAME .keyring systemctl start ceph- FSID @ DAEMON_NAME
systemctl start ceph-b341e254-b165-11ed-a564-ac1f6bb26e8c@mgr.extensa003.exrqql.service
systemctl start ceph- FSID @osd. OSD_ID
systemctl start ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
ceph -s
HEALTH_ERR 1 full osds osd.3 is full at 95%
ceph df
health: HEALTH_WARN 3 backfillfull osd(s) Low space hindering backfill (add storage if this doesn't resolve itself): 32 pgs backfill_toofull
ceph df
ceph osd set-backfillfull-ratio VALUE
ceph osd set-backfillfull-ratio 0.92
HEALTH_WARN 1 nearfull osds osd.2 is near full at 85%
ceph osd df
df
HEALTH_WARN 1/3 in osds are down
ceph health detail HEALTH_WARN 1/3 in osds are down osd.0 is down since epoch 23, last address 192.168.106.220:6800/11080
systemctl restart ceph- FSID @osd. OSD_ID
systemctl restart ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
FAILED assert(0 == "hit suicide timeout")
dmesg
xfs_log_force: error -5 returned
Caught signal (Segmentation fault)
wrongly marked me down heartbeat_check: no reply from osd.2 since back
ceph -w | grep osds 2022-05-05 06:27:20.810535 mon.0 [INF] osdmap e609: 9 osds: 8 up, 9 in 2022-05-05 06:27:24.120611 mon.0 [INF] osdmap e611: 9 osds: 7 up, 9 in 2022-05-05 06:27:25.975622 mon.0 [INF] HEALTH_WARN; 118 pgs stale; 2/9 in osds are down 2022-05-05 06:27:27.489790 mon.0 [INF] osdmap e614: 9 osds: 6 up, 9 in 2022-05-05 06:27:36.540000 mon.0 [INF] osdmap e616: 9 osds: 7 up, 9 in 2022-05-05 06:27:39.681913 mon.0 [INF] osdmap e618: 9 osds: 8 up, 9 in 2022-05-05 06:27:43.269401 mon.0 [INF] osdmap e620: 9 osds: 9 up, 9 in 2022-05-05 06:27:54.884426 mon.0 [INF] osdmap e622: 9 osds: 8 up, 9 in 2022-05-05 06:27:57.398706 mon.0 [INF] osdmap e624: 9 osds: 7 up, 9 in 2022-05-05 06:27:59.669841 mon.0 [INF] osdmap e625: 9 osds: 6 up, 9 in 2022-05-05 06:28:07.043677 mon.0 [INF] osdmap e628: 9 osds: 7 up, 9 in 2022-05-05 06:28:10.512331 mon.0 [INF] osdmap e630: 9 osds: 8 up, 9 in 2022-05-05 06:28:12.670923 mon.0 [INF] osdmap e631: 9 osds: 9 up, 9 in
2022-05-25 03:44:06.510583 osd.50 127.0.0.1:6801/149046 18992 : cluster [WRN] map e600547 wrongly marked me down
2022-05-25 19:00:08.906864 7fa2a0033700 -1 osd.254 609110 heartbeat_check: no reply from osd.2 since back 2021-07-25 19:00:07.444113 front 2021-07-25 18:59:48.311935 (cutoff 2021-07-25 18:59:48.906862)
ceph health detail HEALTH_WARN 30 requests are blocked > 32 sec; 3 osds have slow requests 30 ops are blocked > 268435 sec 1 ops are blocked > 268435 sec on osd.11 1 ops are blocked > 268435 sec on osd.18 28 ops are blocked > 268435 sec on osd.39 3 osds have slow requests
ceph osd tree | grep down
ceph osd set noup ceph osd set nodown
HEALTH_WARN 30 requests are blocked > 32 sec; 3 osds have slow requests 30 ops are blocked > 268435 sec 1 ops are blocked > 268435 sec on osd.11 1 ops are blocked > 268435 sec on osd.18 28 ops are blocked > 268435 sec on osd.39 3 osds have slow requests
2022-05-24 13:18:10.024659 osd.1 127.0.0.1:6812/3032 9 : cluster [WRN] 6 slow requests, 6 included below; oldest blocked for > 61.758455 secs
2022-05-25 03:44:06.510583 osd.50 [WRN] slow request 30.005692 seconds old, received at {date-time}: osd_op(client.4240.0:8 benchmark_data_ceph-1_39426_object7 [write 0~4194304] 0.69848840) v4 currently waiting for subops from [610]
cephadm shell
ceph osd set noout
ceph osd unset noout
HEALTH_WARN 1/3 in osds are down osd.0 is down since epoch 23, last address 192.168.106.220:6800/11080
cephadm shell
ceph osd tree | grep -i down ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF 0 hdd 0.00999 osd.0 down 1.00000 1.00000
ceph osd out OSD_ID .
ceph osd out osd.0 marked out osd.0.
ceph -w | grep backfill 2022-05-02 04:48:03.403872 mon.0 [INF] pgmap v10293282: 431 pgs: 1 active+undersized+degraded+remapped+backfilling, 28 active+undersized+degraded, 49 active+undersized+degraded+remapped+wait_backfill, 59 stale+active+clean, 294 active+clean; 72347 MB data, 101302 MB used, 1624 GB / 1722 GB avail; 227 kB/s rd, 1358 B/s wr, 12 op/s; 10626/35917 objects degraded (29.585%); 6757/35917 objects misplaced (18.813%); 63500 kB/s, 15 objects/s recovering 2022-05-02 04:48:04.414397 mon.0 [INF] pgmap v10293283: 431 pgs: 2 active+undersized+degraded+remapped+backfilling, 75 active+undersized+degraded+remapped+wait_backfill, 59 stale+active+clean, 295 active+clean; 72347 MB data, 101398 MB used, 1623 GB / 1722 GB avail; 969 kB/s rd, 6778 B/s wr, 32 op/s; 10626/35917 objects degraded (29.585%); 10580/35917 objects misplaced (29.457%); 125 MB/s, 31 objects/s recovering 2022-05-02 04:48:00.380063 osd.1 [INF] 0.6f starting backfill to osd.0 from (0'0,0'0] MAX to 2521'166639 2022-05-02 04:48:00.380139 osd.1 [INF] 0.48 starting backfill to osd.0 from (0'0,0'0] MAX to 2513'43079 2022-05-02 04:48:00.380260 osd.1 [INF] 0.d starting backfill to osd.0 from (0'0,0'0] MAX to 2513'136847 2022-05-02 04:48:00.380849 osd.1 [INF] 0.71 starting backfill to osd.0 from (0'0,0'0] MAX to 2331'28496 2022-05-02 04:48:00.381027 osd.1 [INF] 0.51 starting backfill to osd.0 from (0'0,0'0] MAX to 2513'87544
ceph orch daemon stop OSD_ID
ceph orch daemon stop osd.0
ceph orch osd rm OSD_ID --replace
ceph orch osd rm 0 --replace
ceph orch apply osd --all-available-devices
ceph orch apply osd --all-available-devices --unmanaged=true
ceph orch daemon add osd host02:/dev/sdb
ceph osd tree
sysctl -w kernel.pid.max=4194303
kernel.pid.max = 4194303
cephadm shell
ceph osd dump | grep -i full full_ratio 0.95
ceph osd set-full-ratio 0.97
ceph osd dump | grep -i full full_ratio 0.97
ceph -w
ceph osd set-full-ratio 0.95
ceph osd dump | grep -i full full_ratio 0.95
radosgw-admin user info --uid SYNCHRONIZATION_USER, and radosgw-admin zone get
radosgw-admin sync status
radosgw-admin data sync status --shard-id= X --source-zone= ZONE_NAME
radosgw-admin data sync status --shard-id=27 --source-zone=us-east { "shard_id": 27, "marker": { "status": "incremental-sync", "marker": "1_1534494893.816775_131867195.1", "next_step_marker": "", "total_entries": 1, "pos": 0, "timestamp": "0.000000" }, "pending_buckets": [], "recovering_buckets": [ "pro-registry:4ed07bb2-a80b-4c69-aa15-fdc17ae6f5f2.314303.1:26" ] }
radosgw-admin bucket sync status --bucket= X .
radosgw-admin sync error list
ceph --admin-daemon /var/run/ceph/ceph-client.rgw. RGW_ID .asok perf dump data-sync-from- ZONE_NAME
ceph --admin-daemon /var/run/ceph/ceph-client.rgw.host02-rgw0.103.94309060818504.asok perf dump data-sync-from-us-west { "data-sync-from-us-west": { "fetch bytes": { "avgcount": 54, "sum": 54526039885 }, "fetch not modified": 7, "fetch errors": 0, "poll latency": { "avgcount": 41, "sum": 2.533653367, "avgtime": 0.061796423 }, "poll errors": 0 } }
radosgw-admin sync status realm d713eec8-6ec4-4f71-9eaf-379be18e551b (india) zonegroup ccf9e0b2-df95-4e0a-8933-3b17b64c52b7 (shared) zone 04daab24-5bbd-4c17-9cf5-b1981fd7ff79 (primary) current time 2022-09-15T06:53:52Z zonegroup features enabled: resharding metadata sync no sync (zone is master) data sync source: 596319d2-4ffe-4977-ace1-8dd1790db9fb (secondary) syncing full sync: 0/128 shards incremental sync: 128/128 shards data is caught up with source
radosgw-admin data sync init --source-zone primary
ceph orch restart rgw.myrgw
2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 ERROR: failed to decode obj from .rgw.root:periods.91d2a42c-735b-492a-bcf3-05235ce888aa.3 2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 failed reading current period info: (5) Input/output error 2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 ERROR: failed to start notify service ((5) Input/output error 2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 ERROR: failed to init services (ret=(5) Input/output error) couldn't init storage provider
date;radosgw-admin bucket list Mon May 13 09:05:30 UTC 2024 2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 ERROR: failed to decode obj from .rgw.root:periods.91d2a42c-735b-492a-bcf3-05235ce888aa.3 2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 failed reading current period info: (5) Input/output error 2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 ERROR: failed to start notify service ((5) Input/output error 2024-05-13T09:05:30.607+0000 7f4e7c4ea500 0 ERROR: failed to init services (ret=(5) Input/output error) couldn't init storage provider
cephadm shell --radosgw-admin COMMAND
cephadm shell -- radosgw-admin bucket list
HEALTH_WARN 24 pgs stale; 3/300 in osds are down
ceph health detail HEALTH_WARN 24 pgs stale; 3/300 in osds are down pg 2.5 is stuck stale+active+remapped, last acting [2,0] osd.10 is down since epoch 23, last address 192.168.106.220:6800/11080 osd.11 is down since epoch 13, last address 192.168.106.220:6803/11539 osd.12 is down since epoch 24, last address 192.168.106.220:6806/11861
HEALTH_ERR 1 pgs inconsistent; 2 scrub errors pg 0.6 is active+clean+inconsistent, acting [0,1,2] 2 scrub errors
cephadm shell
ceph health detail HEALTH_ERR 1 pgs inconsistent; 2 scrub errors pg 0.6 is active+clean+inconsistent, acting [0,1,2] 2 scrub errors
ceph pg deep-scrub ID
ceph pg deep-scrub 0.6 instructing pg 0.6 on osd.0 to deep-scrub
ceph -w | grep ID
ceph -w | grep 0.6 2022-05-26 01:35:36.778215 osd.106 [ERR] 0.6 deep-scrub stat mismatch, got 636/635 objects, 0/0 clones, 0/0 dirty, 0/0 omap, 0/0 hit_set_archive, 0/0 whiteouts, 1855455/1854371 bytes. 2022-05-26 01:35:36.788334 osd.106 [ERR] 0.6 deep-scrub 1 errors
PG . ID shard OSD : soid OBJECT missing attr , missing attr _ATTRIBUTE_TYPE PG . ID shard OSD : soid OBJECT digest 0 != known digest DIGEST , size 0 != known size SIZE PG . ID shard OSD : soid OBJECT size 0 != known size SIZE PG . ID deep-scrub stat mismatch, got MISMATCH PG . ID shard OSD : soid OBJECT candidate had a read error, digest 0 != known digest DIGEST
PG . ID shard OSD : soid OBJECT digest DIGEST != known digest DIGEST PG . ID shard OSD : soid OBJECT omap_digest DIGEST != known omap_digest DIGEST
HEALTH_WARN 197 pgs stuck unclean
ceph osd tree
HEALTH_WARN 197 pgs stuck inactive
ceph osd tree
HEALTH_ERR 7 pgs degraded; 12 pgs down; 12 pgs peering; 1 pgs recovering; 6 pgs stuck unclean; 114/3300 degraded (3.455%); 1/3 in osds are down pg 0.5 is down+peering pg 1.4 is down+peering osd.1 is down since epoch 69, last address 192.168.106.220:6801/8651
ceph pg ID query
ceph pg 0.5 query { "state": "down+peering", "recovery_state": [ { "name": "Started\/Primary\/Peering\/GetInfo", "enter_time": "2021-08-06 14:40:16.169679", "requested_info_from": []}, { "name": "Started\/Primary\/Peering", "enter_time": "2021-08-06 14:40:16.169659", "probing_osds": [ 0, 1], "blocked": "peering is blocked due to down osds", "down_osds_we_would_probe": [ 1], "peering_blocked_by": [ { "osd": 1, "current_lost_at": 0, "comment": "starting or marking this osd lost may let us proceed"}]}, { "name": "Started", "enter_time": "2021-08-06 14:40:16.169513"} ] }
HEALTH_WARN 1 pgs degraded; 78/3778 unfound (2.065%)
cephadm shell
ceph health detail HEALTH_WARN 1 pgs recovering; 1 pgs stuck unclean; recovery 5/937611 objects degraded (0.001%); 1/312537 unfound (0.000%) pg 3.8a5 is stuck unclean for 803946.712780, current state active+recovering, last acting [320,248,0] pg 3.8a5 is active+recovering, acting [320,248,0], 1 unfound recovery 5/937611 objects degraded (0.001%); **1/312537 unfound (0.000%)**
ceph pg ID query
ceph pg 3.8a5 query { "state": "active+recovering", "epoch": 10741, "up": [ 320, 248, 0], "acting": [ 320, 248, 0], <snip> "recovery_state": [ { "name": "Started\/Primary\/Active", "enter_time": "2021-08-28 19:30:12.058136", "might_have_unfound": [ { "osd": "0", "status": "already probed"}, { "osd": "248", "status": "already probed"}, { "osd": "301", "status": "already probed"}, { "osd": "362", "status": "already probed"}, { "osd": "395", "status": "already probed"}, { "osd": "429", "status": "osd is down"}], "recovery_progress": { "backfill_targets": [], "waiting_on_backfill": [], "last_backfill_started": "0\/\/0\/\/-1", "backfill_info": { "begin": "0\/\/0\/\/-1", "end": "0\/\/0\/\/-1", "objects": []}, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "pull_from_peer": [], "pushing": []}}, "scrub": { "scrubber.epoch_start": "0", "scrubber.active": 0, "scrubber.block_writes": 0, "scrubber.finalizing": 0, "scrubber.waiting_on": 0, "scrubber.waiting_on_whom": []}}, { "name": "Started", "enter_time": "2021-08-28 19:30:11.044020"}],
cephadm shell
ceph pg dump_stuck inactive ceph pg dump_stuck unclean ceph pg dump_stuck stale
rados list-inconsistent-pg POOL --format=json-pretty
rados list-inconsistent-pg data --format=json-pretty [0.6]
rados list-inconsistent-obj PLACEMENT_GROUP_ID
rados list-inconsistent-obj 0.6 { "epoch": 14, "inconsistents": [ { "object": { "name": "image1", "nspace": "", "locator": "", "snap": "head", "version": 1 }, "errors": [ "data_digest_mismatch", "size_mismatch" ], "union_shard_errors": [ "data_digest_mismatch_oi", "size_mismatch_oi" ], "selected_object_info": "0:602f83fe:::foo:head(16'1 client.4110.0:1 dirty|data_digest|omap_digest s 968 uv 1 dd e978e67f od ffffffff alloc_hint [0 0 0])", "shards": [ { "osd": 0, "errors": [], "size": 968, "omap_digest": "0xffffffff", "data_digest": "0xe978e67f" }, { "osd": 1, "errors": [], "size": 968, "omap_digest": "0xffffffff", "data_digest": "0xe978e67f" }, { "osd": 2, "errors": [ "data_digest_mismatch_oi", "size_mismatch_oi" ], "size": 0, "omap_digest": "0xffffffff", "data_digest": "0xffffffff" } ] } ] }
rados list-inconsistent-snapset PLACEMENT_GROUP_ID
rados list-inconsistent-snapset 0.23 --format=json-pretty { "epoch": 64, "inconsistents": [ { "name": "obj5", "nspace": "", "locator": "", "snap": "0x00000001", "headless": true }, { "name": "obj5", "nspace": "", "locator": "", "snap": "0x00000002", "headless": true }, { "name": "obj5", "nspace": "", "locator": "", "snap": "head", "ss_attr_missing": true, "extra_clones": true, "extra clones": [ 2, 1 ] } ]
HEALTH_ERR 1 pgs inconsistent; 2 scrub errors pg 0.6 is active+clean+inconsistent, acting [0,1,2] 2 scrub errors
_PG_._ID_ shard _OSD_: soid _OBJECT_ digest _DIGEST_ != known digest _DIGEST_ _PG_._ID_ shard _OSD_: soid _OBJECT_ omap_digest _DIGEST_ != known omap_digest _DIGEST_
ceph pg repair ID
ceph tell osd.* injectargs '--osd_max_backfills 1 --osd_recovery_max_active 1 --osd_recovery_op_priority 1'
ceph osd set noscrub ceph osd set nodeep-scrub
ceph osd pool set POOL pg_num VALUE
ceph osd pool set data pg_num 4
ceph -s
ceph osd pool set POOL pgp_num VALUE
ceph osd pool set data pgp_num 4
ceph -s
ceph tell osd.* injectargs '--osd_max_backfills 1 --osd_recovery_max_active 3 --osd_recovery_op_priority 3'
ceph osd unset noscrub ceph osd unset nodeep-scrub
systemctl status ceph- FSID @osd. OSD_ID
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --op list
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --op list
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID --op list
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c --op list
ceph-objectstore-tool --data-path PATH_TO_OSD --op list OBJECT_ID
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --op list default.region
systemctl status ceph- FSID @osd. OSD_ID
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --op fix-lost --dry-run
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --op fix-lost --dry-run
ceph-objectstore-tool --data-path PATH_TO_OSD --op fix-lost
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --op fix-lost
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID --op fix-lost
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c --op fix-lost
ceph-objectstore-tool --data-path PATH_TO_OSD --op fix-lost OBJECT_ID
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --op fix-lost default.region
systemctl status ceph- FSID @osd. OSD_ID
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT get-bytes > OBJECT_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-bytes > zone_info.default.backup ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-bytes > zone_info.default.working-copy
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT set-bytes < OBJECT_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-bytes < zone_info.default.working-copy
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT remove
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' remove
systemctl status ceph-osd@ OSD_ID
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT list-omap
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' list-omap
systemctl status ceph- FSID @osd. OSD_ID
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT get-omaphdr > OBJECT_MAP_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-omaphdr > zone_info.default.omaphdr.txt
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT get-omaphdr < OBJECT_MAP_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-omaphdr < zone_info.default.omaphdr.txt
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT get-omap KEY > OBJECT_MAP_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-omap "" > zone_info.default.omap.txt
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT set-omap KEY < OBJECT_MAP_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-omap "" < zone_info.default.omap.txt
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT rm-omap KEY
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-omap ""
systemctl status ceph- FSID @osd. OSD_ID
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT list-attrs
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' list-attrs
systemctl status ceph- FSID @osd. OSD_ID
systemctl status ceph-b404c440-9e4c-11ec-a28a-001a4a0001df@osd.0.service
cephadm shell --name osd. OSD_ID
cephadm shell --name osd.0
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT get-attr KEY > OBJECT_ATTRS_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-attr "oid" > zone_info.default.attr.txt
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT set-attr KEY < OBJECT_ATTRS_FILE_NAME
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-attr "oid"<zone_info.default.attr.txt
ceph-objectstore-tool --data-path PATH_TO_OSD --pgid PG_ID OBJECT rm-attr KEY
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-attr "oid"
ceph orch apply mon --unmanaged Scheduled mon update...
ceph -s mon: 5 daemons, quorum host01, host02, host04, host05 (age 30s), out of quorum: host07
ceph mon set_new_tiebreaker NEW_HOST
ceph mon set_new_tiebreaker host02
ceph mon set_new_tiebreaker host02 Error EINVAL: mon.host02 has location DC1, which matches mons host02 on the datacenter dividing bucket for stretch mode.
ceph mon set_location HOST datacenter= DATACENTER
ceph mon set_location host02 datacenter=DC3
ceph orch daemon rm FAILED_TIEBREAKER_MONITOR --force
ceph orch daemon rm mon.host07 --force Removed mon.host07 from host 'host07'
ceph mon add HOST IP_ADDRESS datacenter= DATACENTER ceph orch daemon add mon HOST
ceph mon add host07 213.222.226.50 datacenter=DC1 ceph orch daemon add mon host07
ceph -s mon: 5 daemons, quorum host01, host02, host04, host05, host07 (age 15s)
ceph mon dump epoch 19 fsid 1234ab78-1234-11ed-b1b1-de456ef0a89d last_changed 2023-01-17T04:12:05.709475+0000 created 2023-01-16T05:47:25.631684+0000 min_mon_release 16 (pacific) election_strategy: 3 stretch_mode_enabled 1 tiebreaker_mon host02 disallowed_leaders host02 0: [v2:132.224.169.63:3300/0,v1:132.224.169.63:6789/0] mon.host02; crush_location {datacenter=DC3} 1: [v2:220.141.179.34:3300/0,v1:220.141.179.34:6789/0] mon.host04; crush_location {datacenter=DC2} 2: [v2:40.90.220.224:3300/0,v1:40.90.220.224:6789/0] mon.host01; crush_location {datacenter=DC1} 3: [v2:60.140.141.144:3300/0,v1:60.140.141.144:6789/0] mon.host07; crush_location {datacenter=DC1} 4: [v2:186.184.61.92:3300/0,v1:186.184.61.92:6789/0] mon.host03; crush_location {datacenter=DC2} dumped monmap epoch 19
ceph orch apply mon --placement=" HOST_1 , HOST_2 , HOST_3 , HOST_4 , HOST_5 "
ceph orch apply mon --placement="host01, host02, host04, host05, host07" Scheduled mon update
ceph mon add NEW_HOST IP_ADDRESS datacenter= DATACENTER
ceph mon add host06 213.222.226.50 datacenter=DC3 adding mon.host06 at [v2:213.222.226.50:3300/0,v1:213.222.226.50:6789/0]
ceph orch apply mon --unmanaged Scheduled mon update...
ceph orch daemon add mon NEW_HOST
ceph orch daemon add mon host06
ceph -s mon: 6 daemons, quorum host01, host02, host04, host05, host06 (age 30s), out of quorum: host07
ceph mon set_new_tiebreaker NEW_HOST
ceph mon set_new_tiebreaker host06
ceph orch daemon rm FAILED_TIEBREAKER_MONITOR --force
ceph orch daemon rm mon.host07 --force Removed mon.host07 from host 'host07'
ceph mon dump epoch 19 fsid 1234ab78-1234-11ed-b1b1-de456ef0a89d last_changed 2023-01-17T04:12:05.709475+0000 created 2023-01-16T05:47:25.631684+0000 min_mon_release 16 (pacific) election_strategy: 3 stretch_mode_enabled 1 tiebreaker_mon host06 disallowed_leaders host06 0: [v2:213.222.226.50:3300/0,v1:213.222.226.50:6789/0] mon.host06; crush_location {datacenter=DC3} 1: [v2:220.141.179.34:3300/0,v1:220.141.179.34:6789/0] mon.host04; crush_location {datacenter=DC2} 2: [v2:40.90.220.224:3300/0,v1:40.90.220.224:6789/0] mon.host01; crush_location {datacenter=DC1} 3: [v2:60.140.141.144:3300/0,v1:60.140.141.144:6789/0] mon.host02; crush_location {datacenter=DC1} 4: [v2:186.184.61.92:3300/0,v1:186.184.61.92:6789/0] mon.host05; crush_location {datacenter=DC2} dumped monmap epoch 19
ceph orch apply mon --placement=" HOST_1 , HOST_2 , HOST_3 , HOST_4 , HOST_5 "
ceph orch apply mon --placement="host01, host02, host04, host05, host06" Scheduled mon update...
ceph osd force_recovery_stretch_mode --yes-i-really-mean-it
ceph osd force_healthy_stretch_mode --yes-i-really-mean-it
ceph config rm osd.X osd_mclock_max_capacity_iops_[hdd|ssd]
ceph config rm osd.X osd_mclock_max_capacity_iops_[hdd|ssd]
ceph config set osd.X osd_mclock_force_run_benchmark_on_init true
ceph config rm osd.X osd_mclock_max_capacity_iops_[hdd|ssd]
ceph config set osd.X osd_mclock_force_run_benchmark_on_init true
subscription-manager repos --enable=rhceph-6-tools-for-rhel-9-x86_64-rpms yum --enable=rhceph-6-tools-for-rhel-9-x86_64-debug-rpms
ceph-base-debuginfo ceph-common-debuginfo ceph-debugsource ceph-fuse-debuginfo ceph-immutable-object-cache-debuginfo ceph-mds-debuginfo ceph-mgr-debuginfo ceph-mon-debuginfo ceph-osd-debuginfo ceph-radosgw-debuginfo cephfs-mirror-debuginfo
dnf install gdb
echo "| /usr/lib/systemd/systemd-coredump %P %u %g %s %t %c %h %e" > /proc/sys/kernel/core_pattern
ls -ltr /var/lib/systemd/coredump total 8232 -rw-r-----. 1 root root 8427548 Jan 22 19:24 core.ceph-osd.167.5ede29340b6c4fe4845147f847514c12.15622.1584573794000000.xz
ps exec -it MONITOR_ID_OR_OSD_ID bash
podman ps podman exec -it ceph-1ca9f6a8-d036-11ec-8263-fa163ee967ad-osd-2 bash
dnf install procps-ng gdb
ps -aef | grep PROCESS | grep -v run
ps -aef | grep ceph-mon | grep -v run ceph 15390 15266 0 18:54 ? 00:00:29 /usr/bin/ceph-mon --cluster ceph --setroot ceph --setgroup ceph -d -i 5 ceph 18110 17985 1 19:40 ? 00:00:08 /usr/bin/ceph-mon --cluster ceph --setroot ceph --setgroup ceph -d -i 2
gcore ID
gcore 18110 warning: target file /proc/18110/cmdline contained unexpected null characters Saved corefile core.18110
ls -ltr total 709772 -rw-r--r--. 1 root root 726799544 Mar 18 19:46 core.18110
cp ceph-mon- MONITOR_ID :/tmp/mon.core. MONITOR_PID /tmp
cephadm shell
ceph config set mgr mgr/cephadm/allow_ptrace true
ceph orch redeploy SERVICE_ID
ceph orch redeploy mgr ceph orch redeploy rgw.rgw.1
exit ssh root@10.0.0.11
podman ps podman exec -it ceph-1ca9f6a8-d036-11ec-8263-fa163ee967ad-rgw-rgw-1-host04 bash
dnf install procps-ng gdb
ps aux | grep rados ceph 6 0.3 2.8 5334140 109052 ? Sl May10 5:25 /usr/bin/radosgw -n client.rgw.rgw.1.host04 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug
gcore PID
gcore 6
ls -ltr total 108798 -rw-r--r--. 1 root root 726799544 Mar 18 19:46 core.6
cp ceph-mon- DAEMON_ID :/tmp/mon.core. PID /tmp