Administration Guide Red Hat Ceph Storage 7 Administration of Red Hat Ceph Storage Red Hat Ceph Storage Documentation Team

Commands:
systemctl --type=service ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service
systemctl start SERVICE_ID
systemctl start ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service
systemctl stop SERVICE_ID
systemctl stop ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service
systemctl restart SERVICE_ID
systemctl restart ceph-499829b4-832f-11eb-8d6d-001a4a000635@mon.host01.service
systemctl list-units "ceph*"
cephadm shell
ceph orch ls NAME RUNNING REFRESHED AGE PLACEMENT IMAGE NAME IMAGE ID alertmanager 1/1 4m ago 4M count:1 registry.redhat.io/openshift4/ose-prometheus-alertmanager:v4.5 b7bae610cd46 crash 3/3 4m ago 4M * registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 grafana 1/1 4m ago 4M count:1 registry.redhat.io/rhceph-alpha/rhceph-6-dashboard-rhel9:latest bd3d7748747b mgr 2/2 4m ago 4M count:2 registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 mon 2/2 4m ago 10w count:2 registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 nfs.foo 0/1 - - count:1 <unknown> <unknown> node-exporter 1/3 4m ago 4M * registry.redhat.io/openshift4/ose-prometheus-node-exporter:v4.5 mix osd.all-available-devices 5/5 4m ago 3M * registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510 prometheus 1/1 4m ago 4M count:1 registry.redhat.io/openshift4/ose-prometheus:v4.6 bebb0ddef7f0 rgw.test_realm.test_zone 2/2 4m ago 3M count:2 registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest c88a5d60f510
ceph orch start SERVICE_ID
ceph orch start node-exporter
ceph orch stop SERVICE_ID
ceph orch stop node-exporter
ceph orch restart SERVICE_ID
ceph orch restart node-exporter
journalctl -u ceph SERVICE_ID
journalctl -u ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.8.service
journalctl -fu SERVICE_ID
journalctl -fu ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.8.service
cephadm shell
ceph -s
ceph fs set FS_NAME max_mds 1 ceph fs fail FS_NAME ceph status ceph fs set FS_NAME joinable false
ceph fs set cephfs max_mds 1 ceph fs fail cephfs ceph status ceph fs set cephfs joinable false
ceph osd set noout ceph osd set norecover ceph osd set norebalance ceph osd set nobackfill ceph osd set nodown ceph osd set pause
systemctl list-units --type target | grep ceph ceph-0b007564-ec48-11ee-b736-525400fd02f8.target loaded active active Ceph cluster 0b007564-ec48-11ee-b736-525400fd02f8 ceph.target loaded active active All Ceph clusters and services
systemctl disable ceph-0b007564-ec48-11ee-b736-525400fd02f8.target Removed "/etc/systemd/system/multi-user.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target". Removed "/etc/systemd/system/ceph.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target".
systemctl stop ceph-0b007564-ec48-11ee-b736-525400fd02f8.target
shutdown Shutdown scheduled for Wed 2024-03-27 11:47:19 EDT, use 'shutdown -c' to cancel.
systemctl enable ceph-0b007564-ec48-11ee-b736-525400fd02f8.target Created symlink /etc/systemd/system/multi-user.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target -> /etc/systemd/system/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target. Created symlink /etc/systemd/system/ceph.target.wants/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target -> /etc/systemd/system/ceph-0b007564-ec48-11ee-b736-525400fd02f8.target.
systemctl start ceph-0b007564-ec48-11ee-b736-525400fd02f8.target
ceph osd unset noout ceph osd unset norecover ceph osd unset norebalance ceph osd unset nobackfill ceph osd unset nodown ceph osd unset pause
ceph fs set FS_NAME joinable true
ceph fs set cephfs joinable true
ceph -s
cephadm shell
ceph -s
ceph fs set FS_NAME max_mds 1 ceph fs fail FS_NAME ceph status ceph fs set FS_NAME joinable false ceph mds fail FS_NAME : N
ceph fs set cephfs max_mds 1 ceph fs fail cephfs ceph status ceph fs set cephfs joinable false ceph mds fail cephfs:1
ceph osd set noout ceph osd set norecover ceph osd set norebalance ceph osd set nobackfill ceph osd set nodown ceph osd set pause
ceph orch ls --service-type mds
ceph orch stop SERVICE-NAME
ceph orch ls --service-type rgw
ceph orch stop SERVICE-NAME
ceph orch stop alertmanager
ceph orch stop node-exporter
ceph orch stop prometheus
ceph orch stop grafana
ceph orch stop crash
ceph orch ps --daemon-type=osd
ceph orch daemon stop osd.1 Scheduled to stop osd.1 on host 'host02'
ceph orch ps --daemon-type mon
systemctl list-units ceph-* | grep mon
systemct stop SERVICE-NAME
cephadm shell
ceph orch ls
ceph -s
ceph osd unset noout ceph osd unset norecover ceph osd unset norebalance ceph osd unset nobackfill ceph osd unset nodown ceph osd unset pause
ceph fs set FS_NAME joinable true
ceph fs set cephfs joinable true
ceph -s
root@host01 ~]# cephadm shell
ceph health HEALTH_OK
ceph status
root@host01 ~]# cephadm shell
ceph -w cluster: id: 8c9b0072-67ca-11eb-af06-001a4a0002a0 health: HEALTH_OK services: mon: 2 daemons, quorum Ceph5-2,Ceph5-adm (age 3d) mgr: Ceph5-1.nqikfh(active, since 3w), standbys: Ceph5-adm.meckej osd: 5 osds: 5 up (since 2d), 5 in (since 8w) rgw: 2 daemons active (test_realm.test_zone.Ceph5-2.bfdwcn, test_realm.test_zone.Ceph5-adm.acndrh) data: pools: 11 pools, 273 pgs objects: 459 objects, 32 KiB usage: 2.6 GiB used, 72 GiB / 75 GiB avail pgs: 273 active+clean io: client: 170 B/s rd, 730 KiB/s wr, 0 op/s rd, 729 op/s wr 2021-06-02 15:45:21.655871 osd.0 [INF] 17.71 deep-scrub ok 2021-06-02 15:45:47.880608 osd.1 [INF] 1.0 scrub ok 2021-06-02 15:45:48.865375 osd.1 [INF] 1.3 scrub ok 2021-06-02 15:45:50.866479 osd.1 [INF] 1.4 scrub ok 2021-06-02 15:45:01.345821 mon.0 [INF] pgmap v41339: 952 pgs: 952 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:45:05.718640 mon.0 [INF] pgmap v41340: 952 pgs: 1 active+clean+scrubbing+deep, 951 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:45:53.997726 osd.1 [INF] 1.5 scrub ok 2021-06-02 15:45:06.734270 mon.0 [INF] pgmap v41341: 952 pgs: 1 active+clean+scrubbing+deep, 951 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:45:15.722456 mon.0 [INF] pgmap v41342: 952 pgs: 952 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail 2021-06-02 15:46:06.836430 osd.0 [INF] 17.75 deep-scrub ok 2021-06-02 15:45:55.720929 mon.0 [INF] pgmap v41343: 952 pgs: 1 active+clean+scrubbing+deep, 951 active+clean; 17130 MB data, 115 GB used, 167 GB / 297 GB avail
ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 5 TiB 2.9 TiB 2.1 TiB 2.1 TiB 42.98 TOTAL 5 TiB 2.9 TiB 2.1 TiB 2.1 TiB 42.98 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL .mgr 1 1 5.3 MiB 3 16 MiB 0 629 GiB .rgw.root 2 32 1.3 KiB 4 48 KiB 0 629 GiB default.rgw.log 3 32 3.6 KiB 209 408 KiB 0 629 GiB default.rgw.control 4 32 0 B 8 0 B 0 629 GiB default.rgw.meta 5 32 1.7 KiB 10 96 KiB 0 629 GiB default.rgw.buckets.index 7 32 5.5 MiB 22 17 MiB 0 629 GiB default.rgw.buckets.data 8 32 807 KiB 3 2.4 MiB 0 629 GiB default.rgw.buckets.non-ec 9 32 1.0 MiB 1 3.1 MiB 0 629 GiB source-ecpool-86 11 32 1.2 TiB 391.13k 2.1 TiB 53.49 1.1 TiB
ceph osd df ID CLASS WEIGHT REWEIGHT SIZE USE DATA OMAP META AVAIL %USE VAR PGS 3 hdd 0.90959 1.00000 931GiB 70.1GiB 69.1GiB 0B 1GiB 861GiB 7.53 2.93 66 4 hdd 0.90959 1.00000 931GiB 1.30GiB 308MiB 0B 1GiB 930GiB 0.14 0.05 59 0 hdd 0.90959 1.00000 931GiB 18.1GiB 17.1GiB 0B 1GiB 913GiB 1.94 0.76 57 MIN/MAX VAR: 0.02/2.98 STDDEV: 2.91
cephadm shell
ceph status
ceph -s
ceph ceph> status cluster: id: 499829b4-832f-11eb-8d6d-001a4a000635 health: HEALTH_WARN 1 stray daemon(s) not managed by cephadm 1/3 mons down, quorum host03,host02 too many PGs per OSD (261 > max 250) services: mon: 3 daemons, quorum host03,host02 (age 3d), out of quorum: host01 mgr: host01.hdhzwn(active, since 9d), standbys: host05.eobuuv, host06.wquwpj osd: 12 osds: 11 up (since 2w), 11 in (since 5w) rgw: 2 daemons active (test_realm.test_zone.host04.hgbvnq, test_realm.test_zone.host05.yqqilm) rgw-nfs: 1 daemon active (nfs.foo.host06-rgw) data: pools: 8 pools, 960 pgs objects: 414 objects, 1.0 MiB usage: 5.7 GiB used, 214 GiB / 220 GiB avail pgs: 960 active+clean io: client: 41 KiB/s rd, 0 B/s wr, 41 op/s rd, 27 op/s wr ceph> health HEALTH_WARN 1 stray daemon(s) not managed by cephadm; 1/3 mons down, quorum host03,host02; too many PGs per OSD (261 > max 250) ceph> mon stat e3: 3 mons at {host01=[v2:10.74.255.0:3300/0,v1:10.74.255.0:6789/0],host02=[v2:10.74.249.253:3300/0,v1:10.74.249.253:6789/0],host03=[v2:10.74.251.164:3300/0,v1:10.74.251.164:6789/0]}, election epoch 6688, leader 1 host03, quorum 1,2 host03,host02
cephadm shell
ceph mon stat
ceph mon dump
ceph quorum_status -f json-pretty
{ "election_epoch": 6686, "quorum": [ 0, 1, 2 ], "quorum_names": [ "host01", "host03", "host02" ], "quorum_leader_name": "host01", "quorum_age": 424884, "features": { "quorum_con": "4540138297136906239", "quorum_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ] }, "monmap": { "epoch": 3, "fsid": "499829b4-832f-11eb-8d6d-001a4a000635", "modified": "2021-03-15T04:51:38.621737Z", "created": "2021-03-12T12:35:16.911339Z", "min_mon_release": 16, "min_mon_release_name": "pacific", "election_strategy": 1, "disallowed_leaders: ": "", "stretch_mode": false, "features": { "persistent": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "optional": [] }, "mons": [ { "rank": 0, "name": "host01", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.255.0:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.255.0:6789", "nonce": 0 } ] }, "addr": "10.74.255.0:6789/0", "public_addr": "10.74.255.0:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 1, "name": "host03", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.251.164:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.251.164:6789", "nonce": 0 } ] }, "addr": "10.74.251.164:6789/0", "public_addr": "10.74.251.164:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 2, "name": "host02", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.253:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.253:6789", "nonce": 0 } ] }, "addr": "10.74.249.253:6789/0", "public_addr": "10.74.249.253:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" } ] } }
Error 111: Connection Refused
cephadm shell
ceph daemon MONITOR_ID COMMAND
ceph daemon mon.host01 help { "add_bootstrap_peer_hint": "add peer address as potential bootstrap peer for cluster bringup", "add_bootstrap_peer_hintv": "add peer address vector as potential bootstrap peer for cluster bringup", "compact": "cause compaction of monitor's leveldb/rocksdb storage", "config diff": "dump diff of current config and default config", "config diff get": "dump diff get <field>: dump diff of current and default config setting <field>", "config get": "config get <field>: get the config value", "config help": "get config setting schema and descriptions", "config set": "config set <field> <val> [<val> ...]: set a config variable", "config show": "dump current config settings", "config unset": "config unset <field>: unset a config variable", "connection scores dump": "show the scores used in connectivity-based elections", "connection scores reset": "reset the scores used in connectivity-based elections", "counter dump": "dump all labeled and non-labeled counters and their values", "counter schema": "dump all labeled and non-labeled counters schemas", "dump_historic_ops": "show recent ops", "dump_historic_slow_ops": "show recent slow ops", "dump_mempools": "get mempool stats", "get_command_descriptions": "list available commands", "git_version": "get git sha1", "heap": "show heap usage info (available only if compiled with tcmalloc)", "help": "list available commands", "injectargs": "inject configuration arguments into running daemon", "log dump": "dump recent log entries to log file", "log flush": "flush log entries to log file", "log reopen": "reopen log file", "mon_status": "report status of monitors", "ops": "show the ops currently in flight", "perf dump": "dump non-labeled counters and their values", "perf histogram dump": "dump perf histogram values", "perf histogram schema": "dump perf histogram schema", "perf reset": "perf reset <name>: perf reset all or one perfcounter name", "perf schema": "dump non-labeled counters schemas", "quorum enter": "force monitor back into quorum", "quorum exit": "force monitor out of the quorum", "sessions": "list existing sessions", "smart": "Query health metrics for underlying device", "sync_force": "force sync of and clear monitor store", "version": "get ceph version" }
ceph daemon mon.host01 mon_status { "name": "host01", "rank": 0, "state": "leader", "election_epoch": 120, "quorum": [ 0, 1, 2 ], "quorum_age": 206358, "features": { "required_con": "2449958747317026820", "required_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "quorum_con": "4540138297136906239", "quorum_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ] }, "outside_quorum": [], "extra_probe_peers": [], "sync_provider": [], "monmap": { "epoch": 3, "fsid": "81a4597a-b711-11eb-8cb8-001a4a000740", "modified": "2021-05-18T05:50:17.782128Z", "created": "2021-05-17T13:13:13.383313Z", "min_mon_release": 16, "min_mon_release_name": "pacific", "election_strategy": 1, "disallowed_leaders: ": "", "stretch_mode": false, "features": { "persistent": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "optional": [] }, "mons": [ { "rank": 0, "name": "host01", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.41:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.41:6789", "nonce": 0 } ] }, "addr": "10.74.249.41:6789/0", "public_addr": "10.74.249.41:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 1, "name": "host02", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.55:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.55:6789", "nonce": 0 } ] }, "addr": "10.74.249.55:6789/0", "public_addr": "10.74.249.55:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 2, "name": "host03", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "10.74.249.49:3300", "nonce": 0 }, { "type": "v1", "addr": "10.74.249.49:6789", "nonce": 0 } ] }, "addr": "10.74.249.49:6789/0", "public_addr": "10.74.249.49:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" } ] }, "feature_map": { "mon": [ { "features": "0x3f01cfb9fffdffff", "release": "luminous", "num": 1 } ], "osd": [ { "features": "0x3f01cfb9fffdffff", "release": "luminous", "num": 3 } ] }, "stretch_mode": false }
ceph daemon /var/run/ceph/ SOCKET_FILE COMMAND
ceph daemon /var/run/ceph/ceph-osd.0.asok status { "cluster_fsid": "9029b252-1668-11ee-9399-001a4a000429", "osd_fsid": "1de9b064-b7a5-4c54-9395-02ccda637d21", "whoami": 0, "state": "active", "oldest_map": 1, "newest_map": 58, "num_pgs": 33 }
ls /var/run/ceph
ceph osd stat
ceph osd dump
eNNNN: x osds: y up, z in
ceph osd tree id weight type name up/down reweight -1 3 pool default -3 3 rack mainrack -2 3 host osd-host 0 1 osd.0 up 1 1 1 osd.1 up 1 2 1 osd.2 up 1
systemctl start CEPH_OSD_SERVICE_ID
systemctl start ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.6.service
cephadm shell
ceph pg dump
ceph pg map PG_NUM
ceph pg map 128
ceph pg stat
vNNNNNN: x pgs: y active+clean; z bytes data, aa MB used, bb GB / cc GB avail
244 active+clean+snaptrim_wait 32 active+clean+snaptrim
POOL_NUM . PG_ID
0.1f
ceph pg dump
ceph pg dump -o FILE_NAME --format=json
ceph pg dump -o test --format=json
ceph pg POOL_NUM . PG_ID query
ceph pg 5.fe query { "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+clean", "epoch": 2449, "up": [ 3, 8, 10 ], "acting": [ 3, 8, 10 ], "acting_recovery_backfill": [ "3", "8", "10" ], "info": { "pgid": "5.ff", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 114, "epoch_pool_created": 82, "last_epoch_started": 2402, "last_interval_started": 2401, "last_epoch_clean": 2402, "last_interval_clean": 2401, "last_epoch_split": 114, "last_epoch_marked_full": 0, "same_up_since": 2401, "same_interval_since": 2401, "same_primary_since": 2086, "last_scrub": "0'0", "last_scrub_stamp": "2021-06-17T01:32:03.763988+0000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2021-06-17T01:32:03.763988+0000", "last_clean_scrub_stamp": "2021-06-17T01:32:03.763988+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "2989", "reported_epoch": "2449", "state": "active+clean", "last_fresh": "2021-06-18T05:16:59.401080+0000", "last_change": "2021-06-17T01:32:03.764162+0000", "last_active": "2021-06-18T05:16:59.401080+0000", .
pg 1.5: up=acting: [0,1,2] ADD_OSD_3 pg 1.5: up: [0,3,1] acting: [0,1,2]
pg 1.5: up=acting: [0,3,1]
ceph pg dump_stuck {inactive|unclean|stale|undersized|degraded [inactive|unclean|stale|undersized|degraded...]} {<int>}
ceph pg dump_stuck stale OK
ceph osd map POOL_NAME OBJECT_NAME
ceph osd map mypool myobject
ceph mon enable_stretch_mode host05 stretch_rule datacenter Error EINVAL: the 2 datacenter instances in the cluster have differing weights 25947 and 15728 but stretch mode currently requires they be the same!
service_type: host addr: host01 hostname: host01 location: root: default datacenter: DC1 labels: - osd - mon - mgr --- service_type: host addr: host02 hostname: host02 location: datacenter: DC1 labels: - osd - mon --- service_type: host addr: host03 hostname: host03 location: datacenter: DC1 labels: - osd - mds - rgw --- service_type: host addr: host04 hostname: host04 location: root: default datacenter: DC2 labels: - osd - mon - mgr --- service_type: host addr: host05 hostname: host05 location: datacenter: DC2 labels: - osd - mon --- service_type: host addr: host06 hostname: host06 location: datacenter: DC2 labels: - osd - mds - rgw --- service_type: host addr: host07 hostname: host07 labels: - mon --- service_type: mon placement: label: "mon" --- service_id: cephfs placement: label: "mds" --- service_type: mgr service_name: mgr placement: label: "mgr" --- service_type: osd service_id: all-available-devices service_name: osd.all-available-devices placement: label: "osd" spec: data_devices: all: true --- service_type: rgw service_id: objectgw service_name: rgw.objectgw placement: count: 2 label: "rgw" spec: rgw_frontend_port: 8080
cephadm bootstrap --apply-spec CONFIGURATION_FILE_NAME --mon-ip MONITOR_IP_ADDRESS --ssh-private-key PRIVATE_KEY --ssh-public-key PUBLIC_KEY --registry-url REGISTRY_URL --registry-username USER_NAME --registry-password PASSWORD
cephadm bootstrap --apply-spec initial-config.yaml --mon-ip 10.10.128.68 --ssh-private-key /home/ceph/.ssh/id_rsa --ssh-public-key /home/ceph/.ssh/id_rsa.pub --registry-url registry.redhat.io --registry-username myuser1 --registry-password mypassword1
ceph osd crush add-bucket BUCKET_NAME BUCKET_TYPE
ceph osd crush add-bucket DC1 datacenter ceph osd crush add-bucket DC2 datacenter
ceph osd crush move BUCKET_NAME root=default
ceph osd crush move DC1 root=default ceph osd crush move DC2 root=default
ceph osd crush move HOST datacenter= DATACENTER
ceph osd crush move host01 datacenter=DC1
ceph mon set_location HOST datacenter= DATACENTER
ceph mon set_location host01 datacenter=DC1 ceph mon set_location host02 datacenter=DC1 ceph mon set_location host04 datacenter=DC2 ceph mon set_location host05 datacenter=DC2 ceph mon set_location host07 datacenter=DC3
ceph osd getcrushmap > COMPILED_CRUSHMAP_FILENAME crushtool -d COMPILED_CRUSHMAP_FILENAME -o DECOMPILED_CRUSHMAP_FILENAME
ceph osd getcrushmap > crush.map.bin crushtool -d crush.map.bin -o crush.map.txt
rule stretch_rule { id 1 1 type replicated min_size 1 max_size 10 step take DC1 2 step chooseleaf firstn 2 type host step emit step take DC2 3 step chooseleaf firstn 2 type host step emit }
rule stretch_rule { id 1 type replicated min_size 1 max_size 10 step take default step choose firstn 0 type datacenter step chooseleaf firstn 2 type host step emit }
crushtool -c DECOMPILED_CRUSHMAP_FILENAME -o COMPILED_CRUSHMAP_FILENAME ceph osd setcrushmap -i COMPILED_CRUSHMAP_FILENAME
crushtool -c crush.map.txt -o crush2.map.bin ceph osd setcrushmap -i crush2.map.bin
ceph mon set election_strategy connectivity
ceph mon set_location HOST datacenter= DATACENTER ceph mon enable_stretch_mode HOST stretch_rule datacenter
ceph mon set_location host07 datacenter=DC3 ceph mon enable_stretch_mode host07 stretch_rule datacenter
Error EINVAL: there are 3 datacenters in the cluster but stretch mode currently only works with 2!
ceph osd dump epoch 361 fsid 1234ab78-1234-11ed-b1b1-de456ef0a89d created 2023-01-16T05:47:28.482717+0000 modified 2023-01-17T17:36:50.066183+0000 flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit crush_version 31 full_ratio 0.95 backfillfull_ratio 0.92 nearfull_ratio 0.85 require_min_compat_client luminous min_compat_client luminous require_osd_release quincy stretch_mode_enabled true stretch_bucket_count 2 degraded_stretch_mode 0 recovering_stretch_mode 0 stretch_mode_bucket 8
ceph mon dump epoch 19 fsid 1234ab78-1234-11ed-b1b1-de456ef0a89d last_changed 2023-01-17T04:12:05.709475+0000 created 2023-01-16T05:47:25.631684+0000 min_mon_release 16 (pacific) election_strategy: 3 stretch_mode_enabled 1 tiebreaker_mon host07 disallowed_leaders host07 0: [v2:132.224.169.63:3300/0,v1:132.224.169.63:6789/0] mon.host07; crush_location {datacenter=DC3} 1: [v2:220.141.179.34:3300/0,v1:220.141.179.34:6789/0] mon.host04; crush_location {datacenter=DC2} 2: [v2:40.90.220.224:3300/0,v1:40.90.220.224:6789/0] mon.host01; crush_location {datacenter=DC1} 3: [v2:60.140.141.144:3300/0,v1:60.140.141.144:6789/0] mon.host02; crush_location {datacenter=DC1} 4: [v2:186.184.61.92:3300/0,v1:186.184.61.92:6789/0] mon.host05; crush_location {datacenter=DC2} dumped monmap epoch 19
ceph orch device ls [--hostname= HOST_1 HOST_2 ] [--wide] [--refresh]
ceph orch device ls
ceph orch daemon add osd HOST : DEVICE_PATH
ceph orch daemon add osd host03:/dev/sdb
ceph orch apply osd --all-available-devices
ceph osd crush move HOST datacenter= DATACENTER
ceph osd crush move host03 datacenter=DC1 ceph osd crush move host06 datacenter=DC2
ceph osd set FLAG
ceph osd set noout
ceph osd unset FLAG
ceph osd unset noout
DAEMON_TYPE 'allow CAPABILITY ' [ DAEMON_TYPE 'allow CAPABILITY ']
mon 'allow rwx` mon 'allow profile osd'
osd 'allow CAPABILITY ' [pool= POOL_NAME ] [namespace= NAMESPACE_NAME ]
ceph auth list installed auth entries: osd.10 key: AQBW7U5gqOsEExAAg/CxSwZ/gSh8iOsDV3iQOA== caps: [mgr] allow profile osd caps: [mon] allow profile osd caps: [osd] allow * osd.11 key: AQBX7U5gtj/JIhAAPsLBNG+SfC2eMVEFkl3vfA== caps: [mgr] allow profile osd caps: [mon] allow profile osd caps: [osd] allow * osd.9 key: AQBV7U5g1XDULhAAKo2tw6ZhH1jki5aVui2v7g== caps: [mgr] allow profile osd caps: [mon] allow profile osd caps: [osd] allow * client.admin key: AQADYEtgFfD3ExAAwH+C1qO7MSLE4TWRfD2g6g== caps: [mds] allow * caps: [mgr] allow * caps: [mon] allow * caps: [osd] allow * client.bootstrap-mds key: AQAHYEtgpbkANBAANqoFlvzEXFwD8oB0w3TF4Q== caps: [mon] allow profile bootstrap-mds client.bootstrap-mgr key: AQAHYEtg3dcANBAAVQf6brq3sxTSrCrPe0pKVQ== caps: [mon] allow profile bootstrap-mgr client.bootstrap-osd key: AQAHYEtgD/QANBAATS9DuP3DbxEl86MTyKEmdw== caps: [mon] allow profile bootstrap-osd client.bootstrap-rbd key: AQAHYEtgjxEBNBAANho25V9tWNNvIKnHknW59A== caps: [mon] allow profile bootstrap-rbd client.bootstrap-rbd-mirror key: AQAHYEtgdE8BNBAAr6rLYxZci0b2hoIgH9GXYw== caps: [mon] allow profile bootstrap-rbd-mirror client.bootstrap-rgw key: AQAHYEtgwGkBNBAAuRzI4WSrnowBhZxr2XtTFg== caps: [mon] allow profile bootstrap-rgw client.crash.host04 key: AQCQYEtgz8lGGhAAy5bJS8VH9fMdxuAZ3CqX5Q== caps: [mgr] profile crash caps: [mon] profile crash client.crash.host02 key: AQDuYUtgqgfdOhAAsyX+Mo35M+HFpURGad7nJA== caps: [mgr] profile crash caps: [mon] profile crash client.crash.host03 key: AQB98E5g5jHZAxAAklWSvmDsh2JaL5G7FvMrrA== caps: [mgr] profile crash caps: [mon] profile crash client.nfs.foo.host03 key: AQCgTk9gm+HvMxAAHbjG+XpdwL6prM/uMcdPdQ== caps: [mon] allow r caps: [osd] allow rw pool=nfs-ganesha namespace=foo client.nfs.foo.host03-rgw key: AQCgTk9g8sJQNhAAPykcoYUuPc7IjubaFx09HQ== caps: [mon] allow r caps: [osd] allow rwx tag rgw *=* client.rgw.test_realm.test_zone.host01.hgbvnq key: AQD5RE9gAQKdCRAAJzxDwD/dJObbInp9J95sXw== caps: [mgr] allow rw caps: [mon] allow * caps: [osd] allow rwx tag rgw *=* client.rgw.test_realm.test_zone.host02.yqqilm key: AQD0RE9gkxA4ExAAFXp3pLJWdIhsyTe2ZR6Ilw== caps: [mgr] allow rw caps: [mon] allow * caps: [osd] allow rwx tag rgw *=* mgr.host01.hdhzwn key: AQAEYEtg3lhIBxAAmHodoIpdvnxK0llWF80ltQ== caps: [mds] allow * caps: [mon] profile mgr caps: [osd] allow * mgr.host02.eobuuv key: AQAn6U5gzUuiABAA2Fed+jPM1xwb4XDYtrQxaQ== caps: [mds] allow * caps: [mon] profile mgr caps: [osd] allow * mgr.host03.wquwpj key: AQAd6U5gIzWsLBAAbOKUKZlUcAVe9kBLfajMKw== caps: [mds] allow * caps: [mon] profile mgr caps: [osd] allow *
ceph auth export TYPE . ID
ceph auth export mgr.host02.eobuuv
ceph auth export TYPE . ID -o FILE_NAME
ceph auth export osd.9 -o filename export auth(key=AQBV7U5g1XDULhAAKo2tw6ZhH1jki5aVui2v7g==)
ceph auth add client.john mon 'allow r' osd 'allow rw pool=mypool' ceph auth get-or-create client.paul mon 'allow r' osd 'allow rw pool=mypool' ceph auth get-or-create client.george mon 'allow r' osd 'allow rw pool=mypool' -o george.keyring ceph auth get-or-create-key client.ringo mon 'allow r' osd 'allow rw pool=mypool' -o ringo.key
ceph auth caps USERTYPE . USERID DAEMON 'allow [r|w|x|*|...] [pool= POOL_NAME ] [namespace= NAMESPACE_NAME ]'
ceph auth caps client.john mon 'allow r' osd 'allow rw pool=mypool' ceph auth caps client.paul mon 'allow rw' osd 'allow rwx pool=mypool' ceph auth caps client.brian-manager mon 'allow *' osd 'allow *'
ceph auth caps client.ringo mon ' ' osd ' '
ceph auth del TYPE . ID
ceph auth del osd.6
ceph auth print-key TYPE . ID
ceph auth print-key osd.6 AQBQ7U5gAry3JRAA3NoPrqBBThpFMcRL6Sr+5w==[ceph: root@host01 /]#
ceph-volume lvm
ceph auth get client. ID -o ceph.client. ID .keyring
ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
ceph-volume lvm prepare --bluestore --data VOLUME_GROUP / LOGICAL_VOLUME
ceph-volume lvm prepare --bluestore --data example_vg/data_lv
ceph-volume lvm prepare --bluestore --block.db BLOCK_DB_DEVICE --block.wal BLOCK_WAL_DEVICE --data DATA_DEVICE
ceph-volume lvm prepare --bluestore --block.db /dev/sda --block.wal /dev/sdb --data /dev/sdc
ceph-volume lvm prepare --bluestore --dmcrypt --data VOLUME_GROUP / LOGICAL_VOLUME
ceph-volume lvm prepare --bluestore --dmcrypt --data example_vg/data_lv
1 devices have fault light turned on
ceph-volume lvm list ====== osd.6 ======= [block] /dev/ceph-83909f70-95e9-4273-880e-5851612cbe53/osd-block-7ce687d9-07e7-4f8f-a34e-d1b0efb89920 block device /dev/ceph-83909f70-95e9-4273-880e-5851612cbe53/osd-block-7ce687d9-07e7-4f8f-a34e-d1b0efb89920 block uuid 4d7gzX-Nzxp-UUG0-bNxQ-Jacr-l0mP-IPD8cX cephx lockbox secret cluster fsid 1ca9f6a8-d036-11ec-8263-fa163ee967ad cluster name ceph crush device class None encrypted 0 osd fsid 7ce687d9-07e7-4f8f-a34e-d1b0efb89920 osd id 6 osdspec affinity all-available-devices type block vdo 0 devices /dev/vdc
ceph device ls-lights { "fault": [ "SEAGATE_ST12000NM002G_ZL2KTGCK0000C149" ], "ident": [] }
ceph device light off DEVICE_NAME FAULT/INDENT --force
ceph device light off SEAGATE_ST12000NM002G_ZL2KTGCK0000C149 fault --force
ceph-volume lvm list
ceph-volume lvm activate --bluestore OSD_ID OSD_FSID
ceph-volume lvm activate --bluestore 10 7ce687d9-07e7-4f8f-a34e-d1b0efb89920
ceph-volume lvm activate --all
ceph-volume lvm trigger SYSTEMD_DATA
ceph-volume lvm trigger 10 7ce687d9-07e7-4f8f-a34e-d1b0efb89920
ceph-volume lvm list
ceph-volume lvm deactivate OSD_ID
ceph-volume lvm deactivate 16
ceph-volume lvm create --bluestore --data VOLUME_GROUP / LOGICAL_VOLUME
ceph-volume lvm create --bluestore --data example_vg/data_lv
cephadm shell
ceph orch daemon stop osd.1
cephadm shell --mount /var/lib/ceph/72436d46-ca06-11ec-9809-ac1f6b5635ee/osd.1:/var/lib/ceph/osd/ceph-1
ceph-volume lvm new-db --osd-id OSD_ID --osd-fsid OSD_FSID --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME
ceph-volume lvm new-db --osd-id 1 --osd-fsid 7ce687d9-07e7-4f8f-a34e-d1b0efb89921 --target vgname/new_db ceph-volume lvm new-wal --osd-id 1 --osd-fsid 7ce687d9-07e7-4f8f-a34e-d1b0efb89921 --target vgname/new_wal
ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME
ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data --target vgname/db
ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME
ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data --target vgname/new_db
ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from db --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME
ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from db --target vgname/new_db
ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data db --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME
ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data db --target vgname/new_db
ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from data db wal --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME
ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from data db --target vgname/new_db
ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_UUID --from db wal --target VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME
ceph-volume lvm migrate --osd-id 1 --osd-fsid 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 --from db wal --target vgname/data
ceph-volume lvm list ====== osd.3 ======= [db] /dev/db-test/db1 block device /dev/test/lv1 block uuid N5zoix-FePe-uExe-UngY-D9YG-BMs0-1tTDyB cephx lockbox secret cluster fsid 1a6112da-ed05-11ee-bacd-525400565cda cluster name ceph crush device class db device /dev/db-test/db1 db uuid 1TUaDY-3mEt-fReP-cyB2-JyZ1-oUPa-hKPfo6 encrypted 0 osd fsid 94ff742c-7bfd-4fb5-8dc4-843d10ac6731 osd id 3 osdspec affinity None type db vdo 0 devices /dev/vdh [block] /dev/test/lv1 block device /dev/test/lv1 block uuid N5zoix-FePe-uExe-UngY-D9YG-BMs0-1tTDyB cephx lockbox secret cluster fsid 1a6112da-ed05-11ee-bacd-525400565cda cluster name ceph crush device class db device /dev/db-test/db1 db uuid 1TUaDY-3mEt-fReP-cyB2-JyZ1-oUPa-hKPfo6 encrypted 0 osd fsid 94ff742c-7bfd-4fb5-8dc4-843d10ac6731 osd id 3 osdspec affinity None type block vdo 0 devices /dev/vdg
vgs VG #PV #LV #SN Attr VSize VFree db-test 1 1 0 wz--n- <200.00g <160.00g test 1 1 0 wz--n- <200.00g <170.00g
systemctl stop host01a6112da-ed05-11ee-bacd-525400565cda@osd.3.service
lvresize -l 100%FREE /dev/db-test/db1 Size of logical volume db-test/db1 changed from 40.00 GiB (10240 extents) to <160.00 GiB (40959 extents). Logical volume db-test/db1 successfully resized.
cephadm shell -m /var/lib/ceph/ CLUSTER_FSID /osd. OSD_ID :/var/lib/ceph/osd/ceph- OSD_ID :z
cephadm shell -m /var/lib/ceph/1a6112da-ed05-11ee-bacd-525400565cda/osd.3:/var/lib/ceph/osd/ceph-3:z
ceph-bluestore-tool show-label --path OSD_DIRECTORY_PATH
ceph-bluestore-tool show-label --path /var/lib/ceph/osd/ceph-3/ inferring bluefs devices from bluestore path { "/var/lib/ceph/osd/ceph-3/block": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 32212254720, "btime": "2024-04-03T08:34:12.742848+0000", "description": "main", "bfm_blocks": "7864320", "bfm_blocks_per_key": "128", "bfm_bytes_per_block": "4096", "bfm_size": "32212254720", "bluefs": "1", "ceph_fsid": "1a6112da-ed05-11ee-bacd-525400565cda", "ceph_version_when_created": "ceph version 19.0.0-2493-gd82c9aa1 (d82c9aa17f09785fe698d262f9601d87bb79f962) squid (dev)", "created_at": "2024-04-03T08:34:15.637253Z", "elastic_shared_blobs": "1", "kv_backend": "rocksdb", "magic": "ceph osd volume v026", "mkfs_done": "yes", "osd_key": "AQCEFA1m9xuwABAAwKEHkASVbgB1GVt5jYC2Sg==", "osdspec_affinity": "None", "ready": "ready", "require_osd_release": "19", "whoami": "3" }, "/var/lib/ceph/osd/ceph-3/block.db": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 40794497536, "btime": "2024-04-03T08:34:12.748816+0000", "description": "bluefs db" } }
ceph-bluestore-tool bluefs-bdev-expand --path OSD_DIRECTORY_PATH
ceph-bluestore-tool bluefs-bdev-expand --path /var/lib/ceph/osd/ceph-3/ inferring bluefs devices from bluestore path 1 : device size 0x27ffbfe000 : using 0x2300000(35 MiB) 2 : device size 0x780000000 : using 0x52000(328 KiB) Expanding DB/WAL 1 : expanding to 0x171794497536 1 : size label updated to 171794497536
ceph-bluestore-tool show-label --path OSD_DIRECTORY_PATH
ceph-bluestore-tool show-label --path /var/lib/ceph/osd/ceph-3/ inferring bluefs devices from bluestore path { "/var/lib/ceph/osd/ceph-3/block": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 32212254720, "btime": "2024-04-03T08:34:12.742848+0000", "description": "main", "bfm_blocks": "7864320", "bfm_blocks_per_key": "128", "bfm_bytes_per_block": "4096", "bfm_size": "32212254720", "bluefs": "1", "ceph_fsid": "1a6112da-ed05-11ee-bacd-525400565cda", "ceph_version_when_created": "ceph version 19.0.0-2493-gd82c9aa1 (d82c9aa17f09785fe698d262f9601d87bb79f962) squid (dev)", "created_at": "2024-04-03T08:34:15.637253Z", "elastic_shared_blobs": "1", "kv_backend": "rocksdb", "magic": "ceph osd volume v026", "mkfs_done": "yes", "osd_key": "AQCEFA1m9xuwABAAwKEHkASVbgB1GVt5jYC2Sg==", "osdspec_affinity": "None", "ready": "ready", "require_osd_release": "19", "whoami": "3" }, "/var/lib/ceph/osd/ceph-3/block.db": { "osd_uuid": "94ff742c-7bfd-4fb5-8dc4-843d10ac6731", "size": 171794497536, "btime": "2024-04-03T08:34:12.748816+0000", "description": "bluefs db" } }
systemctl start host01a6112da-ed05-11ee-bacd-525400565cda@osd.3.service osd.3 host01 running (15s) 0s ago 13m 46.9M 4096M 19.0.0-2493-gd82c9aa1 3714003597ec 02150b3b6877
ceph-volume lvm batch --bluestore PATH_TO_DEVICE [ PATH_TO_DEVICE ]
ceph-volume lvm batch --bluestore /dev/sda /dev/sdb /dev/nvme0n1
ceph-volume lvm zap VOLUME_GROUP_NAME / LOGICAL_VOLUME_NAME [--destroy]
ceph-volume lvm zap osd-vg/data-lv
ceph-volume lvm zap DEVICE_PATH_PARTITION [--destroy]
ceph-volume lvm zap /dev/sdc1
ceph-volume lvm zap DEVICE_PATH --destroy
ceph-volume lvm zap /dev/sdc --destroy
ceph-volume lvm zap --destroy --osd-id OSD_ID
ceph-volume lvm zap --destroy --osd-id 16
ceph-volume lvm zap --destroy --osd-fsid OSD_FSID
ceph-volume lvm zap --destroy --osd-fsid 65d7b6b1-e41a-4a3c-b363-83ade63cb32b
echo 3 | sudo tee /proc/sys/vm/drop_caches && sudo sync
ceph osd pool create testbench 100 100
rados bench -p testbench 10 write --no-cleanup Maintaining 16 concurrent writes of 4194304 bytes for up to 10 seconds or 0 objects Object prefix: benchmark_data_cephn1.home.network_10510 sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 1 16 16 0 0 0 - 0 2 16 16 0 0 0 - 0 3 16 16 0 0 0 - 0 4 16 17 1 0.998879 1 3.19824 3.19824 5 16 18 2 1.59849 4 4.56163 3.87993 6 16 18 2 1.33222 0 - 3.87993 7 16 19 3 1.71239 2 6.90712 4.889 8 16 25 9 4.49551 24 7.75362 6.71216 9 16 25 9 3.99636 0 - 6.71216 10 16 27 11 4.39632 4 9.65085 7.18999 11 16 27 11 3.99685 0 - 7.18999 12 16 27 11 3.66397 0 - 7.18999 13 16 28 12 3.68975 1.33333 12.8124 7.65853 14 16 28 12 3.42617 0 - 7.65853 15 16 28 12 3.19785 0 - 7.65853 16 11 28 17 4.24726 6.66667 12.5302 9.27548 17 11 28 17 3.99751 0 - 9.27548 18 11 28 17 3.77546 0 - 9.27548 19 11 28 17 3.57683 0 - 9.27548 Total time run: 19.505620 Total writes made: 28 Write size: 4194304 Bandwidth (MB/sec): 5.742 Stddev Bandwidth: 5.4617 Max bandwidth (MB/sec): 24 Min bandwidth (MB/sec): 0 Average Latency: 10.4064 Stddev Latency: 3.80038 Max latency: 19.503 Min latency: 3.19824
rados bench -p testbench 10 seq sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 Total time run: 0.804869 Total reads made: 28 Read size: 4194304 Bandwidth (MB/sec): 139.153 Average Latency: 0.420841 Max latency: 0.706133 Min latency: 0.0816332
rados bench -p testbench 10 rand sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 1 16 46 30 119.801 120 0.440184 0.388125 2 16 81 65 129.408 140 0.577359 0.417461 3 16 120 104 138.175 156 0.597435 0.409318 4 15 157 142 141.485 152 0.683111 0.419964 5 16 206 190 151.553 192 0.310578 0.408343 6 16 253 237 157.608 188 0.0745175 0.387207 7 16 287 271 154.412 136 0.792774 0.39043 8 16 325 309 154.044 152 0.314254 0.39876 9 16 362 346 153.245 148 0.355576 0.406032 10 16 405 389 155.092 172 0.64734 0.398372 Total time run: 10.302229 Total reads made: 405 Read size: 4194304 Bandwidth (MB/sec): 157.248 Average Latency: 0.405976 Max latency: 1.00869 Min latency: 0.0378431
rados bench -p testbench 10 write -t 4 --run-name client1 Maintaining 4 concurrent writes of 4194304 bytes for up to 10 seconds or 0 objects Object prefix: benchmark_data_node1_12631 sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 0 0 0 0 0 0 - 0 1 4 4 0 0 0 - 0 2 4 6 2 3.99099 4 1.94755 1.93361 3 4 8 4 5.32498 8 2.978 2.44034 4 4 8 4 3.99504 0 - 2.44034 5 4 10 6 4.79504 4 2.92419 2.4629 6 3 10 7 4.64471 4 3.02498 2.5432 7 4 12 8 4.55287 4 3.12204 2.61555 8 4 14 10 4.9821 8 2.55901 2.68396 9 4 16 12 5.31621 8 2.68769 2.68081 10 4 17 13 5.18488 4 2.11937 2.63763 11 4 17 13 4.71431 0 - 2.63763 12 4 18 14 4.65486 2 2.4836 2.62662 13 4 18 14 4.29757 0 - 2.62662 Total time run: 13.123548 Total writes made: 18 Write size: 4194304 Bandwidth (MB/sec): 5.486 Stddev Bandwidth: 3.0991 Max bandwidth (MB/sec): 8 Min bandwidth (MB/sec): 0 Average Latency: 2.91578 Stddev Latency: 0.956993 Max latency: 5.72685 Min latency: 1.91967
rados -p testbench cleanup
rbd bench --io-type write image01 --pool=testbench bench-write io_size 4096 io_threads 16 bytes 1073741824 pattern seq SEC OPS OPS/SEC BYTES/SEC 2 11127 5479.59 22444382.79 3 11692 3901.91 15982220.33 4 12372 2953.34 12096895.42 5 12580 2300.05 9421008.60 6 13141 2101.80 8608975.15 7 13195 356.07 1458459.94 8 13820 390.35 1598876.60 9 14124 325.46 1333066.62 ..
cd /mnt/ceph-block-device cd /mnt/ceph-file-system
fio --name=randwrite --rw=randwrite --direct=1 --ioengine=libaio --bs=4k --iodepth=32 --size=5G --runtime=60 --group_reporting=1
fio --name=randwrite --rw=randwrite --direct=1 --ioengine=libaio --bs=8k --iodepth=32 --size=5G --runtime=60 --group_reporting=1
time s3cmd put PATH_OF_SOURCE_FILE PATH_OF_DESTINATION_FILE
time s3cmd put /path-to-local-file s3://bucket-name/remote/file
time s3cmd get PATH_OF_DESTINATION_FILE DESTINATION_PATH
time s3cmd get s3://bucket-name/remote/file /path-to-local-destination
time s3cmd ls s3:// BUCKET_NAME
time s3cmd ls s3://bucket-name
ceph daemon DAEMON_NAME perf schema
ceph daemon mon.host01 perf schema
ceph daemon osd.11 perf schema
ceph config set client.rgw.rgw.1.host05 rgw_user_counters_cache true
ceph config set client.rgw.rgw.1.host05 rgw_bucket_counters_cache true
ceph orch restart rgw.rgw.1
s3cmd --configure
s3cmd mb s3:// NAME_OF_THE_BUCKET_FOR_S3
s3cmd mb s3://bucket Bucket 's3://bucket/' created
s3cmd put FILE_NAME s3:// NAME_OF_THE_BUCKET_ON_S3
s3cmd put test.txt s3://bucket upload: 'test.txt' -> 's3://bucket/test.txt' [1 of 1] 21 of 21 100% in 1s 16.75 B/s done
s3cmd ls s3://bucket
config dump ceph daemon DAEMON_ID counter dump
http:// RGW_IP_ADDRESS : CEPH-EXPORTER_PORT /
HELP ceph_rgw_op_per_bucket_put_obj_ops Puts TYPE ceph_rgw_op_per_bucket_put_obj_ops counter ceph_rgw_op_per_bucket_put_obj_ops{bucket="test-bkt1",instance_id="ceph-ck-perf-ej61qj-node5"} 10
HELP ceph_rgw_op_per_user_put_obj_ops Puts TYPE ceph_rgw_op_per_user_put_obj_ops counter ceph_rgw_op_per_user_put_obj_ops{instance_id="ceph-ck-perf-ej61qj-node5",user="ckulal"} 10
http:// RGW_IP_ADDRESS : PROMETHEUS_PORT /
https://10.0.210.100:9283/
ceph daemon DAEMON_NAME perf dump
ceph daemon mon.host01 perf dump
ceph daemon osd.11 perf dump
cephadm shell
ceph config set osd. OSD_ID osd_mclock_profile VALUE
ceph config set osd.0 osd_mclock_profile high_recovery_ops
ceph config set osd osd_mclock_profile VALUE
cephadm shell
ceph config set osd. OSD_ID osd_mclock_profile custom
ceph config set osd.0 osd_mclock_profile custom
ceph config set osd osd_mclock_profile custom
ceph config set osd. OSD_ID MCLOCK_CONFIGURATION_OPTION VALUE
ceph config set osd.0 osd_mclock_scheduler_client_res 0.5
cephadm shell
ceph config set osd osd_mclock_profile MCLOCK_PROFILE
ceph config set osd osd_mclock_profile high_client_ops
ceph config dump
ceph config rm osd MCLOCK_CONFIGURATION_OPTION
ceph config rm osd osd_mclock_scheduler_client_res
ceph config show osd. OSD_ID
ceph config show osd.0
cephadm shell
ceph tell osd. OSD_ID injectargs '-- MCLOCK_CONFIGURATION_OPTION = VALUE '
ceph tell osd.0 injectargs '--osd_mclock_profile=high_recovery_ops'
ceph daemon osd. OSD_ID config set MCLOCK_CONFIGURATION_OPTION VALUE
ceph daemon osd.0 config set osd_mclock_profile high_recovery_ops
cephadm shell
ceph config set osd osd_mclock_override_recovery_settings true
ceph config set osd OPTION VALUE
ceph config set osd osd_max_backfills_ 5
ceph config show osd. OSD_ID_ | grep OPTION
ceph config show osd.0 | grep osd_max_backfills
ceph config set osd osd_mclock_override_recovery_settings false
ceph config show osd.0 osd_mclock_iops_capacity_threshold_hdd 500.000000 ceph config show osd.0 osd_mclock_iops_capacity_threshold_ssd 80000.000000
ceph config show osd. N osd_mclock_max_capacity_iops_[hdd, ssd]
ceph config show osd.0 osd_mclock_max_capacity_iops_ssd
3403 Sep 11 11:52:50 dell-r640-039.dsal.lab.eng.rdu2.redhat.com ceph-osd[70342]: log_channel(cluster) log [WRN] : OSD bench result of 49691.213005 IOPS exceeded the threshold limit of 500.000000 IOPS for osd.27. IOPS capacity is unchanged at 315.000000 IOPS. The recommendation is to establish the osd's IOPS capacity using other benchmark tools (e.g. Fio) and then override osd_mclock_max_capacity_iops_[hdd|ssd].
cephadm shell
ceph config show osd. OSD_ID osd_mclock_max_capacity_iops_[hdd, ssd]
ceph config show osd.0 osd_mclock_max_capacity_iops_ssd 21500.000000
cephadm shell
ceph tell osd. OSD_ID bench [ TOTAL_BYTES ] [ BYTES_PER_WRITE ] [ OBJ_SIZE ] [ NUM_OBJS ]
ceph tell osd.0 bench 12288000 4096 4194304 100 { "bytes_written": 12288000, "blocksize": 4096, "elapsed_sec": 1.3718913019999999, "bytes_per_sec": 8956977.8466311768, "iops": 2186.7621695876896 }
ceph tell osd. OSD_ID cache drop
ceph tell osd.0 cache drop
cephadm shell
ceph tell osd. OSD_ID bench 12288000 4096 4194304 100
ceph tell osd.0 bench 12288000 4096 4194304 100 { "bytes_written": 12288000, "blocksize": 4096, "elapsed_sec": 1.3718913019999999, "bytes_per_sec": 8956977.8466311768, "iops": 2186.7621695876896 1 }
ceph config set osd. OSD_ID bluestore_throttle_bytes 32768 ceph config set osd. OSD_ID bluestore_throttle_deferred_bytes 32768
ceph config set osd.0 bluestore_throttle_bytes 32768 ceph config set osd.0 bluestore_throttle_deferred_bytes 32768
ceph tell osd.0 bench 12288000 4096 4194304 100
cephadm shell
ceph config set osd. OSD_ID osd_mclock_max_capacity_iops_[hdd,ssd] VALUE
ceph config set osd.0 osd_mclock_max_capacity_iops_hdd 350
ceph config set osd. OSD_ID bluestore_min_alloc_size_DEVICE_NAME_ VALUE
ceph config set osd.4 bluestore_min_alloc_size_hdd 8192
systemctl restart SERVICE_ID
systemctl restart ceph-499829b4-832f-11eb-8d6d-001a4a000635@osd.4.service
ceph daemon osd. OSD_ID config get bluestore_min_alloc_size__DEVICE_
ceph daemon osd.4 config get bluestore_min_alloc_size_hdd ceph daemon osd.4 config get bluestore_min_alloc_size { "bluestore_min_alloc_size": "8192" }
cd /usr/share/cephadm-ansible
ansible-playbook -i hosts rocksdb-resharding.yml -e osd_id= OSD_ID -e admin_node= HOST_NAME
ansible-playbook -i hosts rocksdb-resharding.yml -e osd_id=7 -e admin_node=host03 ............ TASK [stop the osd] *********************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:18 +0000 (0:00:00.037) 0:00:03.864 **** changed: [localhost -> host03] TASK [set_fact ceph_cmd] ****************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:32 +0000 (0:00:14.128) 0:00:17.992 **** ok: [localhost -> host03] TASK [check fs consistency with fsck before resharding] *********************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:32 +0000 (0:00:00.041) 0:00:18.034 **** ok: [localhost -> host03] TASK [show current sharding] ************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:43 +0000 (0:00:11.053) 0:00:29.088 **** ok: [localhost -> host03] TASK [reshard] **************************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:45 +0000 (0:00:01.446) 0:00:30.534 **** ok: [localhost -> host03] TASK [check fs consistency with fsck after resharding] ************************************************************************************************************************************************************ Wednesday 29 November 2023 11:25:46 +0000 (0:00:01.479) 0:00:32.014 **** ok: [localhost -> host03] TASK [restart the osd] ******************************************************************************************************************************************************************************************** Wednesday 29 November 2023 11:25:57 +0000 (0:00:10.699) 0:00:42.714 **** changed: [localhost -> host03]
ceph orch daemon stop osd.7
cephadm shell --name osd.7
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-7/ show-sharding m(3) p(3,0-12) O(3,0-13) L P
ceph orch daemon start osd.7
cephadm shell
ceph orch ps
cephadm unit --name OSD_ID stop
cephadm unit --name osd.0 stop
cephadm shell --name OSD_ID
cephadm shell --name osd.0
ceph-bluestore-tool --path/var/lib/ceph/osd/ceph- OSD_ID / fsck
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-0/ fsck fsck success
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph- OSD_ID / show-sharding
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-6/ show-sharding m(3) p(3,0-12) O(3,0-13) L P
ceph-bluestore-tool --log-level 10 -l log.txt --path /var/lib/ceph/osd/ceph- OSD_ID / --sharding="m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P" reshard
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-6/ --sharding="m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P" reshard reshard success
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph- OSD_ID / show-sharding
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-6/ show-sharding m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P
exit
cephadm unit --name OSD_ID start
cephadm unit --name osd.0 start
ceph daemon OSD_ID bluestore allocator score block
ceph daemon osd.123 bluestore allocator score block
ceph daemon OSD_ID bluestore allocator dump block
ceph daemon osd.123 bluestore allocator dump block
cephadm shell --name osd. ID
cephadm shell --name osd.2 Inferring fsid 110bad0a-bc57-11ee-8138-fa163eb9ffc2 Inferring config /var/lib/ceph/110bad0a-bc57-11ee-8138-fa163eb9ffc2/osd.2/config Using ceph image with id `17334f841482` and tag `ceph-7-rhel-9-containers-candidate-59483-20240301201929` created on 2024-03-01 20:22:41 +0000 UTC registry-proxy.engineering.redhat.com/rh-osbs/rhceph@sha256:09fc3e5baf198614d70669a106eb87dbebee16d4e91484375778d4adbccadacd
ceph-bluestore-tool --path PATH_TO_OSD_DATA_DIRECTORY --allocator block free-score
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-score
ceph-bluestore-tool --path PATH_TO_OSD_DATA_DIRECTORY --allocator block free-dump block: { "fragmentation_rating": 0.018290238194701977 }
ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-dump block: { "capacity": 21470642176, "alloc_unit": 4096, "alloc_type": "hybrid", "alloc_name": "block", "extents": [ { "offset": "0x370000", "length": "0x20000" }, { "offset": "0x3a0000", "length": "0x10000" }, { "offset": "0x3f0000", "length": "0x20000" }, { "offset": "0x460000", "length": "0x10000" },
cephadm shell
ceph config get osd bluefs_buffered_io
ceph config get OSD_ID bluefs_buffered_io
ceph config get osd.2 bluefs_buffered_io
ceph config show OSD_ID bluefs_buffered_io
ceph config show osd.3 bluefs_buffered_io
cephadm shell
ceph daemon osd. OSD_ID bluefs stats
ceph daemon osd.1 bluefs stats 1 : device size 0x3bfc00000 : using 0x1a428000(420 MiB) wal_total:0, db_total:15296836403, slow_total:0
ceph daemon osd.1 bluefs stats 0 : 1 : device size 0x1dfbfe000 : using 0x1100000(17 MiB) 2 : device size 0x27fc00000 : using 0x248000(2.3 MiB) RocksDBBlueFSVolumeSelector: wal_total:0, db_total:7646425907, slow_total:10196562739, db_avail:935539507 Usage matrix: DEV/LEV WAL DB SLOW * * REAL FILES LOG 0 B 4 MiB 0 B 0 B 0 B 756 KiB 1 WAL 0 B 4 MiB 0 B 0 B 0 B 3.3 MiB 1 DB 0 B 9 MiB 0 B 0 B 0 B 76 KiB 10 SLOW 0 B 0 B 0 B 0 B 0 B 0 B 0 TOTALS 0 B 17 MiB 0 B 0 B 0 B 0 B 12 MAXIMUMS: LOG 0 B 4 MiB 0 B 0 B 0 B 756 KiB WAL 0 B 4 MiB 0 B 0 B 0 B 3.3 MiB DB 0 B 11 MiB 0 B 0 B 0 B 112 KiB SLOW 0 B 0 B 0 B 0 B 0 B 0 B TOTALS 0 B 17 MiB 0 B 0 B 0 B 0 B
ceph-bluestore-tool COMMAND [ --dev DEVICE ... ] [ -i OSD_ID ] [ --path OSD_PATH ] [ --out-dir DIR ] [ --log-file | -l filename ] [ --deep ] ceph-bluestore-tool fsck|repair --path OSD_PATH [ --deep ] ceph-bluestore-tool qfsck --path OSD_PATH ceph-bluestore-tool allocmap --path OSD_PATH ceph-bluestore-tool restore_cfb --path OSD_PATH ceph-bluestore-tool show-label --dev DEVICE ... ceph-bluestore-tool prime-osd-dir --dev DEVICE --path OSD_PATH ceph-bluestore-tool bluefs-export --path OSD_PATH --out-dir DIR ceph-bluestore-tool bluefs-bdev-new-wal --path OSD_PATH --dev-target NEW_DEVICE ceph-bluestore-tool bluefs-bdev-new-db --path OSD_PATH --dev-target NEW_DEVICE ceph-bluestore-tool bluefs-bdev-migrate --path OSD_PATH --dev-target NEW_DEVICE --devs-source DEVICE1 [--devs-source DEVICE2 ] ceph-bluestore-tool free-dump|free-score --path OSD_PATH [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ] ceph-bluestore-tool reshard --path OSD_PATH --sharding NEW_SHARDING [ --sharding-ctrl CONTROL_STRING ] ceph-bluestore-tool show-sharding --path OSD_PATH
ceph orch daemon stop osd. ID
ceph orch daemon stop osd.2
cephadm shell --name osd. ID
ceph shell --name osd.2
ceph-bluestore-tool bluefs-bdev-new-wal --dev-target /dev/test/newdb --path /var/lib/ceph/osd/ceph-0
ceph orch daemon start osd. ID
ceph orch daemon start osd.2
ceph tell OSD_ID dump_metrics ceph tell OSD_ID dump_metrics reactor_utilization
ceph tell osd.0 dump_metrics ceph tell osd.0 dump_metrics reactor_utilization
cephadm --image quay.ceph.io/ceph-ci/ceph:b682861f8690608d831f58603303388dd7915aa7-crimson bootstrap --mon-ip 10.1.240.54 --allow-fqdn-hostname --initial-dashboard-password Ceph_Crims
cephadm shell
ceph config set global 'enable_experimental_unrecoverable_data_corrupting_features' crimson
ceph osd set-allow-crimson --yes-i-really-mean-it
ceph config set mon osd_pool_default_crimson true
dnf install libnbd git clone git://git.kernel.dk/fio.git cd fio ./configure --enable-libnbd make
cd build ninja crimson-store-nbd
export disk_img=/tmp/disk.img export unix_socket=/tmp/store_nbd_socket.sock rm -f USDdisk_img USDunix_socket truncate -s 512M USDdisk_img ./bin/crimson-store-nbd --device-path USDdisk_img --smp 1 --mkfs true --type transaction_manager --uds-path USD{unix_socket} & --smp is the CPU cores. --mkfs initializes the device first. --type is the backend.
[global] ioengine=nbd uri=nbd+unix:///?socket=USD{unix_socket} rw=randrw time_based runtime=120 group_reporting iodepth=1 size=512M [job0] offset=0
./fio nbd.fio
git checkout master make crimson-osd ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/baseline ../src/test/crimson/cbt/radosbench_4K_read.yaml git checkout topic make crimson-osd ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/yap ../src/test/crimson/cbt/radosbench_4K_read.yaml
~/dev/cbt/compare.py -b /tmp/baseline -a /tmp/yap -v
ceph orch pause
ceph orch set backend '' ceph mgr module disable cephadm
ceph mgr module enable cephadm ceph orch set backend cephadm
ceph orch ls --service_name SERVICE_NAME --format yaml
ceph orch ls --service_name alertmanager --format yaml service_type: alertmanager service_name: alertmanager placement: hosts: - unknown_host status: running: 1 size: 1 events: - 2021-02-01T08:58:02.741162 service:alertmanager [INFO] "service was created" - '2021-02-01T12:09:25.264584 service:alertmanager [ERROR] "Failed to apply: Cannot place <AlertManagerSpec for service_name=alertmanager> on unknown_host: Unknown hosts"'
ceph orch ps --service-name SERVICE_NAME --daemon-id DAEMON_ID --format yaml
ceph orch ps --service-name mds --daemon-id cephfs.hostname.ppdhsz --format yaml daemon_type: mds daemon_id: cephfs.hostname.ppdhsz hostname: hostname status_desc: running events: - 2021-02-01T08:59:43.845866 daemon:mds.cephfs.hostname.ppdhsz [INFO] "Reconfigured mds.cephfs.hostname.ppdhsz on host 'hostname'"
ceph -W cephadm
ceph log last cephadm
cephadm logs --name DAEMON_NAME
cephadm logs --name cephfs.hostname.ppdhsz
cephadm logs --fsid FSID --name DAEMON_NAME
cephadm logs --fsid 2d2fd136-6df1-11ea-ae74-002590e526e8 --name cephfs.hostname.ppdhsz
for name in USD(cephadm ls | python3 -c "import sys, json; [print(i['name']) for i in json.load(sys.stdin)]") ; do cephadm logs --fsid FSID_OF_CLUSTER --name "USDname" > USDname; done
for name in USD(cephadm ls | python3 -c "import sys, json; [print(i['name']) for i in json.load(sys.stdin)]") ; do cephadm logs --fsid 57bddb48-ee04-11eb-9962-001a4a000672 --name "USDname" > USDname; done
[root@host01 ~]USD systemctl status ceph-a538d494-fb2a-48e4-82c8-b91c37bb0684@mon.host01.service
podman ps -a --format json | jq '.[].Image' "docker.io/library/rhel9" "registry.redhat.io/rhceph-alpha/rhceph-6-rhel9@sha256:9aaea414e2c263216f3cdcb7a096f57c3adf6125ec9f4b0f5f65fa8c43987155"
execnet.gateway_bootstrap.HostNotFound: -F /tmp/cephadm-conf-73z09u6g -i /tmp/cephadm-identity-ky7ahp_5 root@10.10.1.2 raise OrchestratorError(msg) from e orchestrator._interface.OrchestratorError: Failed to connect to 10.10.1.2 (10.10.1.2). Please make sure that the host is reachable and accepts connections using the cephadm SSH key
ceph config-key get mgr/cephadm/ssh_identity_key > ~/cephadm_private_key INFO:cephadm:Inferring fsid f8edc08a-7f17-11ea-8707-000c2915dd98 INFO:cephadm:Using recent ceph image docker.io/ceph/ceph:v15 obtained 'mgr/cephadm/ssh_identity_key' chmod 0600 ~/cephadm_private_key
chmod 0600 ~/cephadm_private_key
cat ~/cephadm_private_key | ceph cephadm set-ssk-key -i-
ceph cephadm get-ssh-config
ssh -F config -i ~/cephadm_private_key root@host01
ceph cephadm get-pub-key grep "`cat ~/ceph.pub`" /root/.ssh/authorized_keys
ceph config set host public_network hostnetwork
cephadm enter --name cephfs.hostname.ppdhsz ceph --admin-daemon /var/run/ceph/ceph-cephfs.hostname.ppdhsz.asok config show
cephadm shell
ceph config-key set mgr/cephadm/pause true
ceph auth get-or-create mgr.host01.smfvfd1 mon "profile mgr" osd "allow *" mds "allow *" [mgr.host01.smfvfd1] key = AQDhcORgW8toCRAAlMzlqWXnh3cGRjqYEa9ikw==
ceph config generate-minimal-conf minimal ceph.conf for 8c9b0072-67ca-11eb-af06-001a4a0002a0 [global] fsid = 8c9b0072-67ca-11eb-af06-001a4a0002a0 mon_host = [v2:10.10.200.10:3300/0,v1:10.10.200.10:6789/0] [v2:10.10.10.100:3300/0,v1:10.10.200.100:6789/0]
ceph config get "mgr.host01.smfvfd1" container_image
{ { "config": "# minimal ceph.conf for 8c9b0072-67ca-11eb-af06-001a4a0002a0\n[global]\n\tfsid = 8c9b0072-67ca-11eb-af06-001a4a0002a0\n\tmon_host = [v2:10.10.200.10:3300/0,v1:10.10.200.10:6789/0] [v2:10.10.10.100:3300/0,v1:10.10.200.100:6789/0]\n", "keyring": "[mgr.Ceph5-2.smfvfd1]\n\tkey = AQDhcORgW8toCRAAlMzlqWXnh3cGRjqYEa9ikw==\n" } }
exit
cephadm --image registry.redhat.io/rhceph-alpha/rhceph-6-rhel9:latest deploy --fsid 8c9b0072-67ca-11eb-af06-001a4a0002a0 --name mgr.host01.smfvfd1 --config-json config-json.json
ceph -s
ceph -W cephadm
2022-06-10T17:51:36.335728+0000 mgr.Ceph5-1.nqikfh [INF] refreshing Ceph5-adm facts 2022-06-10T17:51:37.170982+0000 mgr.Ceph5-1.nqikfh [INF] deploying 1 monitor(s) instead of 2 so monitors may achieve consensus 2022-06-10T17:51:37.173487+0000 mgr.Ceph5-1.nqikfh [ERR] It is NOT safe to stop ['mon.Ceph5-adm']: not enough monitors would be available (Ceph5-2) after stopping mons [Ceph5-adm] 2022-06-10T17:51:37.174415+0000 mgr.Ceph5-1.nqikfh [INF] Checking pool "nfs-ganesha" exists for service nfs.foo 2022-06-10T17:51:37.176389+0000 mgr.Ceph5-1.nqikfh [ERR] Failed to apply nfs.foo spec NFSServiceSpec({'placement': PlacementSpec(count=1), 'service_type': 'nfs', 'service_id': 'foo', 'unmanaged': False, 'preview_only': False, 'pool': 'nfs-ganesha', 'namespace': 'nfs-ns'}): Cannot find pool "nfs-ganesha" for service nfs.foo Traceback (most recent call last): File "/usr/share/ceph/mgr/cephadm/serve.py", line 408, in _apply_all_services if self._apply_service(spec): File "/usr/share/ceph/mgr/cephadm/serve.py", line 509, in _apply_service config_func(spec) File "/usr/share/ceph/mgr/cephadm/services/nfs.py", line 23, in config self.mgr._check_pool_exists(spec.pool, spec.service_name()) File "/usr/share/ceph/mgr/cephadm/module.py", line 1840, in _check_pool_exists raise OrchestratorError(f'Cannot find pool "{pool}" for ' orchestrator._interface.OrchestratorError: Cannot find pool "nfs-ganesha" for service nfs.foo 2022-06-10T17:51:37.179658+0000 mgr.Ceph5-1.nqikfh [INF] Found osd claims -> {} 2022-06-10T17:51:37.180116+0000 mgr.Ceph5-1.nqikfh [INF] Found osd claims for drivegroup all-available-devices -> {} 2022-06-10T17:51:37.182138+0000 mgr.Ceph5-1.nqikfh [INF] Applying all-available-devices on host Ceph5-adm 2022-06-10T17:51:37.182987+0000 mgr.Ceph5-1.nqikfh [INF] Applying all-available-devices on host Ceph5-1 2022-06-10T17:51:37.183395+0000 mgr.Ceph5-1.nqikfh [INF] Applying all-available-devices on host Ceph5-2 2022-06-10T17:51:43.373570+0000 mgr.Ceph5-1.nqikfh [INF] Reconfiguring node-exporter.Ceph5-1 (unknown last config time) 2022-06-10T17:51:43.373840+0000 mgr.Ceph5-1.nqikfh [INF] Reconfiguring daemon node-exporter.Ceph5-1 on Ceph5-1
ceph config set mgr mgr/cephadm/log_to_cluster_level debug ceph -W cephadm --watch-debug ceph -W cephadm --verbose
ceph config set mgr mgr/cephadm/log_to_cluster_level info
ceph log last cephadm
journalctl -u ceph-5c5a50ae-272a-455d-99e9-32c6a013e694@host01
ceph config set global log_to_stderr false ceph config set global mon_cluster_log_to_stderr false
ceph config set global log_to_file true ceph config set global mon_cluster_log_to_file true
service_type: grafana service_name: grafana custom_configs: - mount_path: /etc/example.conf content: | setting1 = value1 setting2 = value2 - mount_path: /usr/share/grafana/example.cert content: | -----BEGIN PRIVATE KEY----- V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= -----END CERTIFICATE-----
ceph orch redeploy SERVICE_NAME
ceph orch redeploy grafana
ceph config set mgr mgr/cephadm/config_checks_enabled true
ALL cephadm checks are disabled, use 'ceph config set mgr mgr/cephadm/config_checks_enabled true' to enable
CEPHADM 8/8 checks enabled and executed (0 bypassed, 0 disabled). No issues detected
ceph cephadm config-check status
ceph cephadm config-check ls NAME HEALTHCHECK STATUS DESCRIPTION kernel_security CEPHADM_CHECK_KERNEL_LSM enabled checks SELINUX/Apparmor profiles are consistent across cluster hosts os_subscription CEPHADM_CHECK_SUBSCRIPTION enabled checks subscription states are consistent for all cluster hosts public_network CEPHADM_CHECK_PUBLIC_MEMBERSHIP enabled check that all hosts have a NIC on the Ceph public_netork osd_mtu_size CEPHADM_CHECK_MTU enabled check that OSD hosts share a common MTU setting osd_linkspeed CEPHADM_CHECK_LINKSPEED enabled check that OSD hosts share a common linkspeed network_missing CEPHADM_CHECK_NETWORK_MISSING enabled checks that the cluster/public networks defined exist on the Ceph hosts ceph_release CEPHADM_CHECK_CEPH_RELEASE enabled check for Ceph version consistency - ceph daemons should be on the same release (unless upgrade is active) kernel_version CEPHADM_CHECK_KERNEL_VERSION enabled checks that the MAJ.MIN of the kernel on Ceph hosts is consistent
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi INVENTORY_FILE HOST1 labels="[' LABEL1 ', ' LABEL2 ']" HOST2 labels="[' LABEL1 ', ' LABEL2 ']" HOST3 labels="[' LABEL1 ']" [admin] ADMIN_HOST monitor_address= MONITOR_IP_ADDRESS labels="[' ADMIN_LABEL ', ' LABEL1 ', ' LABEL2 ']"
[ceph-admin@admin cephadm-ansible]USD sudo vi hosts host02 labels="['mon', 'mgr']" host03 labels="['mon', 'mgr']" host04 labels="['osd']" host05 labels="['osd']" host06 labels="['osd']" [admin] host01 monitor_address=10.10.128.68 labels="['_admin', 'mon', 'mgr']"
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs"
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs"
sudo vi PLAYBOOK_FILENAME .yml --- - name: NAME_OF_PLAY hosts: BOOTSTRAP_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: -name: NAME_OF_TASK cephadm_registry_login: state: STATE registry_url: REGISTRY_URL registry_username: REGISTRY_USER_NAME registry_password: REGISTRY_PASSWORD - name: NAME_OF_TASK cephadm_bootstrap: mon_ip: "{{ monitor_address }}" dashboard_user: DASHBOARD_USER dashboard_password: DASHBOARD_PASSWORD allow_fqdn_hostname: ALLOW_FQDN_HOSTNAME cluster_network: NETWORK_CIDR
[ceph-admin@admin cephadm-ansible]USD sudo vi bootstrap.yml --- - name: bootstrap the cluster hosts: host01 become: true gather_facts: false tasks: - name: login to registry cephadm_registry_login: state: login registry_url: registry.redhat.io registry_username: user1 registry_password: mypassword1 - name: bootstrap initial cluster cephadm_bootstrap: mon_ip: "{{ monitor_address }}" dashboard_user: mydashboarduser dashboard_password: mydashboardpassword allow_fqdn_hostname: true cluster_network: 10.10.128.0/28
ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml -vvv
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts bootstrap.yml -vvv
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi INVENTORY_FILE NEW_HOST1 labels="[' LABEL1 ', ' LABEL2 ']" NEW_HOST2 labels="[' LABEL1 ', ' LABEL2 ']" NEW_HOST3 labels="[' LABEL1 ']" [admin] ADMIN_HOST monitor_address= MONITOR_IP_ADDRESS labels="[' ADMIN_LABEL ', ' LABEL1 ', ' LABEL2 ']"
[ceph-admin@admin cephadm-ansible]USD sudo vi hosts host02 labels="['mon', 'mgr']" host03 labels="['mon', 'mgr']" host04 labels="['osd']" host05 labels="['osd']" host06 labels="['osd']" [admin] host01 monitor_address= 10.10.128.68 labels="['_admin', 'mon', 'mgr']"
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit NEWHOST
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit host02
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: HOSTS_OR_HOST_GROUPS become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_host: name: "{{ ansible_facts['hostname'] }}" address: "{{ ansible_facts['default_ipv4']['address'] }}" labels: "{{ labels }}" delegate_to: HOST_TO_DELEGATE_TASK_TO - name: NAME_OF_TASK when: inventory_hostname in groups['admin'] ansible.builtin.shell: cmd: CEPH_COMMAND_TO_RUN register: REGISTER_NAME - name: NAME_OF_TASK when: inventory_hostname in groups['admin'] debug: msg: "{{ REGISTER_NAME .stdout }}"
[ceph-admin@admin cephadm-ansible]USD sudo vi add-hosts.yml --- - name: add additional hosts to the cluster hosts: all become: true gather_facts: true tasks: - name: add hosts to the cluster ceph_orch_host: name: "{{ ansible_facts['hostname'] }}" address: "{{ ansible_facts['default_ipv4']['address'] }}" labels: "{{ labels }}" delegate_to: host01 - name: list hosts in the cluster when: inventory_hostname in groups['admin'] ansible.builtin.shell: cmd: ceph orch host ls register: host_list - name: print current list of hosts when: inventory_hostname in groups['admin'] debug: msg: "{{ host_list.stdout }}"
ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts add-hosts.yml
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: NAME_OF_PLAY hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_host: name: HOST_TO_REMOVE state: STATE - name: NAME_OF_TASK ceph_orch_host: name: HOST_TO_REMOVE state: STATE retries: NUMBER_OF_RETRIES delay: DELAY until: CONTINUE_UNTIL register: REGISTER_NAME - name: NAME_OF_TASK ansible.builtin.shell: cmd: ceph orch host ls register: REGISTER_NAME - name: NAME_OF_TASK debug: msg: "{{ REGISTER_NAME .stdout }}"
[ceph-admin@admin cephadm-ansible]USD sudo vi remove-hosts.yml --- - name: remove host hosts: host01 become: true gather_facts: true tasks: - name: drain host07 ceph_orch_host: name: host07 state: drain - name: remove host from the cluster ceph_orch_host: name: host07 state: absent retries: 20 delay: 1 until: result is succeeded register: result - name: list hosts in the cluster ansible.builtin.shell: cmd: ceph orch host ls register: host_list - name: print current list of hosts debug: msg: "{{ host_list.stdout }}"
ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts remove-hosts.yml
TASK [print current hosts] ****************************************************************************************************** Friday 24 June 2022 14:52:40 -0400 (0:00:03.365) 0:02:31.702 *********** ok: [host01] => msg: |- HOST ADDR LABELS STATUS host01 10.10.128.68 _admin mon mgr host02 10.10.128.69 mon mgr host03 10.10.128.70 mon mgr host04 10.10.128.71 osd host05 10.10.128.72 osd host06 10.10.128.73 osd
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_config: action: GET_OR_SET who: DAEMON_TO_SET_CONFIGURATION_TO option: CEPH_CONFIGURATION_OPTION value: VALUE_OF_PARAMETER_TO_SET - name: NAME_OF_TASK ceph_config: action: GET_OR_SET who: DAEMON_TO_SET_CONFIGURATION_TO option: CEPH_CONFIGURATION_OPTION register: REGISTER_NAME - name: NAME_OF_TASK debug: msg: " MESSAGE_TO_DISPLAY {{ REGISTER_NAME .stdout }}"
[ceph-admin@admin cephadm-ansible]USD sudo vi change_configuration.yml --- - name: set pool delete hosts: host01 become: true gather_facts: false tasks: - name: set the allow pool delete option ceph_config: action: set who: mon option: mon_allow_pool_delete value: true - name: get the allow pool delete setting ceph_config: action: get who: mon option: mon_allow_pool_delete register: verify_mon_allow_pool_delete - name: print current mon_allow_pool_delete setting debug: msg: "the value of 'mon_allow_pool_delete' is {{ verify_mon_allow_pool_delete.stdout }}"
ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts change_configuration.yml
TASK [print current mon_allow_pool_delete setting] ************************************************************* Wednesday 29 June 2022 13:51:41 -0400 (0:00:05.523) 0:00:17.953 ******** ok: [host01] => msg: the value of 'mon_allow_pool_delete' is true
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: HOSTS_OR_HOST_GROUPS become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_apply: spec: | service_type: SERVICE_TYPE service_id: UNIQUE_NAME_OF_SERVICE placement: host_pattern: ' HOST_PATTERN_TO_SELECT_HOSTS ' label: LABEL spec: SPECIFICATION_OPTIONS :
[ceph-admin@admin cephadm-ansible]USD sudo vi deploy_osd_service.yml --- - name: deploy osd service hosts: host01 become: true gather_facts: true tasks: - name: apply osd spec ceph_orch_apply: spec: | service_type: osd service_id: osd placement: host_pattern: '*' label: osd spec: data_devices: all: true
ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts deploy_osd_service.yml
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_daemon: state: STATE_OF_SERVICE daemon_id: DAEMON_ID daemon_type: TYPE_OF_SERVICE
[ceph-admin@admin cephadm-ansible]USD sudo vi restart_services.yml --- - name: start and stop services hosts: host01 become: true gather_facts: false tasks: - name: start osd.0 ceph_orch_daemon: state: started daemon_id: 0 daemon_type: osd - name: stop mon.host02 ceph_orch_daemon: state: stopped daemon_id: host02 daemon_type: mon
ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts restart_services.yml