Storage Strategies Guide Red Hat Ceph Storage 8 Creating storage strategies for Red Hat Ceph Storage clusters Red Hat Ceph Storage Documentation Team

Commands:
ceph osd crush tree --show-shadow ID CLASS WEIGHT TYPE NAME -24 ssd 4.54849 root default~ssd -19 ssd 0.90970 host ceph01~ssd 8 ssd 0.90970 osd.8 -20 ssd 0.90970 host ceph02~ssd 7 ssd 0.90970 osd.7 -21 ssd 0.90970 host ceph03~ssd 3 ssd 0.90970 osd.3 -22 ssd 0.90970 host ceph04~ssd 5 ssd 0.90970 osd.5 -23 ssd 0.90970 host ceph05~ssd 6 ssd 0.90970 osd.6 -2 hdd 50.94173 root default~hdd -4 hdd 7.27739 host ceph01~hdd 10 hdd 7.27739 osd.10 -12 hdd 14.55478 host ceph02~hdd 0 hdd 7.27739 osd.0 12 hdd 7.27739 osd.12 -6 hdd 14.55478 host ceph03~hdd 4 hdd 7.27739 osd.4 11 hdd 7.27739 osd.11 -10 hdd 7.27739 host ceph04~hdd 1 hdd 7.27739 osd.1 -8 hdd 7.27739 host ceph05~hdd 2 hdd 7.27739 osd.2 -1 55.49022 root default -3 8.18709 host ceph01 10 hdd 7.27739 osd.10 8 ssd 0.90970 osd.8 -11 15.46448 host ceph02 0 hdd 7.27739 osd.0 12 hdd 7.27739 osd.12 7 ssd 0.90970 osd.7 -5 15.46448 host ceph03 4 hdd 7.27739 osd.4 11 hdd 7.27739 osd.11 3 ssd 0.90970 osd.3 -9 8.18709 host ceph04 1 hdd 7.27739 osd.1 5 ssd 0.90970 osd.5 -7 8.18709 host ceph05 2 hdd 7.27739 osd.2 6 ssd 0.90970 osd.6
[bucket-type] [bucket-name] { id [a unique negative numeric ID] weight [the relative capacity/capability of the item(s)] alg [the bucket type: uniform | list | tree | straw2 ] hash [the hash type: 0 by default] item [item-name] weight [weight] }
host node1 { id -1 alg straw2 hash 0 item osd.0 weight 1.00 item osd.1 weight 1.00 } host node2 { id -2 alg straw2 hash 0 item osd.2 weight 1.00 item osd.3 weight 1.00 } rack rack1 { id -3 alg straw2 hash 0 item node1 weight 2.00 item node2 weight 2.00 }
root=default row=a rack=a2 chassis=a2a host=a2a1
ceph osd crush add-bucket {name} {type}
ceph osd crush add-bucket ssd-root root ceph osd crush add-bucket hdd-journal-root root ceph osd crush add-bucket hdd-root root
added bucket ssd-root type root to crush map added bucket hdd-journal-root type root to crush map added bucket hdd-root type root to crush map
ceph osd crush add-bucket ssd-row1 row ceph osd crush add-bucket ssd-row1-rack1 rack ceph osd crush add-bucket ssd-row1-rack1-host1 host ceph osd crush add-bucket ssd-row1-rack1-host2 host ceph osd crush add-bucket hdd-row1 row ceph osd crush add-bucket hdd-row1-rack2 rack ceph osd crush add-bucket hdd-row1-rack1-host1 host ceph osd crush add-bucket hdd-row1-rack1-host2 host ceph osd crush add-bucket hdd-row1-rack1-host3 host ceph osd crush add-bucket hdd-row1-rack1-host4 host
ceph osd tree
ceph osd crush move ssd-row1 root=ssd-root ceph osd crush move ssd-row1-rack1 row=ssd-row1 ceph osd crush move ssd-row1-rack1-host1 rack=ssd-row1-rack1 ceph osd crush move ssd-row1-rack1-host2 rack=ssd-row1-rack1
ceph osd tree
ceph osd crush remove {bucket-name}
ceph osd crush rm {bucket-name}
ceph osd crush tree -f json-pretty
[ { "id": -2, "name": "ssd", "type": "root", "type_id": 10, "items": [ { "id": -6, "name": "dell-per630-11-ssd", "type": "host", "type_id": 1, "items": [ { "id": 6, "name": "osd.6", "type": "osd", "type_id": 0, "crush_weight": 0.099991, "depth": 2 } ] }, { "id": -7, "name": "dell-per630-12-ssd", "type": "host", "type_id": 1, "items": [ { "id": 7, "name": "osd.7", "type": "osd", "type_id": 0, "crush_weight": 0.099991, "depth": 2 } ] }, { "id": -8, "name": "dell-per630-13-ssd", "type": "host", "type_id": 1, "items": [ { "id": 8, "name": "osd.8", "type": "osd", "type_id": 0, "crush_weight": 0.099991, "depth": 2 } ] } ] }, { "id": -1, "name": "default", "type": "root", "type_id": 10, "items": [ { "id": -3, "name": "dell-per630-11", "type": "host", "type_id": 1, "items": [ { "id": 0, "name": "osd.0", "type": "osd", "type_id": 0, "crush_weight": 0.449997, "depth": 2 }, { "id": 3, "name": "osd.3", "type": "osd", "type_id": 0, "crush_weight": 0.289993, "depth": 2 } ] }, { "id": -4, "name": "dell-per630-12", "type": "host", "type_id": 1, "items": [ { "id": 1, "name": "osd.1", "type": "osd", "type_id": 0, "crush_weight": 0.449997, "depth": 2 }, { "id": 4, "name": "osd.4", "type": "osd", "type_id": 0, "crush_weight": 0.289993, "depth": 2 } ] }, { "id": -5, "name": "dell-per630-13", "type": "host", "type_id": 1, "items": [ { "id": 2, "name": "osd.2", "type": "osd", "type_id": 0, "crush_weight": 0.449997, "depth": 2 }, { "id": 5, "name": "osd.5", "type": "osd", "type_id": 0, "crush_weight": 0.289993, "depth": 2 } ] } ] } ]
ceph orch daemon add osd HOST :_DEVICE_,[ DEVICE ]
ceph osd crush add ID_OR_NAME WEIGHT [ BUCKET_TYPE = BUCKET_NAME ...]
ceph osd crush add osd.0 1.0 root=default datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1
ceph osd crush set ID_OR_NAME WEIGHT root= POOL_NAME [ BUCKET_TYPE = BUCKET_NAME ...]
ceph osd crush remove NAME
ceph osd crush set-device-class CLASS OSD_ID [ OSD_ID ..]
ceph osd crush set-device-class hdd osd.0 osd.1 ceph osd crush set-device-class ssd osd.2 osd.3 ceph osd crush set-device-class bucket-index osd.4
ceph osd crush rm-device-class CLASS OSD_ID [ OSD_ID ..]
ceph osd crush rm-device-class hdd osd.0 osd.1 ceph osd crush rm-device-class ssd osd.2 osd.3 ceph osd crush rm-device-class bucket-index osd.4
ceph osd crush class rename OLD_NAME NEW_NAME
ceph osd crush class rename hdd sas15k
ceph osd crush class ls
hdd, ssd, bucket-index
ceph osd crush class ls-osd CLASS
ceph osd crush class ls-osd hdd
0 1 2 3 4 5 6
ceph osd crush rule ls-by-class CLASS
ceph osd crush rule ls-by-class hdd
ceph osd crush reweight _NAME_ _WEIGHT_
osd crush reweight-subtree NAME
ceph osd reweight ID WEIGHT
ceph osd reweight-by-utilization [THRESHOLD_] [ WEIGHT_CHANGE_AMOUNT ] [ NUMBER_OF_OSDS ] [--no-increasing]
ceph osd test-reweight-by-utilization 110 .5 4 --no-increasing
osd reweight-by-pg POOL_NAME
osd crush reweight-all
ceph osd primary-affinity OSD_ID WEIGHT
rule <rulename> { id <unique number> type [replicated | erasure] min_size <min-size> max_size <max-size> step take <bucket-type> [class <class-name>] step [choose|chooseleaf] [firstn|indep] <N> <bucket-type> step emit }
ceph osd crush rule list ceph osd crush rule ls
ceph osd crush rule dump NAME
ceph osd crush rule create-simple RUENAME ROOT BUCKET_NAME FIRSTN_OR_INDEP
ceph osd crush rule create-simple deleteme default host firstn
{ "id": 1, "rule_name": "deleteme", "type": 1, "min_size": 1, "max_size": 10, "steps": [ { "op": "take", "item": -1, "item_name": "default"}, { "op": "chooseleaf_firstn", "num": 0, "type": "host"}, { "op": "emit"}]}
ceph osd crush rule create-replicated NAME ROOT FAILURE_DOMAIN CLASS
ceph osd crush rule create-replicated fast default host ssd
ceph osd crush rule create-erasure RULE_NAME PROFILE_NAME
ceph osd crush rule create-erasure default default
ceph osd crush rule rm NAME
ceph osd crush tunables PROFILE
ceph osd crush tunables optimal
ceph osd crush tunables PROFILE
ceph osd crush tunables legacy
ceph osd getcrushmap -o /tmp/crush
crushtool -i /tmp/crush --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 -o /tmp/crush.new
ceph osd setcrushmap -i /tmp/crush.new
crushtool -i /tmp/crush --set-choose-local-tries 2 --set-choose-local-fallback-tries 5 --set-choose-total-tries 19 --set-chooseleaf-descend-once 0 --set-chooseleaf-vary-r 0 -o /tmp/crush.legacy
ceph osd getcrushmap -o COMPILED_CRUSHMAP_FILENAME
crushtool -d COMPILED_CRUSHMAP_FILENAME -o DECOMPILED_CRUSHMAP_FILENAME
ceph osd setcrushmap -i COMPILED_CRUSHMAP_FILENAME
crushtool -c DECOMPILED_CRUSHMAP_FILENAME -o COMPILED_CRUSHMAP_FILENAME
ceph osd crush set-device-class CLASS OSD_ID [ OSD_ID ]
ceph osd crush set-device-class hdd osd.0 osd.1 osd.4 osd.5 ceph osd crush set-device-class ssd osd.2 osd.3 osd.6 osd.7
ceph osd crush rule create-replicated RULENAME ROOT FAILURE_DOMAIN_TYPE DEVICE_CLASS
ceph osd crush rule create-replicated cold default host hdd ceph osd crush rule create-replicated hot default host ssd
ceph osd pool set POOL_NAME crush_rule RULENAME
ceph osd pool set cold crush_rule hdd ceph osd pool set hot crush_rule ssd
device 0 osd.0 class hdd device 1 osd.1 class hdd device 2 osd.2 class ssd device 3 osd.3 class ssd device 4 osd.4 class hdd device 5 osd.5 class hdd device 6 osd.6 class ssd device 7 osd.7 class ssd host ceph-osd-server-1 { id -1 alg straw2 hash 0 item osd.0 weight 1.00 item osd.1 weight 1.00 item osd.2 weight 1.00 item osd.3 weight 1.00 } host ceph-osd-server-2 { id -2 alg straw2 hash 0 item osd.4 weight 1.00 item osd.5 weight 1.00 item osd.6 weight 1.00 item osd.7 weight 1.00 } root default { id -3 alg straw2 hash 0 item ceph-osd-server-1 weight 4.00 item ceph-osd-server-2 weight 4.00 } rule cold { ruleset 0 type replicated min_size 2 max_size 11 step take default class hdd step chooseleaf firstn 0 type host step emit } rule hot { ruleset 1 type replicated min_size 2 max_size 11 step take default class ssd step chooseleaf firstn 0 type host step emit }
osd pool default pg num = 100 osd pool default pgp num = 100
(OSDs * 100) Total PGs = ------------ pool size
(200 * 100) ----------- = 6667. Nearest power of 2: 8192 3
mon pg warn max per osd
ceph osd pool set POOL_NAME pg_autoscale_mode on
ceph osd pool set testpool pg_autoscale_mode on
ceph config set global osd_pool_default_pg_autoscale_mode MODE
ceph config set global osd_pool_default_pg_autoscale_mode on
ceph osd pool create POOL_NAME --bulk
ceph osd pool create testpool --bulk
ceph osd pool set ec_pool_overwrite bulk True Error EINVAL: expecting value 'true', 'false', '0', or '1'
ceph osd pool set POOL_NAME bulk true / false / 1 / 0
ceph osd pool set testpool bulk true
ceph osd pool get POOL_NAME bulk
ceph osd pool get testpool bulk bulk: true
ceph osd pool autoscale-status
POOL SIZE TARGET SIZE RATE RAW CAPACITY RATIO TARGET RATIO EFFECTIVE RATIO BIAS PG_NUM NEW PG_NUM AUTOSCALE BULK device_health_metrics 0 3.0 374.9G 0.0000 1.0 1 on False cephfs.cephfs.meta 24632 3.0 374.9G 0.0000 4.0 32 on False cephfs.cephfs.data 0 3.0 374.9G 0.0000 1.0 32 on False .rgw.root 1323 3.0 374.9G 0.0000 1.0 32 on False default.rgw.log 3702 3.0 374.9G 0.0000 1.0 32 on False default.rgw.control 0 3.0 374.9G 0.0000 1.0 32 on False default.rgw.meta 382 3.0 374.9G 0.0000 4.0 8 on False
ceph config set global mon_target_pg_per_osd number
ceph config set global mon_target_pg_per_osd 150
ceph osd pool get noautoscale
ceph osd pool set noautoscale
ceph osd pool unset noautoscale
ceph osd pool set pool-name target_size_bytes value
ceph osd pool set mypool target_size_bytes 100T
ceph osd pool set pool-name target_size_ratio ratio
ceph osd pool set mypool target_size_ratio 1.0
ceph osd pool set POOL_NAME pg_num PG_NUM
ceph osd pool set POOL_NAME pgp_num PGP_NUM
ceph osd pool get POOL_NAME pg_num
ceph pg dump [--format FORMAT ]
ceph pg dump_stuck {inactive|unclean|stale|undersized|degraded [inactive|unclean|stale|undersized|degraded...]} INTERVAL
ceph pg map PG_ID
ceph pg map 1.6c
osdmap e13 pg 1.6c (1.6c) -> up [1,0] acting [1,0]
ceph pg scrub PG_ID
ceph pg PG_ID mark_unfound_lost revert|delete
ceph osd lspools
ceph config set global osd_pool_default_pg_num 250 ceph config set global osd_pool_default_pgp_num 250
ceph osd pool create POOL_NAME PG_NUM PGP_NUM [replicated] [ CRUSH_RULE_NAME ] [ EXPECTED_NUMBER_OBJECTS ]
ceph osd pool create POOL_NAME PG_NUM PGP_NUM erasure [ ERASURE_CODE_PROFILE ] [ CRUSH_RULE_NAME ] [ EXPECTED_NUMBER_OBJECTS ]
ceph osd pool create POOL_NAME [--bulk]
ceph osd pool set-quota POOL_NAME [max_objects OBJECT_COUNT ] [max_bytes BYTES ]
ceph osd pool set-quota data max_objects 10000
ceph osd pool delete POOL_NAME [ POOL_NAME --yes-i-really-really-mean-it]
ceph osd pool rename CURRENT_POOL_NAME NEW_POOL_NAME
ceph osd pool create NEW_POOL PG_NUM [ <other new pool parameters> ] rados cppool SOURCE_POOL NEW_POOL ceph osd pool rename SOURCE_POOL NEW_SOURCE_POOL_NAME ceph osd pool rename NEW_POOL SOURCE_POOL
ceph osd pool create pool1 250 rados cppool pool2 pool1 ceph osd pool rename pool2 pool3 ceph osd pool rename pool1 pool2
ceph osd pool create NEW_POOL PG_NUM [ <other new pool parameters> ] rados export --create SOURCE_POOL FILE_PATH rados import FILE_PATH NEW_POOL
ceph osd pool create pool1 250 rados export --create pool2 <path of export file> rados import <path of export file> pool1
rados export --workers 5 SOURCE_POOL FILE_PATH rados import --workers 5 FILE_PATH NEW_POOL
rados export --workers 5 pool2 <path of export file> rados import --workers 5 <path of export file> pool1
[ceph: root@host01 /] rados df
ceph osd pool set POOL_NAME KEY VALUE
ceph osd pool get POOL_NAME KEY
ceph osd pool application enable POOL_NAME APP {--yes-i-really-mean-it}
{ "checks": { "POOL_APP_NOT_ENABLED": { "severity": "HEALTH_WARN", "summary": { "message": "application not enabled on 1 pool(s)" }, "detail": [ { "message": "application not enabled on pool '_POOL_NAME_'" }, { "message": "use 'ceph osd pool application enable _POOL_NAME_ _APP_', where _APP_ is 'cephfs', 'rbd', 'rgw', or freeform for custom applications." } ] } }, "status": "HEALTH_WARN", "overall_status": "HEALTH_WARN", "detail": [ "'ceph health' JSON format has changed in luminous. If you see this your monitoring system is scraping the wrong fields. Disable this with 'mon health preluminous compat warning = false'" ] }
ceph osd pool application disable POOL_NAME APP {--yes-i-really-mean-it}
ceph osd pool application set POOL_NAME APP KEY
ceph osd pool application rm POOL_NAME APP KEY
ceph osd pool set POOL_NAME size NUMBER_OF_REPLICAS
ceph osd pool set data size 3
ceph osd pool set data min_size 2
ceph osd dump | grep 'replicated size'
set_req_state_err err_no=95 resorting to 500
ceph config set mon mon_osd_down_out_subtree_limit host ceph config set osd osd_async_recovery_min_cost 1099511627776
ceph osd erasure-code-profile set ec22 k=2 m=2 crush-failure-domain=host
ceph osd erasure-code-profile set ec22 k=2 m=2 crush-failure-domain=host Pool : ceph osd pool create test-ec-22 erasure ec22
ceph osd pool create ecpool 32 32 erasure pool 'ecpool' created echo ABCDEFGHI | rados --pool ecpool put NYAN - rados --pool ecpool get NYAN - ABCDEFGHI
ceph osd erasure-code-profile get default k=2 m=2 plugin=jerasure technique=reed_sol_van
ceph osd erasure-code-profile set myprofile k=4 m=2 crush-failure-domain=rack ceph osd pool create ecpool 12 12 erasure *myprofile* echo ABCDEFGHIJKL | rados --pool ecpool put NYAN - rados --pool ecpool get NYAN - ABCDEFGHIJKL
ceph osd erasure-code-profile set NAME [<directory= DIRECTORY >] [<plugin= PLUGIN >] [<stripe_unit= STRIPE_UNIT >] [<_CRUSH_DEVICE_CLASS_>] [<_CRUSH_FAILURE_DOMAIN_>] [<key=value> ...] [--force]
ceph osd erasure-code-profile rm RULE_NAME
ceph osd erasure-code-profile get NAME
ceph osd erasure-code-profile ls
ceph osd pool set ERASURE_CODED_POOL_NAME allow_ec_overwrites true
ceph osd pool set ec_pool allow_ec_overwrites true
rbd create --size IMAGE_SIZE_M|G|T --data-pool _ERASURE_CODED_POOL_NAME REPLICATED_POOL_NAME / IMAGE_NAME
rbd create --size 1G --data-pool ec_pool rep_pool/image01
ceph osd erasure-code-profile set NAME plugin=jerasure k= DATA_CHUNKS m= DATA_CHUNKS technique= TECHNIQUE [crush-root= ROOT ] [crush-failure-domain= BUCKET_TYPE ] [directory= DIRECTORY ] [--force]
chunk nr 01234567 step 1 _cDD_cDD step 2 cDDD____ step 3 ____cDDD
crush-steps='[ [ "choose", "rack", 2 ], [ "chooseleaf", "host", 4 ] ]'