Installation Guide Red Hat Ceph Storage 8 Installing Red Hat Ceph Storage on Red Hat Enterprise Linux Red Hat Ceph Storage Documentation Team

Commands:
ceph soft nofile unlimited
USER_NAME soft nproc unlimited
subscription-manager register
subscription-manager refresh
subscription-manager list --available --matches ' Red Hat Ceph Storage '
subscription-manager attach --pool= POOL_ID
subscription-manager repos --disable=* subscription-manager repos --enable=rhel-9-for-x86_64-baseos-rpms subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms
dnf update
subscription-manager repos --enable=rhceph-8-tools-for-rhel-9-x86_64-rpms
dnf install cephadm-ansible
cd /usr/share/cephadm-ansible
mkdir -p inventory/staging inventory/production
[defaults] inventory = ./inventory/staging
touch inventory/staging/hosts touch inventory/production/hosts
NODE_NAME_1 NODE_NAME_2 [admin] ADMIN_NODE_NAME_1
host02 host03 host04 [admin] host01
ansible-playbook -i inventory/staging/hosts PLAYBOOK.yml
ansible-playbook -i inventory/production/hosts PLAYBOOK.yml
ssh root@myhostname root@myhostname password: Permission denied, please try again.
echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config.d/01-permitrootlogin.conf
systemctl restart sshd.service
ssh root@ HOST_NAME
ssh root@host01
ssh root@ HOST_NAME
ssh root@host01
adduser USER_NAME
adduser ceph-admin
passwd USER_NAME
passwd ceph-admin
cat << EOF >/etc/sudoers.d/ USER_NAME USDUSER_NAME ALL = (root) NOPASSWD:ALL EOF
cat << EOF >/etc/sudoers.d/ceph-admin ceph-admin ALL = (root) NOPASSWD:ALL EOF
chmod 0440 /etc/sudoers.d/ USER_NAME
chmod 0440 /etc/sudoers.d/ceph-admin
[ceph-admin@admin ~]USD ssh-keygen
ssh-copy-id USER_NAME @ HOST_NAME
[ceph-admin@admin ~]USD ssh-copy-id ceph-admin@host01
[ceph-admin@admin ~]USD touch ~/.ssh/config
Host host01 Hostname HOST_NAME User USER_NAME Host host02 Hostname HOST_NAME User USER_NAME
Host host01 Hostname host01 User ceph-admin Host host02 Hostname host02 User ceph-admin Host host03 Hostname host03 User ceph-admin
[ceph-admin@admin ~]USD chmod 600 ~/.ssh/config
host02 host03 host04 [admin] host01
host02 host03 host04 [admin] host01
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs"
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs"
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit GROUP_NAME | NODE_NAME
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit clients [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit host01
cephadm bootstrap --cluster-network NETWORK_CIDR --mon-ip IP_ADDRESS --registry-url registry.redhat.io --registry-username USER_NAME --registry-password PASSWORD --yes-i-know
cephadm bootstrap --cluster-network 10.10.128.0/24 --mon-ip 10.10.128.68 --registry-url registry.redhat.io --registry-username myuser1 --registry-password mypassword1 --yes-i-know
Ceph Dashboard is now available at: URL: https://host01:8443/ User: admin Password: i8nhu7zham Enabling client.admin keyring and conf on hosts with "admin" label You can access the Ceph CLI with: sudo /usr/sbin/cephadm shell --fsid 266ee7a8-2a05-11eb-b846-5254002d4916 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring Please consider enabling telemetry to help improve Ceph: ceph telemetry on For more information see: https://docs.ceph.com/docs/master/mgr/telemetry/ Bootstrap complete.
cephadm bootstrap --ssh-user USER_NAME --mon-ip IP_ADDRESS --allow-fqdn-hostname --registry-json REGISTRY_JSON
cephadm bootstrap --ssh-user ceph --mon-ip 10.10.128.68 --allow-fqdn-hostname --registry-json /etc/mylogin.json
url:  REGISTRY_URL 
username:  USER_NAME 
password:  PASSWORD 
url: registry.redhat.io
username: myuser1
password: mypassword1
cephadm bootstrap --mon-ip IP_ADDRESS --registry-json /etc/mylogin.json
cephadm bootstrap --mon-ip 10.10.128.68 --registry-json /etc/mylogin.json
service_type: host addr: host01 hostname: host01 --- service_type: host addr: host02 hostname: host02 --- service_type: host addr: host03 hostname: host03 --- service_type: host addr: host04 hostname: host04 --- service_type: mon placement: host_pattern: "host[0-2]" --- service_type: osd service_id: my_osds placement: host_pattern: "host[1-3]" data_devices: all: true
cephadm bootstrap --apply-spec CONFIGURATION_FILE_NAME --mon-ip MONITOR_IP_ADDRESS --registry-url registry.redhat.io --registry-username USER_NAME --registry-password PASSWORD
cephadm bootstrap --apply-spec initial-config.yaml --mon-ip 10.10.128.68 --registry-url registry.redhat.io --registry-username myuser1 --registry-password mypassword1
su - SSH_USER_NAME
su - ceph Last login: Tue Sep 14 12:00:29 EST 2021 on pts/0
[ceph@host01 ~]USD ssh host01 Last login: Tue Sep 14 12:03:29 EST 2021 on pts/0
sudo cephadm bootstrap --ssh-user USER_NAME --mon-ip IP_ADDRESS --ssh-private-key PRIVATE_KEY --ssh-public-key PUBLIC_KEY --registry-url registry.redhat.io --registry-username USER_NAME --registry-password PASSWORD
sudo cephadm bootstrap --ssh-user ceph --mon-ip 10.10.128.68 --ssh-private-key /home/ceph/.ssh/id_rsa --ssh-public-key /home/ceph/.ssh/id_rsa.pub --registry-url registry.redhat.io --registry-username myuser1 --registry-password mypassword1
subscription-manager register
subscription-manager refresh
subscription-manager list --available --all --matches="*Ceph*"
subscription-manager attach --pool= POOL_ID
subscription-manager repos --disable=* subscription-manager repos --enable=rhel-9-for-x86_64-baseos-rpms subscription-manager repos --enable=rhel-9-for-x86_64-appstream-rpms
dnf install -y podman httpd-tools
mkdir -p /opt/registry/{auth,certs,data}
htpasswd -bBc /opt/registry/auth/htpasswd PRIVATE_REGISTRY_USERNAME PRIVATE_REGISTRY_PASSWORD
htpasswd -bBc /opt/registry/auth/htpasswd myregistryusername myregistrypassword1
openssl req -newkey rsa:4096 -nodes -sha256 -keyout /opt/registry/certs/domain.key -x509 -days 365 -out /opt/registry/certs/domain.crt -addext "subjectAltName = DNS: LOCAL_NODE_FQDN "
openssl req -newkey rsa:4096 -nodes -sha256 -keyout /opt/registry/certs/domain.key -x509 -days 365 -out /opt/registry/certs/domain.crt -addext "subjectAltName = DNS:admin.lab.redhat.com"
ln -s /opt/registry/certs/domain.crt /opt/registry/certs/domain.cert
cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/ update-ca-trust trust list | grep -i " LOCAL_NODE_FQDN "
cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/ update-ca-trust trust list | grep -i "admin.lab.redhat.com" label: admin.lab.redhat.com
scp /opt/registry/certs/domain.crt root@host01:/etc/pki/ca-trust/source/anchors/ ssh root@host01 update-ca-trust trust list | grep -i "admin.lab.redhat.com" label: admin.lab.redhat.com
run --restart=always --name NAME_OF_CONTAINER -p 5000:5000 -v /opt/registry/data:/var/lib/registry:z -v /opt/registry/auth:/auth:z -v /opt/registry/certs:/certs:z -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -e "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt" -e "REGISTRY_HTTP_TLS_KEY=/certs/domain.key" -e REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED=true -d registry:2
podman run --restart=always --name myprivateregistry -p 5000:5000 -v /opt/registry/data:/var/lib/registry:z -v /opt/registry/auth:/auth:z -v /opt/registry/certs:/certs:z -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -e "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt" -e "REGISTRY_HTTP_TLS_KEY=/certs/domain.key" -e REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED=true -d registry:2
unqualified-search-registries = ["registry.redhat.io", "registry.access.redhat.com", "registry.fedoraproject.org", "registry.centos.org", "docker.io"]
login registry.redhat.io
run -v / CERTIFICATE_DIRECTORY_PATH :/certs:Z -v / CERTIFICATE_DIRECTORY_PATH /domain.cert:/certs/domain.cert:Z --rm registry.redhat.io/rhel9/skopeo:8.5-8 skopeo copy --remove-signatures --src-creds RED_HAT_CUSTOMER_PORTAL_LOGIN : RED_HAT_CUSTOMER_PORTAL_PASSWORD --dest-cert-dir=./certs/ --dest-creds PRIVATE_REGISTRY_USERNAME : PRIVATE_REGISTRY_PASSWORD docker://registry.redhat.io/ SRC_IMAGE : SRC_TAG docker:// LOCAL_NODE_FQDN :5000/ DST_IMAGE : DST_TAG
podman run -v /opt/registry/certs:/certs:Z -v /opt/registry/certs/domain.cert:/certs/domain.cert:Z --rm registry.redhat.io/rhel9/skopeo skopeo copy --remove-signatures --src-creds myusername:mypassword1 --dest-cert-dir=./certs/ --dest-creds myregistryusername:myregistrypassword1 docker://registry.redhat.io/rhceph/rhceph-8-rhel9:latest docker://admin.lab.redhat.com:5000/rhceph/rhceph-8-rhel9:latest podman run -v /opt/registry/certs:/certs:Z -v /opt/registry/certs/domain.cert:/certs/domain.cert:Z --rm registry.redhat.io/rhel9/skopeo skopeo copy --remove-signatures --src-creds myusername:mypassword1 --dest-cert-dir=./certs/ --dest-creds myregistryusername:myregistrypassword1 docker://registry.redhat.io/openshift4/ose-prometheus-node-exporter:v4.12 docker://admin.lab.redhat.com:5000/openshift4/ose-prometheus-node-exporter:v4.12 podman run -v /opt/registry/certs:/certs:Z -v /opt/registry/certs/domain.cert:/certs/domain.cert:Z --rm registry.redhat.io/rhel9/skopeo skopeo copy --remove-signatures --src-creds myusername:mypassword1 --dest-cert-dir=./certs/ --dest-creds myregistryusername:myregistrypassword1 docker://registry.redhat.io/rhceph/grafana-rhel9:latest docker://admin.lab.redhat.com:5000/rhceph/grafana-rhel9:latest podman run -v /opt/registry/certs:/certs:Z -v /opt/registry/certs/domain.cert:/certs/domain.cert:Z --rm registry.redhat.io/rhel9/skopeo skopeo copy --remove-signatures --src-creds myusername:mypassword1 --dest-cert-dir=./certs/ --dest-creds myregistryusername:myregistrypassword1 docker://registry.redhat.io/openshift4/ose-prometheus:v4.12 docker://admin.lab.redhat.com:5000/openshift4/ose-prometheus:v4.12 podman run -v /opt/registry/certs:/certs:Z -v /opt/registry/certs/domain.cert:/certs/domain.cert:Z --rm registry.redhat.io/rhel9/skopeo skopeo copy --remove-signatures --src-creds myusername:mypassword1 --dest-cert-dir=./certs/ --dest-creds myregistryusername:myregistrypassword1 docker://registry.redhat.io/openshift4/ose-prometheus-alertmanager:v4.12 docker://admin.lab.redhat.com:5000/openshift4/ose-prometheus-alertmanager:v4.12
curl -u PRIVATE_REGISTRY_USERNAME : PRIVATE_REGISTRY_PASSWORD https:// LOCAL_NODE_FQDN :5000/v2/_catalog
curl -u myregistryusername:myregistrypassword1 https://admin.lab.redhat.com:5000/v2/_catalog {"repositories":["openshift4/ose-prometheus","openshift4/ose-prometheus-alertmanager","openshift4/ose-prometheus-node-exporter","rhceph/rhceph-8-dashboard-rhel9","rhceph/rhceph-8-rhel9"]}
host02 host03 host04 [admin] host01
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=custom" -e "custom_repo_url= CUSTOM_REPO_URL "
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=custom" -e "custom_repo_url=http://mycustomrepo.lab.redhat.com/x86_64/os/"
ansible-playbook -vvv -i INVENTORY_HOST_FILE_ cephadm-set-container-insecure-registries.yml -e insecure_registry= REGISTRY_URL
ansible-playbook -vvv -i hosts cephadm-set-container-insecure-registries.yml -e insecure_registry=host01:5050
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=custom" -e "custom_repo_url= CUSTOM_REPO_URL " --limit GROUP_NAME | NODE_NAME
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=custom" -e "custom_repo_url=http://mycustomrepo.lab.redhat.com/x86_64/os/" --limit clients [ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=custom" -e "custom_repo_url=http://mycustomrepo.lab.redhat.com/x86_64/os/" --limit host02
cephadm --image PRIVATE_REGISTRY_NODE_FQDN :5000/ CUSTOM_IMAGE_NAME : IMAGE_TAG bootstrap --mon-ip IP_ADDRESS --registry-url PRIVATE_REGISTRY_NODE_FQDN :5000 --registry-username PRIVATE_REGISTRY_USERNAME --registry-password PRIVATE_REGISTRY_PASSWORD
cephadm --image admin.lab.redhat.com:5000/rhceph-8-rhel9:latest bootstrap --mon-ip 10.10.128.68 --registry-url admin.lab.redhat.com:5000 --registry-username myregistryusername --registry-password myregistrypassword1
Ceph Dashboard is now available at: URL: https://host01:8443/ User: admin Password: i8nhu7zham Enabling client.admin keyring and conf on hosts with "admin" label You can access the Ceph CLI with: sudo /usr/sbin/cephadm shell --fsid 266ee7a8-2a05-11eb-b846-5254002d4916 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring Please consider enabling telemetry to help improve Ceph: ceph telemetry on For more information see: https://docs.ceph.com/docs/master/mgr/telemetry/ Bootstrap complete.
ceph cephadm registry-login --registry-url CUSTOM_REGISTRY_NAME --registry_username REGISTRY_USERNAME --registry_password REGISTRY_PASSWORD
ceph cephadm registry-login --registry-url myregistry --registry_username myregistryusername --registry_password myregistrypassword1
ceph config set mgr mgr/cephadm/ OPTION_NAME CUSTOM_REGISTRY_NAME / CONTAINER_NAME
container_image_prometheus container_image_grafana container_image_alertmanager container_image_node_exporter
ceph config set mgr mgr/cephadm/container_image_prometheus myregistry/mycontainer ceph config set mgr mgr/cephadm/container_image_grafana myregistry/mycontainer ceph config set mgr mgr/cephadm/container_image_alertmanager myregistry/mycontainer ceph config set mgr mgr/cephadm/container_image_node_exporter myregistry/mycontainer
ceph orch redeploy node-exporter
ceph config rm mgr mgr/cephadm/ OPTION_NAME
ceph config rm mgr mgr/cephadm/container_image_prometheus
[ansible@admin ~]USD cd /usr/share/cephadm-ansible
ansible-playbook -i INVENTORY_HOST_FILE cephadm-distribute-ssh-key.yml -e cephadm_ssh_user= USER_NAME -e cephadm_pubkey_path= home/cephadm/ceph.key -e admin_node= ADMIN_NODE_NAME_1
[ansible@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-distribute-ssh-key.yml -e cephadm_ssh_user=ceph-admin -e cephadm_pubkey_path=/home/cephadm/ceph.key -e admin_node=host01 [ansible@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-distribute-ssh-key.yml -e cephadm_ssh_user=ceph-admin -e admin_node=host01
cephadm shell ceph -s
cephadm shell ceph -s
exit
podman ps
cephadm shell ceph -s cluster: id: f64f341c-655d-11eb-8778-fa163e914bcc health: HEALTH_OK services: mon: 3 daemons, quorum host01,host02,host03 (age 94m) mgr: host01.lbnhug(active, since 59m), standbys: host02.rofgay, host03.ohipra mds: 1/1 daemons up, 1 standby osd: 18 osds: 18 up (since 10m), 18 in (since 10m) rgw: 4 daemons active (2 hosts, 1 zones) data: volumes: 1/1 healthy pools: 8 pools, 225 pgs objects: 230 objects, 9.9 KiB usage: 271 MiB used, 269 GiB / 270 GiB avail pgs: 225 active+clean io: client: 85 B/s rd, 0 op/s rd, 0 op/s wr
.Syntax [source,subs="verbatim,quotes"] ---- ceph cephadm registry-login --registry-url _CUSTOM_REGISTRY_NAME_ --registry_username _REGISTRY_USERNAME_ --registry_password _REGISTRY_PASSWORD_ ----
.Example ---- ceph cephadm registry-login --registry-url myregistry --registry_username myregistryusername --registry_password myregistrypassword1 ----
ssh-copy-id -f -i /etc/ceph/ceph.pub user@ NEWHOST
ssh-copy-id -f -i /etc/ceph/ceph.pub root@host02 ssh-copy-id -f -i /etc/ceph/ceph.pub root@host03
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
[ceph-admin@admin ~]USD cat hosts host02 host03 host04 [admin] host01
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit NEWHOST
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit host02
ceph orch host add NEWHOST
ceph orch host add host02 Added host 'host02' with addr '10.10.128.69' ceph orch host add host03 Added host 'host03' with addr '10.10.128.70'
ceph orch host add HOSTNAME IP_ADDRESS
ceph orch host add host02 10.10.128.69 Added host 'host02' with addr '10.10.128.69'
ceph orch host ls
ceph orch host add HOSTNAME IP_ADDR
ceph orch host add host01 10.10.128.68
ceph orch host set-addr HOSTNAME IP_ADDR
ceph orch host set-addr HOSTNAME IPV4_ADDRESS
service_type: host addr: hostname: host02 labels: - mon - osd - mgr --- service_type: host addr: hostname: host03 labels: - mon - osd - mgr --- service_type: host addr: hostname: host04 labels: - mon - osd
ceph orch apply -i hosts.yaml Added host 'host02' with addr '10.10.128.69' Added host 'host03' with addr '10.10.128.70' Added host 'host04' with addr '10.10.128.71'
cephadm shell --mount hosts.yaml -- ceph orch apply -i /mnt/hosts.yaml
ceph orch host ls HOST ADDR LABELS STATUS host02 host02 mon osd mgr host03 host03 mon osd mgr host04 host04 mon osd
cephadm shell
ceph orch host add HOST_NAME HOST_ADDRESS
ceph orch host add host03 10.10.128.70
cephadm shell
ceph orch host ls
ceph orch host drain HOSTNAME
ceph orch host drain host02
ceph orch osd rm status
ceph orch ps HOSTNAME
ceph orch ps host02
ceph orch host rm HOSTNAME
ceph orch host rm host02
cephadm shell
ceph orch host label add HOSTNAME LABEL
ceph orch host label add host02 mon
ceph orch host ls
cephadm shell
ceph orch host label rm HOSTNAME LABEL
ceph orch host label rm host02 mon
ceph orch host ls
cephadm shell
ceph orch host ls HOST ADDR LABELS STATUS host01 _admin mon osd mgr host02 mon osd mgr mylabel
ceph orch apply DAEMON --placement="label: LABEL "
ceph orch apply prometheus --placement="label:mylabel"
vi placement.yml
service_type: prometheus placement: label: "mylabel"
ceph orch apply -i FILENAME
ceph orch apply -i placement.yml Scheduled prometheus update...
ceph orch ps --daemon_type= DAEMON_NAME
ceph orch ps --daemon_type=prometheus NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID prometheus.host02 host02 *:9095 running (2h) 8m ago 2h 85.3M - 2.22.2 ac25aac5d567 ad8c7593d7c0
ceph orch apply mon 5
ceph orch apply mon --unmanaged
ceph orch host label add HOSTNAME mon
ceph orch host label add host01 mon
ceph orch host ls
ceph orch host label add host02 mon ceph orch host label add host03 mon ceph orch host ls HOST ADDR LABELS STATUS host01 mon host02 mon host03 mon host04 host05 host06
ceph orch apply mon label:mon
ceph orch apply mon HOSTNAME1 , HOSTNAME2 , HOSTNAME3
ceph orch apply mon host01,host02,host03
[ceph-admin@admin cephadm-ansible]USD ceph cephadm generate-key
[ceph-admin@admin cephadm-ansible]USD ceph cephadm get-pub-key
[ceph-admin@admin cephadm-ansible]USDceph cephadm clear-key
[ceph-admin@admin cephadm-ansible]USD ceph mgr fail
[ceph-admin@admin cephadm-ansible]USD ceph cephadm set-user <user>
[ceph-admin@admin cephadm-ansible]USD ceph cephadm set-user user
ceph cephadm get-pub-key > ~/ceph.pub
[ceph-admin@admin cephadm-ansible]USD ceph cephadm get-pub-key > ~/ceph.pub
ssh-copy-id -f -i ~/ceph.pub USER @ HOST
[ceph-admin@admin cephadm-ansible]USD ssh-copy-id ceph-admin@host01
ceph orch host ls HOST ADDR LABELS STATUS host01 mon,mgr,_admin host02 mon host03 mon,mgr host04 host05 host06
ceph orch host label add HOSTNAME _admin
ceph orch host label add host03 _admin
ceph orch host ls HOST ADDR LABELS STATUS host01 mon,mgr,_admin host02 mon host03 mon,mgr,_admin host04 host05 host06
ceph orch host label add HOSTNAME mon
ceph orch host label add host02 mon ceph orch host label add host03 mon
ceph orch host ls
ceph orch host ls HOST ADDR LABELS STATUS host01 mon,mgr,_admin host02 mon host03 mon host04 host05 host06
ceph orch apply mon label:mon
ceph orch apply mon HOSTNAME1 , HOSTNAME2 , HOSTNAME3
ceph orch apply mon host01,host02,host03
ceph orch apply mon NODE:IP_ADDRESS_OR_NETWORK_NAME [ NODE:IP_ADDRESS_OR_NETWORK_NAME ...]
ceph orch apply mon host02:10.10.128.69 host03:mynetwork
ceph orch apply mgr NUMBER_OF_DAEMONS
ceph orch apply mgr 3
ceph orch apply mgr --placement " HOSTNAME1 HOSTNAME2 HOSTNAME3 "
ceph orch apply mgr --placement "host02 host03 host04"
ceph orch device ls [--hostname= HOSTNAME1 HOSTNAME2 ] [--wide] [--refresh]
ceph orch device ls --wide --refresh
ceph orch daemon add osd HOSTNAME : DEVICE_PATH
ceph orch daemon add osd host02:/dev/sdb
ceph orch apply osd --all-available-devices
ansible-playbook -i hosts cephadm-clients.yml -extra-vars '{"fsid":" FSID ", "client_group":" ANSIBLE_GROUP_NAME ", "keyring":" PATH_TO_KEYRING ", "conf":" CONFIG_FILE "}'
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-clients.yml --extra-vars '{"fsid":"be3ca2b2-27db-11ec-892b-005056833d58","client_group":"fs_clients","keyring":"/etc/ceph/fs.keyring", "conf": "/etc/ceph/ceph.conf"}'
ceph mgr module disable cephadm
ceph fsid
exit
cephadm rm-cluster --force --zap-osds --fsid FSID
cephadm rm-cluster --force --zap-osds --fsid a6ca415a-cde7-11eb-a41a-002590fc2544
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
host02 host03 host04 [admin] host01 [clients] client01 client02 client03
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --limit CLIENT_GROUP_NAME | CLIENT_NODE_NAME
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --limit clients
ansible-playbook -i INVENTORY_FILE cephadm-clients.yml --extra-vars '{"fsid":" FSID ","keyring":" KEYRING_PATH ","client_group":" CLIENT_GROUP_NAME ","conf":" CEPH_CONFIGURATION_PATH ","keyring_dest":" KEYRING_DESTINATION_PATH "}'
[ceph-admin@host01 cephadm-ansible]USD ansible-playbook -i hosts cephadm-clients.yml --extra-vars '{"fsid":"266ee7a8-2a05-11eb-b846-5254002d4916","keyring":"/etc/ceph/ceph.client.admin.keyring","client_group":"clients","conf":"/etc/ceph/ceph.conf","keyring_dest":"/etc/ceph/custom.name.ceph.keyring"}'
ansible-playbook -i INVENTORY_FILE cephadm-clients.yml --extra-vars '{"fsid":" FSID ","keyring":" KEYRING_PATH ","conf":" CONF_PATH "}'
ls -l /etc/ceph/ -rw-------. 1 ceph ceph 151 Jul 11 12:23 custom.name.ceph.keyring -rw-------. 1 ceph ceph 151 Jul 11 12:23 ceph.keyring -rw-------. 1 ceph ceph 269 Jul 11 12:23 ceph.conf
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi INVENTORY_FILE HOST1 labels="[' LABEL1 ', ' LABEL2 ']" HOST2 labels="[' LABEL1 ', ' LABEL2 ']" HOST3 labels="[' LABEL1 ']" [admin] ADMIN_HOST monitor_address= MONITOR_IP_ADDRESS labels="[' ADMIN_LABEL ', ' LABEL1 ', ' LABEL2 ']"
[ceph-admin@admin cephadm-ansible]USD sudo vi hosts host02 labels="['mon', 'mgr']" host03 labels="['mon', 'mgr']" host04 labels="['osd']" host05 labels="['osd']" host06 labels="['osd']" [admin] host01 monitor_address=10.10.128.68 labels="['_admin', 'mon', 'mgr']"
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs"
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs"
sudo vi PLAYBOOK_FILENAME .yml --- - name: NAME_OF_PLAY hosts: BOOTSTRAP_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: -name: NAME_OF_TASK cephadm_registry_login: state: STATE registry_url: REGISTRY_URL registry_username: REGISTRY_USER_NAME registry_password: REGISTRY_PASSWORD - name: NAME_OF_TASK cephadm_bootstrap: mon_ip: "{{ monitor_address }}" dashboard_user: DASHBOARD_USER dashboard_password: DASHBOARD_PASSWORD allow_fqdn_hostname: ALLOW_FQDN_HOSTNAME cluster_network: NETWORK_CIDR
[ceph-admin@admin cephadm-ansible]USD sudo vi bootstrap.yml --- - name: bootstrap the cluster hosts: host01 become: true gather_facts: false tasks: - name: login to registry cephadm_registry_login: state: login registry_url: registry.redhat.io registry_username: user1 registry_password: mypassword1 - name: bootstrap initial cluster cephadm_bootstrap: mon_ip: "{{ monitor_address }}" dashboard_user: mydashboarduser dashboard_password: mydashboardpassword allow_fqdn_hostname: true cluster_network: 10.10.128.0/28
ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml -vvv
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts bootstrap.yml -vvv
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi INVENTORY_FILE NEW_HOST1 labels="[' LABEL1 ', ' LABEL2 ']" NEW_HOST2 labels="[' LABEL1 ', ' LABEL2 ']" NEW_HOST3 labels="[' LABEL1 ']" [admin] ADMIN_HOST monitor_address= MONITOR_IP_ADDRESS labels="[' ADMIN_LABEL ', ' LABEL1 ', ' LABEL2 ']"
[ceph-admin@admin cephadm-ansible]USD sudo vi hosts host02 labels="['mon', 'mgr']" host03 labels="['mon', 'mgr']" host04 labels="['osd']" host05 labels="['osd']" host06 labels="['osd']" [admin] host01 monitor_address= 10.10.128.68 labels="['_admin', 'mon', 'mgr']"
ansible-playbook -i INVENTORY_FILE cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit NEWHOST
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts cephadm-preflight.yml --extra-vars "ceph_origin=rhcs" --limit host02
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: HOSTS_OR_HOST_GROUPS become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_host: name: "{{ ansible_facts['hostname'] }}" address: "{{ ansible_facts['default_ipv4']['address'] }}" labels: "{{ labels }}" delegate_to: HOST_TO_DELEGATE_TASK_TO - name: NAME_OF_TASK when: inventory_hostname in groups['admin'] ansible.builtin.shell: cmd: CEPH_COMMAND_TO_RUN register: REGISTER_NAME - name: NAME_OF_TASK when: inventory_hostname in groups['admin'] debug: msg: "{{ REGISTER_NAME .stdout }}"
[ceph-admin@admin cephadm-ansible]USD sudo vi add-hosts.yml --- - name: add additional hosts to the cluster hosts: all become: true gather_facts: true tasks: - name: add hosts to the cluster ceph_orch_host: name: "{{ ansible_facts['hostname'] }}" address: "{{ ansible_facts['default_ipv4']['address'] }}" labels: "{{ labels }}" delegate_to: host01 - name: list hosts in the cluster when: inventory_hostname in groups['admin'] ansible.builtin.shell: cmd: ceph orch host ls register: host_list - name: print current list of hosts when: inventory_hostname in groups['admin'] debug: msg: "{{ host_list.stdout }}"
ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts add-hosts.yml
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: NAME_OF_PLAY hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_host: name: HOST_TO_REMOVE state: STATE - name: NAME_OF_TASK ceph_orch_host: name: HOST_TO_REMOVE state: STATE retries: NUMBER_OF_RETRIES delay: DELAY until: CONTINUE_UNTIL register: REGISTER_NAME - name: NAME_OF_TASK ansible.builtin.shell: cmd: ceph orch host ls register: REGISTER_NAME - name: NAME_OF_TASK debug: msg: "{{ REGISTER_NAME .stdout }}"
[ceph-admin@admin cephadm-ansible]USD sudo vi remove-hosts.yml --- - name: remove host hosts: host01 become: true gather_facts: true tasks: - name: drain host07 ceph_orch_host: name: host07 state: drain - name: remove host from the cluster ceph_orch_host: name: host07 state: absent retries: 20 delay: 1 until: result is succeeded register: result - name: list hosts in the cluster ansible.builtin.shell: cmd: ceph orch host ls register: host_list - name: print current list of hosts debug: msg: "{{ host_list.stdout }}"
ansible-playbook -i INVENTORY_FILE PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts remove-hosts.yml
TASK [print current hosts] ****************************************************************************************************** Friday 24 June 2022 14:52:40 -0400 (0:00:03.365) 0:02:31.702 *********** ok: [host01] => msg: |- HOST ADDR LABELS STATUS host01 10.10.128.68 _admin mon mgr host02 10.10.128.69 mon mgr host03 10.10.128.70 mon mgr host04 10.10.128.71 osd host05 10.10.128.72 osd host06 10.10.128.73 osd
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_config: action: GET_OR_SET who: DAEMON_TO_SET_CONFIGURATION_TO option: CEPH_CONFIGURATION_OPTION value: VALUE_OF_PARAMETER_TO_SET - name: NAME_OF_TASK ceph_config: action: GET_OR_SET who: DAEMON_TO_SET_CONFIGURATION_TO option: CEPH_CONFIGURATION_OPTION register: REGISTER_NAME - name: NAME_OF_TASK debug: msg: " MESSAGE_TO_DISPLAY {{ REGISTER_NAME .stdout }}"
[ceph-admin@admin cephadm-ansible]USD sudo vi change_configuration.yml --- - name: set pool delete hosts: host01 become: true gather_facts: false tasks: - name: set the allow pool delete option ceph_config: action: set who: mon option: mon_allow_pool_delete value: true - name: get the allow pool delete setting ceph_config: action: get who: mon option: mon_allow_pool_delete register: verify_mon_allow_pool_delete - name: print current mon_allow_pool_delete setting debug: msg: "the value of 'mon_allow_pool_delete' is {{ verify_mon_allow_pool_delete.stdout }}"
ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts change_configuration.yml
TASK [print current mon_allow_pool_delete setting] ************************************************************* Wednesday 29 June 2022 13:51:41 -0400 (0:00:05.523) 0:00:17.953 ******** ok: [host01] => msg: the value of 'mon_allow_pool_delete' is true
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: HOSTS_OR_HOST_GROUPS become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_apply: spec: | service_type: SERVICE_TYPE service_id: UNIQUE_NAME_OF_SERVICE placement: host_pattern: ' HOST_PATTERN_TO_SELECT_HOSTS ' label: LABEL spec: SPECIFICATION_OPTIONS :
[ceph-admin@admin cephadm-ansible]USD sudo vi deploy_osd_service.yml --- - name: deploy osd service hosts: host01 become: true gather_facts: true tasks: - name: apply osd spec ceph_orch_apply: spec: | service_type: osd service_id: osd placement: host_pattern: '*' label: osd spec: data_devices: all: true
ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts deploy_osd_service.yml
[ceph-admin@admin ~]USD cd /usr/share/cephadm-ansible
sudo vi PLAYBOOK_FILENAME .yml --- - name: PLAY_NAME hosts: ADMIN_HOST become: USE_ELEVATED_PRIVILEGES gather_facts: GATHER_FACTS_ABOUT_REMOTE_HOSTS tasks: - name: NAME_OF_TASK ceph_orch_daemon: state: STATE_OF_SERVICE daemon_id: DAEMON_ID daemon_type: TYPE_OF_SERVICE
[ceph-admin@admin cephadm-ansible]USD sudo vi restart_services.yml --- - name: start and stop services hosts: host01 become: true gather_facts: false tasks: - name: start osd.0 ceph_orch_daemon: state: started daemon_id: 0 daemon_type: osd - name: stop mon.host02 ceph_orch_daemon: state: stopped daemon_id: host02 daemon_type: mon
ansible-playbook -i INVENTORY_FILE _PLAYBOOK_FILENAME .yml
[ceph-admin@admin cephadm-ansible]USD ansible-playbook -i hosts restart_services.yml
cephadm adopt [-h] --name DAEMON_NAME --style STYLE [--cluster CLUSTER ] --legacy-dir [ LEGACY_DIR ] --config-json CONFIG_JSON ] [--skip-firewalld] [--skip-pull]
cephadm adopt --style=legacy --name prometheus.host02
cephadm ceph-volume inventory/simple/raw/lvm [-h] [--fsid FSID ] [--config-json CONFIG_JSON ] [--config CONFIG , -c CONFIG ] [--keyring KEYRING , -k KEYRING ]
cephadm ceph-volume inventory --fsid f64f341c-655d-11eb-8778-fa163e914bcc
cephadm check-host [--expect-hostname HOSTNAME ]
cephadm check-host --expect-hostname host02
cephadm shell deploy DAEMON_TYPE [-h] [--name DAEMON_NAME ] [--fsid FSID ] [--config CONFIG , -c CONFIG ] [--config-json CONFIG_JSON ] [--keyring KEYRING ] [--key KEY ] [--osd-fsid OSD_FSID ] [--skip-firewalld] [--tcp-ports TCP_PORTS ] [--reconfig] [--allow-ptrace] [--memory-request MEMORY_REQUEST ] [--memory-limit MEMORY_LIMIT ] [--meta-json META_JSON ]
cephadm shell deploy mon --fsid f64f341c-655d-11eb-8778-fa163e914bcc
cephadm enter [-h] [--fsid FSID ] --name NAME [command [command ...]]
cephadm enter --name 52c611f2b1d9
cephadm help
cephadm help
cephadm install PACKAGES
cephadm install ceph-common ceph-osd
cephadm --image IMAGE_ID inspect-image
cephadm --image 13ea90216d0be03003d12d7869f72ad9de5cec9e54a27fd308e01e467c0d4a0a inspect-image
cephadm list-networks
cephadm list-networks
cephadm ls [--no-detail] [--legacy-dir LEGACY_DIR ]
cephadm ls --no-detail
cephadm logs [--fsid FSID ] --name DAEMON_NAME cephadm logs [--fsid FSID ] --name DAEMON_NAME -- -n NUMBER # Last N lines cephadm logs [--fsid FSID ] --name DAEMON_NAME -- -f # Follow the logs
cephadm logs --fsid 57bddb48-ee04-11eb-9962-001a4a000672 --name osd.8 cephadm logs --fsid 57bddb48-ee04-11eb-9962-001a4a000672 --name osd.8 -- -n 20 cephadm logs --fsid 57bddb48-ee04-11eb-9962-001a4a000672 --name osd.8 -- -f
cephadm prepare-host [--expect-hostname HOSTNAME ]
cephadm prepare-host cephadm prepare-host --expect-hostname host01
cephadm [-h] [--image IMAGE_ID ] pull
cephadm --image 13ea90216d0be03003d12d7869f72ad9de5cec9e54a27fd308e01e467c0d4a0a pull
cephadm registry-login --registry-url [ REGISTRY_URL ] --registry-username [ USERNAME ] --registry-password [ PASSWORD ] [--fsid FSID ] [--registry-json JSON_FILE ]
cephadm registry-login --registry-url registry.redhat.io --registry-username myuser1 --registry-password mypassword1
cat REGISTRY_FILE { "url":" REGISTRY_URL ", "username":" REGISTRY_USERNAME ", "password":" REGISTRY_PASSWORD " }
cat registry_file { "url":"registry.redhat.io", "username":"myuser", "password":"mypass" } cephadm registry-login -i registry_file
cephadm rm-daemon [--fsid FSID ] [--name DAEMON_NAME ] [--force ] [--force-delete-data]
cephadm rm-daemon --fsid f64f341c-655d-11eb-8778-fa163e914bcc --name osd.8
cephadm rm-cluster [--fsid FSID ] [--force]
cephadm rm-cluster --fsid f64f341c-655d-11eb-8778-fa163e914bcc
ceph mgr module disable cephadm
cephadm rm-repo [-h]
cephadm rm-repo
cephadm run [--fsid FSID ] --name DAEMON_NAME
cephadm run --fsid f64f341c-655d-11eb-8778-fa163e914bcc --name osd.8
cephadm shell [--fsid FSID ] [--name DAEMON_NAME , -n DAEMON_NAME ] [--config CONFIG , -c CONFIG ] [--mount MOUNT , -m MOUNT ] [--keyring KEYRING , -k KEYRING ] [--env ENV , -e ENV ]
cephadm shell -- ceph orch ls cephadm shell
cephadm unit [--fsid FSID ] --name DAEMON_NAME start/stop/restart/enable/disable
cephadm unit --fsid f64f341c-655d-11eb-8778-fa163e914bcc --name osd.8 start
cephadm version
cephadm version