CEPH

Ceph is a software defined storage platform. There are various methods and tools to deploy Ceph, none of which (to my knowledge) work on Alpine. Ceph packages have been available in the community repository since Alpine 3.10. Generally, the current latest and previous versions are made available.
This how-to, explains how to deploy Ceph monitors, managers and OSDs, via apk, manually.
== Installing MONITORs ==
=== Bootstrapping the first MONITOR ===
In this example we use 3 hosts as monitor and manager roles:
MON_HOST1="mon01"
MON_HOST1_IP="%IP_ADDRESS%"
MON_HOST2="mon02"
MON_HOST2_IP="%IP_ADDRESS%"
MON_HOST3="mon03"
MON_HOST3_IP="%IP_ADDRESS%"
FSID=$(cat /proc/sys/kernel/random/uuid) #store this FSID
echo $FSID
CLUSTER_NAME="ceph" # default value if unspecified
HOSTNAME=$(cat /etc/hostname)
PUBLIC_NETWORK="%NETWORK_CIDR%"
MYNET="%NETWORK_CIDR%"
VERSION=17
cat << EOF > /etc/ceph/$CLUSTER_NAME.conf 
[global]
# Cluster unique identifier
fsid = $FSID
mon_initial_members = $MON_HOST1
mon_host = $MON_HOST1_IP, $MON_HOST2_IP, $MON_HOST3_IP
mon_allow_pool_delete = true
ms_bind_ipv4 = false # change as needed
ms_bind_ipv6 = true # change as needed
public_network = $PUBLIC_NETWORK
# Enable authentication
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
# https://docs.ceph.com/en/latest/rados/configuration/pool-pg-config-ref/#pool-pg-and-crush-config-reference
osd_pool_default_size = 3 # Write an object three times
osd_pool_default_min_size = 2 # Accept an I/O operation to a degraded PG that has two copies of an object
osd_pool_default_pg_num = 128 # total number of OSDs * 100 / osd_pool_default_size. Use nearest power of two.
osd_crush_chooseleaf_type = 1
rgw_data = /var/lib/ceph/radosgw/\$cluster-\$id # literal variables
[mon]
mon_data = /var/lib/ceph/mon/\$cluster-$FSID # "cluster" is a literal variable
EOF
ceph-authtool --create-keyring /tmp/$CLUSTER_NAME.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/$CLUSTER_NAME.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/$CLUSTER_NAME.client.admin.keyring
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring
chown ceph:ceph /tmp/$CLUSTER_NAME.mon.keyring
monmaptool --create --add $MON_HOST1 $MON_HOST1_IP --add $MON_HOST2 $MON_HOST2_IP --add $MON_HOST3 $MON_HOST3_IP --fsid $FSID /tmp/monmap
install -d -o ceph /var/lib/ceph/mon/$CLUSTER_NAME-$FSID
sudo -u ceph ceph-mon --cluster $CLUSTER_NAME --mkfs -i $HOSTNAME --inject-monmap /tmp/monmap --keyring /tmp/$CLUSTER_NAME.mon.keyring
Create AWall policies
cat << EOF > /etc/awall/optional/ceph-mon.json
{
 "description": "Ceph cluster monitor component",
 "service": {
 "ceph-mon": { "proto": "tcp", "port": [ 3300, 6789 ] }
 },
 "filter": [
 {
 "src": "\$MYNET",
 "out": "_fw",
 "service": "ceph-mon",
 "action": "accept"
 },
 {
 "in": "_fw",
 "dest": "\$MYNET",
 "service": "ceph-mon",
 "action": "accept"
 }
 ]
}
EOF
cat << EOF > /etc/awall/optional/ceph-client-osd.json
{
 "description": "Ceph cluster OSD client",
 
 "service": {
 "ceph-osd": { "proto": "tcp", "port": "6800-7300" }
 },
 "filter": [
 {
 "in": "_fw",
 "dest": "\$MYNET",
 "service": "ceph-osd",
 "action": "accept"
 }
 ]
}
EOF
awall enable ceph-mon
awall enable ceph-client-osd
awall activate -f
ln -s ceph /etc/init.d/ceph-mon.$HOSTNAME
rc-update add ceph-mon.$HOSTNAME
openrc
Inspect that the first node was bootstrapped correctly running 
=== Adding other MONITOR nodes ===
Copy /etc/ceph/ceph.conf, /tmp/ceph.mon.keyring /etc/ceph/ceph.client.admin.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring from existing monitor to the new monitor node.
install -d --owner ceph --group ceph /etc/ceph /var/lib/ceph/bootstrap-osd
install -d -o ceph /var/lib/ceph/mon/$CLUSTER_NAME-$FSID
chown ceph /tmp/$CLUSTER_NAME.mon.keyring
monmaptool --create --add $MON_HOST1 $MON_HOST1_IP --add $MON_HOST2 $MON_HOST2_IP --add $MON_HOST3 $MON_HOST3_IP --fsid $FSID /tmp/monmap
sudo -u ceph ceph-mon --cluster $CLUSTER_NAME --mkfs -i $HOSTNAME --inject-monmap /tmp/monmap --keyring /tmp/$CLUSTER_NAME.mon.keyring
ln -s ceph /etc/init.d/ceph-mon.$HOSTNAME
rc-update add ceph-mon.$HOSTNAME
openrc
If ceph -s returns mon is allowing insecure global_id reclaim fix with 
== Installing MANAGERs ==
install -d --owner ceph --group ceph /var/lib/ceph/mgr/$CLUSTER_NAME-$HOSTNAME
# note this keyring is unique per manager, does not need to be copied across the cluster!
ceph auth get-or-create mgr.$HOSTNAME mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/$CLUSTER_NAME-$HOSTNAME/keyring
cat << EOF > /etc/awall/optional/ceph-mgr.json
{
 "description": "Ceph cluster Manager component",
 
 "service": {
 "ceph-mgr": { "proto": "tcp", "port": "6800-7300" }
 },
 "filter": [
 {
 "src": "\$MYNET",
 "out": "_fw",
 "service": "ceph-mgr",
 "action": "accept"
 }
 ]
}
EOF
awall enable ceph-mgr
awall activate -f
apk add ceph$VERSION-mgr ceph$VERSION-mgr-dashboard
ln -s ceph /etc/init.d/ceph-mgr.$HOSTNAME
rc-update add ceph-mgr.$HOSTNAME
openrc
Expected issues:
Module 'restful' has failed dependency: PyO3 modules may only be initialized once per interpreter process
https://github.com/bazaah/aur-ceph/issues/20
https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/thread/FB7XE6WYDK3EBJYPABSPX5B2LEILWWJA/#FB7XE6WYDK3EBJYPABSPX5B2LEILWWJA
Workaround: ceph mgr module disable restful
== Installing OSDs ==
setup-devd udev #alpine eudev drop-in replacement for udev
apk add ceph$VERSION-osd ceph$VERSION-osd-daemon ceph$VERSION-osd-tools eudev ceph$VERSION-openrc
mkdir -p /var/lib/ceph/bootstrap-osd /etc/ceph
Copy /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring from a monitor node to /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring
Copy /etc/ceph/ceph.conf from a monitor node to /etc/ceph/ceph.conf
cat << EOF > /etc/awall/optional/ceph-client-mon.json
{
 "description": "Ceph cluster monitor client",
 
 "service": {
 "ceph-mon": { "proto": "tcp", "port": [ 3300, 6789 ] }
 },
 "filter": [
 {
 "in": "_fw",
 "dest": "\$MYNET",
 "service": "ceph-mon",
 "action": "accept"
 }
 ]
}
EOF
cat << EOF > /etc/awall/optional/ceph-osd.json
{
 "description": "Ceph cluster OSD component",
 
 "service": {
 "ceph-osd": { "proto": "tcp", "port": "6800-7300" }
 },
 "filter": [
 {
 "src": "\$MYNET",
 "out": "_fw",
 "service": "ceph-osd",
 "action": "accept"
 },
 {
 "in": "_fw",
 "dest": "\$MYNET",
 "service": "ceph-osd",
 "action": "accept"
 }
 ]
}
EOF
awall enable ceph-client-mon
awall enable ceph-osd
awall activate -f
ln -s /var/lib/ceph/bootstrap-osd/ceph.keyring /etc/ceph/ceph.client.bootstrap-osd.keyring #it seems it wants the keyring in this location
ceph-volume lvm create --data /dev/sdX --no-systemd
== See also ==
* Ceph - ArchWiki
* Ceph - Gentoo Wiki
* A Ceph tutorial - YouTube
Category:Filesystems