Change:
The fix rolled out in lacks the third patch () described in . Without that patch it is still not possible to use ceph-volume with a dmcrypt'd volume. Steps to reproduce the behavior: on a bluestore OSD using LUKS Volume is activated. As there is still a stack trace that can be fixed by applying the above (and in the other issue thread) mentioned patch. Please run and paste the result. Add a :+1: [reaction] to [issues you find important]. [reaction]: [issues you find important]:
Is there any chance you could add a NixOS VM test that exercises cryptsetup/dmcrypt? Then your setup would be guarded by that for any future NixOS changes.
That's on my todo list for when I have enough spoons, but it is there and I will probably get to it within the next few weeks.

Discussion:
ceph-multi-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-multi-node.nix {}; ceph-single-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node.nix {}; ceph-single-node-bluestore = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node-bluestore.nix {}; <ins> ceph-single-node-bluestore-dmcrypt = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node-bluestore-dmcrypt.nix {}; </ins> certmgr = handleTest ./certmgr.nix {}; cfssl = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cfssl.nix {}; cgit = handleTest ./cgit.nix {};

<ins> import ./make-test-python.nix ( { pkgs, lib, ... }: let # the single node ipv6 address ip = "2001:db8:ffff::"; # the global ceph cluster id cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a"; # the fsids of OSDs osd-fsid-map = { "0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa"; "1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c"; "2" = "c90de6c7-86c6-41da-9694-e794096dfc5c"; }; in { name = "basic-single-node-ceph-cluster-bluestore-dmcrypt"; meta = with pkgs.lib.maintainers; { maintainers = [ benaryorg nh2 ]; }; nodes = { ceph = { pkgs, config, ... }: { # disks for bluestore virtualisation.emptyDiskImages = [ 20480 20480 20480 ]; # networking setup (no external connectivity required, only local IPv6) networking.useDHCP = false; systemd.network = { enable = true; wait-online.extraArgs = [ "-i" "lo" ]; networks = { "40-loopback" = { enable = true; name = "lo"; DHCP = "no"; addresses = [ { Address = "${ip}/128"; } ]; }; }; }; # do not start the ceph target by default so we can format the disks first systemd.targets.ceph.wantedBy = lib.mkForce [ ]; # add the packages to systemPackages so the testscript doesn't run into any unexpected issues # this shouldn't be required on production systems which have their required packages in the unit paths only # but it helps in case one needs to actually run the tooling anyway environment.systemPackages = with pkgs; [ ceph cryptsetup lvm2 ]; services.ceph = { enable = true; client.enable = true; extraConfig = { public_addr = ip; cluster_addr = ip; # ipv6 ms_bind_ipv4 = "false"; ms_bind_ipv6 = "true"; # msgr2 settings ms_cluster_mode = "secure"; ms_service_mode = "secure"; ms_client_mode = "secure"; ms_mon_cluster_mode = "secure"; ms_mon_service_mode = "secure"; ms_mon_client_mode = "secure"; # less default modules, cuts down on memory and startup time in the tests mgr_initial_modules = ""; # distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host osd_crush_chooseleaf_type = "0"; }; client.extraConfig."mon.0" = { host = "ceph"; mon_addr = "v2:[${ip}]:3300"; public_addr = "v2:[${ip}]:3300"; }; global = { fsid = cluster; clusterNetwork = "${ip}/64"; publicNetwork = "${ip}/64"; monInitialMembers = "0"; }; mon = { enable = true; daemons = [ "0" ]; }; osd = { enable = true; daemons = builtins.attrNames osd-fsid-map; }; mgr = { enable = true; daemons = [ "ceph" ]; }; }; systemd.services = let osd-name = id: "ceph-osd-${id}"; osd-pre-start = id: [ "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd" "${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}" ]; osd-post-stop = id: [ "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}" ]; map-osd = id: { name = osd-name id; value = { serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id); serviceConfig.ExecStopPost = osd-post-stop id; unitConfig.ConditionPathExists = lib.mkForce [ ]; unitConfig.StartLimitBurst = lib.mkForce 4; path = with pkgs; [ util-linux lvm2 cryptsetup ]; }; }; in lib.pipe config.services.ceph.osd.daemons [ (builtins.map map-osd) builtins.listToAttrs ]; }; }; testScript = { ... }: '' start_all() ceph.wait_for_unit("default.target") # Bootstrap ceph-mon daemon ceph.succeed( "mkdir -p /var/lib/ceph/bootstrap-osd", "ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'", "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'", "ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'", "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring", "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring", "monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap", "mkdir -p /var/lib/ceph/mon/ceph-0", "ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring", "chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph", "systemctl start ceph-mon-0.service", ) ceph.wait_for_unit("ceph-mon-0.service") # should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout ceph.wait_for_open_port(3300, "${ip}") ceph.succeed( # required for HEALTH_OK "ceph config set mon auth_allow_insecure_global_id_reclaim false", # IPv6 "ceph config set global ms_bind_ipv4 false", "ceph config set global ms_bind_ipv6 true", # the new (secure) protocol "ceph config set global ms_bind_msgr1 false", "ceph config set global ms_bind_msgr2 true", # just a small little thing "ceph config set mon mon_compact_on_start true", ) # Can't check ceph status until a mon is up ceph.succeed("ceph -s | grep 'mon: 1 daemons'") # Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory) ceph.succeed( # this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases "ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}", "ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}", "ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}", "sudo ceph-volume lvm deactivate 0", "sudo ceph-volume lvm deactivate 1", "sudo ceph-volume lvm deactivate 2", "chown -R ceph:ceph /var/lib/ceph", ) # Start OSDs (again, argon2id eats memory, so this happens before starting the mgr) ceph.succeed( "systemctl start ceph-osd-0.service", "systemctl start ceph-osd-1.service", "systemctl start ceph-osd-2.service", ) ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'") ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'") # Start the ceph-mgr daemon, after copying in the keyring ceph.succeed( "mkdir -p /var/lib/ceph/mgr/ceph-ceph/", "ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow *' mds 'allow *'", "chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/", "systemctl start ceph-mgr-ceph.service", ) ceph.wait_for_unit("ceph-mgr-ceph") ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'") ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'") ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'") ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") # test the actual storage ceph.succeed( "ceph osd pool create single-node-test 32 32", "ceph osd pool ls | grep 'single-node-test'", # We need to enable an application on the pool, otherwise it will # stay unhealthy in state POOL_APP_NOT_ENABLED. # Creating a CephFS would do this automatically, but we haven't done that here. # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application # We use the custom application name "nixos-test" for this. "ceph osd pool application enable single-node-test nixos-test", "ceph osd pool rename single-node-test single-node-other-test", "ceph osd pool ls | grep 'single-node-other-test'", ) ceph.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'") ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") ceph.wait_until_succeeds("ceph -s | grep '33 active+clean'") ceph.fail( # the old pool should be gone "ceph osd pool ls | grep 'multi-node-test'", # deleting the pool should fail without setting mon_allow_pool_delete "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it", ) # rebooting gets rid of any potential tmpfs mounts or device-mapper devices ceph.shutdown() ceph.start() ceph.wait_for_unit("default.target") # Start it up (again OSDs first due to memory constraints of cryptsetup and mgr) ceph.systemctl("start ceph-mon-0.service") ceph.wait_for_unit("ceph-mon-0") ceph.systemctl("start ceph-osd-0.service") ceph.wait_for_unit("ceph-osd-0") ceph.systemctl("start ceph-osd-1.service") ceph.wait_for_unit("ceph-osd-1") ceph.systemctl("start ceph-osd-2.service") ceph.wait_for_unit("ceph-osd-2") ceph.systemctl("start ceph-mgr-ceph.service") ceph.wait_for_unit("ceph-mgr-ceph") # Ensure the cluster comes back up again ceph.succeed("ceph -s | grep 'mon: 1 daemons'") ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'") ceph.wait_until_succeeds("ceph osd stat | grep -E '3 osds: 3 up[^,]*, 3 in'") ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'") ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") ''; } ) </ins>

name = "ceph-reef-ceph-volume-fix-set_dmcrypt_no_workqueue.patch"; hash = "sha256-r+7hcCz2WF/rJfgKwTatKY9unJlE8Uw3fmOyaY5jVH0="; }) <ins> (fetchpatch { url = "https://github.com/ceph/ceph/commit/607eb34b2c278566c386efcbf3018629cf08ccfd.patch"; name = "ceph-volume-fix-set_dmcrypt_no_workqueue-regex.patch"; hash = "sha256-q28Q7OIyFoMyMBCPXGA+AdNqp+9/6J/XwD4ODjx+JXY="; }) </ins> ]; postPatch = ''

inherit (nixosTests) ceph-multi-node ceph-single-node <del> ceph-single-node-bluestore; </del> <ins> ceph-single-node-bluestore ceph-single-node-bluestore-dmcrypt; </ins> }; }; };