Change:
The goal of this issue is to design a CephFS driver using FUSE client for Kubernetes.</brExisting Ceph kernel client has the following drawbacks:</br• Kernel client does not support client-enforced quotas. • Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated. • Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's security context will not work.
Just a detail. Since we will rely on ceph-fuse, we will need the ceph-fuse client to be available on all kubernetes nodes. We can add a parameter to allow the use of the kernel client as well. I would say better make the fuse client opt-in and keep the kernel one as the default option. The kernel client might support quotas in the future. Thanks
/sig storage
We are also inclined to use fuse instead of kernel client ,any progress on this issue?
I like to continue this job and prepare to send a PR to make kubernetes support cephfs fuse mounting.
/assign I think this task would be valuable. =)
waiting for the PR, do you have a plane to support s3 fuse ?
No, no plan for s3fs-fuse currently
also see and cc
Ceph-fuse support in the command to display the specified secret, it may take some time to support, because it involves LibFUSE(I will follow up this demand). On the other hand, ceph-fuse has stability, quota function, expansion, upgrade and other advantages. Although ceph-fuse only supports the secret file way, but if it can be pushed to the k8s community, there is a very meaningful thing.

Discussion:
import ( "fmt" "os" <ins> "os/exec" "path" "runtime" </ins> "strings" "github.com/golang/glog"

} os.MkdirAll(dir, 0750) <ins> // check whether it belongs to fuse, if not, default to use kernel mount. if cephfsVolume.checkFuseMount() { glog.V(4).Infof("CephFS fuse mount.") err = cephfsVolume.execFuseMount(dir) // cleanup no matter if fuse mount fail. keyringPath := cephfsVolume.GetKeyringPath() _, StatErr := os.Stat(keyringPath) if !os.IsNotExist(StatErr) { os.RemoveAll(keyringPath) } if err == nil { // cephfs fuse mount succeeded. return nil } else { // if cephfs fuse mount failed, fallback to kernel mount. glog.V(4).Infof("CephFS fuse mount failed: %v, fallback to kernel mount.", err) } } glog.V(4).Infof("CephFS kernel mount.") </ins> err = cephfsVolume.execMount(dir) <del> if err == nil { return nil </del> <ins> if err != nil { // cleanup upon failure. util.UnmountPath(dir, cephfsVolume.mounter) return err </ins> } <del> // cleanup upon failure util.UnmountPath(dir, cephfsVolume.mounter) // return error return err </del> <ins> return nil </ins> } type cephfsUnmounter struct {

return cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedNameForDisk(name), cephfsVolume.volName) } <ins> // GetKeyringPath creates cephfuse keyring path func (cephfsVolume *cephfs) GetKeyringPath() string { name := cephfsPluginName volumeDir := cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedNameForDisk(name), cephfsVolume.volName) volumeKeyringDir := volumeDir + "~keyring" return volumeKeyringDir } </ins> func (cephfsVolume *cephfs) execMount(mountpoint string) error { // cephfs mount option ceph_opt := ""

return nil } <ins> func (cephfsMounter *cephfsMounter) checkFuseMount() bool { execute := cephfsMounter.plugin.host.GetExec(cephfsMounter.plugin.GetPluginName()) switch runtime.GOOS { case "linux": retBytes, err := execute.Run("/bin/ls", "/sbin/mount.fuse.ceph") if err == nil && string(retBytes) == "/sbin/mount.fuse.cephn" { glog.V(4).Infof("/sbin/mount.fuse.ceph exists, it should be fuse mount") return true } return false } return false } func (cephfsVolume *cephfs) execFuseMount(mountpoint string) error { // cephfs keyring file keyring_file := "" // override secretfile if secret is provided if cephfsVolume.secret != "" { // TODO: cephfs fuse currently doesn't support secret option, // remove keyring file create once secret option is supported. glog.V(4).Infof("cephfs mount begin using fuse.") keyringPath := cephfsVolume.GetKeyringPath() os.MkdirAll(keyringPath, 0750) payload := make(map[string]util.FileProjection, 1) var fileProjection util.FileProjection keyring := fmt.Sprintf("[client.%s]n", cephfsVolume.id) + "key = " + cephfsVolume.secret + "n" fileProjection.Data = []byte(keyring) fileProjection.Mode = int32(0644) fileName := cephfsVolume.id + ".keyring" payload[fileName] = fileProjection writerContext := fmt.Sprintf("cephfuse:%v.keyring", cephfsVolume.id) writer, err := util.NewAtomicWriter(keyringPath, writerContext) if err != nil { glog.Errorf("failed to create atomic writer: %v", err) return err } err = writer.Write(payload) if err != nil { glog.Errorf("failed to write payload to dir: %v", err) return err } keyring_file = path.Join(keyringPath, fileName) } else { keyring_file = cephfsVolume.secret_file } // build src like mon1:6789,mon2:6789,mon3:6789:/ hosts := cephfsVolume.mon l := len(hosts) // pass all monitors and let ceph randomize and fail over i := 0 src := "" for i = 0; i < l-1; i++ { src += hosts[i] + "," } src += hosts[i] mountArgs := []string{} mountArgs = append(mountArgs, "-k") mountArgs = append(mountArgs, keyring_file) mountArgs = append(mountArgs, "-m") mountArgs = append(mountArgs, src) mountArgs = append(mountArgs, mountpoint) mountArgs = append(mountArgs, "-r") mountArgs = append(mountArgs, cephfsVolume.path) glog.V(4).Infof("Mounting cmd ceph-fuse with arguments (%s)", mountArgs) command := exec.Command("ceph-fuse", mountArgs...) output, err := command.CombinedOutput() if err != nil || !(strings.Contains(string(output), "starting fuse")) { return fmt.Errorf("Ceph-fuse failed: %vnarguments: %snOutput: %sn", err, mountArgs, string(output)) } return nil } </ins> func getVolumeSource(spec *volume.Spec) ([]string, string, string, string, bool, error) { if spec.Volume != nil && spec.Volume.CephFS != nil { mon := spec.Volume.CephFS.Monitors