Repository: ceph/go-ceph
Pull request #655 (2022-02-23 15:22:21)

nfs: admin APIs for managing nfs exports
The initial set of APIs are mostly in place, but the tests are untested.

Fixes: #628

## Checklist
- [ ] Added tests for features and functional changes
- [ ] Public functions and types are documented
- [ ] Standard formatting is applied to Go code
- [ ] Is this a new API? Is this new API marked PREVIEW?


nixpanic: Just noting that ceph/ceph-csi#2948 could use this API. We might be able to test it out with a custom build of the (work-in-progress) NFS provisioner.
phlogistonjohn: @nixpanic @BlaineEXE @ansiwen - based on some chats & verbal discussions this week, I've added a special build tag `ceph_ci_untested` that will allow us to leave have the source code for this in our tree but not be enabled by default or tested by the CI.  The ceph-csi team can then try/test it manually by setting this build tag  w/o go-ceph team "making promises" about this API to the other code bases that consume go-ceph.

Once ceph pacific has the necessary patches I will remove the special build tag and enable the code as a "normal" preview API.  If ceph pacific releases soon enough I can simply do that before our release in approx. 2 weeks. If I learn ceph pacific is going to release around mid-april, I'm willing to delay the release - but only by a week or so. Otherwise, if ceph pacific doesn't release until late April or later we'll probably just release with this feature gated by this build tag.

Any questions/comments/concerns?

If we're generally all good with the above, and CI passes I'll move this PR out of draft state.
phlogistonjohn: CCing @Rakshith-R as well. See my previous comment, please.
nixpanic: > If we're generally all good with the above, and CI passes I'll move this PR out of draft state.

:+1: Thanks, I'm happy with that! Unfortunately the new API does not work with Ceph 16.2.0, but is seems to be fine for 16.2.7 and newer. In case of failure with the API, Ceph-CSI will probably fall-back to the old `ceph` command (which also changed in 16.2.7).
nixpanic: > @nixpanic github is being rather weird and not letting me add you to the reviewers. I'll try again later, but feel free to review anyway.

It seems I am not part of the maintainers for go-ceph anymore?
phlogistonjohn: @ansiwen please review when you have a moment, thanks!

Patch:
diff --git a/Makefile b/Makefile
index 59df296..5688100 100644
--- a/Makefile
+++ b/Makefile
@@ -193,7 +193,9 @@ test-binaries: \
 	cephfs.test \
 	cephfs/admin.test \
 	common/admin/manager.test \
+	common/admin/nfs.test \
 	internal/callbacks.test \
+	internal/commands.test \
 	internal/cutil.test \
 	internal/errutil.test \
 	internal/retry.test \
diff --git a/common/admin/nfs/admin.go b/common/admin/nfs/admin.go
new file mode 100644
index 0000000..2f644d0
--- /dev/null
+++ b/common/admin/nfs/admin.go
@@ -0,0 +1,21 @@
+//go:build !(nautilus || octopus) && ceph_preview && ceph_ci_untested
+// +build !nautilus,!octopus,ceph_preview,ceph_ci_untested
+
+package nfs
+
+import (
+	ccom "github.com/ceph/go-ceph/common/commands"
+)
+
+// Admin is used to administer ceph nfs features.
+type Admin struct {
+	conn ccom.RadosCommander
+}
+
+// NewFromConn creates an new management object from a preexisting
+// rados connection. The existing connection can be rados.Conn or any
+// type implementing the RadosCommander interface.
+//  PREVIEW
+func NewFromConn(conn ccom.RadosCommander) *Admin {
+	return &Admin{conn}
+}
diff --git a/common/admin/nfs/doc.go b/common/admin/nfs/doc.go
new file mode 100644
index 0000000..05fb3a4
--- /dev/null
+++ b/common/admin/nfs/doc.go
@@ -0,0 +1,5 @@
+/*
+Package nfs from common/admin contains a set of APIs used to interact
+with and administer NFS support for ceph clusters.
+*/
+package nfs
diff --git a/common/admin/nfs/export.go b/common/admin/nfs/export.go
new file mode 100644
index 0000000..2309464
--- /dev/null
+++ b/common/admin/nfs/export.go
@@ -0,0 +1,198 @@
+//go:build !(nautilus || octopus) && ceph_preview && ceph_ci_untested
+// +build !nautilus,!octopus,ceph_preview,ceph_ci_untested
+
+package nfs
+
+import (
+	"github.com/ceph/go-ceph/internal/commands"
+)
+
+// SquashMode indicates the kind of user-id squashing performed on an export.
+type SquashMode string
+
+// src: https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/export.txt
+const (
+	// NoneSquash performs no id squashing.
+	NoneSquash SquashMode = "None"
+	// RootSquash performs squashing of root user (with any gid).
+	RootSquash SquashMode = "Root"
+	// AllSquash performs squashing of all users.
+	AllSquash SquashMode = "All"
+	// RootIDSquash performs squashing of root uid/gid.
+	RootIDSquash SquashMode = "RootId"
+	// NoRootSquash is equivalent to NoneSquash
+	NoRootSquash = NoneSquash
+	// Unspecifiedquash
+	Unspecifiedquash SquashMode = ""
+)
+
+// CephFSExportSpec is used to specify the parameters used to create a new
+// CephFS based export.
+type CephFSExportSpec struct {
+	FileSystemName string     `json:"fsname"`
+	ClusterID      string     `json:"cluster_id"`
+	PseudoPath     string     `json:"pseudo_path"`
+	Path           string     `json:"path,omitempty"`
+	ReadOnly       bool       `json:"readonly"`
+	ClientAddr     []string   `json:"client_addr,omitempty"`
+	Squash         SquashMode `json:"squash,omitempty"`
+}
+
+// ExportResult is returned along with newly created exports.
+type ExportResult struct {
+	Bind           string `json:"bind"`
+	FileSystemName string `json:"fs"`
+	Path           string `json:"path"`
+	ClusterID      string `json:"cluster"`
+	Mode           string `json:"mode"`
+}
+
+type cephFSExportFields struct {
+	Prefix string `json:"prefix"`
+	Format string `json:"format"`
+
+	CephFSExportSpec
+}
+
+// FSALInfo describes NFS-Ganesha specific FSAL properties of an export.
+type FSALInfo struct {
+	Name           string `json:"name"`
+	UserID         string `json:"user_id"`
+	FileSystemName string `json:"fs_name"`
+}
+
+// ClientInfo describes per-client parameters of an export.
+type ClientInfo struct {
+	Addresses  []string   `json:"addresses"`
+	AccessType string     `json:"access_type"`
+	Squash     SquashMode `json:"squash"`
+}
+
+// ExportInfo describes an NFS export.
+type ExportInfo struct {
+	ExportID      int64        `json:"export_id"`
+	Path          string       `json:"path"`
+	ClusterID     string       `json:"cluster_id"`
+	PseudoPath    string       `json:"pseudo"`
+	AccessType    string       `json:"access_type"`
+	Squash        SquashMode   `json:"squash"`
+	SecurityLabel bool         `json:"security_label"`
+	Protocols     []int        `json:"protocols"`
+	Transports    []string     `json:"transports"`
+	FSAL          FSALInfo     `json:"fsal"`
+	Clients       []ClientInfo `json:"clients"`
+}
+
+func parseExportResult(res commands.Response) (*ExportResult, error) {
+	r := &ExportResult{}
+	if err := res.NoStatus().Unmarshal(r).End(); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func parseExportsList(res commands.Response) ([]ExportInfo, error) {
+	l := []ExportInfo{}
+	if err := res.NoStatus().Unmarshal(&l).End(); err != nil {
+		return nil, err
+	}
+	return l, nil
+}
+
+func parseExportInfo(res commands.Response) (ExportInfo, error) {
+	i := ExportInfo{}
+	if err := res.NoStatus().Unmarshal(&i).End(); err != nil {
+		return i, err
+	}
+	return i, nil
+}
+
+// CreateCephFSExport will create a new NFS export for a CephFS file system.
+//  PREVIEW
+//
+// Similar To:
+//  ceph nfs export create cephfs
+func (nfsa *Admin) CreateCephFSExport(spec CephFSExportSpec) (
+	*ExportResult, error) {
+	// ---
+	f := &cephFSExportFields{
+		Prefix:           "nfs export create cephfs",
+		Format:           "json",
+		CephFSExportSpec: spec,
+	}
+	return parseExportResult(commands.MarshalMgrCommand(nfsa.conn, f))
+}
+
+const delSucc = "Successfully deleted export"
+
+// RemoveExport will remove an NFS export based on the pseudo-path of the export.
+//  PREVIEW
+//
+// Similar To:
+//  ceph nfs export rm
+func (nfsa *Admin) RemoveExport(clusterID, pseudoPath string) error {
+	m := map[string]string{
+		"prefix":      "nfs export rm",
+		"format":      "json",
+		"cluster_id":  clusterID,
+		"pseudo_path": pseudoPath,
+	}
+	return (commands.MarshalMgrCommand(nfsa.conn, m).
+		FilterBodyPrefix(delSucc).NoData().End())
+}
+
+// ListDetailedExports will return a list of exports with details.
+//  PREVIEW
+//
+// Similar To:
+//  ceph nfs export ls --detailed
+func (nfsa *Admin) ListDetailedExports(clusterID string) ([]ExportInfo, error) {
+	/*
+		NOTE: there is no simple list because based on a quick reading of the code
+		in ceph, the details fetching should not be significantly slower with
+		details than without, and since this is an API call not a CLI its easy
+		enough to ignore the details you don't care about. If I'm wrong, and
+		we discover a major perf. difference in the future we can always add a new
+		simpler list-without-details function.
+	*/
+	m := map[string]string{
+		"prefix":     "nfs export ls",
+		"detailed":   "true",
+		"format":     "json",
+		"cluster_id": clusterID,
+	}
+	return parseExportsList(commands.MarshalMgrCommand(nfsa.conn, m))
+}
+
+// ExportInfo will return a structure describing the export specified by it's
+// pseudo-path.
+//  PREVIEW
+//
+// Similar To:
+//  ceph nfs export info
+func (nfsa *Admin) ExportInfo(clusterID, pseudoPath string) (ExportInfo, error) {
+	m := map[string]string{
+		"prefix":      "nfs export info",
+		"format":      "json",
+		"cluster_id":  clusterID,
+		"pseudo_path": pseudoPath,
+	}
+	return parseExportInfo(commands.MarshalMgrCommand(nfsa.conn, m))
+}
+
+/*
+TODO?
+
+'nfs export apply': cluster_id: str, inbuf: str
+"""Create or update an export by `-i <json_or_ganesha_export_file>`"""
+
+
+'nfs export create rgw':
+	   bucket: str,
+	   cluster_id: str,
+	   pseudo_path: str,
+	   readonly: Optional[bool] = False,
+	   client_addr: Optional[List[str]] = None,
+	   squash: str = 'none',
+"""Create an RGW export"""
+*/
diff --git a/internal/commands/response.go b/internal/commands/response.go
index cafd4ba..f5d3d3c 100644
--- a/internal/commands/response.go
+++ b/internal/commands/response.go
@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -156,6 +157,18 @@ func (r Response) FilterSuffix(s string) Response {
 	return r
 }
 
+// FilterBodyPrefix sets the body value equivalent to an empty string if the
+// body value contains the given prefix string.
+func (r Response) FilterBodyPrefix(p string) Response {
+	if !r.Ok() {
+		return r
+	}
+	if bytes.HasPrefix(r.body, []byte(p)) {
+		return Response{[]byte(""), r.status, r.err}
+	}
+	return r
+}
+
 // FilterDeprecated removes deprecation warnings from the response status.
 // Use it when checking the response from calls that may be deprecated in ceph
 // if you want those calls to continue working if the warning is present.
diff --git a/micro-osd.sh b/micro-osd.sh
index 832c9e6..c6009d0 100755
--- a/micro-osd.sh
+++ b/micro-osd.sh
@@ -72,6 +72,9 @@ osd objectstore = memstore
 osd class load list = *
 osd class default list = *
 
+[mgr.${MGR_NAME}]
+log_file = ${LOG_DIR}/mgr.log
+
 [client.rgw.${RGW_ID}]
 rgw dns name = ${HOSTNAME}
 rgw enable usage log = true


Test patch:
diff --git a/common/admin/nfs/export_test.go b/common/admin/nfs/export_test.go
new file mode 100644
index 0000000..eae2798
--- /dev/null
+++ b/common/admin/nfs/export_test.go
@@ -0,0 +1,299 @@
+//go:build !(nautilus || octopus) && ceph_preview && ceph_ci_untested
+// +build !nautilus,!octopus,ceph_preview,ceph_ci_untested
+
+package nfs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	tsuite "github.com/stretchr/testify/suite"
+
+	"github.com/ceph/go-ceph/internal/admintest"
+	"github.com/ceph/go-ceph/internal/commands"
+	"github.com/ceph/go-ceph/rados"
+)
+
+var radosConnector = admintest.NewConnector()
+
+func TestNFSAdmin(t *testing.T) {
+	tsuite.Run(t, new(NFSAdminSuite))
+}
+
+// NFSAdminSuite is a suite of tests for the nfs admin package.
+// A suite is used because creating/managing NFS has certain expectations for
+// the cluster that we may need to mock, especially when running in the
+// standard go-ceph test container. Using a suite allows us to have suite
+// setup/validate the environment and the tests can largely be ignorant of the
+// environment.
+type NFSAdminSuite struct {
+	tsuite.Suite
+
+	fileSystemName string
+	clusterID      string
+	mockConfig     bool
+}
+
+func (suite *NFSAdminSuite) SetupSuite() {
+	suite.fileSystemName = "cephfs"
+	suite.clusterID = "goceph"
+	suite.mockConfig = true
+
+	mock := os.Getenv("GO_CEPH_TEST_MOCK_NFS")
+	if ok, err := strconv.ParseBool(mock); err == nil {
+		suite.mockConfig = ok
+	}
+
+	if suite.mockConfig {
+		suite.setupMockNFSConfig()
+	}
+}
+
+func (suite *NFSAdminSuite) setupMockNFSConfig() {
+	require := suite.Require()
+	conn := radosConnector.GetConn(suite.T())
+
+	err := conn.MakePool(".nfs")
+	if err != nil && !errors.Is(err, rados.ErrObjectExists) {
+		suite.T().Fatalf("failed to make pool: %v", err)
+	}
+
+	ioctx, err := conn.OpenIOContext(".nfs")
+	require.NoError(err)
+	defer ioctx.Destroy()
+
+	ioctx.SetNamespace(suite.clusterID)
+
+	err = ioctx.Create(
+		fmt.Sprintf("conf-nfs.%s", suite.clusterID),
+		rados.CreateIdempotent)
+	require.NoError(err)
+}
+
+func (suite *NFSAdminSuite) TestCreateDeleteCephFSExport() {
+	require := suite.Require()
+	ra := radosConnector.Get(suite.T())
+	nfsa := NewFromConn(ra)
+
+	res, err := nfsa.CreateCephFSExport(CephFSExportSpec{
+		FileSystemName: suite.fileSystemName,
+		ClusterID:      suite.clusterID,
+		PseudoPath:     "/cheese",
+	})
+	require.NoError(err)
+	require.Equal("/cheese", res.Bind)
+
+	err = nfsa.RemoveExport(suite.clusterID, "/cheese")
+	require.NoError(err)
+}
+
+func (suite *NFSAdminSuite) TestListDetailedExports() {
+	require := suite.Require()
+	ra := radosConnector.Get(suite.T())
+	nfsa := NewFromConn(ra)
+
+	_, err := nfsa.CreateCephFSExport(CephFSExportSpec{
+		FileSystemName: suite.fileSystemName,
+		ClusterID:      suite.clusterID,
+		PseudoPath:     "/01",
+		Path:           "/january",
+	})
+	require.NoError(err)
+
+	defer func() {
+		err = nfsa.RemoveExport(suite.clusterID, "/01")
+		require.NoError(err)
+	}()
+
+	_, err = nfsa.CreateCephFSExport(CephFSExportSpec{
+		FileSystemName: suite.fileSystemName,
+		ClusterID:      suite.clusterID,
+		PseudoPath:     "/02",
+		Path:           "/february",
+	})
+	require.NoError(err)
+
+	defer func() {
+		err = nfsa.RemoveExport(suite.clusterID, "/02")
+		require.NoError(err)
+	}()
+
+	l, err := nfsa.ListDetailedExports(suite.clusterID)
+	require.NoError(err)
+	require.Len(l, 2)
+	var e1, e2 ExportInfo
+	for _, e := range l {
+		if e.PseudoPath == "/01" {
+			e1 = e
+		}
+		if e.PseudoPath == "/02" {
+			e2 = e
+		}
+	}
+	require.Equal(e1.PseudoPath, "/01")
+	require.Equal(e2.PseudoPath, "/02")
+}
+
+func (suite *NFSAdminSuite) TestExportInfo() {
+	require := suite.Require()
+	ra := radosConnector.Get(suite.T())
+	nfsa := NewFromConn(ra)
+
+	_, err := nfsa.CreateCephFSExport(CephFSExportSpec{
+		FileSystemName: suite.fileSystemName,
+		ClusterID:      suite.clusterID,
+		PseudoPath:     "/03",
+		Path:           "/march",
+	})
+	require.NoError(err)
+
+	defer func() {
+		err = nfsa.RemoveExport(suite.clusterID, "/03")
+		require.NoError(err)
+	}()
+
+	e1, err := nfsa.ExportInfo(suite.clusterID, "/03")
+	require.NoError(err)
+	require.Equal(e1.PseudoPath, "/03")
+
+	_, err = nfsa.ExportInfo(suite.clusterID, "/88")
+	require.Error(err)
+}
+
+const resultExport1 = `{
+    "bind": "/cheese",
+    "fs": "cephfs",
+    "path": "/",
+    "cluster": "foobar",
+    "mode": "RW"
+}
+`
+
+func TestParseExportResult(t *testing.T) {
+	t.Run("resultExport", func(t *testing.T) {
+		r := commands.NewResponse([]byte(resultExport1), "", nil)
+		e, err := parseExportResult(r)
+		assert.NoError(t, err)
+		assert.Equal(t, e.Bind, "/cheese")
+	})
+	t.Run("errorSet", func(t *testing.T) {
+		r := commands.NewResponse([]byte(""), "", errors.New("beep"))
+		_, err := parseExportResult(r)
+		assert.Error(t, err)
+	})
+	t.Run("statusSet", func(t *testing.T) {
+		r := commands.NewResponse([]byte(""), "boo", nil)
+		_, err := parseExportResult(r)
+		assert.Error(t, err)
+	})
+}
+
+// # ceph nfs export ls --cluster-id foobar --detailed
+const exportList1 = `[
+  {
+    "export_id": 1,
+    "path": "/",
+    "cluster_id": "foobar",
+    "pseudo": "/cheese",
+    "access_type": "RW",
+    "squash": "none",
+    "security_label": true,
+    "protocols": [
+      4
+    ],
+    "transports": [
+      "TCP"
+    ],
+    "fsal": {
+      "name": "CEPH",
+      "user_id": "nfs.foobar.1",
+      "fs_name": "cephfs"
+    },
+    "clients": []
+  }
+]`
+
+func TestParseExportsList(t *testing.T) {
+	t.Run("exportList1", func(t *testing.T) {
+		r := commands.NewResponse([]byte(exportList1), "", nil)
+		l, err := parseExportsList(r)
+		assert.NoError(t, err)
+		if assert.Len(t, l, 1) {
+			e := l[0]
+			assert.Equal(t, e.Path, "/")
+			assert.Equal(t, e.PseudoPath, "/cheese")
+			if assert.Len(t, e.Protocols, 1) {
+				assert.Equal(t, e.Protocols[0], 4)
+			}
+			if assert.Len(t, e.Transports, 1) {
+				assert.Equal(t, e.Transports[0], "TCP")
+			}
+			assert.Equal(t, e.FSAL.Name, "CEPH")
+		}
+	})
+	t.Run("errorSet", func(t *testing.T) {
+		r := commands.NewResponse([]byte(""), "", errors.New("beep"))
+		_, err := parseExportsList(r)
+		assert.Error(t, err)
+	})
+	t.Run("statusSet", func(t *testing.T) {
+		r := commands.NewResponse([]byte(""), "boo", nil)
+		_, err := parseExportsList(r)
+		assert.Error(t, err)
+	})
+}
+
+// # ceph nfs export info --cluster-id foobar --pseudo-path /cheese
+const exportInfo1 = `{
+  "export_id": 1,
+  "path": "/",
+  "cluster_id": "foobar",
+  "pseudo": "/cheese",
+  "access_type": "RW",
+  "squash": "none",
+  "security_label": true,
+  "protocols": [
+    4
+  ],
+  "transports": [
+    "TCP"
+  ],
+  "fsal": {
+    "name": "CEPH",
+    "user_id": "nfs.foobar.1",
+    "fs_name": "cephfs"
+  },
+  "clients": []
+}
+`
+
+func TestParseExportInfo(t *testing.T) {
+	t.Run("exportInfo1", func(t *testing.T) {
+		r := commands.NewResponse([]byte(exportInfo1), "", nil)
+		e, err := parseExportInfo(r)
+		assert.NoError(t, err)
+		assert.Equal(t, e.Path, "/")
+		assert.Equal(t, e.PseudoPath, "/cheese")
+		if assert.Len(t, e.Protocols, 1) {
+			assert.Equal(t, e.Protocols[0], 4)
+		}
+		if assert.Len(t, e.Transports, 1) {
+			assert.Equal(t, e.Transports[0], "TCP")
+		}
+		assert.Equal(t, e.FSAL.Name, "CEPH")
+	})
+	t.Run("errorSet", func(t *testing.T) {
+		r := commands.NewResponse([]byte(""), "", errors.New("beep"))
+		_, err := parseExportInfo(r)
+		assert.Error(t, err)
+	})
+	t.Run("statusSet", func(t *testing.T) {
+		r := commands.NewResponse([]byte(""), "boo", nil)
+		_, err := parseExportInfo(r)
+		assert.Error(t, err)
+	})
+}
diff --git a/internal/commands/response_test.go b/internal/commands/response_test.go
index 044eae1..d61ab7a 100644
--- a/internal/commands/response_test.go
+++ b/internal/commands/response_test.go
@@ -137,6 +137,43 @@ func TestResponse(t *testing.T) {
 			assert.Contains(t, err.Error(), "not implemented")
 		}
 	})
+
+	t.Run("filterBodyPrefix", func(t *testing.T) {
+		rtemp := Response{
+			body: []byte("No way, no how"),
+		}
+		if assert.True(t, rtemp.Ok()) {
+			r2 := rtemp.FilterBodyPrefix("No way")
+			assert.True(t, r2.Ok())
+			assert.Equal(t, []byte(""), r2.body)
+			err := r2.NoBody().End()
+			assert.NoError(t, err)
+		}
+	})
+	t.Run("filterBodyPrefixEmpty", func(t *testing.T) {
+		rtemp := Response{
+			body: []byte(""),
+		}
+		if assert.True(t, rtemp.Ok()) {
+			r2 := rtemp.FilterBodyPrefix("No way")
+			assert.True(t, r2.Ok())
+			assert.Equal(t, []byte(""), r2.body)
+			err := r2.NoBody().End()
+			assert.NoError(t, err)
+		}
+	})
+	t.Run("filterBodyPrefixNoMatch", func(t *testing.T) {
+		rtemp := Response{
+			body: []byte("No way, no how"),
+		}
+		if assert.True(t, rtemp.Ok()) {
+			r2 := rtemp.FilterBodyPrefix("No foolin")
+			assert.True(t, r2.Ok())
+			assert.Equal(t, []byte("No way, no how"), r2.body)
+			err := r2.NoBody().End()
+			assert.Error(t, err)
+		}
+	})
 }
 
 type myCephError int