Repository: ceph/go-ceph
Pull request #399 (2020-11-04 21:28:23)

rbd: add an initial set of basic mirroring functions
Add functions:
* SetMirrorMode implementing rbd_mirror_mode_set
* GetMirrorMode implementing rbd_mirror_mode_get
* MirrorEnable implementing rbd_mirror_image_enable2
* MirrorDisable implementing rbd_mirror_image_disable
* MirrorPromote implementing rbd_mirror_image_promote
* MirrorDemote implementing rbd_mirror_image_demote
* MirrorResync implementing rbd_mirror_image_resync
* MirrorInstanceId implementing rbd_mirror_image_get_instance_id

For now, these mirroring related functions are only supported on octopus
builds. Right now the demand is for snapshot based mirroring, which is
new in octopus. We can always relax this in the future and add the
necessary support for nautilus, for journaling only based mirroring.

Part of: #393  - however not all functions are added in this PR.


## Checklist
- [x] Added tests for features and functional changes
- [x] Public functions and types are documented
- [x] Standard formatting is applied to Go code


phlogistonjohn: > No need to mention unsupported releases, I think. You might want to change this.
> Surprised you included everything in a single commit (OK, but unexpected).
> 
> Looks good otherwise!

FWIW, many of the functions depended on each other for testing, and individually each function's implementation was pretty simple (barring InstanceId but even that's not too bad). The more complex stuff will have to be added (and tested!) in follow up prs.
phlogistonjohn: @Mergifyio rebase
mergify[bot]: **Command `rebase`: success**
> **Branch has been successfully rebased**
> 

phlogistonjohn: @mergifyio rebase
mergify[bot]: **Command `rebase`: success**
> **Branch has been successfully rebased**
> 

phlogistonjohn: @Mergifyio rebase
mergify[bot]: **Command `rebase`: success**
> **Branch has been successfully rebased**
> 

phlogistonjohn: @agarwal-mudit or @nixpanic PR was updated so this needs re-review. Thanks.
phlogistonjohn: @mergifyio rebase
mergify[bot]: **Command `rebase`: success**
> **Branch has been successfully rebased**
> 

phlogistonjohn: @ansiwen can you please reapprove? You left a LGTM and I think a gh approval but mergify has dissmissed it. I don't think I made any new changes since, but feel free to give it a 2nd look even though I think mergify is in the wrong.

Patch:
diff --git a/rbd/group.go b/rbd/group.go
new file mode 100644
index 0000000..7838458
--- /dev/null
+++ b/rbd/group.go
@@ -0,0 +1,86 @@
+package rbd
+
+/*
+#cgo LDFLAGS: -lrbd
+#include <stdlib.h>
+#include <rbd/librbd.h>
+*/
+import "C"
+
+import (
+	"unsafe"
+
+	"github.com/ceph/go-ceph/internal/cutil"
+	"github.com/ceph/go-ceph/internal/retry"
+	"github.com/ceph/go-ceph/rados"
+)
+
+// GroupCreate is used to create an image group.
+//
+// Implements:
+//  int rbd_group_create(rados_ioctx_t p, const char *name);
+func GroupCreate(ioctx *rados.IOContext, name string) error {
+	cName := C.CString(name)
+	defer C.free(unsafe.Pointer(cName))
+
+	ret := C.rbd_group_create(cephIoctx(ioctx), cName)
+	return getError(ret)
+}
+
+// GroupRemove is used to remove an image group.
+//
+// Implements:
+//  int rbd_group_remove(rados_ioctx_t p, const char *name);
+func GroupRemove(ioctx *rados.IOContext, name string) error {
+	cName := C.CString(name)
+	defer C.free(unsafe.Pointer(cName))
+
+	ret := C.rbd_group_remove(cephIoctx(ioctx), cName)
+	return getError(ret)
+}
+
+// GroupRename will rename an existing image group.
+//
+// Implements:
+//  int rbd_group_rename(rados_ioctx_t p, const char *src_name,
+//                       const char *dest_name);
+func GroupRename(ioctx *rados.IOContext, src, dest string) error {
+	cSrc := C.CString(src)
+	defer C.free(unsafe.Pointer(cSrc))
+	cDest := C.CString(dest)
+	defer C.free(unsafe.Pointer(cDest))
+
+	ret := C.rbd_group_rename(cephIoctx(ioctx), cSrc, cDest)
+	return getError(ret)
+}
+
+// GroupList returns a slice of image group names.
+//
+// Implements:
+//  int rbd_group_list(rados_ioctx_t p, char *names, size_t *size);
+func GroupList(ioctx *rados.IOContext) ([]string, error) {
+	var (
+		buf []byte
+		err error
+		ret C.int
+	)
+	retry.WithSizes(1024, 262144, func(size int) retry.Hint {
+		cSize := C.size_t(size)
+		buf = make([]byte, cSize)
+		ret = C.rbd_group_list(
+			cephIoctx(ioctx),
+			(*C.char)(unsafe.Pointer(&buf[0])),
+			&cSize)
+		err = getErrorIfNegative(ret)
+		return retry.Size(int(cSize)).If(err == errRange)
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// cSize is not set to the expected size when it is sufficiently large
+	// but ret will be set to the size in a non-error condition.
+	groups := cutil.SplitBuffer(buf[:ret])
+	return groups, nil
+}
diff --git a/rbd/mirror.go b/rbd/mirror.go
new file mode 100644
index 0000000..6c49b8e
--- /dev/null
+++ b/rbd/mirror.go
@@ -0,0 +1,167 @@
+// +build !nautilus
+
+// Initially, we're only providing mirroring related functions for octopus as
+// that version of ceph deprecated a number of the functions in nautilus. If
+// you need mirroring on an earlier supported version of ceph please file an
+// issue in our tracker.
+
+package rbd
+
+// #cgo LDFLAGS: -lrbd
+// #include <stdlib.h>
+// #include <rbd/librbd.h>
+import "C"
+
+import (
+	"unsafe"
+
+	"github.com/ceph/go-ceph/internal/retry"
+	"github.com/ceph/go-ceph/rados"
+)
+
+// MirrorMode is used to indicate an approach used for RBD mirroring.
+type MirrorMode int64
+
+const (
+	// MirrorModeDisabled disables mirroring.
+	MirrorModeDisabled = MirrorMode(C.RBD_MIRROR_MODE_DISABLED)
+	// MirrorModeImage enables mirroring on a per-image basis.
+	MirrorModeImage = MirrorMode(C.RBD_MIRROR_MODE_IMAGE)
+	// MirrorModePool enables mirroring on all journaled images.
+	MirrorModePool = MirrorMode(C.RBD_MIRROR_MODE_POOL)
+)
+
+// ImageMirrorMode is used to indicate the mirroring approach for an RBD image.
+type ImageMirrorMode int64
+
+const (
+	// ImageMirrorModeJournal uses journaling to propagate RBD images between
+	// ceph clusters.
+	ImageMirrorModeJournal = ImageMirrorMode(C.RBD_MIRROR_IMAGE_MODE_JOURNAL)
+	// ImageMirrorModeSnapshot uses snapshot RDB images to propagate images
+	// between ceph clusters.
+	ImageMirrorModeSnapshot = ImageMirrorMode(C.RBD_MIRROR_IMAGE_MODE_SNAPSHOT)
+)
+
+// SetMirrorMode is used to enable or disable pool level mirroring with either
+// an automatic or per-image behavior.
+//
+// Implements:
+//  int rbd_mirror_mode_set(rados_ioctx_t io_ctx,
+//                          rbd_mirror_mode_t mirror_mode);
+func SetMirrorMode(ioctx *rados.IOContext, mode MirrorMode) error {
+	ret := C.rbd_mirror_mode_set(
+		cephIoctx(ioctx),
+		C.rbd_mirror_mode_t(mode))
+	return getError(ret)
+}
+
+// GetMirrorMode is used to fetch the current mirroring mode for a pool.
+//
+// Implements:
+//  int rbd_mirror_mode_get(rados_ioctx_t io_ctx,
+//                          rbd_mirror_mode_t *mirror_mode);
+func GetMirrorMode(ioctx *rados.IOContext) (MirrorMode, error) {
+	var mode C.rbd_mirror_mode_t
+
+	ret := C.rbd_mirror_mode_get(
+		cephIoctx(ioctx),
+		&mode)
+	if err := getError(ret); err != nil {
+		return MirrorModeDisabled, err
+	}
+	return MirrorMode(mode), nil
+}
+
+// MirrorEnable will enable mirroring for an image using the specified mode.
+//
+// Implements:
+//  int rbd_mirror_image_enable2(rbd_image_t image,
+//                               rbd_mirror_image_mode_t mode);
+func (image *Image) MirrorEnable(mode ImageMirrorMode) error {
+	if err := image.validate(imageIsOpen); err != nil {
+		return err
+	}
+	ret := C.rbd_mirror_image_enable2(image.image, C.rbd_mirror_image_mode_t(mode))
+	return getError(ret)
+}
+
+// MirrorDisable will disable mirroring for the image.
+//
+// Implements:
+//  int rbd_mirror_image_disable(rbd_image_t image, bool force);
+func (image *Image) MirrorDisable(force bool) error {
+	if err := image.validate(imageIsOpen); err != nil {
+		return err
+	}
+	ret := C.rbd_mirror_image_disable(image.image, C.bool(force))
+	return getError(ret)
+}
+
+// MirrorPromote will promote the image to primary status.
+//
+// Implements:
+//  int rbd_mirror_image_promote(rbd_image_t image, bool force);
+func (image *Image) MirrorPromote(force bool) error {
+	if err := image.validate(imageIsOpen); err != nil {
+		return err
+	}
+	ret := C.rbd_mirror_image_promote(image.image, C.bool(force))
+	return getError(ret)
+}
+
+// MirrorDemote will demote the image to secondary status.
+//
+// Implements:
+//  int rbd_mirror_image_demote(rbd_image_t image);
+func (image *Image) MirrorDemote() error {
+	if err := image.validate(imageIsOpen); err != nil {
+		return err
+	}
+	ret := C.rbd_mirror_image_demote(image.image)
+	return getError(ret)
+}
+
+// MirrorResync is used to manually resolve split-brain status by triggering
+// resynchronization.
+//
+// Implements:
+//  int rbd_mirror_image_resync(rbd_image_t image);
+func (image *Image) MirrorResync() error {
+	if err := image.validate(imageIsOpen); err != nil {
+		return err
+	}
+	ret := C.rbd_mirror_image_resync(image.image)
+	return getError(ret)
+}
+
+// MirrorInstanceID returns a string naming the instance id for the image.
+//
+// Implements:
+//  int rbd_mirror_image_get_instance_id(rbd_image_t image,
+//                                       char *instance_id,
+//                                       size_t *id_max_length);
+func (image *Image) MirrorInstanceID() (string, error) {
+	if err := image.validate(imageIsOpen); err != nil {
+		return "", err
+	}
+	var (
+		err   error
+		buf   []byte
+		cSize C.size_t
+	)
+	retry.WithSizes(1024, 1<<16, func(size int) retry.Hint {
+		cSize = C.size_t(size)
+		buf = make([]byte, cSize)
+		ret := C.rbd_mirror_image_get_instance_id(
+			image.image,
+			(*C.char)(unsafe.Pointer(&buf[0])),
+			&cSize)
+		err = getErrorIfNegative(ret)
+		return retry.Size(int(cSize)).If(err == errRange)
+	})
+	if err != nil {
+		return "", err
+	}
+	return string(buf[:cSize]), nil
+}


Test patch:
diff --git a/rbd/group_test.go b/rbd/group_test.go
new file mode 100644
index 0000000..c4c9f9e
--- /dev/null
+++ b/rbd/group_test.go
@@ -0,0 +1,115 @@
+package rbd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGroupCreateRemove(t *testing.T) {
+	conn := radosConnect(t)
+	require.NotNil(t, conn)
+	defer conn.Shutdown()
+
+	poolname := GetUUID()
+	err := conn.MakePool(poolname)
+	require.NoError(t, err)
+	defer conn.DeletePool(poolname)
+
+	ioctx, err := conn.OpenIOContext(poolname)
+	require.NoError(t, err)
+	defer ioctx.Destroy()
+
+	err = GroupCreate(ioctx, "group1")
+	assert.NoError(t, err)
+
+	err = GroupRemove(ioctx, "group1")
+	assert.NoError(t, err)
+
+	err = GroupRemove(ioctx, "group2")
+	assert.NoError(t, err)
+
+	err = GroupCreate(ioctx, "group2")
+	assert.NoError(t, err)
+	err = GroupCreate(ioctx, "group")
+	assert.NoError(t, err)
+
+	err = GroupRemove(ioctx, "group2")
+	assert.NoError(t, err)
+}
+
+func TestGroupRename(t *testing.T) {
+	conn := radosConnect(t)
+	require.NotNil(t, conn)
+	defer conn.Shutdown()
+
+	poolname := GetUUID()
+	err := conn.MakePool(poolname)
+	require.NoError(t, err)
+	defer conn.DeletePool(poolname)
+
+	ioctx, err := conn.OpenIOContext(poolname)
+	require.NoError(t, err)
+	defer ioctx.Destroy()
+
+	err = GroupCreate(ioctx, "group1")
+	assert.NoError(t, err)
+
+	err = GroupRename(ioctx, "group1", "club1")
+	assert.NoError(t, err)
+
+	err = GroupRemove(ioctx, "club1")
+	assert.NoError(t, err)
+
+	// unlike remove, rename does return an error if the src name
+	// doesn't exist
+	err = GroupRename(ioctx, "club1", "nowhere")
+	assert.Error(t, err)
+}
+
+func TestGroupList(t *testing.T) {
+	conn := radosConnect(t)
+	require.NotNil(t, conn)
+	defer conn.Shutdown()
+
+	poolname := GetUUID()
+	err := conn.MakePool(poolname)
+	require.NoError(t, err)
+	defer conn.DeletePool(poolname)
+
+	ioctx, err := conn.OpenIOContext(poolname)
+	require.NoError(t, err)
+	defer ioctx.Destroy()
+
+	err = GroupCreate(ioctx, "uno")
+	assert.NoError(t, err)
+	err = GroupCreate(ioctx, "dos")
+	assert.NoError(t, err)
+	err = GroupCreate(ioctx, "tres")
+	assert.NoError(t, err)
+
+	l, err := GroupList(ioctx)
+	assert.NoError(t, err)
+	if assert.Len(t, l, 3) {
+		assert.Contains(t, l, "uno")
+		assert.Contains(t, l, "dos")
+		assert.Contains(t, l, "tres")
+	}
+
+	err = GroupRemove(ioctx, "uno")
+	assert.NoError(t, err)
+	err = GroupRemove(ioctx, "dos")
+	assert.NoError(t, err)
+	err = GroupRemove(ioctx, "tres")
+	assert.NoError(t, err)
+
+	l, err = GroupList(ioctx)
+	assert.NoError(t, err)
+	assert.Len(t, l, 0)
+
+	// test that GroupList panics if passed a nil ioctx
+	assert.Panics(t, func() {
+		GroupList(nil)
+	})
+}
diff --git a/rbd/mirror_test.go b/rbd/mirror_test.go
new file mode 100644
index 0000000..fed76cb
--- /dev/null
+++ b/rbd/mirror_test.go
@@ -0,0 +1,184 @@
+// +build !nautilus
+
+// Initially, we're only providing mirroring related functions for octopus as
+// that version of ceph deprecated a number of the functions in nautilus. If
+// you need mirroring on an earlier supported version of ceph please file an
+// issue in our tracker.
+
+package rbd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetMirrorMode(t *testing.T) {
+	conn := radosConnect(t)
+	poolName := GetUUID()
+	err := conn.MakePool(poolName)
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, conn.DeletePool(poolName))
+		conn.Shutdown()
+	}()
+
+	ioctx, err := conn.OpenIOContext(poolName)
+	assert.NoError(t, err)
+	defer func() {
+		ioctx.Destroy()
+	}()
+
+	t.Run("mirrorModeDisabled", func(t *testing.T) {
+		m, err := GetMirrorMode(ioctx)
+		assert.NoError(t, err)
+		assert.Equal(t, m, MirrorModeDisabled)
+	})
+	t.Run("mirrorModeEnabled", func(t *testing.T) {
+		err = SetMirrorMode(ioctx, MirrorModeImage)
+		require.NoError(t, err)
+		m, err := GetMirrorMode(ioctx)
+		assert.NoError(t, err)
+		assert.Equal(t, m, MirrorModeImage)
+	})
+	t.Run("ioctxNil", func(t *testing.T) {
+		assert.Panics(t, func() {
+			GetMirrorMode(nil)
+		})
+	})
+
+}
+
+func TestMirroring(t *testing.T) {
+	conn := radosConnect(t)
+	poolName := GetUUID()
+	err := conn.MakePool(poolName)
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, conn.DeletePool(poolName))
+		conn.Shutdown()
+	}()
+
+	ioctx, err := conn.OpenIOContext(poolName)
+	assert.NoError(t, err)
+	defer func() {
+		ioctx.Destroy()
+	}()
+
+	// verify that mirroring is not enabled on this new pool
+	m, err := GetMirrorMode(ioctx)
+	assert.NoError(t, err)
+	assert.Equal(t, m, MirrorModeDisabled)
+
+	// enable per-image mirroring for this pool
+	err = SetMirrorMode(ioctx, MirrorModeImage)
+	require.NoError(t, err)
+
+	name1 := GetUUID()
+	options := NewRbdImageOptions()
+	assert.NoError(t,
+		options.SetUint64(ImageOptionOrder, uint64(testImageOrder)))
+	err = CreateImage(ioctx, name1, testImageSize, options)
+	require.NoError(t, err)
+
+	t.Run("enableDisable", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		defer func() {
+			assert.NoError(t, img.Close())
+		}()
+
+		err = img.MirrorEnable(ImageMirrorModeSnapshot)
+		assert.NoError(t, err)
+		err = img.MirrorDisable(false)
+		assert.NoError(t, err)
+	})
+	t.Run("enableDisableInvalid", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		assert.NoError(t, img.Close())
+
+		err = img.MirrorEnable(ImageMirrorModeSnapshot)
+		assert.Error(t, err)
+		err = img.MirrorDisable(false)
+		assert.Error(t, err)
+	})
+	t.Run("promoteDemote", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		defer func() {
+			assert.NoError(t, img.Close())
+		}()
+
+		err = img.MirrorEnable(ImageMirrorModeSnapshot)
+		assert.NoError(t, err)
+		err = img.MirrorDemote()
+		assert.NoError(t, err)
+		err = img.MirrorPromote(false)
+		assert.NoError(t, err)
+		err = img.MirrorDisable(false)
+		assert.NoError(t, err)
+	})
+	t.Run("promoteDemoteInvalid", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		assert.NoError(t, img.Close())
+
+		err = img.MirrorDemote()
+		assert.Error(t, err)
+		err = img.MirrorPromote(false)
+		assert.Error(t, err)
+	})
+	t.Run("resync", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		defer func() {
+			assert.NoError(t, img.Close())
+		}()
+
+		err = img.MirrorEnable(ImageMirrorModeSnapshot)
+		assert.NoError(t, err)
+		err = img.MirrorDemote()
+		assert.NoError(t, err)
+		err = img.MirrorResync()
+		assert.NoError(t, err)
+		err = img.MirrorDisable(true)
+		assert.NoError(t, err)
+	})
+	t.Run("resyncInvalid", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		assert.NoError(t, img.Close())
+
+		err = img.MirrorResync()
+		assert.Error(t, err)
+	})
+	t.Run("instanceId", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		defer func() {
+			assert.NoError(t, img.Close())
+		}()
+
+		err = img.MirrorEnable(ImageMirrorModeSnapshot)
+		assert.NoError(t, err)
+		miid, err := img.MirrorInstanceID()
+		// this is not currently testable for the "success" case
+		// see also the ceph tree where nothing is asserted except
+		// that the error is raised.
+		// TODO(?): figure out how to test this
+		assert.Error(t, err)
+		assert.Equal(t, "", miid)
+		err = img.MirrorDisable(false)
+		assert.NoError(t, err)
+	})
+	t.Run("instanceIdInvalid", func(t *testing.T) {
+		img, err := OpenImage(ioctx, name1, NoSnapshot)
+		assert.NoError(t, err)
+		assert.NoError(t, img.Close())
+
+		_, err = img.MirrorInstanceID()
+		assert.Error(t, err)
+	})
+}