Repository: ceph/go-ceph
Pull request #461 (2021-02-24 16:20:07)

rbd: add GetGlobalMirrorStatus implementing rbd_mirror_image_get_global_status

This function is added, along with required return types and some helper
functions, to support fetching the global mirroring status component
that makes up a major part of `rbd mirror image status` command's
output.

Part of: #460 

## Checklist
- [x] Added tests for features and functional changes
- [x] Public functions and types are documented
- [x] Standard formatting is applied to Go code


phlogistonjohn: I have put a DNM label on this for now so that we can revisit the testing once we have ci that supports proper two-cluster mirroring. I do think that the code is fairly close to correct, but I'd love more thorough testing first if possible.
mergify[bot]: This pull request now has conflicts with the target branch. Could you please resolve conflicts and force push the corrected changes? 🙏
phlogistonjohn: @mergifyio rebase
mergify[bot]: **Command `rebase`: success**
> **Branch has been successfully rebased**
> 


Patch:
diff --git a/entrypoint.sh b/entrypoint.sh
index 03a12ff..66519a0 100755
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -259,6 +259,7 @@ test_go_ceph() {
     fi
     if [[ ${MIRROR_CONF} && ${CEPH_VERSION} != nautilus ]]; then
         setup_mirroring
+        export MIRROR_CONF
     fi
     for pkg in "${pkgs[@]}"; do
         test_pkg "${pkg}" || test_failed "${pkg}"
diff --git a/rbd/errors.go b/rbd/errors.go
index 5bfc092..3469333 100644
--- a/rbd/errors.go
+++ b/rbd/errors.go
@@ -71,6 +71,12 @@ var (
 	// revive:enable:exported
 )
 
+// Public general error
+const (
+	// ErrNotExist indicates a non-specific missing resource.
+	ErrNotExist = rbdError(-C.ENOENT)
+)
+
 // Private errors:
 
 const (
diff --git a/rbd/mirror.go b/rbd/mirror.go
index 2d67105..ddfe47e 100644
--- a/rbd/mirror.go
+++ b/rbd/mirror.go
@@ -13,6 +13,8 @@ package rbd
 import "C"
 
 import (
+	"fmt"
+	"math"
 	"unsafe"
 
 	"github.com/ceph/go-ceph/internal/retry"
@@ -228,6 +230,14 @@ type MirrorImageInfo struct {
 	Primary  bool
 }
 
+func convertMirrorImageInfo(cInfo *C.rbd_mirror_image_info_t) MirrorImageInfo {
+	return MirrorImageInfo{
+		GlobalID: C.GoString(cInfo.global_id),
+		State:    MirrorImageState(cInfo.state),
+		Primary:  bool(cInfo.primary),
+	}
+}
+
 // GetMirrorImageInfo fetches the mirroring status information of a RBD image.
 //
 // Implements:
@@ -249,11 +259,7 @@ func (image *Image) GetMirrorImageInfo() (*MirrorImageInfo, error) {
 		return nil, getError(ret)
 	}
 
-	mii := MirrorImageInfo{
-		GlobalID: C.GoString(cInfo.global_id),
-		State:    MirrorImageState(cInfo.state),
-		Primary:  bool(cInfo.primary),
-	}
+	mii := convertMirrorImageInfo(&cInfo)
 
 	// free C memory allocated by C.rbd_mirror_image_get_info call
 	C.rbd_mirror_image_get_info_cleanup(&cInfo)
@@ -273,3 +279,145 @@ func (image *Image) GetImageMirrorMode() (ImageMirrorMode, error) {
 	ret := C.rbd_mirror_image_get_mode(image.image, &mode)
 	return ImageMirrorMode(mode), getError(ret)
 }
+
+// MirrorImageStatusState is used to indicate the state of a mirrored image
+// within the site status info.
+type MirrorImageStatusState int64
+
+const (
+	// MirrorImageStatusStateUnknown is equivalent to MIRROR_IMAGE_STATUS_STATE_UNKNOWN
+	MirrorImageStatusStateUnknown = MirrorImageStatusState(C.MIRROR_IMAGE_STATUS_STATE_UNKNOWN)
+	// MirrorImageStatusStateError is equivalent to MIRROR_IMAGE_STATUS_STATE_ERROR
+	MirrorImageStatusStateError = MirrorImageStatusState(C.MIRROR_IMAGE_STATUS_STATE_ERROR)
+	// MirrorImageStatusStateSyncing is equivalent to MIRROR_IMAGE_STATUS_STATE_SYNCING
+	MirrorImageStatusStateSyncing = MirrorImageStatusState(C.MIRROR_IMAGE_STATUS_STATE_SYNCING)
+	// MirrorImageStatusStateStartingReplay is equivalent to MIRROR_IMAGE_STATUS_STATE_STARTING_REPLAY
+	MirrorImageStatusStateStartingReplay = MirrorImageStatusState(C.MIRROR_IMAGE_STATUS_STATE_STARTING_REPLAY)
+	// MirrorImageStatusStateReplaying is equivalent to MIRROR_IMAGE_STATUS_STATE_REPLAYING
+	MirrorImageStatusStateReplaying = MirrorImageStatusState(C.MIRROR_IMAGE_STATUS_STATE_REPLAYING)
+	// MirrorImageStatusStateStoppingReplay is equivalent to MIRROR_IMAGE_STATUS_STATE_STOPPING_REPLAY
+	MirrorImageStatusStateStoppingReplay = MirrorImageStatusState(C.MIRROR_IMAGE_STATUS_STATE_STOPPING_REPLAY)
+	// MirrorImageStatusStateStopped is equivalent to MIRROR_IMAGE_STATUS_STATE_STOPPED
+	MirrorImageStatusStateStopped = MirrorImageStatusState(C.MIRROR_IMAGE_STATUS_STATE_STOPPED)
+)
+
+// String represents the MirrorImageStatusState as a short string.
+func (state MirrorImageStatusState) String() (s string) {
+	switch state {
+	case MirrorImageStatusStateUnknown:
+		s = "unknown"
+	case MirrorImageStatusStateError:
+		s = "error"
+	case MirrorImageStatusStateSyncing:
+		s = "syncing"
+	case MirrorImageStatusStateStartingReplay:
+		s = "starting_replay"
+	case MirrorImageStatusStateReplaying:
+		s = "replaying"
+	case MirrorImageStatusStateStoppingReplay:
+		s = "stopping_replay"
+	case MirrorImageStatusStateStopped:
+		s = "stopped"
+	default:
+		s = fmt.Sprintf("unknown(%d)", state)
+	}
+	return s
+}
+
+// SiteMirrorImageStatus contains information pertaining to the status of
+// a mirrored image within a site.
+type SiteMirrorImageStatus struct {
+	MirrorUUID  string
+	State       MirrorImageStatusState
+	Description string
+	LastUpdate  int64
+	Up          bool
+}
+
+// GlobalMirrorImageStatus contains information pertaining to the global
+// status of a mirrored image. It contains general information as well
+// as per-site information stored in the SiteStatuses slice.
+type GlobalMirrorImageStatus struct {
+	Name         string
+	Info         MirrorImageInfo
+	SiteStatuses []SiteMirrorImageStatus
+}
+
+// LocalStatus returns one SiteMirrorImageStatus item from the SiteStatuses
+// slice that corresponds to the local site's status. If the local status
+// is not found than the error ErrNotExist will be returned.
+func (gmis GlobalMirrorImageStatus) LocalStatus() (SiteMirrorImageStatus, error) {
+	var (
+		ss  SiteMirrorImageStatus
+		err error = ErrNotExist
+	)
+	for i := range gmis.SiteStatuses {
+		// I couldn't find it explicitly documented, but a site mirror uuid
+		// of an empty string indicates that this is the local site.
+		// This pattern occurs in both the pybind code and ceph c++.
+		if gmis.SiteStatuses[i].MirrorUUID == "" {
+			ss = gmis.SiteStatuses[i]
+			err = nil
+			break
+		}
+	}
+	return ss, err
+}
+
+type siteArray [math.MaxInt32]C.rbd_mirror_image_site_status_t
+
+// GetGlobalMirrorStatus returns status information pertaining to the state
+// of the images's mirroring.
+//
+// Implements:
+//   int rbd_mirror_image_get_global_status(
+//     rbd_image_t image,
+//     rbd_mirror_image_global_status_t *mirror_image_global_status,
+//     size_t status_size);
+func (image *Image) GetGlobalMirrorStatus() (GlobalMirrorImageStatus, error) {
+	if err := image.validate(imageIsOpen); err != nil {
+		return GlobalMirrorImageStatus{}, err
+	}
+
+	s := C.rbd_mirror_image_global_status_t{}
+	ret := C.rbd_mirror_image_get_global_status(
+		image.image,
+		&s,
+		C.sizeof_rbd_mirror_image_global_status_t)
+	if err := getError(ret); err != nil {
+		return GlobalMirrorImageStatus{}, err
+	}
+	defer C.rbd_mirror_image_global_status_cleanup(&s)
+
+	status := GlobalMirrorImageStatus{
+		Name:         C.GoString(s.name),
+		Info:         convertMirrorImageInfo(&s.info),
+		SiteStatuses: make([]SiteMirrorImageStatus, s.site_statuses_count),
+	}
+	// use the "Sven Technique" to treat the C pointer as a go slice temporarily
+	sscs := (*siteArray)(unsafe.Pointer(s.site_statuses))[:s.site_statuses_count:s.site_statuses_count]
+	for i := C.uint32_t(0); i < s.site_statuses_count; i++ {
+		ss := sscs[i]
+		status.SiteStatuses[i] = SiteMirrorImageStatus{
+			MirrorUUID:  C.GoString(ss.mirror_uuid),
+			State:       MirrorImageStatusState(ss.state),
+			Description: C.GoString(ss.description),
+			LastUpdate:  int64(ss.last_update),
+			Up:          bool(ss.up),
+		}
+	}
+	return status, nil
+}
+
+// CreateMirrorSnapshot creates a snapshot for image propagation to mirrors.
+//
+// Implements:
+//  int rbd_mirror_image_create_snapshot(rbd_image_t image,
+//                                       uint64_t *snap_id);
+func (image *Image) CreateMirrorSnapshot() (uint64, error) {
+	var snapID C.uint64_t
+	ret := C.rbd_mirror_image_create_snapshot(
+		image.image,
+		&snapID)
+	return uint64(snapID), getError(ret)
+}


Test patch:
diff --git a/rbd/mirror_test.go b/rbd/mirror_test.go
index 8a77a72..df04baf 100644
--- a/rbd/mirror_test.go
+++ b/rbd/mirror_test.go
@@ -9,7 +9,9 @@ package rbd
 
 import (
 	"fmt"
+	"os"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -262,3 +264,202 @@ func TestMirrorConstantStrings(t *testing.T) {
 		assert.Equal(t, v.s.String(), v.t)
 	}
 }
+
+func TestGetGlobalMirrorStatus(t *testing.T) {
+	conn := radosConnect(t)
+	poolName := GetUUID()
+	err := conn.MakePool(poolName)
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, conn.DeletePool(poolName))
+		conn.Shutdown()
+	}()
+
+	ioctx, err := conn.OpenIOContext(poolName)
+	assert.NoError(t, err)
+	defer func() {
+		ioctx.Destroy()
+	}()
+
+	// enable per-image mirroring for this pool
+	err = SetMirrorMode(ioctx, MirrorModeImage)
+	require.NoError(t, err)
+
+	imgName := GetUUID()
+	options := NewRbdImageOptions()
+	assert.NoError(t, options.SetUint64(ImageOptionOrder, uint64(testImageOrder)))
+	err = CreateImage(ioctx, imgName, testImageSize, options)
+	require.NoError(t, err)
+
+	t.Run("closedImage", func(t *testing.T) {
+		img := GetImage(ioctx, imgName)
+		_, err = img.GetGlobalMirrorStatus()
+		assert.Error(t, err)
+	})
+
+	t.Run("getStatus", func(t *testing.T) {
+		// open image, enable, mirroring.
+		img, err := OpenImage(ioctx, imgName, NoSnapshot)
+		assert.NoError(t, err)
+		defer func() {
+			assert.NoError(t, img.Close())
+		}()
+
+		err = img.MirrorEnable(ImageMirrorModeSnapshot)
+		assert.NoError(t, err)
+		gms, err := img.GetGlobalMirrorStatus()
+		assert.NoError(t, err)
+		assert.NotEqual(t, "", gms.Name)
+		assert.NotEqual(t, "", gms.Info.GlobalID)
+		assert.Equal(t, gms.Info.State, MirrorImageEnabled)
+		assert.Equal(t, gms.Info.Primary, true)
+		if assert.Len(t, gms.SiteStatuses, 1) {
+			ss := gms.SiteStatuses[0]
+			assert.Equal(t, "", ss.MirrorUUID)
+			assert.Equal(t, MirrorImageStatusStateUnknown, ss.State, ss.State)
+			assert.Equal(t, "status not found", ss.Description)
+			assert.Equal(t, int64(0), ss.LastUpdate)
+			assert.False(t, ss.Up)
+			ls, err := gms.LocalStatus()
+			assert.NoError(t, err)
+			assert.Equal(t, ss, ls)
+		}
+	})
+}
+
+func mirrorConfig() string {
+	return os.Getenv("MIRROR_CONF")
+}
+
+func TestGetGlobalMirrorStatusMirroredPool(t *testing.T) {
+	mconfig := mirrorConfig()
+	if mconfig == "" {
+		t.Skip("no mirror config env var set")
+	}
+	conn := radosConnect(t)
+	// this test assumes the rbd pool already exists and is mirrored
+	// this must be set up previously by the CI or manually
+	poolName := "rbd"
+
+	ioctx, err := conn.OpenIOContext(poolName)
+	assert.NoError(t, err)
+	defer func() {
+		ioctx.Destroy()
+	}()
+
+	imgName := GetUUID()
+	options := NewRbdImageOptions()
+	assert.NoError(t, options.SetUint64(ImageOptionOrder, uint64(testImageOrder)))
+	err = CreateImage(ioctx, imgName, testImageSize, options)
+	require.NoError(t, err)
+
+	defer func() {
+		err = RemoveImage(ioctx, imgName)
+		assert.NoError(t, err)
+	}()
+
+	// this next section is not a t.Run because it must be unconditionally
+	// executed. It is wrapped in a func to use defer to close the img.
+	func() {
+		img, err := OpenImage(ioctx, imgName, NoSnapshot)
+		assert.NoError(t, err)
+		defer func() {
+			assert.NoError(t, img.Close())
+		}()
+
+		err = img.MirrorEnable(ImageMirrorModeSnapshot)
+		assert.NoError(t, err)
+
+		mid, err := img.CreateMirrorSnapshot()
+		assert.NoError(t, err)
+		assert.NotEqual(t, 0, mid)
+
+		// wait for site statuses to get updated
+		for i := 0; i < 30; i++ {
+			gms, err := img.GetGlobalMirrorStatus()
+			assert.NoError(t, err)
+			if len(gms.SiteStatuses) > 1 {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
+		gms, err := img.GetGlobalMirrorStatus()
+		assert.NoError(t, err)
+		assert.NotEqual(t, "", gms.Name)
+		assert.NotEqual(t, "", gms.Info.GlobalID)
+		assert.Equal(t, gms.Info.State, MirrorImageEnabled)
+		assert.Equal(t, gms.Info.Primary, true)
+		if assert.Len(t, gms.SiteStatuses, 2) {
+			ss1 := gms.SiteStatuses[0]
+			assert.Equal(t, "", ss1.MirrorUUID)
+			assert.Equal(t, MirrorImageStatusStateStopped, ss1.State, ss1.State)
+			assert.Equal(t, "local image is primary", ss1.Description)
+			assert.Greater(t, ss1.LastUpdate, int64(0))
+			assert.True(t, ss1.Up)
+			ls, err := gms.LocalStatus()
+			assert.NoError(t, err)
+			assert.Equal(t, ss1, ls)
+
+			ss2 := gms.SiteStatuses[1]
+			assert.NotEqual(t, "", ss2.MirrorUUID)
+			assert.Equal(t, MirrorImageStatusStateReplaying, ss2.State, ss2.State)
+			assert.Contains(t, ss2.Description, "replaying,")
+			assert.Greater(t, ss2.LastUpdate, int64(0))
+			assert.True(t, ss2.Up)
+		}
+	}()
+
+	// test the results of GetGlobalMirrorStatus using the "other"
+	// mirror+pool as a source
+	t.Run("fromMirror", func(t *testing.T) {
+		conn := radosConnectConfig(t, mconfig)
+		ioctx2, err := conn.OpenIOContext(poolName)
+		assert.NoError(t, err)
+		defer func() {
+			ioctx2.Destroy()
+		}()
+
+		img, err := OpenImage(ioctx2, imgName, NoSnapshot)
+		assert.NoError(t, err)
+		defer func() {
+			assert.NoError(t, img.Close())
+		}()
+
+		// wait for site statuses to get updated
+		for i := 0; i < 30; i++ {
+			gms, err := img.GetGlobalMirrorStatus()
+			assert.NoError(t, err)
+			if len(gms.SiteStatuses) > 1 {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
+		gms, err := img.GetGlobalMirrorStatus()
+		assert.NoError(t, err)
+		assert.NotEqual(t, "", gms.Name)
+		assert.NotEqual(t, "", gms.Info.GlobalID)
+		assert.Equal(t, gms.Info.State, MirrorImageEnabled)
+		assert.Equal(t, gms.Info.Primary, false)
+		if assert.Len(t, gms.SiteStatuses, 2) {
+			ls, err := gms.LocalStatus()
+			assert.NoError(t, err)
+			assert.Equal(t, "", ls.MirrorUUID)
+			assert.Equal(t, MirrorImageStatusStateReplaying, ls.State, ls.State)
+			assert.Contains(t, ls.Description, "replaying,")
+			assert.Greater(t, ls.LastUpdate, int64(0))
+			assert.True(t, ls.Up)
+
+			assert.Equal(t, ls, gms.SiteStatuses[0])
+
+			ss2 := gms.SiteStatuses[1]
+			assert.NotEqual(t, "", ss2.MirrorUUID)
+			assert.Equal(t, MirrorImageStatusStateStopped, ss2.State, ss2.State)
+
+			assert.Equal(t, "local image is primary", ss2.Description)
+			assert.Greater(t, ss2.LastUpdate, int64(0))
+			assert.True(t, ss2.Up)
+		}
+	})
+}
diff --git a/rbd/rbd_test.go b/rbd/rbd_test.go
index 781bb99..80fa6c0 100644
--- a/rbd/rbd_test.go
+++ b/rbd/rbd_test.go
@@ -37,8 +37,22 @@ func radosConnect(t *testing.T) *rados.Conn {
 	require.NoError(t, err)
 	err = conn.ReadDefaultConfigFile()
 	require.NoError(t, err)
+	waitForRadosConn(t, conn)
+	return conn
+}
 
-	timeout := time.After(time.Second * 5)
+func radosConnectConfig(t *testing.T, p string) *rados.Conn {
+	conn, err := rados.NewConn()
+	require.NoError(t, err)
+	err = conn.ReadConfigFile(p)
+	require.NoError(t, err)
+	waitForRadosConn(t, conn)
+	return conn
+}
+
+func waitForRadosConn(t *testing.T, conn *rados.Conn) {
+	var err error
+	timeout := time.After(time.Second * 15)
 	ch := make(chan error)
 	go func(conn *rados.Conn) {
 		ch <- conn.Connect()
@@ -49,7 +63,6 @@ func radosConnect(t *testing.T) *rados.Conn {
 		err = fmt.Errorf("timed out waiting for connect")
 	}
 	require.NoError(t, err)
-	return conn
 }
 
 func TestImageCreate(t *testing.T) {