Repository: ceph/go-ceph
Pull request #187 (2020-02-25 15:58:22)

cephfs: Initial directory support
This PR conflicts with / obsoletes PR #91.

Start implementing functions related to directories in cephfs. We establish a new directory handle type with associated open and close functions. We add a readdir wrapper. We add a rewinddir wrapper and with readdir create a List convenience function for common use cases that just want to consume all the dir entries in one call and iterate over them. The List function returns a DirEntries type that has a Names() method that returns only the names of the directory entries, once again as a convenience for the common case.


## Checklist
- [x] Added tests for features and functional changes
- [x] Public functions and types are documented
- [x] Standard formatting is applied to Go code


phlogistonjohn: This differs from the other PR in that the Close method is on the dir handle, the functionality is located in its own file and the additional methods provided.
phlogistonjohn: There's a bunch of fields in the underlying dir entry that are not represented in the new struct yet. I did this intentionally to keep this initial PR simple. I do plan on specifically adding support for "d_type" but didn't want to "boil the ocean" with this pr.
isi-lincoln: this will meet my requirements.  Thanks for putting in the effort to revitalize it!
phlogistonjohn: @nixpanic I have removed capitalization, and thus made private, from the list method as well as names and dirEntries type. This retains the logic for any future tests I will write as well as leaving open the possibility of making these convenience functions public again in the future without advertising them right away and locking them in as *the* convenience api. PTAL.

Patch:
diff --git a/cephfs/directory.go b/cephfs/directory.go
new file mode 100644
index 0000000..fddf847
--- /dev/null
+++ b/cephfs/directory.go
@@ -0,0 +1,141 @@
+package cephfs
+
+/*
+#cgo LDFLAGS: -lcephfs
+#cgo CPPFLAGS: -D_FILE_OFFSET_BITS=64
+#include <stdlib.h>
+#include <dirent.h>
+#include <cephfs/libcephfs.h>
+*/
+import "C"
+
+import (
+	"unsafe"
+)
+
+// Directory represents an open directory handle.
+type Directory struct {
+	mount *MountInfo
+	dir   *C.struct_ceph_dir_result
+}
+
+// OpenDir returns a new Directory handle open for I/O.
+//
+// Implements:
+//  int ceph_opendir(struct ceph_mount_info *cmount, const char *name, struct ceph_dir_result **dirpp);
+func (mount *MountInfo) OpenDir(path string) (*Directory, error) {
+	var dir *C.struct_ceph_dir_result
+
+	cPath := C.CString(path)
+	defer C.free(unsafe.Pointer(cPath))
+
+	ret := C.ceph_opendir(mount.mount, cPath, &dir)
+	if ret != 0 {
+		return nil, getError(ret)
+	}
+
+	return &Directory{
+		mount: mount,
+		dir:   dir,
+	}, nil
+}
+
+// Close the open directory handle.
+//
+// Implements:
+//  int ceph_closedir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
+func (dir *Directory) Close() error {
+	return getError(C.ceph_closedir(dir.mount.mount, dir.dir))
+}
+
+// Inode represents an inode number in the file system.
+type Inode uint64
+
+// DirEntry represents an entry within a directory.
+type DirEntry struct {
+	inode Inode
+	name  string
+}
+
+// Name returns the directory entry's name.
+func (d *DirEntry) Name() string {
+	return d.name
+}
+
+// Inode returns the directory entry's inode number.
+func (d *DirEntry) Inode() Inode {
+	return d.inode
+}
+
+// toDirEntry converts a c struct dirent to our go wrapper.
+func toDirEntry(de *C.struct_dirent) *DirEntry {
+	return &DirEntry{
+		inode: Inode(de.d_ino),
+		name:  C.GoString(&de.d_name[0]),
+	}
+}
+
+// ReadDir reads a single directory entry from the open Directory.
+// A nil DirEntry pointer will be returned when the Directory stream has been
+// exhausted.
+//
+// Implements:
+//  int ceph_readdir_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de);
+func (dir *Directory) ReadDir() (*DirEntry, error) {
+	var de C.struct_dirent
+	ret := C.ceph_readdir_r(dir.mount.mount, dir.dir, &de)
+	if ret < 0 {
+		return nil, getError(ret)
+	}
+	if ret == 0 {
+		return nil, nil // End-of-stream
+	}
+	return toDirEntry(&de), nil
+}
+
+// RewindDir sets the directory stream to the beginning of the directory.
+//
+// Implements:
+//  void ceph_rewinddir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
+func (dir *Directory) RewindDir() {
+	C.ceph_rewinddir(dir.mount.mount, dir.dir)
+}
+
+// dirEntries provides a convenient wrapper around slices of DirEntry items.
+// For example, use the Names() call to easily get only the names from a
+// DirEntry slice.
+type dirEntries []*DirEntry
+
+// list returns all the contents of a directory as a dirEntries slice.
+//
+// list is implemented using ReadDir. If any of the calls to ReadDir returns
+// an error List will return an error. However, all previous entries
+// collected will still be returned. Callers of this function may want to check
+// the entries return value even when an error is returned.
+// List rewinds the handle every time it is called to get a full
+// listing of directory contents.
+func (dir *Directory) list() (dirEntries, error) {
+	var (
+		err     error
+		entry   *DirEntry
+		entries = make(dirEntries, 0)
+	)
+	dir.RewindDir()
+	for {
+		entry, err = dir.ReadDir()
+		if err != nil || entry == nil {
+			break
+		}
+		entries = append(entries, entry)
+	}
+	return entries, err
+}
+
+// names returns a slice of only the name fields from dir entries.
+func (entries dirEntries) names() []string {
+	names := make([]string, len(entries))
+	for i, v := range entries {
+		names[i] = v.Name()
+	}
+	return names
+}


Test patch:
diff --git a/cephfs/directory_test.go b/cephfs/directory_test.go
new file mode 100644
index 0000000..46b3cf7
--- /dev/null
+++ b/cephfs/directory_test.go
@@ -0,0 +1,162 @@
+package cephfs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOpenCloseDir(t *testing.T) {
+	mount := fsConnect(t)
+	defer fsDisconnect(t, mount)
+
+	dir1 := "/base"
+	err := mount.MakeDir(dir1, 0755)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, mount.RemoveDir(dir1)) }()
+
+	dir2 := dir1 + "/a"
+	err = mount.MakeDir(dir2, 0755)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, mount.RemoveDir(dir2)) }()
+
+	dir, err := mount.OpenDir(dir1)
+	assert.NoError(t, err)
+	assert.NotNil(t, dir)
+	err = dir.Close()
+	assert.NoError(t, err)
+
+	dir, err = mount.OpenDir(dir2)
+	assert.NoError(t, err)
+	assert.NotNil(t, dir)
+	err = dir.Close()
+	assert.NoError(t, err)
+
+	dir, err = mount.OpenDir("/no.such.dir")
+	assert.Error(t, err)
+	assert.Nil(t, dir)
+}
+
+func TestReadDir(t *testing.T) {
+	mount := fsConnect(t)
+	defer fsDisconnect(t, mount)
+
+	dir1 := "/base"
+	err := mount.MakeDir(dir1, 0755)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, mount.RemoveDir(dir1)) }()
+
+	subdirs := []string{"a", "bb", "ccc", "dddd"}
+	for _, s := range subdirs {
+		spath := dir1 + "/" + s
+		err = mount.MakeDir(spath, 0755)
+		assert.NoError(t, err)
+		defer func(d string) {
+			assert.NoError(t, mount.RemoveDir(d))
+		}(spath)
+	}
+
+	t.Run("root", func(t *testing.T) {
+		dir, err := mount.OpenDir("/")
+		assert.NoError(t, err)
+		assert.NotNil(t, dir)
+		defer func() { assert.NoError(t, dir.Close()) }()
+
+		found := []string{}
+		for {
+			entry, err := dir.ReadDir()
+			assert.NoError(t, err)
+			if entry == nil {
+				break
+			}
+			assert.NotEqual(t, Inode(0), entry.Inode())
+			assert.NotEqual(t, "", entry.Name())
+			found = append(found, entry.Name())
+		}
+		assert.Contains(t, found, "base")
+	})
+	t.Run("dir1", func(t *testing.T) {
+		dir, err := mount.OpenDir(dir1)
+		assert.NoError(t, err)
+		assert.NotNil(t, dir)
+		defer func() { assert.NoError(t, dir.Close()) }()
+
+		found := []string{}
+		for {
+			entry, err := dir.ReadDir()
+			assert.NoError(t, err)
+			if entry == nil {
+				break
+			}
+			assert.NotEqual(t, Inode(0), entry.Inode())
+			assert.NotEqual(t, "", entry.Name())
+			found = append(found, entry.Name())
+		}
+		assert.Subset(t, found, subdirs)
+	})
+}
+
+func TestDirectoryList(t *testing.T) {
+	mount := fsConnect(t)
+	defer fsDisconnect(t, mount)
+
+	dir1 := "/base"
+	err := mount.MakeDir(dir1, 0755)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, mount.RemoveDir(dir1)) }()
+
+	subdirs := []string{"a", "bb", "ccc", "dddd"}
+	for _, s := range subdirs {
+		spath := dir1 + "/" + s
+		err = mount.MakeDir(spath, 0755)
+		assert.NoError(t, err)
+		defer func(d string) {
+			assert.NoError(t, mount.RemoveDir(d))
+		}(spath)
+	}
+
+	t.Run("root", func(t *testing.T) {
+		dir, err := mount.OpenDir("/")
+		assert.NoError(t, err)
+		assert.NotNil(t, dir)
+		defer func() { assert.NoError(t, dir.Close()) }()
+
+		entries, err := dir.list()
+		assert.NoError(t, err)
+		assert.Greater(t, len(entries), 1)
+		found := entries.names()
+		assert.Contains(t, found, "base")
+	})
+	t.Run("dir1", func(t *testing.T) {
+		dir, err := mount.OpenDir(dir1)
+		assert.NoError(t, err)
+		assert.NotNil(t, dir)
+		defer func() { assert.NoError(t, dir.Close()) }()
+
+		entries, err := dir.list()
+		assert.NoError(t, err)
+		assert.Greater(t, len(entries), 1)
+		found := entries.names()
+		assert.Subset(t, found, subdirs)
+	})
+	t.Run("dir1Twice", func(t *testing.T) {
+		dir, err := mount.OpenDir(dir1)
+		assert.NoError(t, err)
+		assert.NotNil(t, dir)
+		defer func() { assert.NoError(t, dir.Close()) }()
+
+		entries, err := dir.list()
+		assert.NoError(t, err)
+		assert.Greater(t, len(entries), 1)
+		found := entries.names()
+		assert.Subset(t, found, subdirs)
+
+		// verify that calling list gives a complete list
+		// even after being used for the same directory already
+		entries, err = dir.list()
+		assert.NoError(t, err)
+		assert.Greater(t, len(entries), 1)
+		found = entries.names()
+		assert.Subset(t, found, subdirs)
+	})
+}