Repository: spf13/afero
Pull request #333 (2021-12-27 18:02:17)

GCS fs: move all gcsfs related implementations to its own package
this way we don't force any application that import afero to include
gcfs deps in its binary


Patch:
diff --git a/gcsfs/file.go b/gcsfs/file.go
index b3b3d89..a3d491e 100644
--- a/gcsfs/file.go
+++ b/gcsfs/file.go
@@ -44,7 +44,7 @@ type GcsFile struct {
 
 func NewGcsFile(
 	ctx context.Context,
-	fs *GcsFs,
+	fs *Fs,
 	obj stiface.ObjectHandle,
 	openFlags int,
 	// Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it
diff --git a/gcsfs/file_info.go b/gcsfs/file_info.go
index 2cccb7c..0a42d38 100644
--- a/gcsfs/file_info.go
+++ b/gcsfs/file_info.go
@@ -37,7 +37,7 @@ type FileInfo struct {
 	fileMode os.FileMode
 }
 
-func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error) {
+func newFileInfo(name string, fs *Fs, fileMode os.FileMode) (*FileInfo, error) {
 	res := &FileInfo{
 		name:     name,
 		size:     folderSize,
diff --git a/gcsfs/file_resource.go b/gcsfs/file_resource.go
index b25d0e8..c25ad51 100644
--- a/gcsfs/file_resource.go
+++ b/gcsfs/file_resource.go
@@ -40,7 +40,7 @@ const (
 type gcsFileResource struct {
 	ctx context.Context
 
-	fs *GcsFs
+	fs *Fs
 
 	obj      stiface.ObjectHandle
 	name     string
diff --git a/gcsfs/fs.go b/gcsfs/fs.go
index c4d899c..dc099b1 100644
--- a/gcsfs/fs.go
+++ b/gcsfs/fs.go
@@ -33,8 +33,8 @@ const (
 	gsPrefix        = "gs://"
 )
 
-// GcsFs is a Fs implementation that uses functions provided by google cloud storage
-type GcsFs struct {
+// Fs is a Fs implementation that uses functions provided by google cloud storage
+type Fs struct {
 	ctx       context.Context
 	client    stiface.Client
 	separator string
@@ -45,12 +45,12 @@ type GcsFs struct {
 	autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs)
 }
 
-func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs {
+func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
 	return NewGcsFsWithSeparator(ctx, client, "/")
 }
 
-func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs {
-	return &GcsFs{
+func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs {
+	return &Fs{
 		ctx:           ctx,
 		client:        client,
 		separator:     folderSep,
@@ -61,17 +61,17 @@ func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep
 }
 
 // normSeparators will normalize all "\\" and "/" to the provided separator
-func (fs *GcsFs) normSeparators(s string) string {
+func (fs *Fs) normSeparators(s string) string {
 	return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
 }
 
-func (fs *GcsFs) ensureTrailingSeparator(s string) string {
+func (fs *Fs) ensureTrailingSeparator(s string) string {
 	if len(s) > 0 && !strings.HasSuffix(s, fs.separator) {
 		return s + fs.separator
 	}
 	return s
 }
-func (fs *GcsFs) ensureNoLeadingSeparator(s string) string {
+func (fs *Fs) ensureNoLeadingSeparator(s string) string {
 	if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
 		s = s[len(fs.separator):]
 	}
@@ -94,13 +94,13 @@ func validateName(s string) error {
 }
 
 // Splits provided name into bucket name and path
-func (fs *GcsFs) splitName(name string) (bucketName string, path string) {
+func (fs *Fs) splitName(name string) (bucketName string, path string) {
 	splitName := strings.Split(name, fs.separator)
 
 	return splitName[0], strings.Join(splitName[1:], fs.separator)
 }
 
-func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
+func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) {
 	bucket := fs.buckets[name]
 	if bucket == nil {
 		bucket = fs.client.Bucket(name)
@@ -112,7 +112,7 @@ func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
 	return bucket, nil
 }
 
-func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
+func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) {
 	bucketName, path := fs.splitName(name)
 
 	bucket, err := fs.getBucket(bucketName)
@@ -123,9 +123,9 @@ func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
 	return bucket.Object(path), nil
 }
 
-func (fs *GcsFs) Name() string { return "GcsFs" }
+func (fs *Fs) Name() string { return "GcsFs" }
 
-func (fs *GcsFs) Create(name string) (*GcsFile, error) {
+func (fs *Fs) Create(name string) (*GcsFile, error) {
 	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
 	if err := validateName(name); err != nil {
 		return nil, err
@@ -156,7 +156,7 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
 	return file, nil
 }
 
-func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
+func (fs *Fs) Mkdir(name string, _ os.FileMode) error {
 	name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
 	if err := validateName(name); err != nil {
 		return err
@@ -179,7 +179,7 @@ func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
 	return w.Close()
 }
 
-func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
+func (fs *Fs) MkdirAll(path string, perm os.FileMode) error {
 	path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
 	if err := validateName(path); err != nil {
 		return err
@@ -216,11 +216,11 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
 	return nil
 }
 
-func (fs *GcsFs) Open(name string) (*GcsFile, error) {
+func (fs *Fs) Open(name string) (*GcsFile, error) {
 	return fs.OpenFile(name, os.O_RDONLY, 0)
 }
 
-func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
+func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
 	var file *GcsFile
 	var err error
 
@@ -277,7 +277,7 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
 	return file, nil
 }
 
-func (fs *GcsFs) Remove(name string) error {
+func (fs *Fs) Remove(name string) error {
 	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
 	if err := validateName(name); err != nil {
 		return err
@@ -318,7 +318,7 @@ func (fs *GcsFs) Remove(name string) error {
 	return obj.Delete(fs.ctx)
 }
 
-func (fs *GcsFs) RemoveAll(path string) error {
+func (fs *Fs) RemoveAll(path string) error {
 	path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
 	if err := validateName(path); err != nil {
 		return err
@@ -351,7 +351,7 @@ func (fs *GcsFs) RemoveAll(path string) error {
 	return fs.Remove(path)
 }
 
-func (fs *GcsFs) Rename(oldName, newName string) error {
+func (fs *Fs) Rename(oldName, newName string) error {
 	oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
 	if err := validateName(oldName); err != nil {
 		return err
@@ -378,7 +378,7 @@ func (fs *GcsFs) Rename(oldName, newName string) error {
 	return src.Delete(fs.ctx)
 }
 
-func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
+func (fs *Fs) Stat(name string) (os.FileInfo, error) {
 	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
 	if err := validateName(name); err != nil {
 		return nil, err
@@ -387,14 +387,14 @@ func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
 	return newFileInfo(name, fs, defaultFileMode)
 }
 
-func (fs *GcsFs) Chmod(_ string, _ os.FileMode) error {
+func (fs *Fs) Chmod(_ string, _ os.FileMode) error {
 	return errors.New("method Chmod is not implemented in GCS")
 }
 
-func (fs *GcsFs) Chtimes(_ string, _, _ time.Time) error {
+func (fs *Fs) Chtimes(_ string, _, _ time.Time) error {
 	return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly")
 }
 
-func (fs *GcsFs) Chown(_ string, _, _ int) error {
+func (fs *Fs) Chown(_ string, _, _ int) error {
 	return errors.New("method Chown is not implemented for GCS")
 }
diff --git a/gcs-fake-service-account.json b/gcsfs/gcs-fake-service-account.json
similarity index 100%
rename from gcs-fake-service-account.json
rename to gcsfs/gcs-fake-service-account.json
diff --git a/gcs.go b/gcsfs/gcs.go
similarity index 88%
rename from gcs.go
rename to gcsfs/gcs.go
index 1135822..78cc924 100644
--- a/gcs.go
+++ b/gcsfs/gcs.go
@@ -14,29 +14,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package afero
+package gcsfs
 
 import (
 	"context"
 	"os"
 	"time"
 
-	"github.com/spf13/afero/gcsfs"
-
 	"cloud.google.com/go/storage"
 	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
+	"github.com/spf13/afero"
 
 	"google.golang.org/api/option"
 )
 
 type GcsFs struct {
-	source *gcsfs.GcsFs
+	source *Fs
 }
 
 // NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
 // You can provide additional options to be passed to the client creation, as per
 // cloud.google.com/go/storage documentation
-func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
+func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (afero.Fs, error) {
 	if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" {
 		opts = append(opts, option.WithCredentialsJSON([]byte(json)))
 	}
@@ -49,7 +48,7 @@ func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
 }
 
 // NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
-func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
+func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (afero.Fs, error) {
 	client, err := storage.NewClient(ctx, opts...)
 	if err != nil {
 		return nil, err
@@ -59,17 +58,17 @@ func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...
 }
 
 // NewGcsFSFromClient creates a GCS file system from a given storage client
-func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
+func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (afero.Fs, error) {
 	c := stiface.AdaptClient(client)
 
-	return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
+	return &GcsFs{NewGcsFs(ctx, c)}, nil
 }
 
 // NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
-func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
+func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (afero.Fs, error) {
 	c := stiface.AdaptClient(client)
 
-	return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
+	return &GcsFs{NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
 }
 
 // Wraps gcs.GcsFs and convert some return types to afero interfaces.
@@ -77,7 +76,7 @@ func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client
 func (fs *GcsFs) Name() string {
 	return fs.source.Name()
 }
-func (fs *GcsFs) Create(name string) (File, error) {
+func (fs *GcsFs) Create(name string) (afero.File, error) {
 	return fs.source.Create(name)
 }
 func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
@@ -86,10 +85,10 @@ func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
 func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
 	return fs.source.MkdirAll(path, perm)
 }
-func (fs *GcsFs) Open(name string) (File, error) {
+func (fs *GcsFs) Open(name string) (afero.File, error) {
 	return fs.source.Open(name)
 }
-func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
 	return fs.source.OpenFile(name, flag, perm)
 }
 func (fs *GcsFs) Remove(name string) error {
diff --git a/gcs_mocks.go b/gcsfs/gcs_mocks.go
similarity index 91%
rename from gcs_mocks.go
rename to gcsfs/gcs_mocks.go
index 57e0da5..272eb9f 100644
--- a/gcs_mocks.go
+++ b/gcsfs/gcs_mocks.go
@@ -9,7 +9,7 @@
 // switching over to a real bucket - and then the mocks have to be adjusted to match the
 // implementation.
 
-package afero
+package gcsfs
 
 import (
 	"context"
@@ -17,10 +17,9 @@ import (
 	"os"
 	"strings"
 
-	"github.com/spf13/afero/gcsfs"
-
 	"cloud.google.com/go/storage"
 	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
+	"github.com/spf13/afero"
 	"google.golang.org/api/iterator"
 )
 
@@ -31,11 +30,11 @@ func normSeparators(s string) string {
 
 type clientMock struct {
 	stiface.Client
-	fs Fs
+	fs afero.Fs
 }
 
 func newClientMock() *clientMock {
-	return &clientMock{fs: NewMemMapFs()}
+	return &clientMock{fs: afero.NewMemMapFs()}
 }
 
 func (m *clientMock) Bucket(name string) stiface.BucketHandle {
@@ -47,7 +46,7 @@ type bucketMock struct {
 
 	bucketName string
 
-	fs Fs
+	fs afero.Fs
 }
 
 func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) {
@@ -66,7 +65,7 @@ type objectMock struct {
 	stiface.ObjectHandle
 
 	name string
-	fs   Fs
+	fs   afero.Fs
 }
 
 func (o *objectMock) NewWriter(_ context.Context) stiface.Writer {
@@ -75,7 +74,7 @@ func (o *objectMock) NewWriter(_ context.Context) stiface.Writer {
 
 func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (stiface.Reader, error) {
 	if o.name == "" {
-		return nil, gcsfs.ErrEmptyObjectName
+		return nil, ErrEmptyObjectName
 	}
 
 	file, err := o.fs.Open(o.name)
@@ -104,14 +103,14 @@ func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (st
 
 func (o *objectMock) Delete(_ context.Context) error {
 	if o.name == "" {
-		return gcsfs.ErrEmptyObjectName
+		return ErrEmptyObjectName
 	}
 	return o.fs.Remove(o.name)
 }
 
 func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) {
 	if o.name == "" {
-		return nil, gcsfs.ErrEmptyObjectName
+		return nil, ErrEmptyObjectName
 	}
 
 	info, err := o.fs.Stat(o.name)
@@ -130,7 +129,7 @@ func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) {
 
 	if info.IsDir() {
 		// we have to mock it here, because of FileInfo logic
-		return nil, gcsfs.ErrObjectDoesNotExist
+		return nil, ErrObjectDoesNotExist
 	}
 
 	return res, nil
@@ -140,14 +139,14 @@ type writerMock struct {
 	stiface.Writer
 
 	name string
-	fs   Fs
+	fs   afero.Fs
 
-	file File
+	file afero.File
 }
 
 func (w *writerMock) Write(p []byte) (n int, err error) {
 	if w.name == "" {
-		return 0, gcsfs.ErrEmptyObjectName
+		return 0, ErrEmptyObjectName
 	}
 
 	if w.file == nil {
@@ -162,7 +161,7 @@ func (w *writerMock) Write(p []byte) (n int, err error) {
 
 func (w *writerMock) Close() error {
 	if w.name == "" {
-		return gcsfs.ErrEmptyObjectName
+		return ErrEmptyObjectName
 	}
 	if w.file == nil {
 		var err error
@@ -187,7 +186,7 @@ func (w *writerMock) Close() error {
 type readerMock struct {
 	stiface.Reader
 
-	file File
+	file afero.File
 
 	buf []byte
 }
@@ -212,9 +211,9 @@ type objectItMock struct {
 	stiface.ObjectIterator
 
 	name string
-	fs   Fs
+	fs   afero.Fs
 
-	dir   File
+	dir   afero.File
 	infos []*storage.ObjectAttrs
 }
 
@@ -227,7 +226,7 @@ func (it *objectItMock) Next() (*storage.ObjectAttrs, error) {
 		}
 
 		var isDir bool
-		isDir, err = IsDir(it.fs, it.name)
+		isDir, err = afero.IsDir(it.fs, it.name)
 		if err != nil {
 			return nil, err
 		}


Test patch:
diff --git a/gcs_test.go b/gcsfs/gcs_test.go
similarity index 98%
rename from gcs_test.go
rename to gcsfs/gcs_test.go
index 37c2cf5..671787c 100644
--- a/gcs_test.go
+++ b/gcsfs/gcs_test.go
@@ -3,7 +3,7 @@
 // Most of the tests are "derived" from the Afero's own tarfs implementation.
 // Write-oriented tests and/or checks have been added on top of that
 
-package afero
+package gcsfs
 
 import (
 	"context"
@@ -19,10 +19,9 @@ import (
 
 	"golang.org/x/oauth2/google"
 
-	"github.com/spf13/afero/gcsfs"
-
 	"cloud.google.com/go/storage"
 	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
+	"github.com/spf13/afero"
 )
 
 const (
@@ -62,7 +61,7 @@ var dirs = []struct {
 	{"testDir1", []string{"testFile"}},
 }
 
-var gcsAfs *Afero
+var gcsAfs *afero.Afero
 
 func TestMain(m *testing.M) {
 	ctx := context.Background()
@@ -120,7 +119,7 @@ func TestMain(m *testing.M) {
 	mockClient := newClientMock()
 	mockClient.Client = client
 
-	gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}}
+	gcsAfs = &afero.Afero{Fs: &GcsFs{NewGcsFs(ctx, mockClient)}}
 
 	// Uncomment to use the real, not mocked, client
 	//gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}}
@@ -137,7 +136,7 @@ func createFiles(t *testing.T) {
 		if !f.isdir && f.exists {
 			name := filepath.Join(bucketName, f.name)
 
-			var freshFile File
+			var freshFile afero.File
 			freshFile, err = gcsAfs.Create(name)
 			if err != nil {
 				t.Fatalf("failed to create a file \"%s\": %s", f.name, err)
@@ -481,7 +480,7 @@ func TestGcsOpenFile(t *testing.T) {
 			file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400)
 			if !f.exists {
 				if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
-					(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
+					(f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
 					t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
 				}
 
@@ -524,7 +523,7 @@ func TestGcsFsStat(t *testing.T) {
 			fi, err := gcsAfs.Stat(name)
 			if !f.exists {
 				if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
-					(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
+					(f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
 					t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
 				}
 
@@ -698,7 +697,7 @@ func TestGcsGlob(t *testing.T) {
 		}
 
 		for i, prefixedGlob := range prefixedGlobs {
-			entries, err := Glob(gcsAfs.Fs, prefixedGlob)
+			entries, err := afero.Glob(gcsAfs.Fs, prefixedGlob)
 			if err != nil {
 				t.Error(err)
 			}