Repository: fsouza/fake-gcs-server
Pull request #1017 (2022-12-23 09:16:49)

Do not escape the object name in the fs backend
Now fake-gcs-server implicitly creates parent directories when creating an object `/foo/bar/baz`.  This allows creating paths longer than `NAME_MAX` (256) and interoperating with existing filesystems.  This approach does not allow some kinds of objects to exist, e.g., `/foo/` with a non-zero object size, although these should be rare.

gaul: Sorry had some trouble with Windows.

Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index de11d4e..e421cf4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -75,7 +75,7 @@ jobs:
 
       - name: staticcheck cache
         id: staticcheck-cache
-        uses: actions/cache@v3.2.0
+        uses: actions/cache@v3.2.2
         with:
           path: ~/.cache/staticcheck
           key: "${{ steps.get-staticcheck-version.outputs.version }}"
diff --git a/fakestorage/object.go b/fakestorage/object.go
index e413f4c..6cf298f 100644
--- a/fakestorage/object.go
+++ b/fakestorage/object.go
@@ -810,6 +810,9 @@ func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("X-Goog-Generation", strconv.FormatInt(obj.Generation, 10))
 	w.Header().Set("X-Goog-Hash", fmt.Sprintf("crc32c=%s,md5=%s", obj.Crc32c, obj.Md5Hash))
 	w.Header().Set("Last-Modified", obj.Updated.Format(http.TimeFormat))
+	for name, value := range obj.Metadata {
+		w.Header().Set("X-Goog-Meta-"+name, value)
+	}
 
 	if ranged && !satisfiable {
 		status = http.StatusRequestedRangeNotSatisfiable
diff --git a/internal/backend/fs.go b/internal/backend/fs.go
index 01f1cce..27a3a5f 100644
--- a/internal/backend/fs.go
+++ b/internal/backend/fs.go
@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -178,7 +179,16 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St
 		return StreamingObject{}, PreConditionFailed
 	}
 
-	path := filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), url.PathEscape(obj.Name))
+	path := filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), obj.Name)
+	if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return StreamingObject{}, err
+	}
+
+	// Nothing to do if this operation only creates directories
+	if strings.HasSuffix(obj.Name, "/") {
+		// TODO: populate Crc32c, Md5Hash, and Etag
+		return StreamingObject{obj.ObjectAttrs, noopSeekCloser{bytes.NewReader([]byte{})}}, nil
+	}
 
 	var buf bytes.Buffer
 	hasher := checksum.NewStreamingHasher()
@@ -217,28 +227,31 @@ func (s *storageFS) ListObjects(bucketName string, prefix string, versions bool)
 	s.mtx.RLock()
 	defer s.mtx.RUnlock()
 
-	infos, err := os.ReadDir(filepath.Join(s.rootDir, url.PathEscape(bucketName)))
-	if err != nil {
-		return nil, err
-	}
 	objects := []ObjectAttrs{}
-	for _, info := range infos {
+	// TODO: WalkDir more efficient?
+	bucketPath := filepath.Join(s.rootDir, url.PathEscape(bucketName))
+	if err := filepath.Walk(bucketPath, func(path string, info fs.FileInfo, _ error) error {
+		// Rel() should never return error since path always descend from bucketPath
+		objName, _ := filepath.Rel(bucketPath, path)
+		// TODO: should this check path?
 		if s.mh.isSpecialFile(info.Name()) {
-			continue
+			return nil
 		}
-		unescaped, err := url.PathUnescape(info.Name())
-		if err != nil {
-			return nil, fmt.Errorf("failed to unescape object name %s: %w", info.Name(), err)
+		if info.IsDir() {
+			return nil
 		}
-		if prefix != "" && !strings.HasPrefix(unescaped, prefix) {
-			continue
+		if prefix != "" && !strings.HasPrefix(objName, prefix) {
+			return nil
 		}
-		object, err := s.getObject(bucketName, unescaped)
+		object, err := s.getObject(bucketName, objName)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		object.Close()
 		objects = append(objects, object.ObjectAttrs)
+		return nil
+	}); err != nil {
+		return nil, err
 	}
 	return objects, nil
 }
@@ -264,7 +277,7 @@ func (s *storageFS) GetObjectWithGeneration(bucketName, objectName string, gener
 }
 
 func (s *storageFS) getObject(bucketName, objectName string) (StreamingObject, error) {
-	path := filepath.Join(s.rootDir, url.PathEscape(bucketName), url.PathEscape(objectName))
+	path := filepath.Join(s.rootDir, url.PathEscape(bucketName), objectName)
 
 	encoded, err := s.mh.read(path)
 	if err != nil {
@@ -303,7 +316,7 @@ func (s *storageFS) DeleteObject(bucketName, objectName string) error {
 	if objectName == "" {
 		return errors.New("can't delete object with empty name")
 	}
-	path := filepath.Join(s.rootDir, url.PathEscape(bucketName), url.PathEscape(objectName))
+	path := filepath.Join(s.rootDir, url.PathEscape(bucketName), objectName)
 	if err := s.mh.remove(path); err != nil {
 		return err
 	}
diff --git a/internal/config/config.go b/internal/config/config.go
index d0d8ce9..0bc72a7 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -40,6 +40,7 @@ type Config struct {
 	bucketLocation      string
 	certificateLocation string
 	privateKeyLocation  string
+	LogLevel            logrus.Level
 }
 
 type EventConfig struct {
@@ -55,6 +56,7 @@ func Load(args []string) (Config, error) {
 	var cfg Config
 	var allowedCORSHeaders string
 	var eventList string
+	var givenLogLevel string
 
 	fs := flag.NewFlagSet("fake-gcs-server", flag.ContinueOnError)
 	fs.StringVar(&cfg.backend, "backend", filesystemBackend, "storage backend (memory or filesystem)")
@@ -73,12 +75,18 @@ func Load(args []string) (Config, error) {
 	fs.StringVar(&cfg.bucketLocation, "location", "US-CENTRAL1", "location for buckets")
 	fs.StringVar(&cfg.certificateLocation, "cert-location", "", "location for server certificate")
 	fs.StringVar(&cfg.privateKeyLocation, "private-key-location", "", "location for private key")
+	fs.StringVar(&givenLogLevel, "log-level", "info", "level for logging. Options same as for logrus: trace, debug, info, warn, error, fatal, and panic")
 
 	err := fs.Parse(args)
 	if err != nil {
 		return cfg, err
 	}
 
+	cfg.LogLevel, err = logrus.ParseLevel(givenLogLevel)
+	if err != nil {
+		return cfg, err
+	}
+
 	if allowedCORSHeaders != "" {
 		cfg.allowedCORSHeaders = strings.Split(allowedCORSHeaders, ",")
 	}
@@ -157,8 +165,9 @@ func (c *Config) ToFakeGcsOptions() fakestorage.Options {
 			}
 		}
 	}
-
-	return fakestorage.Options{
+	logger := logrus.New()
+	logger.SetLevel(c.LogLevel)
+	opts := fakestorage.Options{
 		StorageRoot:         storageRoot,
 		Scheme:              c.scheme,
 		Host:                c.host,
@@ -166,10 +175,11 @@ func (c *Config) ToFakeGcsOptions() fakestorage.Options {
 		PublicHost:          c.publicHost,
 		ExternalURL:         c.externalURL,
 		AllowedCORSHeaders:  c.allowedCORSHeaders,
-		Writer:              logrus.New().Writer(),
+		Writer:              logger.Writer(),
 		EventOptions:        eventOptions,
 		BucketsLocation:     c.bucketLocation,
 		CertificateLocation: c.certificateLocation,
 		PrivateKeyLocation:  c.privateKeyLocation,
 	}
+	return opts
 }
diff --git a/main.go b/main.go
index db69daa..df6b63b 100644
--- a/main.go
+++ b/main.go
@@ -29,6 +29,7 @@ func main() {
 		log.Fatal(err)
 	}
 	logger := logrus.New()
+	logger.SetLevel(cfg.LogLevel)
 
 	var emptyBuckets []string
 	opts := cfg.ToFakeGcsOptions()


Test patch:
diff --git a/fakestorage/object_test.go b/fakestorage/object_test.go
index 4d3f3b1..19f270d 100644
--- a/fakestorage/object_test.go
+++ b/fakestorage/object_test.go
@@ -721,11 +721,18 @@ func TestServerClientObjectReaderError(t *testing.T) {
 
 func TestServerClientObjectReadBucketCNAME(t *testing.T) {
 	url := "https://mybucket.mydomain.com:4443/files/txt/text-01.txt"
-	expectedHeaders := map[string]string{"accept-ranges": "bytes", "content-length": "9"}
+	expectedHeaders := map[string]string{"accept-ranges": "bytes", "content-length": "9", "x-goog-meta-marco": "Polo"}
 	expectedBody := "something"
 	opts := Options{
 		InitialObjects: []Object{
-			{ObjectAttrs: ObjectAttrs{BucketName: "mybucket.mydomain.com", Name: "files/txt/text-01.txt"}, Content: []byte("something")},
+			{
+				ObjectAttrs: ObjectAttrs{
+					BucketName: "mybucket.mydomain.com",
+					Name:       "files/txt/text-01.txt",
+					Metadata:   map[string]string{"Marco": "Polo"},
+				},
+				Content: []byte("something"),
+			},
 		},
 	}
 	server, err := NewServerWithOptions(opts)
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index fd7ed67..6da0b59 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/fsouza/fake-gcs-server/internal/notification"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/sirupsen/logrus"
 )
 
 func TestLoadConfig(t *testing.T) {
@@ -38,6 +39,7 @@ func TestLoadConfig(t *testing.T) {
 				"-event.object-prefix", "uploads/",
 				"-event.list", "finalize,delete,metadataUpdate,archive",
 				"-location", "US-EAST1",
+				"-log-level", "warn",
 			},
 			expectedConfig: Config{
 				Seed:               "/var/gcs",
@@ -56,6 +58,7 @@ func TestLoadConfig(t *testing.T) {
 					list:            []string{"finalize", "delete", "metadataUpdate", "archive"},
 				},
 				bucketLocation: "US-EAST1",
+				LogLevel:       logrus.WarnLevel,
 			},
 		},
 		{
@@ -74,6 +77,7 @@ func TestLoadConfig(t *testing.T) {
 					list: []string{"finalize"},
 				},
 				bucketLocation: "US-CENTRAL1",
+				LogLevel:       logrus.InfoLevel,
 			},
 		},
 		{
@@ -111,6 +115,11 @@ func TestLoadConfig(t *testing.T) {
 			args:      []string{"-event.list", "invalid,stuff", "-event.pubsub-topic", "gcs-events", "-event.pubsub-project-id", "test-project"},
 			expectErr: true,
 		},
+		{
+			name:      "invalid log level",
+			args:      []string{"-log-level", "non-existent-level"},
+			expectErr: true,
+		},
 	}
 	for _, test := range tests {
 		test := test