Repository: peak/s5cmd (go)

Issue:
Add latest AWS S3 storage class types
Can you please add the latest storage class types for S3:
'STANDARD','REDUCED_REDUNDANCY','STANDARD_IA','ONEZONE_IA','INTELLIGENT_TIERING','GLACIER','DEEP_ARCHIVE'
https://github.com/peak/s5cmd/blob/e1bf6cf824684f2d7f7c3f6296c265ac825a1c02/command/cp.go#L96

--storage-class (string) The type of storage to use for the object. Valid choices are: STANDARD | REDUCED_REDUNDANCY | STANDARD_IA | ONEZONE_IA | INTELLIGENT_TIERING | GLACIER | DEEP_ARCHIVE. Defaults to 'STANDARD'
https://docs.aws.amazon.com/cli/latest/reference/s3/cp.html

Maybe it is also an idea to add the latest google cloud storage classes (standard, nearline, coldline):
https://cloud.google.com/storage/docs/storage-classes

Pull request:
storage/s3: allow any storage-class value
- e2e/cp: add --storage-class=GLACIER test
- also show 2 character storage class short code in 'ls' output

If you pass an invalid storage class, S3 will return:

```
s5cmd cp --storage-class=invalidstorageclass file s3://bucket/
ERROR "cp file s3://bucket/file": InvalidStorageClass: The storage class you specified is not valid status code: 400...
```

Fixes #160

Patch:
diff --git a/command/cp.go b/command/cp.go
index 3599140..07de400 100644
--- a/command/cp.go
+++ b/command/cp.go
@@ -131,7 +131,7 @@ var copyCommand = &cli.Command{
 			ifSourceNewer:  c.Bool("if-source-newer"),
 			flatten:        c.Bool("flatten"),
 			followSymlinks: !c.Bool("no-follow-symlinks"),
-			storageClass:   storage.LookupClass(c.String("storage-class")),
+			storageClass:   storage.StorageClass(c.String("storage-class")),
 			concurrency:    c.Int("concurrency"),
 			partSize:       c.Int64("part-size") * megabytes,
 		}.Run(c.Context)
@@ -214,7 +214,7 @@ func (c Copy) Run(ctx context.Context) error {
 			continue
 		}
 
-		if object.StorageClass == storage.StorageGlacier {
+		if object.StorageClass.IsGlacier() {
 			err := fmt.Errorf("object '%v' is on Glacier storage", object)
 			printError(c.fullCommand, c.op, err)
 			continue
@@ -588,11 +588,6 @@ func validate(c *cli.Context) error {
 		return fmt.Errorf("expected source and destination arguments")
 	}
 
-	storageClass := storage.LookupClass(c.String("storage-class"))
-	if storageClass == storage.StorageInvalid {
-		return fmt.Errorf("invalid storage class")
-	}
-
 	ctx := c.Context
 	src := c.Args().Get(0)
 	dst := c.Args().Get(1)
diff --git a/command/ls.go b/command/ls.go
index 0e08975..8d4c0f9 100644
--- a/command/ls.go
+++ b/command/ls.go
@@ -159,11 +159,11 @@ const (
 
 // String returns the string representation of ListMessage.
 func (l ListMessage) String() string {
-	var listFormat = "%19s %1s %-1s %12s %s"
+	var listFormat = "%19s %2s %-1s %12s %s"
 	var etag string
 	if l.showEtag {
 		etag = l.Object.Etag
-		listFormat = "%19s %1s %-38s %12s %s"
+		listFormat = "%19s %2s %-38s %12s %s"
 	}
 
 	if l.Object.Type.IsDir() {
diff --git a/command/mv.go b/command/mv.go
index f71dae8..88c4924 100644
--- a/command/mv.go
+++ b/command/mv.go
@@ -52,7 +52,7 @@ var moveCommand = &cli.Command{
 			ifSizeDiffer:  c.Bool("if-size-differ"),
 			ifSourceNewer: c.Bool("if-source-newer"),
 			flatten:       c.Bool("flatten"),
-			storageClass:  storage.LookupClass(c.String("storage-class")),
+			storageClass:  storage.StorageClass(c.String("storage-class")),
 		}
 
 		return copyCommand.Run(c.Context)
diff --git a/parallel/global.go b/parallel/global.go
new file mode 100644
index 0000000..e51a580
--- /dev/null
+++ b/parallel/global.go
@@ -0,0 +1,44 @@
+package parallel
+
+import "syscall"
+
+const (
+	minOpenFilesLimit = 1000
+)
+
+var global *Manager
+
+// Init tries to increase the soft limit of open files and
+// creates new global ParallelManager.
+func Init(workercount int) {
+	adjustOpenFilesLimit()
+	global = New(workercount)
+}
+
+// Close waits all jobs to finish and
+// closes the semaphore of global ParallelManager.
+func Close() { global.Close() }
+
+// Run runs global ParallelManager.
+func Run(task Task, waiter *Waiter) { global.Run(task, waiter) }
+
+// adjustOpenFilesLimit tries to increase the soft limit of open files.
+func adjustOpenFilesLimit() {
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return
+	}
+
+	if rLimit.Cur >= minOpenFilesLimit {
+		return
+	}
+
+	if rLimit.Max < minOpenFilesLimit {
+		return
+	}
+
+	rLimit.Cur = minOpenFilesLimit
+
+	_ = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+}
diff --git a/parallel/parallel.go b/parallel/parallel.go
index e41971d..40bd07d 100644
--- a/parallel/parallel.go
+++ b/parallel/parallel.go
@@ -3,31 +3,15 @@ package parallel
 import (
 	"runtime"
 	"sync"
-	"syscall"
 )
 
-const minNumWorkers = 2
-const minOpenFilesLimit = 1000
-
-var global *Manager
+const (
+	minNumWorkers = 2
+)
 
 // Task is a function type for parallel manager.
 type Task func() error
 
-// Init tries to increase the soft limit of open files and
-// creates new global ParallelManager.
-func Init(workercount int) {
-	adjustOpenFilesLimit()
-	global = New(workercount)
-}
-
-// Close waits all jobs to finish and
-// closes the semaphore of global ParallelManager.
-func Close() { global.Close() }
-
-// Run runs global ParallelManager.
-func Run(task Task, waiter *Waiter) { global.Run(task, waiter) }
-
 // Manager is a structure for running tasks in parallel.
 type Manager struct {
 	wg        *sync.WaitGroup
@@ -107,24 +91,3 @@ func (w *Waiter) Wait() {
 func (w *Waiter) Err() <-chan error {
 	return w.errch
 }
-
-// adjustOpenFilesLimit tries to increase the soft limit of open files.
-func adjustOpenFilesLimit() {
-	var rLimit syscall.Rlimit
-	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-	if err != nil {
-		return
-	}
-
-	if rLimit.Cur >= minOpenFilesLimit {
-		return
-	}
-
-	if rLimit.Max < minOpenFilesLimit {
-		return
-	}
-
-	rLimit.Cur = minOpenFilesLimit
-
-	_ = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-}
diff --git a/storage/s3.go b/storage/s3.go
index 2dd3862..bcb4689 100644
--- a/storage/s3.go
+++ b/storage/s3.go
@@ -353,23 +353,25 @@ func (s *S3) Put(
 	concurrency int,
 	partSize int64,
 ) error {
-	storageClass := metadata["StorageClass"]
-	if storageClass == "" {
-		storageClass = string(StorageStandard)
-	}
 
 	contentType := metadata["ContentType"]
 	if contentType == "" {
 		contentType = "application/octet-stream"
 	}
 
-	_, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
-		Bucket:       aws.String(to.Bucket),
-		Key:          aws.String(to.Path),
-		Body:         reader,
-		ContentType:  aws.String(contentType),
-		StorageClass: aws.String(storageClass),
-	}, func(u *s3manager.Uploader) {
+	input := &s3manager.UploadInput{
+		Bucket:      aws.String(to.Bucket),
+		Key:         aws.String(to.Path),
+		Body:        reader,
+		ContentType: aws.String(contentType),
+	}
+
+	storageClass := metadata["StorageClass"]
+	if storageClass != "" {
+		input.StorageClass = aws.String(storageClass)
+	}
+
+	_, err := s.uploader.UploadWithContext(ctx, input, func(u *s3manager.Uploader) {
 		u.PartSize = partSize
 		u.Concurrency = concurrency
 	})
diff --git a/storage/storage.go b/storage/storage.go
index c60dc20..f8e9845 100644
--- a/storage/storage.go
+++ b/storage/storage.go
@@ -161,57 +161,33 @@ func (b Bucket) JSON() string {
 // StorageClass represents the storage used to store an object.
 type StorageClass string
 
-// ShortCode returns the short code of Storage Class.
+// ShortCode returns the short code of Amazon S3 Storage Class.
 func (s StorageClass) ShortCode() string {
 	var code string
 	switch s {
-	case StorageStandard:
-		code = ""
-	case StorageGlacier:
-		code = "G"
-	case StorageReducedRedundancy:
-		code = "R"
-	case StorageStandardIA:
-		code = "I"
-	default:
-		code = "?"
-	}
-	return code
-}
-
-// LookupClass looks up given class.
-func LookupClass(s string) StorageClass {
-	switch s {
-	case "", "STANDARD":
-		return StorageStandard
+	case "STANDARD":
+		code = "ST"
 	case "REDUCED_REDUNDANCY":
-		return StorageReducedRedundancy
-	case "GLACIER":
-		return StorageGlacier
+		code = "RR"
 	case "STANDARD_IA":
-		return StorageStandardIA
+		code = "SI"
+	case "ONEZONE_IA":
+		code = "OI"
+	case "INTELLIGENT_TIERING":
+		code = "IT"
+	case "GLACIER":
+		code = "GL"
+	case "DEEP_ARCHIVE":
+		code = "DA"
 	default:
-		return StorageInvalid
+		code = " ?"
 	}
+	return code
 }
 
-const (
-	// StorageInvalid is a placeholder class if the class is not valid.
-	StorageInvalid StorageClass = "UNKNOWN"
-
-	// StorageStandard is a standard storage class type.
-	StorageStandard StorageClass = "STANDARD"
-
-	// StorageReducedRedundancy is a reduced redundancy storage class type.
-	StorageReducedRedundancy StorageClass = "REDUCED_REDUNDANCY"
-
-	// StorageGlacier is a glacier storage class type.
-	StorageGlacier StorageClass = "GLACIER"
-
-	// StorageStandardIA is a Standard Infrequent-Access storage
-	// class type.
-	StorageStandardIA StorageClass = "STANDARD_IA"
-)
+func (s StorageClass) IsGlacier() bool {
+	return s == "GLACIER"
+}
 
 // notImplemented is a structure which is used on the unsupported operations.
 type notImplemented struct {


Test patch:
diff --git a/e2e/cp_test.go b/e2e/cp_test.go
index 1d3b45c..1917926 100644
--- a/e2e/cp_test.go
+++ b/e2e/cp_test.go
@@ -737,8 +737,7 @@ func TestCopySingleFileToS3JSON(t *testing.T) {
 			"destination": "s3://%v/testfile1.txt",
 			"object": {
 				"type": "file",
-				"size":19,
-				"storage_class": "STANDARD"
+				"size":19
 			}
 		}
 	`
@@ -804,6 +803,47 @@ func TestCopyDirToS3(t *testing.T) {
 	assert.Assert(t, ensureS3Object(s3client, bucket, "c/file2.txt", "this is the second test file"))
 }
 
+// cp --storage-class=GLACIER file s3://bucket/
+func TestCopySingleFileToS3WithStorageClassGlacier(t *testing.T) {
+	t.Parallel()
+
+	bucket := s3BucketFromTestName(t)
+
+	s3client, s5cmd, cleanup := setup(t)
+	defer cleanup()
+
+	createBucket(t, s3client, bucket)
+
+	const (
+		// make sure that Put reads the file header, not the extension
+		filename             = "index.txt"
+		content              = "content"
+		expectedStorageClass = "GLACIER"
+	)
+
+	workdir := fs.NewDir(t, bucket, fs.WithFile(filename, content))
+	defer workdir.Remove()
+
+	srcpath := workdir.Join(filename)
+	dstpath := fmt.Sprintf("s3://%v/", bucket)
+
+	cmd := s5cmd("cp", "--storage-class=GLACIER", srcpath, dstpath)
+	result := icmd.RunCmd(cmd)
+
+	result.Assert(t, icmd.Success)
+
+	assertLines(t, result.Stdout(), map[int]compareFunc{
+		0: suffix(`cp %v %v%v`, srcpath, dstpath, filename),
+	})
+
+	// assert local filesystem
+	expected := fs.Expected(t, fs.WithFile(filename, content))
+	assert.Assert(t, fs.Equal(workdir.Path(), expected))
+
+	// assert S3
+	assert.Assert(t, ensureS3Object(s3client, bucket, filename, content, ensureStorageClass(expectedStorageClass)))
+}
+
 // cp --flatten dir/ s3://bucket/
 func TestFlattenCopyDirToS3(t *testing.T) {
 	t.Parallel()
@@ -1373,8 +1413,7 @@ func TestCopySingleS3ObjectToS3JSON(t *testing.T) {
 			"destination":"%v",
 			"object": {
 				"key": "%v",
-				"type":"file",
-				"storage_class":"STANDARD"
+				"type":"file"
 			}
 		}
 	`, src, dst, dst)
@@ -1727,8 +1766,7 @@ func TestCopyMultipleS3ObjectsToS3JSON(t *testing.T) {
 				"destination": "s3://%v/dst/readme.md",
 				"object": {
 					"key": "s3://%v/dst/readme.md",
-					"type": "file",
-					"storage_class": "STANDARD"
+					"type": "file"
 				}
 			}
 		`, bucket, bucket, bucket),
@@ -1740,8 +1778,7 @@ func TestCopyMultipleS3ObjectsToS3JSON(t *testing.T) {
 				"destination": "s3://%v/dst/testfile1.txt",
 				"object": {
 					"key": "s3://%v/dst/testfile1.txt",
-					"type": "file",
-					"storage_class": "STANDARD"
+					"type": "file"
 				}
 			}
 		`, bucket, bucket, bucket),
diff --git a/e2e/util_test.go b/e2e/util_test.go
index 7d6692e..b1eddfc 100644
--- a/e2e/util_test.go
+++ b/e2e/util_test.go
@@ -184,7 +184,8 @@ func createBucket(t *testing.T, client *s3.S3, bucket string) {
 var errS3NoSuchKey = fmt.Errorf("s3: no such key")
 
 type ensureOpts struct {
-	contentType *string
+	contentType  *string
+	storageClass *string
 }
 
 type ensureOption func(*ensureOpts)
@@ -195,6 +196,12 @@ func ensureContentType(contentType string) ensureOption {
 	}
 }
 
+func ensureStorageClass(expected string) ensureOption {
+	return func(opts *ensureOpts) {
+		opts.storageClass = &expected
+	}
+}
+
 func ensureS3Object(
 	client *s3.S3,
 	bucket string,
@@ -235,7 +242,13 @@ func ensureS3Object(
 
 	if opts.contentType != nil {
 		if diff := cmp.Diff(opts.contentType, output.ContentType); diff != "" {
-			return fmt.Errorf("s3 %v/%v: (-want +got):\n%v", bucket, key, diff)
+			return fmt.Errorf("content-type of %v/%v: (-want +got):\n%v", bucket, key, diff)
+		}
+	}
+
+	if opts.storageClass != nil {
+		if diff := cmp.Diff(opts.storageClass, output.StorageClass); diff != "" {
+			return fmt.Errorf("storage-class of %v/%v: (-want +got):\n%v", bucket, key, diff)
 		}
 	}