Repository: peak/s5cmd
Pull request #280 (2021-04-01 15:35:59)

Implement #279
Implement Issue #279 

### Issue:
S3 key may contain special characters as mention in this document. In this case, downloading S3 object throws 404 error.

### Solution:
The S3 key need to be URL encoded before download.


Patch:
diff --git a/go.mod b/go.mod
index 0dfb313..f27254a 100644
--- a/go.mod
+++ b/go.mod
@@ -22,4 +22,4 @@ require (
 	gotest.tools/v3 v3.0.2
 )
 
-replace github.com/johannesboyne/gofakes3 => github.com/igungor/gofakes3 v0.0.5
\ No newline at end of file
+replace github.com/johannesboyne/gofakes3 => github.com/igungor/gofakes3 v0.0.6
diff --git a/go.sum b/go.sum
index 6058b42..1dddcb9 100644
--- a/go.sum
+++ b/go.sum
@@ -1,10 +1,6 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/aws/aws-sdk-go v1.17.4 h1:L2KFocQhg48kIzEAV98SnSz3nmIZ3UDFP+vU647KO3c=
 github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.29.14 h1:NToqC5ZQ2RaxxSPp9szuQimWQWPG++ITwXbklq/FN7c=
-github.com/aws/aws-sdk-go v1.29.14/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
-github.com/aws/aws-sdk-go v1.34.12 h1:7UbBEYDUa4uW0YmRnOd806MS1yoJMcaodBWDzvBShAI=
-github.com/aws/aws-sdk-go v1.34.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go v1.35.13 h1:Y49GifH2czbooBMkVpoXwokur1JRBFKVLVCQzO0YsW8=
 github.com/aws/aws-sdk-go v1.35.13/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
@@ -14,7 +10,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -24,21 +19,14 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8=
 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
-github.com/igungor/gofakes3 v0.0.4 h1:6ngDgSIOQBxuUimMberClVfXLtz3ldmmBrH9+Lk+nms=
-github.com/igungor/gofakes3 v0.0.4/go.mod h1:TXY9jeWWVeGPkjO1xspOjbhOMSCld99woAEP0kxgFds=
-github.com/igungor/gofakes3 v0.0.5 h1:ae+qLhH8O3iBNLwC+cO9oIlC2HbO/lfUE+KDOcIBcsk=
-github.com/igungor/gofakes3 v0.0.5/go.mod h1:TXY9jeWWVeGPkjO1xspOjbhOMSCld99woAEP0kxgFds=
+github.com/igungor/gofakes3 v0.0.6 h1:rSznIcTjex2tSPQFQy7PfcXKrHefg0AVL/VhhK7/E60=
+github.com/igungor/gofakes3 v0.0.6/go.mod h1:TXY9jeWWVeGPkjO1xspOjbhOMSCld99woAEP0kxgFds=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/johannesboyne/gofakes3 v0.0.0-20200605082314-e89baa38acc1 h1:e8Y1zSwTOaRHF+CzcCn99gUdIlbVivfI9xyK5IEqJx4=
-github.com/johannesboyne/gofakes3 v0.0.0-20200605082314-e89baa38acc1/go.mod h1:fNiSoOiEI5KlkWXn26OwKnNe58ilTIkpBlgOrt7Olu8=
-github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc h1:JJPhSHowepOF2+ElJVyb9jgt5ZyBkPMkPuhS0uODSFs=
-github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc/go.mod h1:fNiSoOiEI5KlkWXn26OwKnNe58ilTIkpBlgOrt7Olu8=
 github.com/karrick/godirwalk v1.15.3 h1:0a2pXOgtB16CqIqXTiT7+K9L73f74n/aNQUnH6Ortew=
 github.com/karrick/godirwalk v1.15.3/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
@@ -70,8 +58,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1aZOlG32uyE6xHpLdKhZzcTEktz5wM=
 github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs=
 github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
@@ -85,7 +71,6 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjut
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa h1:lqti/xP+yD/6zH5TqEwx2MilNIJY5Vbc6Qr8J3qyPIQ=
 golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/storage/s3.go b/storage/s3.go
index 9eababd..98c02b4 100644
--- a/storage/s3.go
+++ b/storage/s3.go
@@ -316,7 +316,7 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err
 	}
 
 	// SDK expects CopySource like "bucket[/key]"
-	copySource := strings.TrimPrefix(from.String(), "s3://")
+	copySource := from.EscapedPath()
 
 	input := &s3.CopyObjectInput{
 		Bucket:     aws.String(to.Bucket),
diff --git a/storage/url/url.go b/storage/url/url.go
index 5d4813f..01c1ac6 100644
--- a/storage/url/url.go
+++ b/storage/url/url.go
@@ -4,6 +4,7 @@ package url
 import (
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -347,3 +348,12 @@ func parseNonBatch(prefix string, key string) string {
 func hasGlobCharacter(s string) bool {
 	return strings.ContainsAny(s, globCharacters)
 }
+
+func (u *URL) EscapedPath() string {
+	sourceKey := strings.TrimPrefix(u.String(), "s3://")
+	sourceKeyElements := strings.Split(sourceKey, "/")
+	for i, element := range sourceKeyElements {
+		sourceKeyElements[i] = url.QueryEscape(element)
+	}
+	return strings.Join(sourceKeyElements, "/")
+}
diff --git a/vendor/github.com/johannesboyne/gofakes3/gofakes3.go b/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
index 9ca4cd9..eedd925 100644
--- a/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
+++ b/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
@@ -553,6 +553,11 @@ func (g *GoFakeS3) createObject(bucket, object string, w http.ResponseWriter, r
 		srcBucket := parts[0]
 		srcKey := parts[1]
 
+		srcKey, err = url.QueryUnescape(srcKey)
+		if err != nil {
+			return err
+		}
+
 		src, err := g.storage.GetObject(srcBucket, srcKey, nil)
 		if err != nil {
 			return err
@@ -625,6 +630,11 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h
 	srcBucket := parts[0]
 	srcKey := strings.SplitN(parts[1], "?", 2)[0]
 
+	srcKey, err = url.QueryUnescape(srcKey)
+	if err != nil {
+		return err
+	}
+
 	srcObj, err := g.storage.GetObject(srcBucket, srcKey, nil)
 	if err != nil {
 		return err
diff --git a/vendor/modules.txt b/vendor/modules.txt
index f1058a1..3e1d768 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -68,7 +68,7 @@ github.com/hashicorp/go-multierror
 github.com/iancoleman/strcase
 # github.com/jmespath/go-jmespath v0.4.0
 github.com/jmespath/go-jmespath
-# github.com/johannesboyne/gofakes3 v0.0.0-20191228161223-9aee1c78a252 => github.com/igungor/gofakes3 v0.0.5
+# github.com/johannesboyne/gofakes3 v0.0.0-20191228161223-9aee1c78a252 => github.com/igungor/gofakes3 v0.0.6
 github.com/johannesboyne/gofakes3
 github.com/johannesboyne/gofakes3/backend/s3bolt
 github.com/johannesboyne/gofakes3/backend/s3mem


Test patch:
diff --git a/e2e/cp_test.go b/e2e/cp_test.go
index a8742c9..b59cf43 100644
--- a/e2e/cp_test.go
+++ b/e2e/cp_test.go
@@ -1646,6 +1646,47 @@ func TestCopySingleS3ObjectIntoAnotherBucketWithPrefix(t *testing.T) {
 	assert.Assert(t, ensureS3Object(s3client, bucket, "prefix/"+filename, content))
 }
 
+// cp s3://bucket/* s3://dstbucket/
+func TestCopyAllObjectsIntoAnotherBucketIncludingSpecialCharacter(t *testing.T) {
+	t.Parallel()
+
+	const (
+		srcbucket = "bucket"
+		dstbucket = "dstbucket"
+	)
+
+	s3client, s5cmd, cleanup := setup(t)
+	defer cleanup()
+
+	createBucket(t, s3client, srcbucket)
+	createBucket(t, s3client, dstbucket)
+
+	filesToContent := map[string]string{
+		"sub&@$/test+1.txt":           "this is a test file 1",
+		"sub:,?/test; =2.txt":         "this is a test file 2",
+		"test&@$:,?;= 3.txt":          "this is a test file 3",
+		"sub/this-is-normal-file.txt": "this is a normal file",
+	}
+
+	for filename, content := range filesToContent {
+		putFile(t, s3client, srcbucket, filename, content)
+	}
+
+	src := fmt.Sprintf("s3://%v/*", srcbucket)
+	dst := fmt.Sprintf("s3://%v/", dstbucket)
+
+	cmd := s5cmd("cp", src, dst)
+	result := icmd.RunCmd(cmd)
+
+	result.Assert(t, icmd.Success)
+	assertLines(t, result.Stdout(), map[int]compareFunc{
+		0: equals(`cp s3://%v/sub&@$/test+1.txt s3://%v/sub&@$/test+1.txt`, srcbucket, dstbucket),
+		1: equals(`cp s3://%v/sub/this-is-normal-file.txt s3://%v/sub/this-is-normal-file.txt`, srcbucket, dstbucket),
+		2: equals(`cp s3://%v/sub:,?/test; =2.txt s3://%v/sub:,?/test; =2.txt`, srcbucket, dstbucket),
+		3: equals(`cp s3://%v/test&@$:,?;= 3.txt s3://%v/test&@$:,?;= 3.txt`, srcbucket, dstbucket),
+	}, sortInput(true))
+}
+
 // cp s3://bucket/* s3://bucket/prefix/
 func TestCopyMultipleS3ObjectsToS3WithPrefix(t *testing.T) {
 	t.Parallel()