Repository: peak/s5cmd (go)

Issue:
Error in `sync` command when using `--delete` flag with 2.1.0 release
Hi 👋

Firstly I want to thank you for this great tool!

After updating from `2.1.0-beta.1` to `2.1.0` we've encountered a strange behavior when `sync`ing a local folder to our S3 buckets. We're not using AWS, but I could reproduce this issue with a LocalStack S3 bucket locally.

Using s5cmd in version `2.1.0-beta.1` syncs a local folder without any issues:
```
❯ ls -l
total 0
-rw-r--r--  1 x  staff  0 21 Jun 07:47 Test1.txt
-rw-r--r--  1 x  staff  0 21 Jun 07:47 Test2.txt
❯ s5cmd-2.1.0-beta version
v2.1.0-beta.1-3e08061
❯ s5cmd-2.1.0-beta ls s3://test-bucket/\*
ERROR "ls s3://test-bucket/*": no object found
❯ s5cmd-2.1.0-beta --stat --log debug sync --delete ./ s3://test-bucket/
cp Test2.txt s3://test-bucket/Test2.txt
cp Test1.txt s3://test-bucket/Test1.txt

Operation	Total	Error	Success
sync		1	0	1
cp		2	0	2

❯ s5cmd-2.1.0-beta ls s3://test-bucket/\*
2023/06/21 05:54:06                 0 Test1.txt
2023/06/21 05:54:06                 0 Test2.txt
❯ s5cmd-2.1.0-beta --stat --log debug sync --delete ./ s3://test-bucket/
DEBUG "sync Test1.txt s3://test-bucket/Test1.txt": object is newer or same age and object size matches
DEBUG "sync Test2.txt s3://test-bucket/Test2.txt": object is newer or same age and object size matches

Operation	Total	Error	Success
sync		1	0	1
```

Running the exact same commands with s5cmd in version `2.1.0` results in errors (`ERROR "rm --raw=true": expected at least 1 object to remove`) when using the `--delete` flag with the `sync` command:
```
❯ ls -l
total 0
-rw-r--r--  1 x  staff  0 21 Jun 07:47 Test1.txt
-rw-r--r--  1 x  staff  0 21 Jun 07:47 Test2.txt
❯ s5cmd version
v2.1.0-3efbbe8
❯ s5cmd ls s3://test-bucket/\*
ERROR "ls s3://test-bucket/*": no object found
❯ s5cmd --stat --log debug sync --delete ./ s3://test-bucket/
ERROR "rm --raw=true": expected at least 1 object to remove
cp Test2.txt s3://test-bucket/Test2.txt
cp Test1.txt s3://test-bucket/Test1.txt

Operation	Total	Error	Success
cp		2	0	2
sync		1	1	0

❯ s5cmd ls s3://test-bucket/\*
2023/06/21 05:49:47                 0  Test1.txt
2023/06/21 05:49:47                 0  Test2.txt
❯ s5cmd --stat --log debug sync --delete ./ s3://test-bucket/
DEBUG "sync Test1.txt s3://test-bucket/Test1.txt": object is newer or same age and object size matches
DEBUG "sync Test2.txt s3://test-bucket/Test2.txt": object is newer or same age and object size matches
ERROR "rm --raw=true": expected at least 1 object to remove

Operation	Total	Error	Success
sync		1	1	0
```

Is there something we need to change in CLI usage after updating to version `2.1.0`? I don't see any changes regarding the `sync` command and `--delete` flag in the release notes. As the `sync` command is now failing we cannot update s5cmd in our CI/CD pipelines.

If you need anything further, feel free to let me know.

Thank you!

Pull request:
command/sync: fix delete option with empty target
Fixes #576,
Thanks to @Kucukaslan for pointing it out.

Patch:
diff --git a/command/sync.go b/command/sync.go
index bb7db04..2c64994 100644
--- a/command/sync.go
+++ b/command/sync.go
@@ -393,7 +393,6 @@ func (s Sync) getSourceAndDestinationObjects(ctx context.Context, srcurl, dsturl
 				printError(s.fullCommand, s.op, err)
 			}
 		}()
-
 	}()
 
 	return sourceObjects, destObjects, nil
@@ -460,27 +459,36 @@ func (s Sync) planRun(
 	}()
 
 	// only in destination
-	if s.delete {
-		// unfortunately we need to read them all!
-		// or rewrite generateCommand function?
-		dstURLs := make([]*url.URL, 0, extsortChunkSize)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if s.delete {
+			// unfortunately we need to read them all!
+			// or rewrite generateCommand function?
+			dstURLs := make([]*url.URL, 0, extsortChunkSize)
 
-		for d := range onlyDest {
-			dstURLs = append(dstURLs, d)
-		}
-		command, err := generateCommand(c, "rm", defaultFlags, dstURLs...)
-		if err != nil {
-			printDebug(s.op, err, dstURLs...)
-			return
-		}
-		fmt.Fprintln(w, command)
-	} else {
-		// we only need  to consume them from the channel so that rest of the objects
-		// can be sent to channel.
-		for d := range onlyDest {
-			_ = d
+			for d := range onlyDest {
+				dstURLs = append(dstURLs, d)
+			}
+
+			if len(dstURLs) == 0 {
+				return
+			}
+
+			command, err := generateCommand(c, "rm", defaultFlags, dstURLs...)
+			if err != nil {
+				printDebug(s.op, err, dstURLs...)
+				return
+			}
+			fmt.Fprintln(w, command)
+		} else {
+			// we only need  to consume them from the channel so that rest of the objects
+			// can be sent to channel.
+			for d := range onlyDest {
+				_ = d
+			}
 		}
-	}
+	}()
 
 	wg.Wait()
 }


Test patch:
diff --git a/e2e/sync_test.go b/e2e/sync_test.go
index e4dc000..3cc2894 100644
--- a/e2e/sync_test.go
+++ b/e2e/sync_test.go
@@ -1130,6 +1130,52 @@ func TestSyncS3BucketToLocalWithDelete(t *testing.T) {
 	}
 }
 
+// sync --delete s3://bucket/* .
+func TestSyncS3BucketToEmptyLocalWithDelete(t *testing.T) {
+	t.Parallel()
+	s3client, s5cmd := setup(t)
+
+	bucket := s3BucketFromTestName(t)
+	createBucket(t, s3client, bucket)
+
+	S3Content := map[string]string{
+		"contributing.md": "S: this is a readme file",
+	}
+
+	for filename, content := range S3Content {
+		putFile(t, s3client, bucket, filename, content)
+	}
+
+	workdir := fs.NewDir(t, "somedir")
+	defer workdir.Remove()
+
+	src := fmt.Sprintf("s3://%v/", bucket)
+	dst := fmt.Sprintf("%v/", workdir.Path())
+	dst = filepath.ToSlash(dst)
+
+	cmd := s5cmd("sync", "--delete", src+"*", dst)
+	result := icmd.RunCmd(cmd)
+
+	result.Assert(t, icmd.Success)
+	stdout := result.Stdout()
+	assertLines(t, stdout, map[int]compareFunc{
+		0: equals(`cp %vcontributing.md %vcontributing.md`, src, dst),
+	}, sortInput(true))
+
+	expectedFolderLayout := []fs.PathOp{
+		fs.WithFile("contributing.md", "S: this is a readme file"),
+	}
+
+	// assert local filesystem
+	expected := fs.Expected(t, expectedFolderLayout...)
+	assert.Assert(t, fs.Equal(workdir.Path(), expected))
+
+	// assert s3
+	for key, content := range S3Content {
+		assert.Assert(t, ensureS3Object(s3client, bucket, key, content))
+	}
+}
+
 // sync --delete folder/ s3://bucket/*
 func TestSyncLocalToS3BucketWithDelete(t *testing.T) {
 	t.Parallel()
@@ -1199,6 +1245,53 @@ func TestSyncLocalToS3BucketWithDelete(t *testing.T) {
 	}
 }
 
+// sync --delete folder/ s3://bucket/*
+func TestSyncLocalToEmptyS3BucketWithDelete(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	s3client, s5cmd := setup(t)
+
+	bucket := s3BucketFromTestName(t)
+	createBucket(t, s3client, bucket)
+
+	folderLayout := []fs.PathOp{
+		fs.WithFile("contributing.md", "S: this is a readme file", fs.WithTimestamps(now, now)),
+	}
+
+	workdir := fs.NewDir(t, "somedir", folderLayout...)
+	defer workdir.Remove()
+
+	src := fmt.Sprintf("%v/", workdir.Path())
+	src = filepath.ToSlash(src)
+	dst := fmt.Sprintf("s3://%v/", bucket)
+
+	cmd := s5cmd("sync", "--delete", src, dst)
+	result := icmd.RunCmd(cmd)
+
+	result.Assert(t, icmd.Success)
+
+	assertLines(t, result.Stdout(), map[int]compareFunc{
+		0: equals(`cp %vcontributing.md %vcontributing.md`, src, dst),
+	}, sortInput(true))
+
+	// assert local filesystem
+	expectedFiles := []fs.PathOp{
+		fs.WithFile("contributing.md", "S: this is a readme file"),
+	}
+	expected := fs.Expected(t, expectedFiles...)
+	assert.Assert(t, fs.Equal(workdir.Path(), expected))
+
+	expectedS3Content := map[string]string{
+		"contributing.md": "S: this is a readme file",
+	}
+
+	// assert s3 objects
+	for key, content := range expectedS3Content {
+		assert.Assert(t, ensureS3Object(s3client, bucket, key, content))
+	}
+}
+
 // sync --delete s3://bucket/* s3://destbucket/
 func TestSyncS3BucketToS3BucketWithDelete(t *testing.T) {
 	t.Parallel()
@@ -1634,7 +1727,7 @@ func TestSyncLocalDirectoryToS3WithExcludeFilter(t *testing.T) {
 				assert.Assert(t, ensureS3Object(s3client, bucket, key, content))
 			}
 
-			//assert objects should not be in S3.
+			// assert objects should not be in S3.
 			for key, content := range nonExpectedS3Content {
 				err := ensureS3Object(s3client, bucket, key, content)
 				assertError(t, err, errS3NoSuchKey)