Repository: qiniu/goc
Pull request #146 (2020-12-21 05:21:14)

Improve performance of getting filelist to copy
`arr = append(arr, 1)` may allocate a new space,
in this PR, it improve performance by allocating new space only one time.

original perf data:
```
goos: darwin
goarch: amd64
pkg: github.com/qiniu/goc/pkg/build
BenchmarkGetFileList-4           1535029               778 ns/op            1488 B/op          5 allocs/op
```
new perf data:
```
goos: darwin
goarch: amd64
pkg: github.com/qiniu/goc/pkg/build
BenchmarkGetFileList-4           3884557               298 ns/op             576 B/op          1 allocs/op
```

The new function use less time and fewer memory.


Patch:
diff --git a/pkg/build/legacy.go b/pkg/build/legacy.go
index 4b47fda..9d539d3 100644
--- a/pkg/build/legacy.go
+++ b/pkg/build/legacy.go
@@ -20,10 +20,9 @@ import (
 	"path/filepath"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
-
-	"github.com/otiai10/copy"
+	cc "github.com/otiai10/copy" // conflit with builtin copy
 	"github.com/qiniu/goc/pkg/cover"
+	log "github.com/sirupsen/logrus"
 )
 
 func (b *Build) cpProject() {
@@ -48,13 +47,13 @@ func (b *Build) cpProject() {
 			if v.Name == "main" {
 				dst := filepath.Join(b.TmpDir, "go.mod")
 				src := filepath.Join(v.Module.Dir, "go.mod")
-				if err := copy.Copy(src, dst); err != nil {
+				if err := cc.Copy(src, dst); err != nil {
 					log.Errorf("Failed to Copy the go mod file from %v to %v, the error is: %v ", src, dst, err)
 				}
 
 				dst = filepath.Join(b.TmpDir, "go.sum")
 				src = filepath.Join(v.Module.Dir, "go.sum")
-				if err := copy.Copy(src, dst); err != nil && !strings.Contains(err.Error(), "no such file or directory") {
+				if err := cc.Copy(src, dst); err != nil && !strings.Contains(err.Error(), "no such file or directory") {
 					log.Errorf("Failed to Copy the go mod file from %v to %v, the error is: %v ", src, dst, err)
 				}
 				break
@@ -68,18 +67,7 @@ func (b *Build) cpProject() {
 func (b *Build) copyDir(pkg *cover.Package) error {
 	fileList := []string{}
 	dir := pkg.Dir
-	fileList = append(fileList, pkg.GoFiles...)
-	fileList = append(fileList, pkg.CompiledGoFiles...)
-	fileList = append(fileList, pkg.IgnoredGoFiles...)
-	fileList = append(fileList, pkg.CFiles...)
-	fileList = append(fileList, pkg.CXXFiles...)
-	fileList = append(fileList, pkg.MFiles...)
-	fileList = append(fileList, pkg.HFiles...)
-	fileList = append(fileList, pkg.FFiles...)
-	fileList = append(fileList, pkg.SFiles...)
-	fileList = append(fileList, pkg.SwigCXXFiles...)
-	fileList = append(fileList, pkg.SwigFiles...)
-	fileList = append(fileList, pkg.SysoFiles...)
+	fileList = getFileListNeedsCopy(pkg)
 	for _, file := range fileList {
 		p := filepath.Join(dir, file)
 		var src, root string
@@ -90,10 +78,69 @@ func (b *Build) copyDir(pkg *cover.Package) error {
 		}
 		src = strings.TrimPrefix(pkg.Dir, root)   // get the relative path of the files
 		dst := filepath.Join(b.TmpDir, src, file) // it will adapt the case where src is ""
-		if err := copy.Copy(p, dst); err != nil {
+		if err := cc.Copy(p, dst); err != nil {
 			log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
 			return err
 		}
 	}
 	return nil
 }
+
+// original implementation
+// goos: darwin
+// goarch: amd64
+// pkg: github.com/qiniu/goc/pkg/build
+// BenchmarkGetFileList-4           1535029               778 ns/op            1488 B/op          5 allocs/op
+// func getFileListNeedsCopy(pkg *cover.Package) []string {
+// 	fileList := []string{}
+
+// 	fileList = append(fileList, pkg.GoFiles...)
+// 	fileList = append(fileList, pkg.CompiledGoFiles...)
+// 	fileList = append(fileList, pkg.IgnoredGoFiles...)
+// 	fileList = append(fileList, pkg.CFiles...)
+// 	fileList = append(fileList, pkg.CXXFiles...)
+// 	fileList = append(fileList, pkg.MFiles...)
+// 	fileList = append(fileList, pkg.HFiles...)
+// 	fileList = append(fileList, pkg.FFiles...)
+// 	fileList = append(fileList, pkg.SFiles...)
+// 	fileList = append(fileList, pkg.SwigCXXFiles...)
+// 	fileList = append(fileList, pkg.SwigFiles...)
+// 	fileList = append(fileList, pkg.SysoFiles...)
+
+// 	return fileList
+// }
+
+// new implementation
+// goos: darwin
+// goarch: amd64
+// pkg: github.com/qiniu/goc/pkg/build
+// BenchmarkGetFileList-4           3884557               298 ns/op             576 B/op          1 allocs/op
+func getFileListNeedsCopy(pkg *cover.Package) []string {
+	fileSlices := [][]string{
+		pkg.GoFiles,
+		pkg.CompiledGoFiles,
+		pkg.IgnoredGoFiles,
+		pkg.CFiles,
+		pkg.CXXFiles,
+		pkg.MFiles,
+		pkg.HFiles,
+		pkg.FFiles,
+		pkg.SFiles,
+		pkg.SwigCXXFiles,
+		pkg.SwigFiles,
+		pkg.SysoFiles,
+	}
+
+	var totalLen int
+	for _, s := range fileSlices {
+		totalLen += len(s)
+	}
+
+	tmp := make([]string, totalLen)
+	var i int
+	for _, s := range fileSlices {
+		i += copy(tmp[i:], s)
+	}
+
+	return tmp
+}


Test patch:
diff --git a/pkg/build/legacy_test.go b/pkg/build/legacy_test.go
index 48bfd6d..0800847 100644
--- a/pkg/build/legacy_test.go
+++ b/pkg/build/legacy_test.go
@@ -17,12 +17,12 @@
 package build
 
 import (
+	"os"
 	"strings"
 	"testing"
 
 	"github.com/qiniu/goc/pkg/cover"
 	"github.com/stretchr/testify/assert"
-	"os"
 )
 
 // copy in cpProject of invalid src, dst name
@@ -80,3 +80,34 @@ func TestCopyDir(t *testing.T) {
 	defer os.RemoveAll(tmpDir)
 	assert.NoError(t, b.copyDir(pkg))
 }
+
+func initSlicesForTest() *cover.Package {
+	var pkg cover.Package
+	pkg.GoFiles = []string{"a1.go", "b1.go", "c1.go"}
+	pkg.CompiledGoFiles = []string{"a2.go", "b2.go", "c2.go"}
+	pkg.IgnoredGoFiles = []string{"a3.go", "b3.go", "c3.go"}
+	pkg.CFiles = []string{"a4.go", "b4.go", "c4.go"}
+	pkg.CXXFiles = []string{"a5.go", "b5.go", "c5.go"}
+	pkg.MFiles = []string{"a6.go", "b6.go", "c6.go"}
+	pkg.HFiles = []string{"a7.go", "b7.go", "c7.go"}
+	pkg.FFiles = []string{"a8.go", "b8.go", "c8.go"}
+	pkg.SFiles = []string{"a9.go", "b9.go", "c9.go"}
+	pkg.SwigCXXFiles = []string{"a10.go", "b10.go", "c10.go"}
+	pkg.SwigFiles = []string{"a11.go", "b11.go", "c11.go"}
+	pkg.SysoFiles = []string{"a12.go", "b12.go", "c12.go"}
+
+	return &pkg
+}
+
+// benchmark getFileListNeedsCopy
+// goos: darwin
+// goarch: amd64
+// pkg: github.com/qiniu/goc/pkg/build
+// BenchmarkGetFileList-4           3884557               298 ns/op             576 B/op          1 allocs/op
+func BenchmarkGetFileList(b *testing.B) {
+	//var files []string
+	pkg := initSlicesForTest()
+	for n := 0; n < b.N; n++ {
+		getFileListNeedsCopy(pkg)
+	}
+}