Repository: qiniu/goc (go)

Issue:
File copy strategy of `goc build` needs to be optimized
Some files like `unix socket` is not allowed to be copied. When project files contians these file, `goc build .` will return an error. The file copy strategy needs optimize.

Pull request:
optimize the file copy strategy of `goc build` and fix bugs
fix #126 and #144

Patch:
diff --git a/go.mod b/go.mod
index ad4cbc3..b9b96d1 100644
--- a/go.mod
+++ b/go.mod
@@ -9,13 +9,13 @@ require (
 	github.com/julienschmidt/httprouter v1.2.0
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/olekukonko/tablewriter v0.0.4
-	github.com/otiai10/copy v1.0.2
 	github.com/qiniu/api.v7/v7 v7.5.0
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.6.2
 	github.com/stretchr/testify v1.5.1
+	github.com/tongjingran/copy v1.3.3
 	golang.org/x/mod v0.3.0
 	golang.org/x/net v0.0.0-20200625001655-4c5254603344
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
diff --git a/go.sum b/go.sum
index 63fe015..caa1d5b 100644
--- a/go.sum
+++ b/go.sum
@@ -325,6 +325,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -578,8 +579,12 @@ github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
 github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
+github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -714,6 +719,8 @@ github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tongjingran/copy v1.3.3 h1:Oha32tKfDrLTSXGaxW+jaTSmGc+hiHFRf/STZ4x9b4g=
+github.com/tongjingran/copy v1.3.3/go.mod h1:Njma1OR5OuzB8pLAmQSzonHXzba+DDiPVmMSonpSpy4=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@@ -739,7 +746,6 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
@@ -962,8 +968,6 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200214144324-88be01311a71/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe h1:Kh3iY7o/2bMfQXZdwLdL9jDMU1k9HoVn0P1mGCfoFLc=
 golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65 h1:1KSbntBked74wYsKq0jzXYy7ZwcjAUtrl7EmPE97Iiw=
-golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200730221956-1ac65761fe2c h1:c5JjBOQWM2pRemVbog00sS+oAdi8tTR+NNRFDwUOrTQ=
 golang.org/x/tools v0.0.0-20200730221956-1ac65761fe2c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/pkg/build/build.go b/pkg/build/build.go
index 052c00d..4fcbe40 100644
--- a/pkg/build/build.go
+++ b/pkg/build/build.go
@@ -133,8 +133,11 @@ func (b *Build) determineOutputDir(outputDir string) (string, error) {
 	targetName := ""
 	for _, pkg := range b.Pkgs {
 		if pkg.Name == "main" {
-			_, file := filepath.Split(pkg.Target)
-			targetName = file
+			if pkg.Target != "" {
+				targetName = filepath.Base(pkg.Target)
+			} else {
+				targetName = filepath.Base(pkg.Dir)
+			}
 			break
 		}
 	}
diff --git a/pkg/build/gomodules.go b/pkg/build/gomodules.go
index 32c9b79..82081f7 100644
--- a/pkg/build/gomodules.go
+++ b/pkg/build/gomodules.go
@@ -20,8 +20,8 @@ import (
 	"io/ioutil"
 	"path/filepath"
 
-	"github.com/otiai10/copy"
 	log "github.com/sirupsen/logrus"
+	"github.com/tongjingran/copy"
 	"golang.org/x/mod/modfile"
 )
 
@@ -31,7 +31,7 @@ func (b *Build) cpGoModulesProject() {
 			dst := b.TmpDir
 			src := v.Module.Dir
 
-			if err := copy.Copy(src, dst); err != nil {
+			if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
 				log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
 			}
 			break
diff --git a/pkg/build/legacy.go b/pkg/build/legacy.go
index 72f5c55..5bcd009 100644
--- a/pkg/build/legacy.go
+++ b/pkg/build/legacy.go
@@ -19,11 +19,12 @@ package build
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/otiai10/copy"
 	"github.com/qiniu/goc/pkg/cover"
+	"github.com/tongjingran/copy"
 )
 
 func (b *Build) cpLegacyProject() {
@@ -37,7 +38,7 @@ func (b *Build) cpLegacyProject() {
 			continue
 		}
 
-		if err := copy.Copy(src, dst); err != nil {
+		if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
 			log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
 		}
 
@@ -66,7 +67,7 @@ func (b *Build) cpDepPackages(pkg *cover.Package, visited map[string]bool) {
 
 		dst := filepath.Join(b.TmpDir, "src", dep)
 
-		if err := copy.Copy(src, dst); err != nil {
+		if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
 			log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
 		}
 
@@ -80,7 +81,7 @@ func (b *Build) cpNonStandardLegacy() {
 			dst := b.TmpDir
 			src := v.Dir
 
-			if err := copy.Copy(src, dst); err != nil {
+			if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
 				log.Printf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
 			}
 			break
@@ -89,3 +90,17 @@ func (b *Build) cpNonStandardLegacy() {
 		}
 	}
 }
+
+// skipCopy skip copy .git dir and irregular files
+func skipCopy(src string, info os.FileInfo) (bool, error) {
+	irregularModeType := os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular
+	if strings.HasSuffix(src, "/.git") {
+		log.Infof("Skip .git dir [%s]", src)
+		return true, nil
+	}
+	if info.Mode()&irregularModeType != 0 {
+		log.Warnf("Skip file [%s], the file mode is [%s]", src, info.Mode().String())
+		return true, nil
+	}
+	return false, nil
+}
diff --git a/pkg/build/tmpfolder.go b/pkg/build/tmpfolder.go
index 9cc8fbe..89b6ea2 100644
--- a/pkg/build/tmpfolder.go
+++ b/pkg/build/tmpfolder.go
@@ -115,8 +115,6 @@ func (b *Build) mvProjectsToTmp() error {
 	} else if b.IsMod == false && b.Root == "" {
 		b.TmpWorkingDir = b.TmpDir
 		b.cpNonStandardLegacy()
-	} else {
-		return fmt.Errorf("unknown project type: %w", ErrShouldNotReached)
 	}
 
 	log.Infof("New workingdir in tmp directory in: %v", b.TmpWorkingDir)


Test patch:
diff --git a/pkg/build/legacy_test.go b/pkg/build/legacy_test.go
index 53f9cc1..ae2e5cc 100644
--- a/pkg/build/legacy_test.go
+++ b/pkg/build/legacy_test.go
@@ -17,9 +17,11 @@
 package build
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/qiniu/goc/pkg/cover"
 	"github.com/stretchr/testify/assert"
@@ -69,3 +71,63 @@ func TestDepPackagesCopyWithInvalidDir(t *testing.T) {
 	})
 	assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
 }
+
+type MockFile struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+	isDir   bool
+}
+
+func (m MockFile) Name() string {
+	return m.name
+}
+
+func (m MockFile) Size() int64 {
+	return m.size
+}
+
+func (m MockFile) Mode() os.FileMode {
+	return m.mode
+}
+
+func (m MockFile) ModTime() time.Time {
+	return m.modTime
+}
+
+func (m MockFile) IsDir() bool {
+	return m.isDir
+}
+
+func (m MockFile) Sys() interface{} {
+	return nil
+}
+
+// skipCopy verify
+func TestSkipCopy(t *testing.T) {
+	testCases := map[string]struct {
+		inputSrc  string
+		inputInfo MockFile
+		expected  bool
+	}{
+		"src with /.git suffix":    {inputSrc: "/test/.git", inputInfo: MockFile{mode: 0}, expected: true},
+		"src with ./git suffix":    {inputSrc: "/test.git", inputInfo: MockFile{mode: 0}, expected: false},
+		"src with /.gita suffix":   {inputSrc: "/test/.gita", inputInfo: MockFile{mode: 0}, expected: false},
+		"src with /.git in middle": {inputSrc: "/test/.git/test", inputInfo: MockFile{mode: 0}, expected: false},
+		"irregular file":           {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeIrregular}, expected: true},
+		"dir file":                 {inputSrc: "/test", inputInfo: MockFile{isDir: true, mode: os.ModeDir}, expected: false},
+		"temporary file":           {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeTemporary}, expected: false},
+		"symlink file":             {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSymlink}, expected: false},
+		"device file":              {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeDevice}, expected: true},
+		"named pipe file":          {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeNamedPipe}, expected: true},
+		"socket file":              {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSocket}, expected: true},
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			output, err := skipCopy(tc.inputSrc, tc.inputInfo)
+			assert.NoError(t, err)
+			assert.Equal(t, output, tc.expected)
+		})
+	}
+}
diff --git a/pkg/cover/cover_test.go b/pkg/cover/cover_test.go
index c096990..e48e048 100644
--- a/pkg/cover/cover_test.go
+++ b/pkg/cover/cover_test.go
@@ -27,8 +27,8 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
-	"github.com/otiai10/copy"
 	"github.com/stretchr/testify/assert"
+	"github.com/tongjingran/copy"
 )
 
 func testCoverage() (c *Coverage) {
diff --git a/tests/build.bats b/tests/build.bats
index 7b8ae9e..055d8c6 100755
--- a/tests/build.bats
+++ b/tests/build.bats
@@ -98,5 +98,18 @@ setup() {
     info build5 output: $output
     [ "$status" -eq 0 ]
 
+    wait $profile_pid
+}
+
+@test "test goc build on reference other package project" {
+    cd samples/reference_other_package_project/app
+
+    wait_profile_backend "build6" &
+    profile_pid=$!
+
+    run gocc build --debug --debugcisyncfile ci-sync.bak;
+    info build5 output: $output
+    [ "$status" -eq 0 ]
+
     wait $profile_pid
 }
\ No newline at end of file
diff --git a/tests/samples/reference_other_package_project/app/main.go b/tests/samples/reference_other_package_project/app/main.go
new file mode 100644
index 0000000..f52a380
--- /dev/null
+++ b/tests/samples/reference_other_package_project/app/main.go
@@ -0,0 +1,8 @@
+package main
+
+import "example.com/reference_other_package_project/foo"
+
+func main() {
+	foo.PrintFoo()
+	return
+}
diff --git a/tests/samples/reference_other_package_project/foo/foo.go b/tests/samples/reference_other_package_project/foo/foo.go
new file mode 100644
index 0000000..8159465
--- /dev/null
+++ b/tests/samples/reference_other_package_project/foo/foo.go
@@ -0,0 +1,7 @@
+package foo
+
+import "fmt"
+
+func PrintFoo() {
+	fmt.Println("hello, world.")
+}
diff --git a/tests/samples/reference_other_package_project/go.mod b/tests/samples/reference_other_package_project/go.mod
new file mode 100644
index 0000000..9de588a
--- /dev/null
+++ b/tests/samples/reference_other_package_project/go.mod
@@ -0,0 +1,3 @@
+module example.com/reference_other_package_project
+
+go 1.11