Repository: google/ko
Pull request #154 (2020-04-30 01:54:50)

[modules] Dependent command packages can now be built with ko 
Thus if you have a (in)direct command package as a dependency
say `myhost.com/go/package/cmd/run` you can now publish this
with the following ko command

```bash
$  ko publish myhost.com/go/package/cmd/run
```

This follows go's module/vendor semantics. Thus go1.14 by default will use vendored modules. With go1.13 you need the explicit flags to use vendored deps

```bash
$  GOFLAGS="-mod=vendor" ko publish myhost.com/go/package/cmd/run
```

Subsequently in yaml files you can drop the superfluous prefixes

ie. this (in a non-module world)  

```yaml
apiVersion: batch/v1
kind: Job
spec:
  containers:
  - name: vegeta
    image: knative.dev/serving/vendor/github.com/tsenart/vegeta
```
becomes this in the module world

```yaml
apiVersion: batch/v1
kind: Job
spec:
  containers:
  - name: vegeta
    image: github.com/tsenart/vegeta
```

dprotaso: cc https://github.com/google/ko/issues/152
ImJasonH: I'm not sure I understand what this is trying to achieve. Is the effect that you can `ko build` a non-`package main` package?
dprotaso: whoops needs a rebase - one minute
dprotaso: Failure in CI is go1.12 

> pkg/build/gobuild.go:134: Errorf format %w has unknown verb w

1.12 is EOL so I made https://github.com/google/ko/pull/156
mattmoor: @ImJasonH If I'm reading the description correctly, this is the `ko publish` complement of my change.
mattmoor: Ok, so reading through the change, I want to make sure I understand...

With my change, a vendored main module `foo` will work if it is written as `ko://foo`, but what you are doing is dumping all of the referenced modules, so that just `foo` still works.

Since in `ko publish` qualify local import turns `./vendor/foo` into `foo` with modules, this unblocks letting `ko publish ./vendor/foo` or `ko publish foo` from working.

@dprotaso is that right?


dprotaso: > this unblocks letting ko publish ./vendor/foo or ko publish foo from working.

This unblocks `ko publish foo`
This _does not unblock_ `ko publish ./vendor/foo`
mattmoor: I wonder, would simply having `qualifyLocalImport` slap a `ko://` on the front of what it produces work?

dprotaso: 🤷‍♂️ 
mattmoor: This just makes me want to deprecated non-`ko://` faster, but it seems like an improvement of the status quo.  thanks for chasing this down.

Patch:
diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go
index ca12ca0a..231e4ee5 100644
--- a/pkg/build/gobuild.go
+++ b/pkg/build/gobuild.go
@@ -46,12 +46,17 @@ const (
 type GetBase func(string) (v1.Image, error)
 type builder func(context.Context, string, v1.Platform, bool) (string, error)
 
+type buildContext interface {
+	Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error)
+}
+
 type gobuild struct {
 	getBase              GetBase
 	creationTime         v1.Time
 	build                builder
 	disableOptimizations bool
-	mod                  *modInfo
+	mod                  *modules
+	buildContext         buildContext
 }
 
 // Option is a functional option for NewGo.
@@ -62,7 +67,8 @@ type gobuildOpener struct {
 	creationTime         v1.Time
 	build                builder
 	disableOptimizations bool
-	mod                  *modInfo
+	mod                  *modules
+	buildContext         buildContext
 }
 
 func (gbo *gobuildOpener) Open() (Interface, error) {
@@ -75,38 +81,80 @@ func (gbo *gobuildOpener) Open() (Interface, error) {
 		build:                gbo.build,
 		disableOptimizations: gbo.disableOptimizations,
 		mod:                  gbo.mod,
+		buildContext:         gbo.buildContext,
 	}, nil
 }
 
 // https://golang.org/pkg/cmd/go/internal/modinfo/#ModulePublic
+type modules struct {
+	main *modInfo
+	deps map[string]*modInfo
+}
+
 type modInfo struct {
 	Path string
 	Dir  string
+	Main bool
 }
 
 // moduleInfo returns the module path and module root directory for a project
 // using go modules, otherwise returns nil.
 //
 // Related: https://github.com/golang/go/issues/26504
-func moduleInfo() *modInfo {
-	output, err := exec.Command("go", "list", "-mod=readonly", "-m", "-json").Output()
+func moduleInfo() (*modules, error) {
+	modules := modules{
+		deps: make(map[string]*modInfo),
+	}
+
+	// TODO we read all the output as a single byte array - it may
+	// be possible & more efficient to stream it
+	output, err := exec.Command("go", "list", "-mod=readonly", "-json", "-m", "all").Output()
 	if err != nil {
-		return nil
+		return nil, nil
 	}
-	var info modInfo
-	if err := json.Unmarshal(output, &info); err != nil {
-		return nil
+
+	dec := json.NewDecoder(bytes.NewReader(output))
+
+	for {
+		var info modInfo
+
+		err := dec.Decode(&info)
+		if err == io.EOF {
+			// all done
+			break
+		}
+
+		modules.deps[info.Path] = &info
+
+		if info.Main {
+			modules.main = &info
+		}
+
+		if err != nil {
+			return nil, fmt.Errorf("error reading module data %w", err)
+		}
+	}
+
+	if modules.main == nil {
+		return nil, fmt.Errorf("couldn't find main module")
 	}
-	return &info
+
+	return &modules, nil
 }
 
 // NewGo returns a build.Interface implementation that:
 //  1. builds go binaries named by importpath,
 //  2. containerizes the binary on a suitable base,
 func NewGo(options ...Option) (Interface, error) {
+	module, err := moduleInfo()
+	if err != nil {
+		return nil, err
+	}
+
 	gbo := &gobuildOpener{
-		build: build,
-		mod:   moduleInfo(),
+		build:        build,
+		mod:          module,
+		buildContext: &gb.Default,
 	}
 
 	for _, option := range options {
@@ -141,7 +189,7 @@ func (g *gobuild) IsSupportedReference(s string) bool {
 // Note that we will fall back to GOPATH if the project isn't using go modules.
 func (g *gobuild) importPackage(ref reference) (*gb.Package, error) {
 	if g.mod == nil {
-		return gb.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment)
+		return g.buildContext.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment)
 	}
 
 	// If we're inside a go modules project, try to use the module's directory
@@ -149,8 +197,11 @@ func (g *gobuild) importPackage(ref reference) (*gb.Package, error) {
 	// * any strict reference we get
 	// * paths that match module path prefix (they should be in this project)
 	// * relative paths (they should also be in this project)
-	if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.Path) || gb.IsLocalImport(ref.Path()) {
-		return gb.Import(ref.Path(), g.mod.Dir, gb.ImportComment)
+	// * path is a module
+
+	_, isDep := g.mod.deps[ref.Path()]
+	if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.main.Path) || gb.IsLocalImport(ref.Path()) || isDep {
+		return g.buildContext.Import(ref.Path(), g.mod.main.Dir, gb.ImportComment)
 	}
 
 	return nil, fmt.Errorf("unmatched importPackage %q with gomodules", ref.String())
diff --git a/pkg/build/options.go b/pkg/build/options.go
index 4bcee0db..66ddf4ca 100644
--- a/pkg/build/options.go
+++ b/pkg/build/options.go
@@ -57,9 +57,16 @@ func withBuilder(b builder) Option {
 // withModulePath is a functional option for overriding the module path for
 // the current ko invocation.
 // This is exposed for testing.
-func withModuleInfo(mi *modInfo) Option {
+func withModuleInfo(m *modules) Option {
 	return func(gbo *gobuildOpener) error {
-		gbo.mod = mi
+		gbo.mod = m
+		return nil
+	}
+}
+
+func withBuildContext(b buildContext) Option {
+	return func(gbo *gobuildOpener) error {
+		gbo.buildContext = b
 		return nil
 	}
 }


Test patch:
diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go
index db320481..0c236f50 100644
--- a/pkg/build/gobuild_test.go
+++ b/pkg/build/gobuild_test.go
@@ -17,6 +17,7 @@ package build
 import (
 	"archive/tar"
 	"context"
+	gb "go/build"
 	"io"
 	"io/ioutil"
 	"path"
@@ -69,12 +70,31 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) {
 	if err != nil {
 		t.Fatalf("random.Image() = %v", err)
 	}
-	mod := &modInfo{
-		Path: "github.com/google/ko/cmd/ko/test",
-		Dir:  ".",
+
+	mods := &modules{
+		main: &modInfo{
+			Path: "github.com/google/ko/cmd/ko/test",
+			Dir:  ".",
+		},
+		deps: map[string]*modInfo{
+			"github.com/some/module/cmd": &modInfo{
+				Path: "github.com/some/module/cmd",
+				Dir:  ".",
+			},
+		},
+	}
+
+	opts := []Option{
+		WithBaseImages(func(string) (v1.Image, error) { return base, nil }),
+		withModuleInfo(mods),
+		withBuildContext(stubBuildContext{
+			// make all referenced deps commands
+			"github.com/google/ko/cmd/ko/test": &gb.Package{Name: "main"},
+			"github.com/some/module/cmd":       &gb.Package{Name: "main"},
+		}),
 	}
 
-	ng, err := NewGo(WithBaseImages(func(string) (v1.Image, error) { return base, nil }), withModuleInfo(mod))
+	ng, err := NewGo(opts...)
 	if err != nil {
 		t.Fatalf("NewGo() = %v", err)
 	}
@@ -82,6 +102,7 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) {
 	// Supported import paths.
 	for _, importpath := range []string{
 		"github.com/google/ko/cmd/ko/test", // ko can build the test package.
+		"github.com/some/module/cmd",       // ko can build commands in dependent modules
 	} {
 		t.Run(importpath, func(t *testing.T) {
 			if !ng.IsSupportedReference(importpath) {
@@ -376,3 +397,9 @@ func TestGoBuild(t *testing.T) {
 		}
 	})
 }
+
+type stubBuildContext map[string]*gb.Package
+
+func (s stubBuildContext) Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error) {
+	return s[path], nil
+}