Repository: openllb/hlb
Pull request #48 (2020-03-25 19:18:29)

Implement skeleton for producing solve request trees
Phase 1 for #47

Allows executing of multiple targets from command line. In phase 2, we need change the compiler to emit solve requests instead of `llb.State`. In phase 3, we need to add to the `parser/cst` to support the `group` type and add the appropriate support in the checker/compiler.

We also need to change the interface between the compiler and solver from `llb.State` to `pb.Definition`. In the future when we want to support constraints, the compiler will need to marshal the `llb.State` with different constraints (for macOS workers, etc).


Patch:
diff --git a/cmd/hlb/command/run.go b/cmd/hlb/command/run.go
index 83f2b30..7b53574 100644
--- a/cmd/hlb/command/run.go
+++ b/cmd/hlb/command/run.go
@@ -2,9 +2,11 @@ package command
 
 import (
 	"context"
+	"encoding/csv"
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/docker/buildx/util/progress"
 	"github.com/mattn/go-isatty"
@@ -21,39 +23,21 @@ var runCommand = &cli.Command{
 	Usage:     "compiles and runs a hlb program",
 	ArgsUsage: "<*.hlb>",
 	Flags: []cli.Flag{
-		&cli.StringFlag{
+		&cli.StringSliceFlag{
 			Name:    "target",
 			Aliases: []string{"t"},
 			Usage:   "specify target filesystem to solve",
-			Value:   "default",
+			Value:   cli.NewStringSlice("default"),
 		},
 		&cli.BoolFlag{
 			Name:  "debug",
 			Usage: "jump into a source level debugger for hlb",
 		},
-		&cli.StringFlag{
-			Name:    "download",
-			Aliases: []string{"d"},
-			Usage:   "downloads the solved hlb filesystem to a directory",
-		},
-		&cli.BoolFlag{
-			Name:  "tarball",
-			Usage: "downloads the solved hlb filesystem as a tarball and writes to stdout",
-		},
-		&cli.StringFlag{
-			Name:  "docker-tarball",
-			Usage: "specify a image name for downloading the solved hlb as a docker image tarball and writes to stdout",
-		},
 		&cli.StringFlag{
 			Name:  "log-output",
 			Usage: "set type of log output (auto, tty, plain, json, raw)",
 			Value: "auto",
 		},
-		&cli.StringFlag{
-			Name:    "push",
-			Aliases: []string{"p"},
-			Usage:   "push the solved hlb filesystem to a docker registry",
-		},
 	},
 	Action: func(c *cli.Context) error {
 		rc, err := ModuleReadCloser(c.Args().Slice())
@@ -69,34 +53,26 @@ var runCommand = &cli.Command{
 		}
 
 		return Run(ctx, cln, rc, RunOptions{
-			Debug:         c.Bool("debug"),
-			DockerTarball: c.String("docker-tarball"),
-			Download:      c.String("download"),
-			Target:        c.String("target"),
-			LLB:           c.Bool("llb"),
-			LogOutput:     c.String("log-output"),
-			Output:        os.Stdout,
-			Push:          c.String("push"),
-			Tarball:       c.Bool("tarball"),
+			Debug:     c.Bool("debug"),
+			Targets:   c.StringSlice("target"),
+			LLB:       c.Bool("llb"),
+			LogOutput: c.String("log-output"),
+			Output:    os.Stdout,
 		})
 	},
 }
 
 type RunOptions struct {
-	Debug         bool
-	DockerTarball string
-	Download      string
-	Target        string
-	LLB           bool
-	LogOutput     string
-	Output        io.WriteCloser
-	Push          string
-	Tarball       bool
+	Debug     bool
+	Targets   []string
+	LLB       bool
+	LogOutput string
+	Output    io.WriteCloser
 }
 
 func Run(ctx context.Context, cln *client.Client, rc io.ReadCloser, opts RunOptions) error {
-	if opts.Target == "" {
-		opts.Target = "default"
+	if len(opts.Targets) == 0 {
+		opts.Targets = []string{"default"}
 	}
 	if opts.Output == nil {
 		opts.Output = os.Stdout
@@ -140,7 +116,35 @@ func Run(ctx context.Context, cln *client.Client, rc io.ReadCloser, opts RunOpti
 		mw = p.MultiWriter()
 	}
 
-	st, info, err := hlb.Compile(ctx, cln, mw, opts.Target, rc)
+	targets := []hlb.Target{}
+	for _, target := range opts.Targets {
+		r := csv.NewReader(strings.NewReader(target))
+		fields, err := r.Read()
+		if err != nil {
+			return err
+		}
+		t := hlb.Target{
+			Name:   fields[0],
+			Output: opts.Output,
+		}
+		for _, field := range fields[1:] {
+			switch {
+			case strings.HasPrefix(field, "download="):
+				t.Download = strings.TrimPrefix(field, "download=")
+			case field == "tarball":
+				t.Tarball = true
+			case strings.HasPrefix(field, "dockerTarball="):
+				t.DockerTarball = strings.TrimPrefix(field, "dockerTarball=")
+			case strings.HasPrefix(field, "push="):
+				t.Push = strings.TrimPrefix(field, "push=")
+			default:
+				return fmt.Errorf("Unknown target option %q for target %q", field, t.Name)
+			}
+		}
+		targets = append(targets, t)
+	}
+
+	solveReq, err := hlb.Compile(ctx, cln, mw, targets, rc)
 	if err != nil {
 		// Ignore early exits from the debugger.
 		if err == codegen.ErrDebugExit {
@@ -153,34 +157,13 @@ func Run(ctx context.Context, cln *client.Client, rc io.ReadCloser, opts RunOpti
 		return nil
 	}
 
-	var solveOpts []solver.SolveOption
-	for id, path := range info.Locals {
-		solveOpts = append(solveOpts, solver.WithLocal(id, path))
-	}
-	for id, path := range info.Secrets {
-		solveOpts = append(solveOpts, solver.WithSecret(id, path))
-	}
-
-	if opts.Download != "" {
-		solveOpts = append(solveOpts, solver.WithDownload(opts.Download))
-	}
-	if opts.Tarball {
-		solveOpts = append(solveOpts, solver.WithDownloadTarball(opts.Output))
-	}
-	if opts.DockerTarball != "" {
-		solveOpts = append(solveOpts, solver.WithDownloadDockerTarball(opts.DockerTarball, opts.Output))
-	}
-	if opts.Push != "" {
-		solveOpts = append(solveOpts, solver.WithPushImage(opts.Push))
-	}
-
 	if p == nil {
-		return solver.Solve(ctx, cln, nil, st, solveOpts...)
+		return solveReq.Solve(ctx, cln, nil)
 	}
 
-	p.WithPrefix("solve", func(ctx context.Context, pw progress.Writer) error {
+	p.Go(func() error {
 		defer p.Release()
-		return solver.Solve(ctx, cln, pw, st, solveOpts...)
+		return solveReq.Solve(ctx, cln, p.MultiWriter())
 	})
 
 	return p.Wait()
diff --git a/hlb.go b/hlb.go
index b47dba3..1a22163 100644
--- a/hlb.go
+++ b/hlb.go
@@ -11,12 +11,12 @@ import (
 	"github.com/docker/buildx/util/progress"
 	isatty "github.com/mattn/go-isatty"
 	"github.com/moby/buildkit/client"
-	"github.com/moby/buildkit/client/llb"
 	"github.com/openllb/hlb/checker"
 	"github.com/openllb/hlb/codegen"
 	"github.com/openllb/hlb/module"
 	"github.com/openllb/hlb/parser"
 	"github.com/openllb/hlb/report"
+	"github.com/openllb/hlb/solver"
 )
 
 func DefaultParseOpts() []ParseOption {
@@ -27,12 +27,19 @@ func DefaultParseOpts() []ParseOption {
 	return opts
 }
 
-func Compile(ctx context.Context, cln *client.Client, mw *progress.MultiWriter, target string, r io.Reader) (llb.State, *codegen.CodeGen, error) {
-	st := llb.Scratch()
+type Target struct {
+	Name          string
+	Download      string
+	Tarball       bool
+	DockerTarball string
+	Push          string
+	Output        io.WriteCloser
+}
 
+func Compile(ctx context.Context, cln *client.Client, mw *progress.MultiWriter, targets []Target, r io.Reader) (solver.Request, error) {
 	mod, ib, err := Parse(r, DefaultParseOpts()...)
 	if err != nil {
-		return st, nil, err
+		return nil, err
 	}
 	ibs := map[string]*report.IndexedBuffer{
 		mod.Pos.Filename: ib,
@@ -40,64 +47,97 @@ func Compile(ctx context.Context, cln *client.Client, mw *progress.MultiWriter,
 
 	err = checker.Check(mod)
 	if err != nil {
-		return st, nil, err
-	}
-
-	obj := mod.Scope.Lookup(target)
-	if obj == nil {
-		name := lexer.NameOfReader(r)
-		if name == "" {
-			name = "<stdin>"
-		}
-		return st, nil, fmt.Errorf("target %q is not defined in %s", target, name)
+		return nil, err
 	}
 
 	resolver, err := module.NewResolver(cln, mw)
 	if err != nil {
-		return st, nil, err
+		return nil, err
 	}
 
 	res, err := module.NewLocalResolved(mod)
 	if err != nil {
-		return st, nil, err
+		return nil, err
 	}
 	defer res.Close()
 
 	err = module.ResolveGraph(ctx, resolver, res, mod, nil)
 	if err != nil {
-		return st, nil, err
+		return nil, err
 	}
 
-	call := &parser.CallStmt{
-		Func: parser.NewIdentExpr(target),
-	}
+	request := solver.NewEmptyRequest()
 
-	var (
-		cg   *codegen.CodeGen
-		opts []codegen.CodeGenOption
-	)
+	for _, target := range targets {
+		var commonSolveOpts []solver.SolveOption
+		if target.Download != "" {
+			commonSolveOpts = append(commonSolveOpts, solver.WithDownload(target.Download))
+		}
+		if target.Tarball && target.Output != nil {
+			commonSolveOpts = append(commonSolveOpts, solver.WithDownloadTarball(target.Output))
+		}
+		if target.DockerTarball != "" && target.Output != nil {
+			commonSolveOpts = append(commonSolveOpts, solver.WithDownloadDockerTarball(target.DockerTarball, target.Output))
+		}
+		if target.Push != "" {
+			commonSolveOpts = append(commonSolveOpts, solver.WithPushImage(target.Push))
+		}
 
-	gen := func() error {
-		var err error
-		cg, err = codegen.New(opts...)
-		if err != nil {
-			return err
+		obj := mod.Scope.Lookup(target.Name)
+		if obj == nil {
+			name := lexer.NameOfReader(r)
+			if name == "" {
+				name = "<stdin>"
+			}
+			return nil, fmt.Errorf("target %q is not defined in %s", target.Name, name)
 		}
 
-		st, err = cg.Generate(ctx, call, mod)
-		return err
-	}
+		call := &parser.CallStmt{
+			Func: parser.NewIdentExpr(target.Name),
+		}
 
-	if mw == nil {
-		r := bufio.NewReader(os.Stdin)
-		opts = append(opts, codegen.WithDebugger(codegen.NewDebugger(cln, os.Stderr, r, ibs)))
-		err = gen()
-	} else {
-		pw := mw.WithPrefix("codegen", false)
-		defer close(pw.Status())
+		var (
+			cg   *codegen.CodeGen
+			opts []codegen.CodeGenOption
+		)
+
+		gen := func() error {
+			var err error
+			cg, err = codegen.New(opts...)
+			if err != nil {
+				return err
+			}
+
+			st, err := cg.Generate(ctx, call, mod)
+			if err != nil {
+				return err
+			}
+
+			var solveOpts []solver.SolveOption
+			for id, path := range cg.Locals {
+				solveOpts = append(solveOpts, solver.WithLocal(id, path))
+			}
+			for id, path := range cg.Secrets {
+				solveOpts = append(solveOpts, solver.WithSecret(id, path))
+			}
+
+			request = request.Peer(
+				solver.NewRequest(st, append(commonSolveOpts, solveOpts...)...),
+			)
+
+			return nil
+		}
 
-		progress.Write(pw, fmt.Sprintf("compiling %s", mod.Pos.Filename), gen)
-	}
+		if mw == nil {
+			r := bufio.NewReader(os.Stdin)
+			opts = append(opts, codegen.WithDebugger(codegen.NewDebugger(cln, os.Stderr, r, ibs)))
+			err = gen()
+		} else {
+			pw := mw.WithPrefix("codegen", false)
+			defer close(pw.Status())
 
-	return st, cg, err
+			progress.Write(pw, fmt.Sprintf("compiling target %s", target.Name), gen)
+		}
+	}
+	return request, err
 }
diff --git a/solver/progress.go b/solver/progress.go
index d0d0a6c..c095506 100644
--- a/solver/progress.go
+++ b/solver/progress.go
@@ -30,6 +30,45 @@ func WithLogOutput(logOutput LogOutput) ProgressOption {
 	}
 }
 
+// NewProgress returns a Progress that presents all the progress on multiple
+// solves to the terminal stdout.
+//
+// Calling (*Progress).WithPrefix creates a progress writer for a callback
+// function, giving each solve its independent progress writer (which is
+// eventually closed by the solve).
+//
+// When all work has been completed, calling (*Progress).Release will start
+// the process for closing out the progress UI. Note that because of the
+// refresh rate of the interactive UI, we need to also call (*Progress).Wait
+// to ensure it has exited cleanly.
+//
+// Example usage without error handling:
+// ```go
+// p, _ := NewProgress(ctx)
+//
+// p.WithPrefix("work", func(ctx context.Context, pw progress.Writer) error {
+// 	defer p.Release()
+//	return workFunc(ctx, pw)
+// })
+//
+// return p.Wait()
+// ```
+//
+// If your work function also needs to dynamically spawn progress writers, then
+// you can call (*Progress).Go to create a goroutine sharing the same errgroup.
+// Then you can share the underlying multiwriter by calling
+// (*Progress).MultiWriter().
+//
+// ```go
+// p, _ := progress.NewProgress(ctx)
+//
+// p.Go(func() {
+// 	defer p.Release()
+// 	return workFunc(ctx, p.MultiWriter())
+// })
+//
+// return p.Wait()
+// ```
 func NewProgress(ctx context.Context, opts ...ProgressOption) (*Progress, error) {
 	info := &ProgressInfo{}
 	for _, opt := range opts {
diff --git a/solver/request.go b/solver/request.go
new file mode 100644
index 0000000..77a0a6d
--- /dev/null
+++ b/solver/request.go
@@ -0,0 +1,130 @@
+package solver
+
+import (
+	"context"
+
+	"github.com/docker/buildx/util/progress"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
+	"golang.org/x/sync/errgroup"
+)
+
+// Request is a node in the solve request tree produced by the compiler. The
+// solve request tree has peer nodes that should be executed in parallel, and
+// next nodes that should be executed sequentially. These can be intermingled
+// to produce a complex build pipeline.
+type Request interface {
+	// Solve sends the request and its children to BuildKit. The request passes
+	// down the progress.MultiWriter for them to spawn their own progress writers
+	// for each independent solve.
+	Solve(ctx context.Context, cln *client.Client, mw *progress.MultiWriter) error
+
+	// Next adds a sequential solve to this request. The added request will only
+	// execute after this request has completed.
+	Next(n Request) Request
+
+	// Peer adds a parallel solve to this request. The added request will execute
+	// in parallel with this request.
+	Peer(p Request) Request
+}
+
+// NewRequest returns a single solve request.
+func NewRequest(st llb.State, opts ...SolveOption) Request {
+	return &singleRequest{
+		st:   st,
+		opts: opts,
+	}
+}
+
+// NewEmptyRequest returns an empty request, which can be used as the root of
+// a solve request tree.
+func NewEmptyRequest() Request {
+	return &nullRequest{}
+}
+
+type nullRequest struct{}
+
+func (r *nullRequest) Solve(ctx context.Context, cln *client.Client, mw *progress.MultiWriter) error {
+	return nil
+}
+
+func (r *nullRequest) Next(n Request) Request {
+	return n
+}
+
+func (r *nullRequest) Peer(p Request) Request {
+	return p
+}
+
+type singleRequest struct {
+	st   llb.State
+	opts []SolveOption
+}
+
+func (r *singleRequest) Solve(ctx context.Context, cln *client.Client, mw *progress.MultiWriter) error {
+	pw := mw.WithPrefix("", false)
+	return Solve(ctx, cln, pw, r.st, r.opts...)
+}
+
+func (r *singleRequest) Next(n Request) Request {
+	return &sequentialRequest{
+		reqs: []Request{r, n},
+	}
+}
+
+func (r *singleRequest) Peer(p Request) Request {
+	return &parallelRequest{
+		reqs: []Request{r, p},
+	}
+}
+
+type parallelRequest struct {
+	reqs []Request
+}
+
+func (r *parallelRequest) Solve(ctx context.Context, cln *client.Client, mw *progress.MultiWriter) error {
+	g, ctx := errgroup.WithContext(ctx)
+	for _, req := range r.reqs {
+		req := req
+		g.Go(func() error {
+			return req.Solve(ctx, cln, mw)
+		})
+	}
+	return g.Wait()
+}
+
+func (r *parallelRequest) Next(n Request) Request {
+	return &sequentialRequest{
+		reqs: []Request{r, n},
+	}
+}
+
+func (r *parallelRequest) Peer(p Request) Request {
+	r.reqs = append(r.reqs, p)
+	return r
+}
+
+type sequentialRequest struct {
+	reqs []Request
+}
+
+func (r *sequentialRequest) Solve(ctx context.Context, cln *client.Client, mw *progress.MultiWriter) error {
+	for _, req := range r.reqs {
+		err := req.Solve(ctx, cln, mw)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *sequentialRequest) Next(n Request) Request {
+	r.reqs = append(r.reqs, n)
+	return r
+}
+
+func (r *sequentialRequest) Peer(p Request) Request {
+	return &parallelRequest{
+		reqs: []Request{r, p},
+	}
+}


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 379a18a..935ae29 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -47,6 +47,6 @@ jobs:
       run: ./build/hlb --addr docker-container://buildkitd run --log-output plain -t lint ./source.hlb
 
     - name: Crosscompile
-      run: ./build/hlb --addr docker-container://buildkitd run --log-output plain --download build/dist ./source.hlb
+      run: ./build/hlb --addr docker-container://buildkitd run --log-output plain -t default,download=build/dist ./source.hlb
 
 
diff --git a/compile_test.go b/compile_test.go
index 225584e..87a474d 100644
--- a/compile_test.go
+++ b/compile_test.go
@@ -12,6 +12,7 @@ import (
 
 type compileTestCase struct {
 	name    string
+	targets []string
 	input   string
 	errType interface{}
 }
@@ -19,6 +20,7 @@ type compileTestCase struct {
 func TestCompile(t *testing.T) {
 	for _, tc := range []compileTestCase{{
 		"empty",
+		[]string{"default"},
 		`
 		fs default() {
 			scratch
@@ -27,6 +29,7 @@ func TestCompile(t *testing.T) {
 		nil,
 	}, {
 		"image",
+		[]string{"default"},
 		`
 		fs default() {
 			image "busybox:latest"
@@ -35,6 +38,7 @@ func TestCompile(t *testing.T) {
 		nil,
 	}, {
 		"second source from function",
+		[]string{"default"},
 		`
 		fs default() {
 			scratch
@@ -47,6 +51,7 @@ func TestCompile(t *testing.T) {
 		checker.ErrOnlyFirstSource{},
 	}, {
 		"single named option",
+		[]string{"default"},
 		`
 		option::run myopt() {
 			dir "/tmp"
@@ -59,6 +64,7 @@ func TestCompile(t *testing.T) {
 		nil,
 	}, {
 		"combine named option",
+		[]string{"default"},
 		`
 		option::run myopt() {
 			dir "/tmp"
@@ -72,16 +78,38 @@ func TestCompile(t *testing.T) {
 		}
 		`,
 		nil,
+	}, {
+		"multiple targets",
+		[]string{"foo", "bar"},
+		`
+		fs foo() {
+			image "busybox:latest"
+			run "echo hello"
+		}
+		fs bar() {
+			image "busybox:latest"
+			run "echo bar"
+		}
+		`,
+		nil,
 	}} {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
+
 			ctx := context.Background()
 			p, err := solver.NewProgress(ctx, solver.WithLogOutput(solver.LogOutputPlain))
 			require.NoError(t, err)
+
 			mw := p.MultiWriter()
 			in := strings.NewReader(cleanup(tc.input))
-			_, _, err = Compile(ctx, nil, mw, "default", in)
+
+			var targets []Target
+			for _, target := range tc.targets {
+				targets = append(targets, Target{Name: target})
+			}
+
+			_, err = Compile(ctx, nil, mw, targets, in)
 			if tc.errType == nil {
 				require.NoError(t, err)
 			} else {