Repository: openllb/hlb
Pull request #107 (2020-04-18 02:02:52)

[#101] implement heredoc support
Adds support for:
```
value <<EOM
  this
    is 
  literal
EOM

# undent with <<-
value <<-EOM
  this
    is
  undented
EOM

# fold to single line with <<~
value <<~EOM
  this 
  is
  folded
EOM
```


Patch:
diff --git a/checker/checker.go b/checker/checker.go
index 1cce89c..896f8ef 100644
--- a/checker/checker.go
+++ b/checker/checker.go
@@ -518,7 +518,7 @@ func (c *checker) checkIdentArg(scope *parser.Scope, typ parser.ObjType, ident *
 func (c *checker) checkBasicLitArg(typ parser.ObjType, lit *parser.BasicLit) error {
 	switch typ {
 	case parser.Str:
-		if lit.Str == nil {
+		if lit.Str == nil && lit.HereDoc == nil {
 			return ErrWrongArgType{lit.Pos, typ, lit.ObjType()}
 		}
 	case parser.Int:
diff --git a/codegen/expr.go b/codegen/expr.go
index c9fe17d..d38d552 100644
--- a/codegen/expr.go
+++ b/codegen/expr.go
@@ -30,7 +30,10 @@ func (cg *CodeGen) EmitStringExpr(ctx context.Context, scope *parser.Scope, expr
 			return "", errors.WithStack(ErrCodeGen{expr, errors.Errorf("unknown obj type")})
 		}
 	case expr.BasicLit != nil:
-		return *expr.BasicLit.Str, nil
+		if expr.BasicLit.Str != nil {
+			return expr.BasicLit.Str.Unquoted(), nil
+		}
+		return expr.BasicLit.HereDoc.Value, nil
 	case expr.FuncLit != nil:
 		return cg.EmitStringBlock(ctx, scope, expr.FuncLit.Body.NonEmptyStmts(), nil)
 	default:
diff --git a/go.mod b/go.mod
index a6ab92c..3f0d16b 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/docker/cli v1.14.0-0.20190523191156-ab688a9a79a1
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kr/pretty v0.2.0 // indirect
+	github.com/lithammer/dedent v1.1.0
 	github.com/logrusorgru/aurora v0.0.0-20191116043053-66b7ad493a23
 	github.com/mattn/go-isatty v0.0.11
 	github.com/mitchellh/go-homedir v1.1.0
diff --git a/go.sum b/go.sum
index 8c614a1..90661a3 100644
--- a/go.sum
+++ b/go.sum
@@ -264,6 +264,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/logrusorgru/aurora v0.0.0-20191116043053-66b7ad493a23 h1:Wp7NjqGKGN9te9N/rvXYRhlVcrulGdxnz8zadXWs7fc=
 github.com/logrusorgru/aurora v0.0.0-20191116043053-66b7ad493a23/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
diff --git a/module/resolve.go b/module/resolve.go
index f5fe545..fa66362 100644
--- a/module/resolve.go
+++ b/module/resolve.go
@@ -394,7 +394,7 @@ func ResolveGraph(ctx context.Context, resolver Resolver, res Resolved, mod *par
 					filename = ModuleFilename
 				case n.ImportPath != nil:
 					importRes = res
-					filename = n.ImportPath.Path
+					filename = n.ImportPath.Path.Unquoted()
 				}
 
 				rc, err := importRes.Open(filename)
diff --git a/module/tree.go b/module/tree.go
index 6219e64..7800eaa 100644
--- a/module/tree.go
+++ b/module/tree.go
@@ -53,9 +53,9 @@ func NewTree(ctx context.Context, cln *client.Client, mw *progress.MultiWriter,
 			value = filepath.Join(prefix, ModuleFilename)
 		case decl.ImportPath != nil:
 			if prefix == "" {
-				value = decl.ImportPath.Path
+				value = decl.ImportPath.Path.Unquoted()
 			} else {
-				value = filepath.Join(prefix, decl.ImportPath.Path)
+				value = filepath.Join(prefix, decl.ImportPath.Path.Unquoted())
 			}
 		}
 
diff --git a/module/vendor.go b/module/vendor.go
index 48f3c71..00ec944 100644
--- a/module/vendor.go
+++ b/module/vendor.go
@@ -102,7 +102,7 @@ func Vendor(ctx context.Context, cln *client.Client, mw *progress.MultiWriter, m
 			case decl.ImportFunc != nil:
 				filename = ModuleFilename
 			case decl.ImportPath != nil:
-				filename = decl.ImportPath.Path
+				filename = decl.ImportPath.Path.Unquoted()
 			}
 
 			f, err := os.Create(filepath.Join(vp, filename))
diff --git a/parser/cst.go b/parser/cst.go
index 86789c1..bbc456e 100644
--- a/parser/cst.go
+++ b/parser/cst.go
@@ -1,20 +1,23 @@
 package parser
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"strconv"
 	"strings"
 
 	"github.com/alecthomas/participle"
 	"github.com/alecthomas/participle/lexer"
 	"github.com/alecthomas/participle/lexer/regex"
+	"github.com/lithammer/dedent"
 )
 
 var (
 	// Lexer lexes HLB into tokens for the parser.
 	Lexer = lexer.Must(regex.New(fmt.Sprintf(`
-	        whitespace = [\r\t ]+
-
+		Whitespace = [\r\t ]+
+		HereDoc = <<[-~]?
 		Keyword  = \b(with|as|import|export|from)\b
 		Modifier = \b(variadic)\b
 		Type     = \b(string|int|bool|fs|option|group)(::[a-z][a-z]*)?\b
@@ -24,20 +27,74 @@ var (
 		Bool     = \b(true|false)\b
 		Selector = \b[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*\b
 		Ident    = \b[a-zA-Z_][a-zA-Z0-9_]*\b
-	        Newline  = \n
+		Newline  = \n
 		Operator = {|}|\(|\)|,|;
-	        Comment  = #[^\n]*\n
-	        Bad      = .*
+		Comment  = #[^\n]*\n
+		Bad      = .*
 	`)))
 
 	// Parser parses HLB into a concrete syntax tree rooted from a Module.
 	Parser = participle.MustBuild(
 		&Module{},
-		participle.Lexer(Lexer),
-		participle.Unquote(),
+		participle.Lexer(&hereDocDefinition{Lexer}),
 	)
 )
 
+// hereDocDefinition allows for a lexer that optionally
+// emits whitespace depending on if it lexing the contents
+// of a here-doc
+type hereDocDefinition struct {
+	def lexer.Definition
+}
+
+func (s *hereDocDefinition) Lex(r io.Reader) (lexer.Lexer, error) {
+	lex, err := s.def.Lex(r)
+	if err != nil {
+		return lex, err
+	}
+	return &hereDocLexer{lex: lex, sym: s.def.Symbols()}, err
+}
+
+func (s *hereDocDefinition) Symbols() map[string]rune {
+	return s.def.Symbols()
+}
+
+// hereDocLexer will only emit whitespace tokens if processing
+// a here-doc statement, otherwise, whitespace tokens will be
+// ignored.
+type hereDocLexer struct {
+	lex       lexer.Lexer
+	sym       map[string]rune
+	keepSpace bool
+	foundHere bool
+	hereIdent lexer.Token
+}
+
+func (s *hereDocLexer) Next() (lexer.Token, error) {
+	t, err := s.lex.Next()
+	switch t.Type {
+	case s.sym["Whitespace"]:
+		if !s.keepSpace {
+			// not preserving spsace, so move on to next token
+			return s.Next()
+		}
+	case s.sym["HereDoc"]:
+		s.foundHere = true
+	case s.sym["Ident"]:
+		if s.foundHere && s.hereIdent.Value == t.Value {
+			// end of heredoc
+			s.foundHere = false
+			s.keepSpace = false
+			s.hereIdent = lexer.Token{}
+		} else if s.foundHere && s.hereIdent.Value == "" {
+			s.hereIdent = t
+			s.keepSpace = true
+		}
+	}
+
+	return t, err
+}
+
 // Node is implemented by all nodes in the CST.
 type Node interface {
 	// Stringer is implemented to unparse a node back into formatted HLB.
@@ -166,7 +223,7 @@ func (f *From) End() lexer.Position      { return shiftPosition(f.Pos, len(f.Key
 // ImportPath represents the relative path to a local import.
 type ImportPath struct {
 	Pos  lexer.Position
-	Path string `parser:"@String"`
+	Path QuotedString `parser:"@@"`
 }
 
 func (i *ImportPath) Position() lexer.Position { return i.Pos }
@@ -407,25 +464,139 @@ func (s *Selector) Capture(tokens []string) error {
 	return nil
 }
 
+type QuotedString string
+
+func (qs *QuotedString) Parse(lex *lexer.PeekingLexer) error {
+	token, err := lex.Peek(0)
+	if err != nil {
+		return err
+	}
+	if token.Type != Lexer.Symbols()["String"] {
+		return participle.NextMatch
+	}
+	token, err = lex.Next()
+	if err != nil {
+		return err
+	}
+	s, err := strconv.Unquote(token.Value)
+	*qs = QuotedString(s)
+	return err
+}
+
 // BasicLit represents a literal of basic type.
 type BasicLit struct {
 	Pos     lexer.Position
-	Str     *string     `parser:"( @String"`
-	Decimal *int        `parser:"| @Decimal"`
-	Numeric *NumericLit `parser:"| @Numeric"`
-	Bool    *bool       `parser:"| @Bool )"`
+	Str     *QuotedString `parser:"( @@"`
+	HereDoc *HereDoc      `parser:"| @@"`
+	Decimal *int          `parser:"| @Decimal"`
+	Numeric *NumericLit   `parser:"| @Numeric"`
+	Bool    *bool         `parser:"| @Bool )"`
 }
 
 func (l *BasicLit) Position() lexer.Position { return l.Pos }
 func (l *BasicLit) End() lexer.Position {
 	switch {
-	case l.Str != nil, l.Decimal != nil, l.Numeric != nil, l.Bool != nil:
+	case l.Str != nil, l.HereDoc != nil, l.Decimal != nil, l.Numeric != nil, l.Bool != nil:
 		return shiftPosition(l.Pos, len(l.String()), 0)
 	default:
 		return shiftPosition(l.Pos, 1, 0)
 	}
 }
 
+// HereDoc represents a multiline here-doc type.
+type HereDoc struct {
+	Pos      lexer.Position
+	Value    string
+	raw      string
+	operator string
+	ident    string
+}
+
+func (h *HereDoc) Position() lexer.Position { return h.Pos }
+func (h *HereDoc) End() lexer.Position {
+	return shiftPosition(h.Pos, len(h.raw), 0)
+}
+
+func (h *HereDoc) Parse(lex *lexer.PeekingLexer) error {
+	token, err := lex.Peek(0)
+	if err != nil {
+		return err
+	}
+
+	if token.Type != Lexer.Symbols()["HereDoc"] {
+		return participle.NextMatch
+	}
+
+	// Consume heredoc token.
+	heredoc, err := lex.Next()
+	if err != nil {
+		return err
+	}
+
+	h.operator = heredoc.Value
+
+	// Consume heredoc identifier.
+	ident, err := lex.Next()
+	if err != nil {
+		return err
+	}
+
+	if ident.Type != Lexer.Symbols()["Ident"] {
+		return participle.NextMatch
+	}
+
+	h.ident = ident.Value
+
+	var next lexer.Token
+
+	heredocTokens := []lexer.Token{}
+
+	for {
+		next, err = lex.Next()
+		if err != nil {
+			return err
+		}
+		if next.Type == Lexer.Symbols()["Ident"] && next.Value == h.ident {
+			break
+		}
+		heredocTokens = append(heredocTokens, next)
+	}
+
+	// trim leading whitespace to first newline
+	for i := range heredocTokens {
+		if heredocTokens[i].Type == Lexer.Symbols()["Newline"] {
+			heredocTokens = heredocTokens[i+1:]
+			break
+		}
+	}
+	// trim trailing whitespace from last line
+	for i := len(heredocTokens) - 1; i >= 0; i-- {
+		if heredocTokens[i].Type == Lexer.Symbols()["Newline"] {
+			heredocTokens = heredocTokens[:i]
+			break
+		}
+	}
+	for _, t := range heredocTokens {
+		h.raw += t.Value
+	}
+
+	switch h.operator {
+	case "<<-": // dedent
+		h.Value = dedent.Dedent(h.raw)
+	case "<<~": // fold
+		s := bufio.NewScanner(strings.NewReader(h.raw))
+		lines := []string{}
+		for s.Scan() {
+			lines = append(lines, strings.TrimSpace(s.Text()))
+		}
+		h.Value = strings.Join(lines, " ")
+	default:
+		h.Value = h.raw
+	}
+
+	return nil
+}
+
 // NumericLit represents a number literal with a non-decimal base.
 type NumericLit struct {
 	Pos   lexer.Position
@@ -460,7 +631,7 @@ func (l *NumericLit) Capture(tokens []string) error {
 // ObjType returns the type of the basic literal.
 func (l *BasicLit) ObjType() ObjType {
 	switch {
-	case l.Str != nil:
+	case l.Str != nil, l.HereDoc != nil:
 		return Str
 	case l.Decimal != nil, l.Numeric != nil:
 		return Int
@@ -471,9 +642,10 @@ func (l *BasicLit) ObjType() ObjType {
 }
 
 func NewStringExpr(v string) *Expr {
+	qs := QuotedString(v)
 	return &Expr{
 		BasicLit: &BasicLit{
-			Str: &v,
+			Str: &qs,
 		},
 	}
 }
diff --git a/parser/unparse.go b/parser/unparse.go
index 7eff881..7df5299 100644
--- a/parser/unparse.go
+++ b/parser/unparse.go
@@ -128,7 +128,7 @@ func (f *From) String() string {
 }
 
 func (ip *ImportPath) String() string {
-	return strconv.Quote(ip.Path)
+	return ip.Path.String()
 }
 
 func (d *ExportDecl) String() string {
@@ -201,10 +201,20 @@ func (i *Ident) String() string {
 	return i.Name
 }
 
+func (qs *QuotedString) String() string {
+	return strconv.Quote(string(*qs))
+}
+
+func (qs *QuotedString) Unquoted() string {
+	return string(*qs)
+}
+
 func (l *BasicLit) String() string {
 	switch {
 	case l.Str != nil:
-		return strconv.Quote(*l.Str)
+		return l.Str.String()
+	case l.HereDoc != nil:
+		return l.HereDoc.String()
 	case l.Decimal != nil:
 		return strconv.Itoa(*l.Decimal)
 	case l.Numeric != nil:
@@ -227,6 +237,13 @@ func (l *NumericLit) String() string {
 	panic("unknown numeric lit")
 }
 
+func (h *HereDoc) String() string {
+	// this is a hack, the unparsing will automatically add tabs for newline separated statements,
+	// but for heredocs we need to preserve the literal input.  So we manually add some markers
+	// to the response that is trimmed out when serializing the block statement.
+	return fmt.Sprintf("%s%s\n<HLB-HEREDOC-MARKER>\n%s\n<HLB-HEREDOC-MARKER>\n%s", h.operator, h.ident, h.raw, h.ident)
+}
+
 func (l *FuncLit) String() string {
 	return fmt.Sprintf("%s %s", l.Type, l.Body)
 }
@@ -348,8 +365,15 @@ func (s *BlockStmt) String() string {
 			}
 		}
 
+		insideHeredoc := false
 		for i := 1; i < len(stmts); i++ {
-			if len(stmts[i]) > 0 {
+			if stmts[i] == "<HLB-HEREDOC-MARKER>" {
+				stmts = append(stmts[:i], stmts[i+1:]...)
+				i--
+				insideHeredoc = !insideHeredoc
+				continue
+			}
+			if len(stmts[i]) > 0 && !insideHeredoc {
 				stmts[i] = fmt.Sprintf("\t%s", stmts[i])
 			}
 		}
diff --git a/solver/request.go b/solver/request.go
index a0492a3..93707ae 100644
--- a/solver/request.go
+++ b/solver/request.go
@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	// LopcalPathDescriptionKey is the key name in the metadata description map for the input path to a local fs.
+	// LocalPathDescriptionKey is the key name in the metadata description map for the input path to a local fs.
 	LocalPathDescriptionKey = "hlb.local.path"
 )
 


Test patch:
diff --git a/codegen/codegen_test.go b/codegen/codegen_test.go
index a21fa40..177df32 100644
--- a/codegen/codegen_test.go
+++ b/codegen/codegen_test.go
@@ -210,6 +210,36 @@ func TestCodeGen(t *testing.T) {
 				Expect(t, llb.Image("busybox:stable")),
 			)
 		},
+	}, {
+		"here doc processing",
+		[]string{"default"},
+		`
+		fs default() {
+			image "busybox"
+
+			run <<~EOM
+			echo
+			hi
+			EOM
+
+			run <<-EOM
+			echo hi
+			EOM
+
+			run <<EOM
+			echo hi
+			EOM
+		}
+		`,
+		func(t *testing.T, cg *CodeGen) solver.Request {
+			return Expect(t, llb.Image("busybox").Run(
+				llb.Shlex("echo hi"),
+			).Run(
+				llb.Shlex("echo hi"),
+			).Run(
+				llb.Shlex("\techo hi"),
+			).Root())
+		},
 	}} {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
diff --git a/parse_test.go b/parse_test.go
index 9bc59a0..7f5346e 100644
--- a/parse_test.go
+++ b/parse_test.go
@@ -85,6 +85,24 @@ var (
 				}
 			} "src" "dst"
 		}
+
+		string heredocTest() {
+			value <<-EOM
+			this
+			  should
+			dedent
+			EOM
+			value <<~EOM
+			this 
+			  should
+			fold
+			EOM
+			value <<EOM
+			this
+			  is
+			literal
+			EOM
+		}
 	`
 )
 
diff --git a/unparse_test.go b/unparse_test.go
index cdf4643..8cc72a3 100644
--- a/unparse_test.go
+++ b/unparse_test.go
@@ -476,6 +476,46 @@ func TestUnparse(t *testing.T) {
 
 			fs bar() { scratch; }
 			`,
+		}, {
+			`heredoc`,
+			`
+			string heredocTest() {
+				value <<-EOM
+				this
+				  should
+				dedent
+				EOM
+				value <<~EOM
+				this 
+				  should
+				fold
+				EOM
+				value <<EOM
+				this
+				  is
+				literal
+				EOM
+			}
+			`,
+			`
+			string heredocTest() {
+				value <<-EOM
+				this
+				  should
+				dedent
+				EOM
+				value <<~EOM
+				this 
+				  should
+				fold
+				EOM
+				value <<EOM
+				this
+				  is
+				literal
+				EOM
+			}
+			`,
 		},
 	} {
 		tc := tc