Repository: go-go-golems/glazed
Pull request #372 (2023-10-07 03:44:38)

Added 'from-markdown' flag to load YAMLs from markdown code snippets

This pull request introduces a new flag, 'from-markdown', to the YamlCommand in the 'yaml.go' file. This flag allows the loading of YAMLs directly from code snippets in markdown files. 

Key changes include:

- Addition of the 'from-markdown' flag to the YamlCommand parameters in 'yaml.go'.
- Modification of the 'Run' function in 'yaml.go' to handle the 'from-markdown' flag.
- Modification of the 'Clean' function in 'yaml.go' and 'yaml_test.go' to handle the 'from-markdown' flag.
- Addition of a regular expression to remove trailing whitespaces in the 'Clean' function in 'yaml.go'.
- Modification of the 'getIndentLevel' function in 'yaml.go' to return a boolean indicating if a line is whitespace.




Patch:
diff --git a/cmd/glaze/cmds/yaml.go b/cmd/glaze/cmds/yaml.go
index 53a970b..84497b1 100644
--- a/cmd/glaze/cmds/yaml.go
+++ b/cmd/glaze/cmds/yaml.go
@@ -45,6 +45,12 @@ func NewYamlCommand() (*YamlCommand, error) {
 					parameters.WithHelp("Sanitize input (very hacky, meant for LLM cleanup)"),
 					parameters.WithDefault(false),
 				),
+				parameters.NewParameterDefinition(
+					"from-markdown",
+					parameters.ParameterTypeBool,
+					parameters.WithHelp("Input is markdown"),
+					parameters.WithDefault(false),
+				),
 			),
 			cmds.WithArguments(
 				parameters.NewParameterDefinition(
@@ -81,6 +87,11 @@ func (y *YamlCommand) Run(
 		return fmt.Errorf("input-files is not a string list")
 	}
 
+	fromMarkdown, ok := ps["from-markdown"].(bool)
+	if !ok {
+		return fmt.Errorf("from-markdown flag is not a bool")
+	}
+
 	for _, arg := range inputFiles {
 		if arg == "-" {
 			arg = "/dev/stdin"
@@ -88,12 +99,12 @@ func (y *YamlCommand) Run(
 		var f io.Reader
 		var err error
 
-		if sanitize {
+		if sanitize || fromMarkdown {
 			// read in file
 			data, err := os.ReadFile(arg)
 			cobra.CheckErr(err)
 
-			cleanData := yaml2.Clean(string(data))
+			cleanData := yaml2.Clean(string(data), fromMarkdown)
 			f = strings.NewReader(cleanData)
 		} else {
 			f, err = os.Open(arg)
diff --git a/pkg/helpers/yaml/yaml.go b/pkg/helpers/yaml/yaml.go
index 68db284..44b93b0 100644
--- a/pkg/helpers/yaml/yaml.go
+++ b/pkg/helpers/yaml/yaml.go
@@ -1,13 +1,14 @@
 package yaml
 
 import (
+	"regexp"
 	"strings"
 )
 
 // Clean tries to cleanup YAML that might be invalid, for example
 // if coming out of a LLM.
 // This is quite a hacky solution, and will only work for simple YAML.
-func Clean(s string) string {
+func Clean(s string, fromMarkdown bool) string {
 	// split by lines and iterate over them
 	lines := strings.Split(s, "\n")
 	ret := []string{}
@@ -16,6 +17,8 @@ func Clean(s string) string {
 	isInQuotedString := false
 	quoteIndent := 0
 
+	re := regexp.MustCompile(`\s+$`)
+
 	for _, line := range lines {
 		// check if the first string is a ``` markdown marker
 		if strings.HasPrefix(line, "```") {
@@ -24,17 +27,23 @@ func Clean(s string) string {
 			continue
 		}
 
+		if fromMarkdown && !isInMarkdown {
+			continue
+		}
+
 		if isInQuotedString {
 			// if quoteIndent is 0, we just started a multiline string
 			if quoteIndent == 0 {
-				quoteIndent = getIndentLevel(line)
+				quoteIndent, _ = getIndentLevel(line)
 			}
 
 			// count start indent
-			indent := getIndentLevel(line)
+			indent, isWhitespace := getIndentLevel(line)
 
 			// check if we are still in the quoted string
-			if indent >= quoteIndent {
+			if indent >= quoteIndent || isWhitespace {
+				// chop off rightmost whitespace because otherwise yaml encoder won't output it as multiline string
+				line = re.ReplaceAllString(line, "")
 				ret = append(ret, line)
 				continue
 			} else {
@@ -95,14 +104,16 @@ func Clean(s string) string {
 	return ret_
 }
 
-func getIndentLevel(line string) int {
+func getIndentLevel(line string) (int, bool) {
 	indent := 0
+	isWhitespace := true
 	for _, c := range line {
 		if c == ' ' {
 			indent++
 		} else {
+			isWhitespace = false
 			break
 		}
 	}
-	return indent
+	return indent, isWhitespace
 }


Test patch:
diff --git a/pkg/helpers/yaml/yaml_test.go b/pkg/helpers/yaml/yaml_test.go
index 00250ae..b1b3e33 100644
--- a/pkg/helpers/yaml/yaml_test.go
+++ b/pkg/helpers/yaml/yaml_test.go
@@ -17,7 +17,7 @@ b: 2
 c: 3
 `
 
-	actualYAML := Clean(inputYAML)
+	actualYAML := Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 }
 
@@ -32,7 +32,7 @@ a: 1
 b: 2
 c: 3`
 
-	actualYAML := Clean(inputYAML)
+	actualYAML := Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 
 	inputYAML = "```\n" + `---
@@ -40,7 +40,7 @@ a: 1
 b: 2
 c: 3` + "\n```"
 
-	actualYAML = Clean(inputYAML)
+	actualYAML = Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 }
 
@@ -57,7 +57,7 @@ b: "foobar: blabla"
 invalidRightHandSide: "&foobar"
 `
 
-	actualYAML := Clean(inputYAML)
+	actualYAML := Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 }
 
@@ -72,7 +72,7 @@ a: 1
 b: "foobar: blabla"
 `
 
-	actualYAML := Clean(inputYAML)
+	actualYAML := Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 
 	inputYAML = `---
@@ -85,7 +85,7 @@ a: 1
 b: 'foobar: blabla'
 `
 
-	actualYAML = Clean(inputYAML)
+	actualYAML = Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 
 	inputYAML = `---
@@ -100,7 +100,7 @@ b: 'foobar: blabla'
 	foobar: blabla
 `
 
-	actualYAML = Clean(inputYAML)
+	actualYAML = Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 }
 
@@ -133,7 +133,7 @@ b:
   g: 2
 `
 
-	actualYAML := Clean(inputYAML)
+	actualYAML := Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 
 }
@@ -156,7 +156,7 @@ c: |
 d: "&lkjsld"
 `
 
-	actualYAML := Clean(inputYAML)
+	actualYAML := Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 }
 
@@ -173,6 +173,6 @@ meta_title: "Buy Evergreen Trees Online - Create Year-Round Privacy and Beauty |
 meta_description: "Shop a variety of evergreen trees that stay green all year at The Tree Center. Create year-round privacy and beauty with our selection of evergreen trees. Find the perfect evergreen for your garden today!"
 `
 
-	actualYAML := Clean(inputYAML)
+	actualYAML := Clean(inputYAML, false)
 	assert.Equal(t, expectedYAML, actualYAML)
 }