Repository: go-go-golems/glazed
Pull request #373 (2023-10-07 18:29:12)

:sparkles: Add markdown code utilities
This adds support for extracting markdown code blocks, code comments, and code
style from markdown content. A new `markdown` package is introduced that can
parse markdown text into a structured representation of code and non-code
blocks. Code blocks are extracted into `MarkdownBlock` structs containing the
language and content. Additional utilities are included for generating comments
in various languages and inline text blocks as comments within code samples. 

- Create new `markdown` package  
- Add `ExtractAllBlocks` to parse markdown into normal and code blocks
- Store code blocks in `MarkdownBlock` struct with type, language and content
- Add `ExtractQuotedBlocks` to extract just code blocks
- Update `ExtractJSON` to use new markdown extractor 
- Add `ExtractCodeBlocksWithComments` to extract code blocks and inline preceding text as comments
- Support commenting text blocks in Go, Python, JavaScript, etc.
- Add `GenerateComment` in `strings` package to generate comments in different languages
- Add tests for markdown block and comment extraction
- Refactor state machine logic into separate functions




Patch:
diff --git a/cmd/examples/markdown-code-blocks/main.go b/cmd/examples/markdown-code-blocks/main.go
new file mode 100644
index 0000000..880bb2c
--- /dev/null
+++ b/cmd/examples/markdown-code-blocks/main.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+	_ "embed"
+	"fmt"
+	"github.com/go-go-golems/glazed/pkg/helpers/markdown"
+	"github.com/go-go-golems/glazed/pkg/helpers/strings"
+)
+
+//go:embed "test.md"
+var md string
+
+func main() {
+	// Extract all blocks
+	fmt.Println("Extract all blocks")
+	fmt.Println("==================")
+	blocks := markdown.ExtractAllBlocks(md)
+	for _, block := range blocks {
+		fmt.Println("---")
+		fmt.Println(block.Type, block.Language, block.Content)
+	}
+	fmt.Println()
+
+	// Extract quoted blocks
+	fmt.Println("Extract quoted blocks")
+	fmt.Println("====================")
+	quotedBlocks := markdown.ExtractQuotedBlocks(md, true)
+	for _, block := range quotedBlocks {
+		fmt.Println("---")
+		fmt.Println(block)
+	}
+	fmt.Println()
+
+	// Extract code blocks with comments
+	fmt.Println("Extract code blocks with comments")
+	fmt.Println("================================")
+	codeBlocksWithComments := markdown.ExtractCodeBlocksWithComments(md, true)
+	for _, block := range codeBlocksWithComments {
+		fmt.Println("---")
+		fmt.Println(block)
+	}
+	fmt.Println()
+
+	// Generate comment
+	fmt.Println("Generate comment")
+	fmt.Println("================")
+	comment := strings.GenerateComment("This is a comment.", strings.GoLang)
+	fmt.Println(comment)
+	fmt.Println()
+
+	// Markdown code block to language
+	fmt.Println("Markdown code block to language")
+	fmt.Println("==============================")
+	lang := strings.MarkdownCodeBlockToLanguage("go")
+	fmt.Println(lang)
+	fmt.Println()
+}
diff --git a/go.mod b/go.mod
index 4d6e115..16d3694 100644
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/kopoli/go-terminal-size v0.0.0-20170219200355-5c97524c8b54
 	github.com/mattn/go-isatty v0.0.17
+	github.com/muesli/reflow v0.3.0
 	github.com/nishanths/exhaustive v0.9.5
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.29.0
@@ -63,7 +64,6 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
-	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.13.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
diff --git a/pkg/doc/topics/11-markdown-code-blocks.md b/pkg/doc/topics/11-markdown-code-blocks.md
new file mode 100644
index 0000000..6f284af
--- /dev/null
+++ b/pkg/doc/topics/11-markdown-code-blocks.md
@@ -0,0 +1,157 @@
+---
+Title: Markdown and Code Blocks Parsing
+Slug: markdown-code-blocks
+Short: |
+  This document explains the API and usage of the Markdown and Code parsing features in the application.
+Topics:
+- Code Parsing
+- Markdown
+IsTopLevel: false
+ShowPerDefault: false
+SectionType: GeneralTopic
+---
+
+# Markdown and Code Parsing
+
+The Glazed package offers a comprehensive suite of functions designed to extract code blocks from markdown files. This
+functionality is particularly useful for post-processing code generated by Low-Level Virtual Machines (LLVMs). By
+parsing markdown files and isolating code blocks, Glazed allows for more granular control over code execution, testing,
+and debugging. This feature enhances the efficiency of working with LLVM-generated code, streamlining the process and
+making it more manageable.
+
+This document explains the API and usage of the Markdown and Code parsing features in the application.
+
+## GenerateComment
+
+The `GenerateComment` function converts a given string into a comment based on the specified programming language.
+
+**Usage**:
+
+Call this function with a string and a language to get the string formatted as a comment in that language.
+
+**Example**:
+
+```go
+comment := GenerateComment("This is a comment.", GoLang)
+```
+
+**Output**:
+
+```go
+// This is a comment.
+```
+
+## MarkdownCodeBlockToLanguage
+
+The `MarkdownCodeBlockToLanguage` function translates a markdown code block language specifier to its corresponding Language type.
+
+**Usage**:
+
+Use this function to identify the language of a code block in markdown.
+
+**Example**:
+
+```go
+lang := MarkdownCodeBlockToLanguage("go")
+```
+
+**Output**:
+
+```go
+Go
+```
+
+## ExtractAllBlocks
+
+The `ExtractAllBlocks` function processes a given markdown string to split it into a series of blocks.
+
+**Usage**:
+
+Call this function with a markdown string as the argument. It will return a slice of MarkdownBlock structs, each representing a block of content (either normal or code).
+
+**Example**:
+
+Using the markdown string:
+
+```
+# Title
+
+This is a markdown document.
+
+```go
+fmt.Println("Hello, world!")
+.```
+
+This is a comment.
+
+.```python
+print("Hello, world!")
+.```
+```
+
+**Output**:
+
+```
+Normal  # Title
+
+This is a markdown document.
+
+Code go fmt.Println("Hello, world!")
+Normal  
+This is a comment.
+
+Code python print("Hello, world!")
+```
+
+## ExtractQuotedBlocks
+
+The `ExtractQuotedBlocks` function extracts only the code blocks from a markdown string.
+
+**Usage**:
+
+This function takes in a markdown string and a boolean `withQuotes`. If `withQuotes` is set to true, the output will include the enclosing ``` for each code block. Otherwise, only the inner content of the code block is returned.
+
+**Example**:
+
+Using the same markdown string...
+
+**Output**:
+
+```
+.```go
+fmt.Println("Hello, world!")
+.```
+.```python
+print("Hello, world!")
+.```
+```
+
+## ExtractCodeBlocksWithComments
+
+The `ExtractCodeBlocksWithComments` function processes a markdown string to return only code blocks. Any non-code content preceding a code block is added as a comment within the respective code block.
+
+**Usage**:
+
+Call this function with a markdown string and a boolean `withQuotes`. If `withQuotes` is set to true, the output will include the enclosing ``` for each code block. Otherwise, only the inner content of the code block (and the preceding comments) is returned.
+
+**Example**:
+
+Using the markdown same markdown string...
+
+**Output**:
+
+```
+.```go
+// # Title
+// 
+// This is a markdown document.
+// 
+fmt.Println("Hello, world!")
+.```
+.```python
+# 
+# This is a comment.
+# 
+print("Hello, world!")
+.```
+```
\ No newline at end of file
diff --git a/pkg/helpers/json/extract.go b/pkg/helpers/json/extract.go
index 65ece33..3bc8779 100644
--- a/pkg/helpers/json/extract.go
+++ b/pkg/helpers/json/extract.go
@@ -1,10 +1,9 @@
 package json
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/json"
-	"strings"
+	"github.com/go-go-golems/glazed/pkg/helpers/markdown"
 )
 
 // ExtractJSON extracts potential JSON blocks from the provided input string.
@@ -24,7 +23,7 @@ func ExtractJSON(input string) []string {
 	}
 
 	// If that fails, scan for quoted blocks
-	quotedBlocks := ExtractQuotedBlocks(input)
+	quotedBlocks := markdown.ExtractQuotedBlocks(input, false)
 	for _, block := range quotedBlocks {
 		var temp map[string]interface{}
 		block = SanitizeJSONString(block)
@@ -36,42 +35,6 @@ func ExtractJSON(input string) []string {
 	return result
 }
 
-type State int
-
-const (
-	OutsideBlock State = iota
-	InsideBlock
-)
-
-// ExtractQuotedBlocks extracts blocks enclosed by ``` using a state machine.
-func ExtractQuotedBlocks(input string) []string {
-	var result []string
-	state := OutsideBlock
-	var blockLines []string
-
-	scanner := bufio.NewScanner(strings.NewReader(input))
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		switch state {
-		case OutsideBlock:
-			if strings.HasPrefix(line, "```") {
-				state = InsideBlock
-				blockLines = nil // reset blockLines
-			}
-		case InsideBlock:
-			if strings.HasPrefix(line, "```") {
-				state = OutsideBlock
-				result = append(result, strings.Join(blockLines, "\n"))
-			} else {
-				blockLines = append(blockLines, line)
-			}
-		}
-	}
-
-	return result
-}
-
 func SanitizeJSONString(input string) string {
 	var result bytes.Buffer
 
diff --git a/pkg/helpers/markdown/markdown.go b/pkg/helpers/markdown/markdown.go
new file mode 100644
index 0000000..bc92cb6
--- /dev/null
+++ b/pkg/helpers/markdown/markdown.go
@@ -0,0 +1,172 @@
+package markdown
+
+import (
+	"bufio"
+	strings2 "github.com/go-go-golems/glazed/pkg/helpers/strings"
+	"strings"
+)
+
+type State int
+
+const (
+	OutsideBlock State = iota
+	InsideBlock
+)
+
+type BlockType string
+
+const (
+	Normal BlockType = "Normal"
+	Code   BlockType = "Code"
+)
+
+type MarkdownBlock struct {
+	Type     BlockType
+	Language string
+	Content  string
+}
+
+// ExtractAllBlocks processes a given markdown string to split it into a series of blocks.
+//
+// **Usage**:
+// Call this function with a markdown string as the argument. It will return a slice
+// of MarkdownBlock structs, each representing a block of content (either normal or code).
+//
+// **Inner Workings**:
+// The function employs a state machine approach to parse through the markdown content.
+// It recognizes and separates out code blocks (enclosed with ```) and normal text blocks.
+// For code blocks, it also identifies the optional language specifier.
+func ExtractAllBlocks(input string) []MarkdownBlock {
+	var result []MarkdownBlock
+	state := OutsideBlock
+	var blockLines []string
+
+	scanner := bufio.NewScanner(strings.NewReader(input))
+	language := ""
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		switch state {
+		case OutsideBlock:
+			if strings.HasPrefix(line, "```") {
+				state = InsideBlock
+				language = strings.TrimPrefix(line, "```")
+				if len(blockLines) > 0 {
+					// For normal blocks
+					result = append(result, MarkdownBlock{Type: Normal, Content: strings.Join(blockLines, "\n")})
+				}
+				blockLines = nil // reset blockLines
+				continue
+			}
+			blockLines = append(blockLines, line)
+		case InsideBlock:
+			if strings.HasPrefix(line, "```") {
+				state = OutsideBlock
+				content := strings.Join(blockLines, "\n") // excluding the language line
+				result = append(result, MarkdownBlock{Type: Code, Language: language, Content: content})
+				blockLines = nil // reset blockLines
+				language = ""
+			} else {
+				blockLines = append(blockLines, line)
+			}
+		}
+	}
+
+	// Handle any remaining normal content outside of code blocks
+	if len(blockLines) > 0 && state == OutsideBlock {
+		result = append(result, MarkdownBlock{Type: Normal, Content: strings.Join(blockLines, "\n")})
+	}
+
+	return result
+}
+
+// ExtractQuotedBlocks extracts only the code blocks from a markdown string.
+//
+// **Usage**:
+// This function takes in a markdown string and a boolean `withQuotes`. If `withQuotes`
+// is set to true, the output will include the enclosing ``` for each code block.
+// Otherwise, only the inner content of the code block is returned.
+//
+// **Inner Workings**:
+// This function leverages the `ExtractAllBlocks` function to first get all blocks
+// from the markdown content. It then filters out only the code blocks and processes
+// them based on the `withQuotes` parameter to decide on the inclusion of the enclosing ``` marks.
+func ExtractQuotedBlocks(input string, withQuotes bool) []string {
+	blocks := ExtractAllBlocks(input)
+	var result []string
+	for _, block := range blocks {
+		if block.Type != Code {
+			continue
+		}
+		if withQuotes {
+			result = append(result, "```"+block.Language+"\n"+block.Content+"\n```")
+		} else {
+			result = append(result, block.Content)
+		}
+	}
+
+	return result
+}
+
+// ExtractCodeBlocksWithComments processes a markdown string to return only code blocks.
+// Any non-code content preceding a code block is added as a comment within the respective code block.
+// If there are non-code contents left at the end without a following code block, they are appended
+// as comments to the last code block.
+//
+// **Usage**:
+// Call this function with a markdown string and a boolean `withQuotes`. If `withQuotes` is set to true,
+// the output will include the enclosing ``` for each code block. Otherwise, only the inner content of
+// the code block (and the preceding comments) is returned.
+func ExtractCodeBlocksWithComments(input string, withQuotes bool) []string {
+	blocks := ExtractAllBlocks(input)
+	var result []string
+	var pendingComments []string
+	lastLanguage := strings2.None
+	lastBlockLanguage := ""
+	codeContent := ""
+
+	for _, block := range blocks {
+		if block.Type == Code {
+			// If there are pending comments (non-code blocks), convert them to comments in the current language
+			codeContent = block.Content
+			language := strings2.MarkdownCodeBlockToLanguage(block.Language)
+			lastLanguage = language
+			lastBlockLanguage = block.Language
+
+			if len(pendingComments) > 0 {
+
+				// only append comments for languages we know how to generate comments for
+				if language != strings2.None {
+					commentedText := strings2.GenerateComment(strings.Join(pendingComments, "\n"), language)
+					codeContent = commentedText + "\n" + codeContent
+				}
+				pendingComments = nil
+			}
+
+			if withQuotes {
+				result = append(result, "```"+block.Language+"\n"+codeContent+"\n```")
+			} else {
+				result = append(result, codeContent)
+			}
+		} else {
+			// Accumulate non-code blocks to be commented later
+			pendingComments = append(pendingComments, block.Content)
+		}
+	}
+
+	// If there's any pending comment without a following code block, append to the previous block
+	if len(pendingComments) > 0 {
+		if lastLanguage != strings2.None && len(result) > 0 {
+			codeContent += "\n" + strings2.GenerateComment(strings.Join(pendingComments, "\n"), lastLanguage)
+			res := ""
+			if withQuotes {
+				res = "```" + lastBlockLanguage + "\n" + codeContent + "\n```"
+			} else {
+				res = codeContent
+			}
+			result[len(result)-1] = res
+		}
+	}
+
+	return result
+}
diff --git a/pkg/helpers/strings/code.go b/pkg/helpers/strings/code.go
new file mode 100644
index 0000000..2f5deae
--- /dev/null
+++ b/pkg/helpers/strings/code.go
@@ -0,0 +1,222 @@
+package strings
+
+import (
+	"github.com/muesli/reflow/wordwrap"
+	"strings"
+)
+
+// CommentType represents the type of comment (block or line)
+type CommentType int
+
+const (
+	Block CommentType = iota
+	Line
+)
+
+// Language represents the supported languages
+type Language string
+
+const (
+	GoLang       Language = "Go"
+	C            Language = "C"
+	CPP          Language = "C++"
+	Java         Language = "Java"
+	Python       Language = "Python"
+	JavaScript   Language = "JavaScript"
+	Ruby         Language = "Ruby"
+	Perl         Language = "Perl"
+	Shell        Language = "Shell"
+	PHP          Language = "PHP"
+	Swift        Language = "Swift"
+	Rust         Language = "Rust"
+	Haskell      Language = "Haskell"
+	HTML         Language = "HTML"
+	XML          Language = "XML"
+	CSS          Language = "CSS"
+	SQL          Language = "SQL"
+	R            Language = "Language"
+	Scala        Language = "Scala"
+	Kotlin       Language = "Kotlin"
+	TypeScript   Language = "TypeScript"
+	Matlab       Language = "Matlab"
+	Assembly     Language = "Assembly"
+	Lua          Language = "Lua"
+	Fortran      Language = "Fortran"
+	Bash         Language = "Bash"
+	Groovy       Language = "Groovy"
+	Prolog       Language = "Prolog"
+	YAML         Language = "YAML"
+	Markdown     Language = "Markdown"
+	Dart         Language = "Dart"
+	CoffeeScript Language = "CoffeeScript"
+	FSharp       Language = "FSharp"
+	Pascal       Language = "Pascal"
+	Lisp         Language = "Lisp"
+	Erlang       Language = "Erlang"
+	Elixir       Language = "Elixir"
+	HCL          Language = "HCL"
+)
+
+// CommentDelimiters represents the delimiters for each language
+type CommentDelimiters struct {
+	Type   CommentType
+	Start  string
+	End    string
+	Inline string
+}
+
+// CommentDict holds the delimiters for each supported language
+var CommentDict = map[Language]CommentDelimiters{
+	GoLang:     {Line, "", "", "//"},
+	C:          {Block, "/*", "*/", "//"},
+	CPP:        {Block, "/*", "*/", "//"},
+	Java:       {Block, "/*", "*/", "//"},
+	Python:     {Line, "", "", "#"},
+	JavaScript: {Block, "/*", "*/", "//"},
+	Ruby:       {Line, "", "", "#"},
+
+	Perl:         {Line, "", "", "#"},
+	Shell:        {Line, "", "", "#"},
+	PHP:          {Block, "/*", "*/", "//"},
+	Swift:        {Block, "/*", "*/", "//"},
+	Rust:         {Block, "/*", "*/", "//"},
+	Haskell:      {Line, "", "", "--"},
+	HTML:         {Block, "<!--", "-->", ""},
+	XML:          {Block, "<!--", "-->", ""},
+	CSS:          {Block, "/*", "*/", ""},
+	SQL:          {Line, "", "", "--"},
+	R:            {Line, "", "", "#"},
+	Scala:        {Block, "/*", "*/", "//"},
+	Kotlin:       {Block, "/*", "*/", "//"},
+	TypeScript:   {Block, "/*", "*/", "//"},
+	Matlab:       {Block, "%{", "%}", "%"},
+	Assembly:     {Line, "", "", ";"},
+	Lua:          {Block, "--[[", "--]]", "--"},
+	Fortran:      {Line, "", "", "!"},
+	Bash:         {Line, "", "", "#"},
+	Groovy:       {Block, "/*", "*/", "//"},
+	Prolog:       {Line, "", "", "%"},
+	YAML:         {Line, "", "", "#"},
+	Markdown:     {Line, "", "", "//"},
+	Dart:         {Block, "/*", "*/", "//"},
+	CoffeeScript: {Line, "", "", "#"},
+	FSharp:       {Line, "", "", "//"},
+	Pascal:       {Block, "{", "}", ""},
+	Lisp:         {Line, "", "", ";;"},
+	Erlang:       {Block, "%{", "%}", "%"},
+	Elixir:       {Line, "", "", "#"},
+	HCL:          {Line, "", "", "#"},
+}
+
+// GenerateComment converts a given string into a comment based on the specified programming language.
+//
+// **Usage**:
+// Call this function with a string and a language to get the string formatted
+// as a comment in that language. For example: `comment := GenerateComment("TODO: Implement this", GoLang)`.
+//
+// **Inner Workings**:
+// The function uses the CommentDict map to determine the appropriate comment delimiters
+// for the provided language. It then formats the string as a block or inline comment
+// based on the language's conventions.
+func GenerateComment(comment string, language Language) string {
+	delimiters, exists := CommentDict[language]
+	if !exists {
+		return "Unsupported language"
+	}
+
+	if delimiters.Type == Line {
+		lines := strings.Split(comment, "\n")
+		lines_ := []string{}
+		for _, line := range lines {
+			// wordwrap the line
+			ww := wordwrap.NewWriter(100)
+			_, err := ww.Write([]byte(line))
+			// if there is an error, we just stick with line
+			if err != nil {
+				lines_ = append(lines_, delimiters.Inline+" "+line)
+				continue
+			}
+
+			_ = ww.Close()
+			line = ww.String()
+			for _, line := range strings.Split(line, "\n") {
+				lines_ = append(lines_, delimiters.Inline+" "+line)
+			}
+		}
+		return strings.Join(lines_, "\n")
+	} else {
+		return delimiters.Start + "\n" + comment + "\n" + delimiters.End
+	}
+}
+
+const (
+	None Language = "None"
+)
+
+var markdownToLanguageMap = map[string]Language{
+	"go":           GoLang,
+	"c":            C,
+	"cpp":          CPP,
+	"c++":          CPP,
+	"java":         Java,
+	"python":       Python,
+	"py":           Python,
+	"js":           JavaScript,
+	"javascript":   JavaScript,
+	"ruby":         Ruby,
+	"rb":           Ruby,
+	"perl":         Perl,
+	"shell":        Shell,
+	"sh":           Shell,
+	"bash":         Bash,
+	"php":          PHP,
+	"swift":        Swift,
+	"rust":         Rust,
+	"haskell":      Haskell,
+	"html":         HTML,
+	"xml":          XML,
+	"css":          CSS,
+	"sql":          SQL,
+	"r":            R,
+	"scala":        Scala,
+	"kotlin":       Kotlin,
+	"kt":           Kotlin,
+	"typescript":   TypeScript,
+	"ts":           TypeScript,
+	"matlab":       Matlab,
+	"assembly":     Assembly,
+	"asm":          Assembly,
+	"lua":          Lua,
+	"fortran":      Fortran,
+	"f":            Fortran,
+	"groovy":       Groovy,
+	"prolog":       Prolog,
+	"yaml":         YAML,
+	"yml":          YAML,
+	"md":           Markdown,
+	"dart":         Dart,
+	"coffeescript": CoffeeScript,
+	"coffee":       CoffeeScript,
+	"fsharp":       FSharp,
+	"fs":           FSharp,
+	"pascal":       Pascal,
+	"lisp":         Lisp,
+	"erlang":       Erlang,
+	"elixir":       Elixir,
+	"ex":           Elixir,
+	"hcl":          HCL,
+}
+
+// MarkdownCodeBlockToLanguage translates a markdown code block language specifier to its corresponding Language type.
+//
+// **Usage**:
+// Use this function to identify the language of a code block in markdown.
+// For example, given the string "py", the function will return the Python constant.
+func MarkdownCodeBlockToLanguage(codeBlock string) Language {
+	codeBlock = strings.ToLower(codeBlock)
+	language, exists := markdownToLanguageMap[codeBlock]
+	if !exists {
+		return None
+	}
+	return language
+}


Test patch:
diff --git a/cmd/examples/markdown-code-blocks/test.md b/cmd/examples/markdown-code-blocks/test.md
new file mode 100644
index 0000000..23b579e
--- /dev/null
+++ b/cmd/examples/markdown-code-blocks/test.md
@@ -0,0 +1,13 @@
+# Title
+
+This is a markdown document.
+
+```go
+fmt.Println("Hello, world!")
+```
+
+This is a comment.
+
+```python
+print("Hello, world!")
+```
diff --git a/misc/test-data/unsanitized-yaml.md b/misc/test-data/unsanitized-yaml.md
new file mode 100644
index 0000000..df2607d
--- /dev/null
+++ b/misc/test-data/unsanitized-yaml.md
@@ -0,0 +1,32 @@
+```yaml
+- name: batched_order_item_cogs
+  description: |
+    The `batched_order_item_cogs` DBT model offers a comprehensive view of the 
+    cost of goods sold (COGS) for batched order items. It combines data from 
+    `batched_order_item_truck_items` and `orders` models, providing a snapshot 
+    of each order with associated batched order item details. Key metrics 
+    include SKU, cost, quantity, truck ID, ground inventory level, and putaway 
+    quantity, which help track inventory flow and its impact on COGS. 
+
+    Detailed Insights:
+    - Detailed view of each order including order number, date, and time.
+    - Key metrics such as SKU, cost, quantity, truck ID, ground inventory 
+      level, and putaway quantity.
+    - Data combined from `batched_order_item_truck_items` and `orders` models 
+      using SQL join operation.
+
+    Important Caveats:
+    - Assumes ground inventory level is greater than the floor quantity and 
+      less than or equal to the ceiling quantity.
+    - Does not account for changes in the status of orders or batches post 
+      model creation. For real-time analysis, frequent model refresh is 
+      recommended.
+    - Does not include data on add-on items or deleted batches.
+```
+
+Omitted Information:
+1. **Redundant**: Mention of the model being a strategic tool for cost analysis, and its optimization for performance. These are inherent qualities of a DBT model.
+2. **Redundant**: Detailed explanation of each column. This information is typically available in the schema or data dictionary.
+3. **Not Important**: The model's dependency on the accuracy and reliability of the `batched_order_item_truck_items` and `orders` models. This is a general caveat applicable to all models.
+4. **Redundant**: Mention of the model being crucial for understanding the cost implications of inventory flow and order management. This is already covered in the overview.
+5. **Redundant**: Mention of the model ensuring that the cost analysis is aligned with the order details. This is a given for any model that combines data from multiple sources.
diff --git a/pkg/helpers/markdown/markdown_test.go b/pkg/helpers/markdown/markdown_test.go
new file mode 100644
index 0000000..ad0ac60
--- /dev/null
+++ b/pkg/helpers/markdown/markdown_test.go
@@ -0,0 +1,219 @@
+package markdown
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestExtractQuotedBlocks(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected []MarkdownBlock
+	}{
+		{
+			name:  "Basic Code Block Extraction",
+			input: "```go\nfmt.Println(\"Hello, World!\")\n```",
+			expected: []MarkdownBlock{
+				{Type: Code, Language: "go", Content: "fmt.Println(\"Hello, World!\")"},
+			},
+		},
+		{
+			name:  "Multiple Code Block Extractions",
+			input: "```go\nfmt.Println(\"Hello, Go!\")\n```\n```python\nprint(\"Hello, Python!\")\n```",
+			expected: []MarkdownBlock{
+				{Type: Code, Language: "go", Content: "fmt.Println(\"Hello, Go!\")"},
+				{Type: Code, Language: "python", Content: "print(\"Hello, Python!\")"},
+			},
+		},
+		{
+			name:  "Language Detection",
+			input: "```rust\nprintln!(\"Hello, Rust!\");\n```",
+			expected: []MarkdownBlock{
+				{Type: Code, Language: "rust", Content: "println!(\"Hello, Rust!\");"},
+			},
+		},
+		{
+			name:  "Mixed Content Extraction",
+			input: "This is a normal text\n```js\nconsole.log(\"Hello, JavaScript!\");\n```\nThis is another normal text",
+			expected: []MarkdownBlock{
+				{Type: Normal, Content: "This is a normal text"},
+				{Type: Code, Language: "js", Content: "console.log(\"Hello, JavaScript!\");"},
+				{Type: Normal, Content: "This is another normal text"},
+			},
+		},
+
+		{
+			name:  "No Code Block",
+			input: "This is just a normal text without any code block.",
+			expected: []MarkdownBlock{
+				{Type: Normal, Content: "This is just a normal text without any code block."},
+			},
+		},
+		{
+			name:     "Empty Input",
+			input:    "",
+			expected: []MarkdownBlock{},
+		},
+
+		{
+			name:  "Incomplete Code Block",
+			input: "```python\nprint(\"Hello, World!\"",
+			expected: []MarkdownBlock{
+				{Type: Normal, Content: "```python"},
+				{Type: Normal, Content: "print(\"Hello, World!\""},
+			},
+		},
+		{
+			name:  "Code Block with No Language",
+			input: "```\nconsole.log('No language specified');\n```",
+			expected: []MarkdownBlock{
+				{Type: Code, Language: "", Content: "console.log('No language specified');"},
+			},
+		},
+		{
+			name:  "Normal Text Before and After Code Blocks",
+			input: "This is normal text.\n```go\nfmt.Println(\"Hello\")\n```\nThis is also normal text.",
+			expected: []MarkdownBlock{
+				{Type: Normal, Content: "This is normal text."},
+				{Type: Code, Language: "go", Content: "fmt.Println(\"Hello\")"},
+				{Type: Normal, Content: "This is also normal text."},
+			},
+		},
+		{
+			name:  "Multiple Lines Within Blocks",
+			input: "This is a normal block\nwith multiple lines.\n```python\nprint(\"Line 1\")\nprint(\"Line 2\")\n```",
+			expected: []MarkdownBlock{
+				{Type: Normal, Content: "This is a normal block\nwith multiple lines."},
+				{Type: Code, Language: "python", Content: "print(\"Line 1\")\nprint(\"Line 2\")"},
+			},
+		},
+		{
+			name:  "Special Characters in Code Block",
+			input: "```js\nconsole.log(\"<>&*\");\n```",
+			expected: []MarkdownBlock{
+				{Type: Code, Language: "js", Content: "console.log(\"<>&*\");"},
+			},
+		},
+		{
+			name:  "Whitespace Handling",
+			input: "   This has leading spaces.\n```python\n   print(\"Indented Python Code\")\n```",
+			expected: []MarkdownBlock{
+				{Type: Normal, Content: "   This has leading spaces."},
+				{Type: Code, Language: "python", Content: "   print(\"Indented Python Code\")"},
+			},
+		},
+		{
+			name:  "Blocks with Only Language Specified",
+			input: "```python\n```",
+			expected: []MarkdownBlock{
+				{Type: Code, Language: "python", Content: ""},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ExtractAllBlocks(tt.input)
+			// compare in a loop
+			for index, block := range got {
+				if index >= len(tt.expected) {
+					t.Errorf("Expected: none: Got: %v", block)
+					continue
+				}
+				if !reflect.DeepEqual(block, tt.expected[index]) {
+					t.Errorf("Expected: %v, Got: %v", tt.expected[index], block)
+				}
+			}
+		})
+	}
+}
+
+func TestExtractCodeBlocksWithComments(t *testing.T) {
+	tests := []struct {
+		name       string
+		input      string
+		withQuotes bool
+		expected   []string
+	}{
+		{
+			name:       "Basic Functionality",
+			input:      "This is a description.\n```Go\nfmt.Println(\"Hello, World!\")\n```",
+			withQuotes: true,
+			expected:   []string{"```Go\n// This is a description.\nfmt.Println(\"Hello, World!\")\n```"},
+		},
+		{
+			name:       "Multiple Code Blocks",
+			input:      "Description 1.\n```Go\nfmt.Println(\"Hello, World!\")\n```\nDescription 2.\n```Python\nprint(\"Hello, World!\")\n```",
+			withQuotes: true,
+			expected: []string{
+				"```Go\n// Description 1.\nfmt.Println(\"Hello, World!\")\n```",
+				"```Python\n# Description 2.\nprint(\"Hello, World!\")\n```",
+			},
+		},
+		{
+			name:       "End Non-Code Content",
+			input:      "```Go\nfmt.Println(\"Hello, World!\")\n```\nThis is a description.",
+			withQuotes: true,
+			expected:   []string{"```Go\nfmt.Println(\"Hello, World!\")\n// This is a description.\n```"},
+		},
+		{
+			name:       "No Code Block",
+			input:      "This is just a text description without any code.",
+			withQuotes: true,
+			expected:   nil, // no code blocks to return
+		},
+
+		{
+			name:       "Unknown Language",
+			input:      "This is a description.\n```bliss\nSome bliss code here\n```",
+			withQuotes: true,
+			expected:   []string{"```bliss\nSome bliss code here\n```"}, // Description is not added as a comment since "bliss" is unknown
+		},
+		{
+			name:       "Empty Language",
+			input:      "This is a description.\n```\nSome code without language\n```",
+			withQuotes: true,
+			expected:   []string{"```\nSome code without language\n```"}, // Description is not added as a comment since language is unspecified
+		},
+		{
+			name:       "Non-Code Content Between Multiple Code Blocks",
+			input:      "Description 1.\n```Go\nfmt.Println(\"Hello from Go!\")\n```\nDescription 2.\n```Python\nprint(\"Hello from Python!\")\n```",
+			withQuotes: true,
+			expected: []string{
+				"```Go\n// Description 1.\nfmt.Println(\"Hello from Go!\")\n```",
+				"```Python\n# Description 2.\nprint(\"Hello from Python!\")\n```",
+			},
+		},
+		{
+			name:       "Code Block Without Preceding Text",
+			input:      "```Go\nfmt.Println(\"Hello, World!\")\n```",
+			withQuotes: true,
+			expected:   []string{"```Go\nfmt.Println(\"Hello, World!\")\n```"}, // No preceding text, so the code block is unchanged
+		},
+		{
+			name:       "Multiple Non-Code Content",
+			input:      "Description 1.\nDescription 2.\n```Go\nfmt.Println(\"Hello, World!\")\n```",
+			withQuotes: true,
+			expected:   []string{"```Go\n// Description 1.\n// Description 2.\nfmt.Println(\"Hello, World!\")\n```"},
+		},
+		{
+			name:       "Mixed Known and Unknown Languages",
+			input:      "Description 1.\n```Go\nfmt.Println(\"Hello from Go!\")\n```\nDescription 2.\n```bliss\nSome bliss code here\n```",
+			withQuotes: true,
+			expected: []string{
+				"```Go\n// Description 1.\nfmt.Println(\"Hello from Go!\")\n```",
+				"```bliss\nSome bliss code here\n```", // Description 2 is not added as a comment since "bliss" is unknown
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			output := ExtractCodeBlocksWithComments(tt.input, tt.withQuotes)
+			if !reflect.DeepEqual(output, tt.expected) {
+				t.Errorf("got %v, want %v", output, tt.expected)
+			}
+		})
+	}
+}