Repository: mikefarah/yq

Issue:
Feature Request: Add `--shell-key-separator` flag for customizable shell output format separator
## Problem / Use Case

When using `--output-format=shell` (or `-o=shell`), yq collapses nested keys using a single underscore `_` as the separator. For example:

```yaml
my_app:
  database_config:
    host: localhost
```

becomes:

```sh
my_app_database_config_host=localhost
```

This creates ambiguity when the original YAML keys themselves contain underscores. It's impossible to distinguish between:
- A nested structure like `my.app.database` (3 keys)  
- A flat structure like `my_app.database` (2 keys, where first key contains `_`)
- A single key like `my_app_database` (1 key)

This is particularly problematic when:
1. Working with YAML files that use snake_case naming conventions
2. Sourcing the shell variables and then trying to reconstruct the original YAML structure
3. Using the shell output in scripts that need to parse the variable names

## Proposed Solution

Add a `--shell-key-separator` flag (similar to existing `--properties-separator` and `--csv-separator` flags) that allows users to specify a custom separator for collapsing nested keys in shell output format.

**Default behavior:** `_` (single underscore) - maintains backward compatibility

**Example usage:**
```bash
yq -o=shell --shell-key-separator="__" file.yaml
```

## Examples

Given `sample.yml`:
```yaml
my_app:
  db_config:
    host: localhost
    port: 5432
```

**Current behavior** (`yq -o=shell sample.yml`):
```sh
my_app_db_config_host=localhost
my_app_db_config_port=5432
```
*Problem: Three underscores - hard to tell where the original key boundaries are*

**Proposed behavior** (`yq -o=shell --shell-key-separator="__" sample.yml`):
```sh
my_app__db_config__host=localhost
my_app__db_config__port=5432
```
*Clear: Double underscores indicate nesting, single underscores are part of the original key names*

## Implementation Notes

Following the pattern of similar flags in yq:

1. **Add ShellVariablesPreferences struct** (similar to `PropertiesPreferences`, `CSVPreferences`, etc.):
   ```go
   type ShellVariablesPreferences struct {
       KeySeparator string
   }
   
   func NewDefaultShellVariablesPreferences() ShellVariablesPreferences {
       return ShellVariablesPreferences{
           KeySeparator: "_",
       }
   }
   
   var ConfiguredShellVariablesPreferences = NewDefaultShellVariablesPreferences()
   ```

2. **Update shellVariablesEncoder** to accept preferences:
   ```go
   func NewShellVariablesEncoder() Encoder {
       return &shellVariablesEncoder{
           prefs: ConfiguredShellVariablesPreferences,
       }
   }
   ```

3. **Update appendPath function** (line 129 in `encoder_shellvariables.go`):
   ```go
   // Current:
   return cookedPath + "_" + key
   
   // Proposed:
   return cookedPath + pe.prefs.KeySeparator + key
   ```

4. **Add CLI flag** in `cmd/root.go`:
   ```go
   rootCmd.PersistentFlags().StringVar(
       &yqlib.ConfiguredShellVariablesPreferences.KeySeparator, 
       "shell-key-separator", 
       yqlib.ConfiguredShellVariablesPreferences.KeySeparator, 
       "separator for shell variable key paths",
   )
   ```

## Backward Compatibility

✅ **Fully backward compatible** - default value of `_` maintains existing behavior

## Related Flags

This follows the same pattern as existing separator flags:
- `--properties-separator` - separator between keys and values in properties format (default `" = "`)
- `--csv-separator` - CSV separator character (default `,`)

## Validation Considerations

The separator should be validated to ensure it produces valid shell variable names:
- Must only contain characters allowed in shell variable names: `[a-zA-Z0-9_]`
- Should not be empty
- Suggested validation in `Set()` method or when flag is parsed

## Additional Benefits

This flag would also help when:
- Generating shell variables for consumption by other tools expecting specific naming conventions
- Working with configuration management systems that have their own naming standards
- Debugging - easier to visually parse variable names with custom separators



Discussion:
@mikefarah: Fixed in 4.48.2

Solution patch:
diff --git a/Dockerfile b/Dockerfile
index f99059a..8c2f0d0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.25.2 AS builder
+FROM golang:1.25.4 AS builder
 
 WORKDIR /go/src/mikefarah/yq
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index 8fc50e8..4f6986b 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM golang:1.25.2
+FROM golang:1.25.4
 
 RUN apt-get update && \
     apt-get install -y npm && \
diff --git a/cmd/root.go b/cmd/root.go
index 268ff16..7a669da 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -168,6 +168,11 @@ yq -P -oy sample.json
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
 
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "shell-key-separator", yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "separator for shell variable key paths")
+	if err = rootCmd.RegisterFlagCompletionFunc("shell-key-separator", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+
 	rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
 
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
diff --git a/pkg/yqlib/doc/usage/shellvariables.md b/pkg/yqlib/doc/usage/shellvariables.md
index ba50315..455e669 100644
--- a/pkg/yqlib/doc/usage/shellvariables.md
+++ b/pkg/yqlib/doc/usage/shellvariables.md
@@ -84,3 +84,23 @@ will output
 name='Miles O'"'"'Brien'
 ```
 
+## Encode shell variables: custom separator
+Use --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores.
+
+Given a sample.yml file of:
+```yaml
+my_app:
+  db_config:
+    host: localhost
+    port: 5432
+```
+then
+```bash
+yq -o=shell --shell-key-separator="__" sample.yml
+```
+will output
+```sh
+my_app__db_config__host=localhost
+my_app__db_config__port=5432
+```
+
diff --git a/pkg/yqlib/encoder_shellvariables.go b/pkg/yqlib/encoder_shellvariables.go
index be54efc..3923670 100644
--- a/pkg/yqlib/encoder_shellvariables.go
+++ b/pkg/yqlib/encoder_shellvariables.go
@@ -12,10 +12,13 @@ import (
 )
 
 type shellVariablesEncoder struct {
+	prefs ShellVariablesPreferences
 }
 
 func NewShellVariablesEncoder() Encoder {
-	return &shellVariablesEncoder{}
+	return &shellVariablesEncoder{
+		prefs: ConfiguredShellVariablesPreferences,
+	}
 }
 
 func (pe *shellVariablesEncoder) CanHandleAliases() bool {
@@ -58,7 +61,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, pat
 		return err
 	case SequenceNode:
 		for index, child := range node.Content {
-			err := pe.doEncode(w, child, appendPath(path, index))
+			err := pe.doEncode(w, child, pe.appendPath(path, index))
 			if err != nil {
 				return err
 			}
@@ -68,7 +71,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, pat
 		for index := 0; index < len(node.Content); index = index + 2 {
 			key := node.Content[index]
 			value := node.Content[index+1]
-			err := pe.doEncode(w, value, appendPath(path, key.Value))
+			err := pe.doEncode(w, value, pe.appendPath(path, key.Value))
 			if err != nil {
 				return err
 			}
@@ -81,7 +84,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, pat
 	}
 }
 
-func appendPath(cookedPath string, rawKey interface{}) string {
+func (pe *shellVariablesEncoder) appendPath(cookedPath string, rawKey interface{}) string {
 
 	// Shell variable names must match
 	//    [a-zA-Z_]+[a-zA-Z0-9_]*
@@ -126,7 +129,7 @@ func appendPath(cookedPath string, rawKey interface{}) string {
 		}
 		return key
 	}
-	return cookedPath + "_" + key
+	return cookedPath + pe.prefs.KeySeparator + key
 }
 
 func quoteValue(value string) string {
diff --git a/pkg/yqlib/shellvariables.go b/pkg/yqlib/shellvariables.go
new file mode 100644
index 0000000..634fc6f
--- /dev/null
+++ b/pkg/yqlib/shellvariables.go
@@ -0,0 +1,14 @@
+package yqlib
+
+type ShellVariablesPreferences struct {
+	KeySeparator string
+}
+
+func NewDefaultShellVariablesPreferences() ShellVariablesPreferences {
+	return ShellVariablesPreferences{
+		KeySeparator: "_",
+	}
+}
+
+var ConfiguredShellVariablesPreferences = NewDefaultShellVariablesPreferences()
+



Test patch:
diff --git a/pkg/yqlib/encoder_shellvariables_test.go b/pkg/yqlib/encoder_shellvariables_test.go
index 3d3d905..d41ad66 100644
--- a/pkg/yqlib/encoder_shellvariables_test.go
+++ b/pkg/yqlib/encoder_shellvariables_test.go
@@ -91,3 +91,47 @@ func TestShellVariablesEncoderEmptyMap(t *testing.T) {
 func TestShellVariablesEncoderScalarNode(t *testing.T) {
 	assertEncodesTo(t, "some string", "value='some string'")
 }
+
+func assertEncodesToWithSeparator(t *testing.T, yaml string, shellvars string, separator string) {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	// Save the original separator
+	originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator
+	defer func() {
+		ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator
+	}()
+
+	// Set the custom separator
+	ConfiguredShellVariablesPreferences.KeySeparator = separator
+
+	var encoder = NewShellVariablesEncoder()
+	inputs, err := readDocuments(strings.NewReader(yaml), "test.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode)
+	err = encoder.Encode(writer, node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	test.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), "\n"))
+}
+
+func TestShellVariablesEncoderCustomSeparator(t *testing.T) {
+	assertEncodesToWithSeparator(t, "a:\n b: Lewis\n c: Carroll", "a__b=Lewis\na__c=Carroll", "__")
+}
+
+func TestShellVariablesEncoderCustomSeparatorNested(t *testing.T) {
+	assertEncodesToWithSeparator(t, "my_app:\n db_config:\n  host: localhost", "my_app__db_config__host=localhost", "__")
+}
+
+func TestShellVariablesEncoderCustomSeparatorArray(t *testing.T) {
+	assertEncodesToWithSeparator(t, "a: [{n: Alice}, {n: Bob}]", "a__0__n=Alice\na__1__n=Bob", "__")
+}
+
+func TestShellVariablesEncoderCustomSeparatorSingleChar(t *testing.T) {
+	assertEncodesToWithSeparator(t, "a:\n b: value", "aXb=value", "X")
+}
diff --git a/pkg/yqlib/shellvariables_test.go b/pkg/yqlib/shellvariables_test.go
index 5a7f56f..d123f4b 100644
--- a/pkg/yqlib/shellvariables_test.go
+++ b/pkg/yqlib/shellvariables_test.go
@@ -54,12 +54,33 @@ var shellVariablesScenarios = []formatScenario{
 		input:          "name: Miles O'Brien",
 		expected:       `name='Miles O'"'"'Brien'` + "\n",
 	},
+	{
+		description:    "Encode shell variables: custom separator",
+		subdescription: "Use --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores.",
+		input: "" +
+			"my_app:" + "\n" +
+			"  db_config:" + "\n" +
+			"    host: localhost" + "\n" +
+			"    port: 5432",
+		expected: "" +
+			"my_app__db_config__host=localhost" + "\n" +
+			"my_app__db_config__port=5432" + "\n",
+		scenarioType: "shell-separator",
+	},
 }
 
 func TestShellVariableScenarios(t *testing.T) {
 	for _, s := range shellVariablesScenarios {
 		//fmt.Printf("\t<%s> <%s>\n", s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description)
+		if s.scenarioType == "shell-separator" {
+			// Save and restore the original separator
+			originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator
+			ConfiguredShellVariablesPreferences.KeySeparator = "__"
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description)
+			ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator
+		} else {
+			test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description)
+		}
 	}
 	genericScenarios := make([]interface{}, len(shellVariablesScenarios))
 	for i, s := range shellVariablesScenarios {
@@ -87,12 +108,22 @@ func documentShellVariableScenario(_ *testing.T, w *bufio.Writer, i interface{})
 
 	expression := s.expression
 
-	if expression != "" {
+	if s.scenarioType == "shell-separator" {
+		writeOrPanic(w, "```bash\nyq -o=shell --shell-key-separator=\"__\" sample.yml\n```\n")
+	} else if expression != "" {
 		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=shell '%v' sample.yml\n```\n", expression))
 	} else {
 		writeOrPanic(w, "```bash\nyq -o=shell sample.yml\n```\n")
 	}
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())))
+	if s.scenarioType == "shell-separator" {
+		// Save and restore the original separator
+		originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator
+		ConfiguredShellVariablesPreferences.KeySeparator = "__"
+		writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())))
+		ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator
+	} else {
+		writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())))
+	}
 }