Repository: mikefarah/yq
Pull request #1556 (2023-02-14 16:06:25)

Allow build without json and xml support
When using yqlib from a project only for the YAML support, it is not needed with JSON support or XML support.

The `encoding/json` and `encoding/xml` makes the library almost twice as big, so allow compiling without those.

`go build -tags "yq_nojson yq_noxml" -o yq`

Trying to use them, will report error instead.

----


Top 10 libraries, yaml (1813K) ~ json + xml (2077K)

```
1813	gopkg.in/yaml.v3
1478	syscall
1083	encoding/json
994	encoding/xml
917	time
830	os
773	fmt
677	regexp
650	github.com/magiconair/properties
552	math

```


Patch:
diff --git a/Dockerfile b/Dockerfile
index 2d10528c..b5fd237f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20.0 as builder
+FROM golang:1.20.1 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index ef1f17eb..1bd9517c 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM golang:1.20.0
+FROM golang:1.20.1
 
 COPY scripts/devtools.sh /opt/devtools.sh
 
diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go
index 63b68871..fff64fa5 100644
--- a/cmd/evaluate_all_command.go
+++ b/cmd/evaluate_all_command.go
@@ -84,7 +84,10 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder := configureEncoder(format)
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
+	}
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go
index 53108165..dab803c3 100644
--- a/cmd/evalute_sequence_command.go
+++ b/cmd/evalute_sequence_command.go
@@ -93,7 +93,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder := configureEncoder(format)
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
+	}
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
diff --git a/cmd/utils.go b/cmd/utils.go
index 8dad7c30..671dd86b 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -61,7 +61,15 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 	if err != nil {
 		return nil, err
 	}
-	switch yqlibInputFormat {
+	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
+	if yqlibDecoder == nil {
+		return nil, fmt.Errorf("no support for %s input format", inputFormat)
+	}
+	return yqlibDecoder, err
+}
+
+func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
+	switch format {
 	case yqlib.XMLInputFormat:
 		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
 	case yqlib.PropertiesInputFormat:
@@ -72,10 +80,12 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 		return yqlib.NewCSVObjectDecoder(','), nil
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
+	case yqlib.YamlInputFormat:
+		prefs := yqlib.ConfiguredYamlPreferences
+		prefs.EvaluateTogether = evaluateTogether
+		return yqlib.NewYamlDecoder(prefs), nil
 	}
-	prefs := yqlib.ConfiguredYamlPreferences
-	prefs.EvaluateTogether = evaluateTogether
-	return yqlib.NewYamlDecoder(prefs), nil
+	return nil, fmt.Errorf("invalid decoder: %v", format)
 }
 
 func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@@ -95,22 +105,34 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 	return printerWriter, nil
 }
 
-func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
+func configureEncoder() (yqlib.Encoder, error) {
+	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return nil, err
+	}
+	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
+	if yqlibEncoder == nil {
+		return nil, fmt.Errorf("no support for %s output format", outputFormat)
+	}
+	return yqlibEncoder, err
+}
+
+func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
+		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder(unwrapScalar)
+		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
 	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(',')
+		return yqlib.NewCsvEncoder(','), nil
 	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t')
+		return yqlib.NewCsvEncoder('\t'), nil
 	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
 	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
+		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
 	}
-	panic("invalid encoder")
+	return nil, fmt.Errorf("invalid encoder: %v", format)
 }
 
 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
diff --git a/cmd/version.go b/cmd/version.go
index 05977638..14b17304 100644
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.30.8"
+	Version = "v4.31.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
diff --git a/go.mod b/go.mod
index 1d3c9613..d340e9f5 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/a8m/envsubst v1.4.1
+	github.com/a8m/envsubst v1.4.2
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
@@ -14,7 +14,7 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.1.1-0.20221104162952-702349b0e862
+	golang.org/x/net v0.7.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -23,8 +23,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
diff --git a/go.sum b/go.sum
index cf7fa672..68287a2a 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/a8m/envsubst v1.4.1 h1:RShc2OKbQpIIp5qR2glVsBOJCImTVHFrDkJvmGh0Qi0=
-github.com/a8m/envsubst v1.4.1/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
+github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
@@ -54,20 +54,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 h1:KrLJ+iz8J6j6VVr/OCfULAcK+xozUmWE43fKpMR4MlI=
-golang.org/x/net v0.1.1-0.20221104162952-702349b0e862/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
diff --git a/pkg/yqlib/decoder_json.go b/pkg/yqlib/decoder_json.go
index a8e1e605..35da6467 100644
--- a/pkg/yqlib/decoder_json.go
+++ b/pkg/yqlib/decoder_json.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go
index ac70e403..bfb18e12 100644
--- a/pkg/yqlib/decoder_xml.go
+++ b/pkg/yqlib/decoder_xml.go
@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go
index e9fae6af..e0ecaccf 100644
--- a/pkg/yqlib/encoder.go
+++ b/pkg/yqlib/encoder.go
@@ -1,10 +1,6 @@
 package yqlib
 
 import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
 	"io"
 
 	yaml "gopkg.in/yaml.v3"
@@ -17,162 +13,17 @@ type Encoder interface {
 	CanHandleAliases() bool
 }
 
-// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
-// order of keys and values in a map or an object.
-type orderedMap struct {
-	// if this is an object, kv != nil. If this is not an object, kv == nil.
-	kv     []orderedMapKV
-	altVal interface{}
-}
-
-type orderedMapKV struct {
-	K string
-	V orderedMap
-}
-
-func (o *orderedMap) UnmarshalJSON(data []byte) error {
-	switch data[0] {
-	case '{':
-		// initialise so that even if the object is empty it is not nil
-		o.kv = []orderedMapKV{}
-
-		// create decoder
-		dec := json.NewDecoder(bytes.NewReader(data))
-		_, err := dec.Token() // open object
-		if err != nil {
-			return err
-		}
-
-		// cycle through k/v
-		var tok json.Token
-		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
-			// we can expect two types: string or Delim. Delim automatically means
-			// that it is the closing bracket of the object, whereas string means
-			// that there is another key.
-			if _, ok := tok.(json.Delim); ok {
-				break
-			}
-			kv := orderedMapKV{
-				K: tok.(string),
-			}
-			if err := dec.Decode(&kv.V); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, kv)
-		}
-		// unexpected error
-		if err != nil && !errors.Is(err, io.EOF) {
-			return err
-		}
-		return nil
-	case '[':
-		var res []*orderedMap
-		if err := json.Unmarshal(data, &res); err != nil {
-			return err
-		}
-		o.altVal = res
-		o.kv = nil
-		return nil
-	}
-
-	return json.Unmarshal(data, &o.altVal)
-}
+func mapKeysToStrings(node *yaml.Node) {
 
-func (o orderedMap) MarshalJSON() ([]byte, error) {
-	buf := new(bytes.Buffer)
-	enc := json.NewEncoder(buf)
-	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
-	if o.kv == nil {
-		if err := enc.Encode(o.altVal); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	}
-	buf.WriteByte('{')
-	for idx, el := range o.kv {
-		if err := enc.Encode(el.K); err != nil {
-			return nil, err
-		}
-		buf.WriteByte(':')
-		if err := enc.Encode(el.V); err != nil {
-			return nil, err
-		}
-		if idx != len(o.kv)-1 {
-			buf.WriteByte(',')
-		}
-	}
-	buf.WriteByte('}')
-	return buf.Bytes(), nil
-}
-
-func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
-	switch node.Kind {
-	case yaml.DocumentNode:
-		if len(node.Content) == 0 {
-			return nil
-		}
-		return o.UnmarshalYAML(node.Content[0])
-	case yaml.AliasNode:
-		return o.UnmarshalYAML(node.Alias)
-	case yaml.ScalarNode:
-		return node.Decode(&o.altVal)
-	case yaml.MappingNode:
-		// set kv to non-nil
-		o.kv = []orderedMapKV{}
-		for i := 0; i < len(node.Content); i += 2 {
-			var key string
-			var val orderedMap
-			if err := node.Content[i].Decode(&key); err != nil {
-				return err
+	if node.Kind == yaml.MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
 			}
-			if err := node.Content[i+1].Decode(&val); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, orderedMapKV{
-				K: key,
-				V: val,
-			})
-		}
-		return nil
-	case yaml.SequenceNode:
-		// note that this has to be a pointer, so that nulls can be represented.
-		var res []*orderedMap
-		if err := node.Decode(&res); err != nil {
-			return err
 		}
-		o.altVal = res
-		o.kv = nil
-		return nil
-	case 0:
-		// null
-		o.kv = nil
-		o.altVal = nil
-		return nil
-	default:
-		return fmt.Errorf("orderedMap: invalid yaml node")
 	}
-}
 
-func (o *orderedMap) MarshalYAML() (interface{}, error) {
-	// fast path: kv is nil, use altVal
-	if o.kv == nil {
-		return o.altVal, nil
-	}
-	content := make([]*yaml.Node, 0, len(o.kv)*2)
-	for _, val := range o.kv {
-		n := new(yaml.Node)
-		if err := n.Encode(val.V); err != nil {
-			return nil, err
-		}
-		content = append(content, &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: val.K,
-		}, n)
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
 	}
-	return &yaml.Node{
-		Kind:    yaml.MappingNode,
-		Tag:     "!!map",
-		Content: content,
-	}, nil
 }
diff --git a/pkg/yqlib/encoder_json.go b/pkg/yqlib/encoder_json.go
index 9f5b71ac..53c6bd06 100644
--- a/pkg/yqlib/encoder_json.go
+++ b/pkg/yqlib/encoder_json.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
@@ -14,21 +16,6 @@ type jsonEncoder struct {
 	UnwrapScalar bool
 }
 
-func mapKeysToStrings(node *yaml.Node) {
-
-	if node.Kind == yaml.MappingNode {
-		for index, child := range node.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range node.Content {
-		mapKeysToStrings(child)
-	}
-}
-
 func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
 	var indentString = ""
 
diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go
index 1d2e7798..d60ede8f 100644
--- a/pkg/yqlib/encoder_xml.go
+++ b/pkg/yqlib/encoder_xml.go
@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 8b61eae3..65389263 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -53,7 +53,7 @@ var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, P
 
 var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
 var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
-var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: assignVariableOperator}
+var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
 var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
 var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
 var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
diff --git a/pkg/yqlib/no_json.go b/pkg/yqlib/no_json.go
new file mode 100644
index 00000000..ae9d531a
--- /dev/null
+++ b/pkg/yqlib/no_json.go
@@ -0,0 +1,11 @@
+//go:build yq_nojson
+
+package yqlib
+
+func NewJSONDecoder() Decoder {
+	return nil
+}
+
+func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
+	return nil
+}
diff --git a/pkg/yqlib/no_xml.go b/pkg/yqlib/no_xml.go
new file mode 100644
index 00000000..d3f96bb6
--- /dev/null
+++ b/pkg/yqlib/no_xml.go
@@ -0,0 +1,11 @@
+//go:build yq_noxml
+
+package yqlib
+
+func NewXMLDecoder(prefs XmlPreferences) Decoder {
+	return nil
+}
+
+func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
+	return nil
+}
diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go
index 255b0664..1c9582d1 100644
--- a/pkg/yqlib/operator_encoder_decoder.go
+++ b/pkg/yqlib/operator_encoder_decoder.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"container/list"
+	"errors"
 	"regexp"
 	"strings"
 
@@ -39,6 +40,9 @@ func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string,
 	log.Debug("printing with indent: %v", prefs.indent)
 
 	encoder := configureEncoder(prefs.format, prefs.indent)
+	if encoder == nil {
+		return "", errors.New("no support for output format")
+	}
 
 	printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
 	err := printer.PrintResults(candidate.AsList())
@@ -98,13 +102,11 @@ type decoderPreferences struct {
 	format InputFormat
 }
 
-/* takes a string and decodes it back into an object */
-func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
-
+func createDecoder(format InputFormat) Decoder {
 	var decoder Decoder
-	switch preferences.format {
+	switch format {
+	case JsonInputFormat:
+		decoder = NewJSONDecoder()
 	case YamlInputFormat:
 		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
 	case XMLInputFormat:
@@ -120,6 +122,18 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	case UriInputFormat:
 		decoder = NewUriDecoder()
 	}
+	return decoder
+}
+
+/* takes a string and decodes it back into an object */
+func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
+
+	decoder := createDecoder(preferences.format)
+	if decoder == nil {
+		return Context{}, errors.New("no support for input format")
+	}
 
 	var results = list.New()
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go
index 7d083d5b..678854f6 100644
--- a/pkg/yqlib/operator_load.go
+++ b/pkg/yqlib/operator_load.go
@@ -34,6 +34,9 @@ func loadString(filename string) (*CandidateNode, error) {
 }
 
 func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
+	if decoder == nil {
+		return nil, fmt.Errorf("could not load %s", filename)
+	}
 
 	file, err := os.Open(filename) // #nosec
 	if err != nil {
diff --git a/pkg/yqlib/operator_pipe.go b/pkg/yqlib/operator_pipe.go
index 77813c67..e1a90d91 100644
--- a/pkg/yqlib/operator_pipe.go
+++ b/pkg/yqlib/operator_pipe.go
@@ -2,9 +2,9 @@ package yqlib
 
 func pipeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 
-	//lhs may update the variable context, we should pass that into the RHS
-	// BUT we still return the original context back (see jq)
-	// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...
+	if expressionNode.LHS.Operation.OperationType == assignVariableOpType {
+		return variableLoop(d, context, expressionNode)
+	}
 
 	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
 	if err != nil {
diff --git a/pkg/yqlib/operator_variables.go b/pkg/yqlib/operator_variables.go
index 5550bded..d9baded9 100644
--- a/pkg/yqlib/operator_variables.go
+++ b/pkg/yqlib/operator_variables.go
@@ -19,24 +19,81 @@ type assignVarPreferences struct {
 	IsReference bool
 }
 
-func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.LHS)
+func useWithPipe(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	return Context{}, fmt.Errorf("must use variable with a pipe, e.g. `exp as $x | ...`")
+}
+
+// variables are like loops in jq
+// https://stedolan.github.io/jq/manual/#Variable
+func variableLoop(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	log.Debug("variable loop!")
+	results := list.New()
+	var evaluateAllTogether = true
+	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
+		if !evaluateAllTogether {
+			break
+		}
+	}
+	if evaluateAllTogether {
+		return variableLoopSingleChild(d, context, originalExp)
+	}
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		result, err := variableLoopSingleChild(d, context.SingleChildContext(el.Value.(*CandidateNode)), originalExp)
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(result.MatchingNodes)
+	}
+	return context.ChildContext(results), nil
+}
+
+func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+
+	variableExp := originalExp.LHS
+	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), variableExp.LHS)
 	if err != nil {
 		return Context{}, err
 	}
-	if expressionNode.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
+	if variableExp.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
 		return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo")
 	}
-	variableName := expressionNode.RHS.Operation.StringValue
+	variableName := variableExp.RHS.Operation.StringValue
+
+	prefs := variableExp.Operation.Preferences.(assignVarPreferences)
 
-	prefs := expressionNode.Operation.Preferences.(assignVarPreferences)
+	results := list.New()
 
-	var variableValue *list.List
-	if prefs.IsReference {
-		variableValue = lhs.MatchingNodes
-	} else {
-		variableValue = lhs.DeepClone().MatchingNodes
+	// now we loop over lhs, set variable to each result and calculate originalExp.Rhs
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		log.Debug("PROCESSING VARIABLE: ", NodeToString(el.Value.(*CandidateNode)))
+		var variableValue = list.New()
+		if prefs.IsReference {
+			variableValue.PushBack(el.Value)
+		} else {
+			candidateCopy, err := el.Value.(*CandidateNode).Copy()
+			if err != nil {
+				return Context{}, err
+			}
+			variableValue.PushBack(candidateCopy)
+		}
+		newContext := context.ChildContext(context.MatchingNodes)
+		newContext.SetVariable(variableName, variableValue)
+
+		rhs, err := d.GetMatchingNodes(newContext, originalExp.RHS)
+		log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(rhs.MatchingNodes)
 	}
-	context.SetVariable(variableName, variableValue)
-	return context, nil
+
+	// if there is no LHS - then I guess we just calculate originalExp.Rhs
+	if lhs.MatchingNodes.Len() == 0 {
+		return d.GetMatchingNodes(context, originalExp.RHS)
+	}
+
+	return context.ChildContext(results), nil
+
 }
diff --git a/pkg/yqlib/ordered_map.go b/pkg/yqlib/ordered_map.go
new file mode 100644
index 00000000..a783423b
--- /dev/null
+++ b/pkg/yqlib/ordered_map.go
@@ -0,0 +1,14 @@
+package yqlib
+
+// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
+// order of keys and values in a map or an object.
+type orderedMap struct {
+	// if this is an object, kv != nil. If this is not an object, kv == nil.
+	kv     []orderedMapKV
+	altVal interface{}
+}
+
+type orderedMapKV struct {
+	K string
+	V orderedMap
+}
diff --git a/pkg/yqlib/ordered_map_json.go b/pkg/yqlib/ordered_map_json.go
new file mode 100644
index 00000000..94a1d780
--- /dev/null
+++ b/pkg/yqlib/ordered_map_json.go
@@ -0,0 +1,83 @@
+package yqlib
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+)
+
+func (o *orderedMap) UnmarshalJSON(data []byte) error {
+	switch data[0] {
+	case '{':
+		// initialise so that even if the object is empty it is not nil
+		o.kv = []orderedMapKV{}
+
+		// create decoder
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
+			}
+			kv := orderedMapKV{
+				K: tok.(string),
+			}
+			if err := dec.Decode(&kv.V); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, kv)
+		}
+		// unexpected error
+		if err != nil && !errors.Is(err, io.EOF) {
+			return err
+		}
+		return nil
+	case '[':
+		var res []*orderedMap
+		if err := json.Unmarshal(data, &res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	}
+
+	return json.Unmarshal(data, &o.altVal)
+}
+
+func (o orderedMap) MarshalJSON() ([]byte, error) {
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+	if o.kv == nil {
+		if err := enc.Encode(o.altVal); err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
+	buf.WriteByte('{')
+	for idx, el := range o.kv {
+		if err := enc.Encode(el.K); err != nil {
+			return nil, err
+		}
+		buf.WriteByte(':')
+		if err := enc.Encode(el.V); err != nil {
+			return nil, err
+		}
+		if idx != len(o.kv)-1 {
+			buf.WriteByte(',')
+		}
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
+}
diff --git a/pkg/yqlib/ordered_map_yaml.go b/pkg/yqlib/ordered_map_yaml.go
new file mode 100644
index 00000000..b3c49442
--- /dev/null
+++ b/pkg/yqlib/ordered_map_yaml.go
@@ -0,0 +1,79 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) == 0 {
+			return nil
+		}
+		return o.UnmarshalYAML(node.Content[0])
+	case yaml.AliasNode:
+		return o.UnmarshalYAML(node.Alias)
+	case yaml.ScalarNode:
+		return node.Decode(&o.altVal)
+	case yaml.MappingNode:
+		// set kv to non-nil
+		o.kv = []orderedMapKV{}
+		for i := 0; i < len(node.Content); i += 2 {
+			var key string
+			var val orderedMap
+			if err := node.Content[i].Decode(&key); err != nil {
+				return err
+			}
+			if err := node.Content[i+1].Decode(&val); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, orderedMapKV{
+				K: key,
+				V: val,
+			})
+		}
+		return nil
+	case yaml.SequenceNode:
+		// note that this has to be a pointer, so that nulls can be represented.
+		var res []*orderedMap
+		if err := node.Decode(&res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	case 0:
+		// null
+		o.kv = nil
+		o.altVal = nil
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
+	}
+}
+
+func (o *orderedMap) MarshalYAML() (interface{}, error) {
+	// fast path: kv is nil, use altVal
+	if o.kv == nil {
+		return o.altVal, nil
+	}
+	content := make([]*yaml.Node, 0, len(o.kv)*2)
+	for _, val := range o.kv {
+		n := new(yaml.Node)
+		if err := n.Encode(val.V); err != nil {
+			return nil, err
+		}
+		content = append(content, &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: val.K,
+		}, n)
+	}
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: content,
+	}, nil
+}
diff --git a/release_notes.txt b/release_notes.txt
index 3cc78097..f4aab574 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,3 +1,12 @@
+4.31.1:
+  - Added shuffle command #1503
+  - Added ability to sort by multiple fields #1541
+  - Added @sh encoder #1526
+  - Added @uri/@urid encoder/decoder #1529
+  - Fixed date comparison with string date #1537
+  - Added from_unix/to_unix Operators
+  - Bumped dependency versions
+
 4.30.8:
  - Log info message when unable to chown file in linux (e.g. snap confinement) #1521
 
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index d6c978ae..5d0b528b 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.30.8'
+version: 'v4.31.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.


Test patch:
diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go
index a17c51a9..cb384562 100644
--- a/pkg/yqlib/encoder_test.go
+++ b/pkg/yqlib/encoder_test.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go
index ad1b6bb8..b2eddb58 100644
--- a/pkg/yqlib/json_test.go
+++ b/pkg/yqlib/json_test.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go
index 99a84845..eeabffde 100644
--- a/pkg/yqlib/operator_add_test.go
+++ b/pkg/yqlib/operator_add_test.go
@@ -34,7 +34,7 @@ var addOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a + .b) as $x",
+		expression: "(.a + .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_alternative_test.go b/pkg/yqlib/operator_alternative_test.go
index 74106639..e70b8b25 100644
--- a/pkg/yqlib/operator_alternative_test.go
+++ b/pkg/yqlib/operator_alternative_test.go
@@ -16,7 +16,7 @@ var alternativeOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		expression: `(.b // "hello") as $x`,
+		expression: `(.b // "hello") as $x | .`,
 		document:   `a: bridge`,
 		expected: []string{
 			"D0, P[], (doc)::a: bridge\n",
diff --git a/pkg/yqlib/operator_booleans_test.go b/pkg/yqlib/operator_booleans_test.go
index f5735bb2..dc219ad1 100644
--- a/pkg/yqlib/operator_booleans_test.go
+++ b/pkg/yqlib/operator_booleans_test.go
@@ -102,7 +102,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `any_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::[{pet: cat}]\n",
 		},
@@ -110,9 +110,17 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `all_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | $c`,
 		expected: []string{
-			"D0, P[], (doc)::[{pet: cat}]\n",
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c | $c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
@@ -185,7 +193,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b or .c) as $x`,
+		expression: `(.a.b or .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -193,7 +201,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b and .c) as $x`,
+		expression: `(.a.b and .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go
index 13448adf..00441f73 100644
--- a/pkg/yqlib/operator_encoder_decoder_test.go
+++ b/pkg/yqlib/operator_encoder_decoder_test.go
@@ -8,15 +8,17 @@ var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
 
 var encoderDecoderOperatorScenarios = []expressionScenario{
 	{
-		description: "Encode value as json string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_json)`,
+		requiresFormat: "json",
+		description:    "Encode value as json string",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_json)`,
 		expected: []string{
 			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
 `,
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Encode value as json string, on one line",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -27,6 +29,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Encode value as json string, on one line shorthand",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -37,6 +40,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Decode a json encoded string",
 		subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
 		document:       `a: '{"cool":"thing"}'`,
@@ -193,33 +197,37 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Encode value as xml string",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `.a | to_xml`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `.a | to_xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\">\n  <foo>bar</foo>\n</cool>\n\n",
 		},
 	},
 	{
-		description: "Encode value as xml string on a single line",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `.a | @xml`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string on a single line",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `.a | @xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
 		},
 	},
 	{
-		description: "Encode value as xml string with custom indentation",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `{"cat": .a | to_xml(1)}`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string with custom indentation",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `{"cat": .a | to_xml(1)}`,
 		expected: []string{
 			"D0, P[], (!!map)::cat: |\n    <cool id=\"hi\">\n     <foo>bar</foo>\n    </cool>\n",
 		},
 	},
 	{
-		description: "Decode a xml encoded string",
-		document:    `a: "<foo>bar</foo>"`,
-		expression:  `.b = (.a | from_xml)`,
+		requiresFormat: "xml",
+		description:    "Decode a xml encoded string",
+		document:       `a: "<foo>bar</foo>"`,
+		expression:     `.b = (.a | from_xml)`,
 		expected: []string{
 			"D0, P[], (doc)::a: \"<foo>bar</foo>\"\nb:\n    foo: bar\n",
 		},
@@ -303,9 +311,10 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "empty xml decode",
-		skipDoc:     true,
-		expression:  `"" | @xmld`,
+		requiresFormat: "xml",
+		description:    "empty xml decode",
+		skipDoc:        true,
+		expression:     `"" | @xmld`,
 		expected: []string{
 			"D0, P[], (!!null)::\n",
 		},
diff --git a/pkg/yqlib/operator_equals_test.go b/pkg/yqlib/operator_equals_test.go
index 37fe5005..7a821647 100644
--- a/pkg/yqlib/operator_equals_test.go
+++ b/pkg/yqlib/operator_equals_test.go
@@ -47,7 +47,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a == .b) as $x",
+		expression: "(.a == .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -63,7 +63,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a != .b) as $x",
+		expression: "(.a != .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_has_test.go b/pkg/yqlib/operator_has_test.go
index 3a631330..47504048 100644
--- a/pkg/yqlib/operator_has_test.go
+++ b/pkg/yqlib/operator_has_test.go
@@ -16,7 +16,7 @@ var hasOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `a: hello`,
-		expression: `has(.b) as $c`,
+		expression: `has(.b) as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::a: hello\n",
 		},
diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go
index b27e8f70..5bb30321 100644
--- a/pkg/yqlib/operator_load_test.go
+++ b/pkg/yqlib/operator_load_test.go
@@ -74,9 +74,10 @@ var loadScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Load from XML",
-		document:    "cool: things",
-		expression:  `.more_stuff = load_xml("../../examples/small.xml")`,
+		requiresFormat: "xml",
+		description:    "Load from XML",
+		document:       "cool: things",
+		expression:     `.more_stuff = load_xml("../../examples/small.xml")`,
 		expected: []string{
 			"D0, P[], (doc)::cool: things\nmore_stuff:\n    this: is some xml\n",
 		},
diff --git a/pkg/yqlib/operator_subtract_test.go b/pkg/yqlib/operator_subtract_test.go
index aaaa91f6..e5134b06 100644
--- a/pkg/yqlib/operator_subtract_test.go
+++ b/pkg/yqlib/operator_subtract_test.go
@@ -8,7 +8,7 @@ var subtractOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a - .b) as $x",
+		expression: "(.a - .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go
index 405a523e..8d3e94ad 100644
--- a/pkg/yqlib/operator_traverse_path_test.go
+++ b/pkg/yqlib/operator_traverse_path_test.go
@@ -151,7 +151,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `c: dog`,
-		expression: `.[.a.b] as $x`,
+		expression: `.[.a.b] as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::c: dog\n",
 		},
diff --git a/pkg/yqlib/operator_union_test.go b/pkg/yqlib/operator_union_test.go
index ce12f19a..4b98ac06 100644
--- a/pkg/yqlib/operator_union_test.go
+++ b/pkg/yqlib/operator_union_test.go
@@ -8,7 +8,7 @@ var unionOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: `(.a, .b.c) as $x`,
+		expression: `(.a, .b.c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_variables_test.go b/pkg/yqlib/operator_variables_test.go
index 7ab9ee30..64643ab0 100644
--- a/pkg/yqlib/operator_variables_test.go
+++ b/pkg/yqlib/operator_variables_test.go
@@ -8,7 +8,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `.a.b as $foo`,
+		expression: `.a.b as $foo | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -16,7 +16,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		document:      "a: [cat]",
 		skipDoc:       true,
-		expression:    "(.[] | {.name: .}) as $item",
+		expression:    "(.[] | {.name: .}) as $item | .",
 		expectedError: `cannot index array with 'name' (strconv.ParseInt: parsing "name": invalid syntax)`,
 	},
 	{
@@ -36,6 +36,22 @@ var variableOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::dog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `.[] | . as $f | select($f == 2)`,
+		expected: []string{
+			"D0, P[1], (!!int)::2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `[.[] | . as $f | $f + 1]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 2\n- 3\n",
+		},
+	},
 	{
 		description:    "Using variables as a lookup",
 		subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)",
diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go
index 79757895..c040cbed 100644
--- a/pkg/yqlib/operators_test.go
+++ b/pkg/yqlib/operators_test.go
@@ -28,6 +28,7 @@ type expressionScenario struct {
 	skipDoc               bool
 	expectedError         string
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
+	requiresFormat        string
 }
 
 func TestMain(m *testing.M) {
@@ -103,6 +104,23 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		return
 	}
 
+	if s.requiresFormat != "" {
+		format := s.requiresFormat
+		inputFormat, err := InputFormatFromString(format)
+		if err != nil {
+			t.Error(err)
+		}
+		if decoder := createDecoder(inputFormat); decoder == nil {
+			t.Skipf("no support for %s input format", format)
+		}
+		outputFormat, err := OutputFormatFromString(format)
+		if err != nil {
+			t.Error(err)
+		}
+		if encoder := configureEncoder(outputFormat, 4); encoder == nil {
+			t.Skipf("no support for %s output format", format)
+		}
+	}
 	if err != nil {
 		t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression))
 		return
diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go
index cc7b3dbe..f3be6401 100644
--- a/pkg/yqlib/printer_test.go
+++ b/pkg/yqlib/printer_test.go
@@ -314,7 +314,11 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer))
+	encoder := NewJSONEncoder(0, false, false)
+	if encoder == nil {
+		t.Skipf("no support for %s output format", "json")
+	}
+	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go
index 2e9badf4..ec9f5b67 100644
--- a/pkg/yqlib/xml_test.go
+++ b/pkg/yqlib/xml_test.go
@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (