Repository: hashicorp/terraform (Go)

Issue:
Terraform is "magically" running `jsonencode()` on strings that are not JSON and then producing strange plan output
### Terraform Version

```shell
$ terraform version
Terraform v1.7.5
on darwin_arm64
+ provider registry.terraform.io/hashicorp/local v2.5.1
```


### Terraform Configuration Files

```terraform
terraform {
  required_providers {
    local = {
      source  = "hashicorp/local"
    }
  }
}

resource "local_file" "foo" {
  count = 10
  content  = "[${10 - count.index}] bottles of beer on the wall!"
  filename = "${path.module}/bottle-${count.index}.txt"
}
```


### Debug Output

https://gist.github.com/spkane/d9e51054cece7963b399ebb2a233c118

### Expected Behavior

- **Note** the `content` field for each one in the plan. It shows the correct output and the resulting files have the proper contents.

```sh
$ terraform apply
local_file.foo[6]: Refreshing state... [id=11d3bf33e314b6fc59fa9f272f7ec5ffe32986ac]
local_file.foo[8]: Refreshing state... [id=0fa535957605bfa166d5d69f4eaecaf9d2eb8dcd]
local_file.foo[7]: Refreshing state... [id=138d25bf0c2d7a80006e868956c59a160c875c46]
local_file.foo[3]: Refreshing state... [id=2112975818324e2c29e485d1458bde09ee214b43]
local_file.foo[9]: Refreshing state... [id=4c58ad59d1b67477fece488b112b4be9388ebbc8]
local_file.foo[5]: Refreshing state... [id=dd46a6f8e2db4e554b43e0046a20d46f337b8f0c]
local_file.foo[1]: Refreshing state... [id=8403b3f70bf511aa48bf4a3ff16e474de75ea141]
local_file.foo[2]: Refreshing state... [id=c6fc8e0f2d5b43f5313ddd7203c0e3542a5a3e22]
local_file.foo[4]: Refreshing state... [id=7ad3f3125fcc2550a582f939229baff52e32a7e1]
local_file.foo[0]: Refreshing state... [id=fb61ffaa3d986de2ac8c827ec15ff672606904f1]

Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with
the following symbols:
  + create

Terraform will perform the following actions:

  # local_file.foo[0] will be created
  + resource "local_file" "foo" {
      + content              = "10 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-0.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[1] will be created
  + resource "local_file" "foo" {
      + content              = "9 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-1.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[2] will be created
  + resource "local_file" "foo" {
      + content              = "8 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-2.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[3] will be created
  + resource "local_file" "foo" {
      + content              = "7 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-3.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[4] will be created
  + resource "local_file" "foo" {
      + content              = "6 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-4.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[5] will be created
  + resource "local_file" "foo" {
      + content              = "5 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-5.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[6] will be created
  + resource "local_file" "foo" {
      + content              = "4 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-6.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[7] will be created
  + resource "local_file" "foo" {
      + content              = "3 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-7.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[8] will be created
  + resource "local_file" "foo" {
      + content              = "2 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-8.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[9] will be created
  + resource "local_file" "foo" {
      + content              = "1 bottles of beer on the wall!"
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-9.txt"
      + id                   = (known after apply)
    }

Plan: 10 to add, 0 to change, 0 to destroy.

Do you want to perform these actions?
  Terraform will perform the actions described above.
  Only 'yes' will be accepted to approve.

  Enter a value: yes

local_file.foo[2]: Creating...
local_file.foo[2]: Creation complete after 0s [id=c6fc8e0f2d5b43f5313ddd7203c0e3542a5a3e22]
local_file.foo[7]: Creating...
local_file.foo[1]: Creating...
local_file.foo[3]: Creating...
local_file.foo[0]: Creating...
local_file.foo[8]: Creating...
local_file.foo[4]: Creating...
local_file.foo[5]: Creating...
local_file.foo[6]: Creating...
local_file.foo[9]: Creating...
local_file.foo[7]: Creation complete after 0s [id=138d25bf0c2d7a80006e868956c59a160c875c46]
local_file.foo[1]: Creation complete after 0s [id=8403b3f70bf511aa48bf4a3ff16e474de75ea141]
local_file.foo[6]: Creation complete after 0s [id=11d3bf33e314b6fc59fa9f272f7ec5ffe32986ac]
local_file.foo[9]: Creation complete after 0s [id=4c58ad59d1b67477fece488b112b4be9388ebbc8]
local_file.foo[8]: Creation complete after 0s [id=0fa535957605bfa166d5d69f4eaecaf9d2eb8dcd]
local_file.foo[0]: Creation complete after 0s [id=fb61ffaa3d986de2ac8c827ec15ff672606904f1]
local_file.foo[5]: Creation complete after 0s [id=dd46a6f8e2db4e554b43e0046a20d46f337b8f0c]
local_file.foo[3]: Creation complete after 0s [id=2112975818324e2c29e485d1458bde09ee214b43]
local_file.foo[4]: Creation complete after 1s [id=7ad3f3125fcc2550a582f939229baff52e32a7e1]

Apply complete! Resources: 10 added, 0 changed, 0 destroyed.

$ cat bottle-*.txt
10 bottles of beer on the wall!9 bottles of beer on the wall!8 bottles of beer on the wall!7 bottles of beer on the wall!6 bottles of beer on the wall!5 bottles of beer on the wall!4 bottles of beer on the wall!3 bottles of beer on the wall!2 bottles of beer on the wall!1 bottles of beer on the wall!
```

### Actual Behavior

- **Note** the `content` field for each one in the plan. It doesn't show the correct output, even though the files actually end up with the proper contents.

```sh
$ terraform apply

Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with
the following symbols:
  + create

Terraform will perform the following actions:

  # local_file.foo[0] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 10,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-0.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[1] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 9,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-1.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[2] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 8,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-2.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[3] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 7,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-3.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[4] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 6,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-4.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[5] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 5,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-5.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[6] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 4,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-6.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[7] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 3,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-7.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[8] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 2,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-8.txt"
      + id                   = (known after apply)
    }

  # local_file.foo[9] will be created
  + resource "local_file" "foo" {
      + content              = jsonencode(
            [
              + 1,
            ]
        )
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0777"
      + file_permission      = "0777"
      + filename             = "./bottle-9.txt"
      + id                   = (known after apply)
    }

Plan: 10 to add, 0 to change, 0 to destroy.

Do you want to perform these actions?
  Terraform will perform the actions described above.
  Only 'yes' will be accepted to approve.

  Enter a value: yes

local_file.foo[6]: Creating...
local_file.foo[9]: Creating...
local_file.foo[3]: Creating...
local_file.foo[0]: Creating...
local_file.foo[4]: Creating...
local_file.foo[8]: Creating...
local_file.foo[2]: Creating...
local_file.foo[1]: Creating...
local_file.foo[5]: Creating...
local_file.foo[7]: Creating...
local_file.foo[3]: Creation complete after 0s [id=d62929a9e54fd41f8de1ff3ac173ab71ef91e1b9]
local_file.foo[2]: Creation complete after 0s [id=3b8793de14163267aeb7c8bc3064500b4fc8f7da]
local_file.foo[7]: Creation complete after 0s [id=6ee42a9cc9426d794a52796cc159119d0bf77a56]
local_file.foo[0]: Creation complete after 0s [id=cce6a69dc470f95d8414846ff43b77cd4a5d3cd1]
local_file.foo[4]: Creation complete after 0s [id=99e3ae889667322b35c34f42471dfe99d3d5e4b8]
local_file.foo[6]: Creation complete after 0s [id=e2a718e41e12255876f422e4fe31d50790a05834]
local_file.foo[1]: Creation complete after 0s [id=e57e70c06defe67bca0e0ebc793bc440ff3ab378]
local_file.foo[8]: Creation complete after 0s [id=2c58683c3cd686a2fb52829cdc9267388d987b70]
local_file.foo[5]: Creation complete after 0s [id=f9985e10e2ac285f588cb7e39a40c601e7f2bb40]
local_file.foo[9]: Creation complete after 0s [id=2184d8655539704ff6bb338892d7bc2754683ae4]

Apply complete! Resources: 10 added, 0 changed, 0 destroyed.

$ cat bottle-*.txt

[10] bottles of beer on the wall![9] bottles of beer on the wall![8] bottles of beer on the wall![7] bottles of beer on the wall![6] bottles of beer on the wall![5] bottles of beer on the wall![4] bottles of beer on the wall![3] bottles of beer on the wall![2] bottles of beer on the wall![1] bottles of beer on the wall!
```

### Steps to Reproduce

1. `terraform init`
2. `terraform plan`
3. `terraform apply -auto-approve`

### Additional Context

I think that this is being triggered by the fact that the string starts with `[`. Removing these square brackets from the string removes this issue completely. It is possible/likely that other things used in JSON, like '{}', would also trigger this.

### References

N/A


Discussion:
Hi @spkane! Thanks for reporting this.

In the original design of the modern plan renderer the intended heuristic here was that the _entire_ string needed to parse as JSON and needed to appear to be either a JSON array or JSON object.

It seems like the current implementation is not quite getting that right: I'd guess that it's treating it as JSON if a _prefix_ of the string can parse as a JSON value. That's definitely a bug if so.

Indeed, that does seem to be the problem:

https://github.com/hashicorp/terraform/blob/81555f5bb3bbc1ef2378d746e3457be24b873a43/internal/command/jsonformat/computed/renderers/string.go#L32-L44

This is using the "decoder" API from Go's `encoding/json`, which is a streaming decoder that can potentially read multiple values from a stream. Therefore it isn't sufficient to just call `Decode` once: we must also check whether there's anything other than spaces after what was decoded.

I think probably the smallest change to what's already present is to follow the `Decode` call with a call to [`Decoder.Token`](https://pkg.go.dev/encoding/json#Decoder.Token). If that returns anything other than the error `io.EOF` then we should not treat the data as JSON.

Thanks again for reporting this!

> Thanks again for reporting this!

Sure thing. Glad to help out.
Here's some additional context I dug up, which I'm sharing in the hope that it'll be useful to someone working on this bug in the future...

The earlier implementation of this took quite a different strategy:

https://github.com/hashicorp/terraform/blob/44993602b566efcb831be6bfa21dd2aecf25abef/internal/command/format/diff.go#L1028-L1051

Notably:
- This uses `cty`'s JSON decoder instead of the Go one, which means that the result is expressed in the same type system that Terraform uses for everything else. The current implementation instead decodes using Go's rules, which might have some other quirks beyond just this one issue. Perhaps we should consider switching back to using cty here, and saving the decoded result as a `cty.Value` instead of as a Go interface value.
- I assume we are using `json.Decoder` here so that we can override its handling of numbers, because Terraform interprets numbers differently than Go does. Using `cty`'s JSON decoder would also take care of that, since it decodes numbers in the `cty` way, which is also the Terraform way, since Terraform uses `cty` as its type system. Doing that would also fix this bug as a side-effect, because `ctyjson.Unmarshal` wants the entire input to contain a single JSON value, unlike `json.Decoder` which is designed for streaming.

With all of that said, I'll leave it up to whoever picks this up to decide if they want to go for a small tactical solution like the one I mentioned in my previous comment, or if they would prefer to tackle this as a larger change to use `cty/json` as the JSON decoder here, which might fix some other as-yet-unknown quirks as a side-effect.

🥇 
I'm going to lock this issue because it has been closed for _30 days_ ⏳. This helps our maintainers find and focus on the active issues.
If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.



Patch:
diff --git a/internal/command/jsonformat/computed/renderers/string.go b/internal/command/jsonformat/computed/renderers/string.go
index c3abadea7440..9d5ed899f80f 100644
--- a/internal/command/jsonformat/computed/renderers/string.go
+++ b/internal/command/jsonformat/computed/renderers/string.go
@@ -4,11 +4,11 @@
 package renderers
 
 import (
-	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/hashicorp/terraform/internal/command/jsonformat/computed"
+	"github.com/hashicorp/terraform/internal/command/jsonformat/structured"
 )
 
 type evaluatedString struct {
@@ -30,12 +30,8 @@ func evaluatePrimitiveString(value interface{}, opts computed.RenderHumanOpts) e
 	str := value.(string)
 
 	if strings.HasPrefix(str, "{") || strings.HasPrefix(str, "[") {
-
-		decoder := json.NewDecoder(strings.NewReader(str))
-		decoder.UseNumber()
-
-		var jv interface{}
-		if err := decoder.Decode(&jv); err == nil {
+		jv, err := structured.ParseJson(strings.NewReader(str))
+		if err == nil {
 			return evaluatedString{
 				String: str,
 				Json:   jv,
diff --git a/internal/command/jsonformat/structured/change.go b/internal/command/jsonformat/structured/change.go
index 5a225be84204..169841666823 100644
--- a/internal/command/jsonformat/structured/change.go
+++ b/internal/command/jsonformat/structured/change.go
@@ -6,6 +6,8 @@ package structured
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
+	"io"
 	"reflect"
 
 	"github.com/hashicorp/terraform/internal/command/jsonformat/structured/attribute_path"
@@ -282,11 +284,8 @@ func unmarshalGeneric(raw json.RawMessage) interface{} {
 		return nil
 	}
 
-	decoder := json.NewDecoder(bytes.NewReader(raw))
-	decoder.UseNumber() // We support very big (> 2^64) numbers.
-
-	var out interface{}
-	if err := decoder.Decode(&out); err != nil {
+	out, err := ParseJson(bytes.NewReader(raw))
+	if err != nil {
 		panic("unrecognized json type: " + err.Error())
 	}
 	return out
@@ -299,3 +298,21 @@ func unwrapAttributeValues(values jsonstate.AttributeValues) map[string]interfac
 	}
 	return out
 }
+
+func ParseJson(reader io.Reader) (interface{}, error) {
+	decoder := json.NewDecoder(reader)
+	decoder.UseNumber()
+
+	var jv interface{}
+	if err := decoder.Decode(&jv); err != nil {
+		return nil, err
+	}
+
+	// The JSON decoder should have consumed the entire input stream, so
+	// we should be at EOF now.
+	if token, err := decoder.Token(); err != io.EOF {
+		return nil, fmt.Errorf("unexpected token after valid JSON: %v", token)
+	}
+
+	return jv, nil
+}


Test patch:
diff --git a/internal/command/jsonformat/computed/renderers/renderer_test.go b/internal/command/jsonformat/computed/renderers/renderer_test.go
index 0a54167e2b8e..b6c9c804b1f6 100644
--- a/internal/command/jsonformat/computed/renderers/renderer_test.go
+++ b/internal/command/jsonformat/computed/renderers/renderer_test.go
@@ -111,6 +111,13 @@ null -> jsonencode(
     )
 `,
 		},
+		"primitive_create_fake_json": {
+			diff: computed.Diff{
+				Renderer: Primitive(nil, "[\"hello\"] and some more", cty.String),
+				Action:   plans.Create,
+			},
+			expected: `"[\"hello\"] and some more"`,
+		},
 		"primitive_create_null_string": {
 			diff: computed.Diff{
 				Renderer: Primitive(nil, nil, cty.String),