Repository: GoogleCloudPlatform/terraform-validator
Pull request #503 (2022-02-02 19:35:05)

Moving to a third party tool for diff
Currently TFV test logs are hard to parse because the diff is output as a giant object. It would be nice to reduce the noice here and assist the developer in pinpointing the exact issues.

ScottSuarez: Hi sorry for the delay here. Lot has been happening on my end.. I'll look at this a soon as possible

Patch:
diff --git a/go.mod b/go.mod
index 1103c441..d461519d 100644
--- a/go.mod
+++ b/go.mod
@@ -18,6 +18,7 @@ require (
 	github.com/onsi/gomega v1.17.0 // indirect
 	github.com/open-policy-agent/opa v0.36.1 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/r3labs/diff/v2 v2.14.6
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.3.0
diff --git a/go.sum b/go.sum
index 1c1e5f74..293102a8 100644
--- a/go.sum
+++ b/go.sum
@@ -1062,6 +1062,8 @@ github.com/quasilyte/go-ruleguard/dsl v0.3.2/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQP
 github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mod h1:7JTjp89EGyU1d6XfBiXihJNG37wB2VRkd125Q1u7Plc=
 github.com/quasilyte/go-ruleguard/rules v0.0.0-20210203162857-b223e0831f88/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50=
 github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
+github.com/r3labs/diff/v2 v2.14.6 h1:jd/9mrYWG3rQLhdqGEAChzbV8HOk9iL2b+LPre8yqzU=
+github.com/r3labs/diff/v2 v2.14.6/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=


Test patch:
diff --git a/test/cli_test.go b/test/cli_test.go
index f3b7f3ca..459fd05d 100644
--- a/test/cli_test.go
+++ b/test/cli_test.go
@@ -22,7 +22,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
 )
@@ -229,7 +229,9 @@ type compareConvertOutputFunc func(t *testing.T, expected []google.Asset, actual
 func compareUnmergedConvertOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
 	expectedAssets := normalizeAssets(t, expected, offline)
 	actualAssets := normalizeAssets(t, actual, offline)
-	require.ElementsMatch(t, expectedAssets, actualAssets)
+	change := getDiff(t, actualAssets, expectedAssets)
+	assert.Equal(t, len(change), 0, "There should not no difference")
+
 }
 
 // For merged IAM members, only consider whether the expected members are present.
@@ -270,7 +272,8 @@ func compareMergedIamMemberOutput(t *testing.T, expected []google.Asset, actual
 
 	expectedAssets := normalizeAssets(t, expected, offline)
 	actualAssets := normalizeAssets(t, normalizedActual, offline)
-	require.ElementsMatch(t, expectedAssets, actualAssets)
+	change := getDiff(t, actualAssets, expectedAssets)
+	assert.Equal(t, len(change), 0, "There should not no difference")
 }
 
 // For merged IAM bindings, only consider whether the expected bindings are as expected.
@@ -300,5 +303,6 @@ func compareMergedIamBindingOutput(t *testing.T, expected []google.Asset, actual
 
 	expectedAssets := normalizeAssets(t, expected, offline)
 	actualAssets := normalizeAssets(t, normalizedActual, offline)
-	require.ElementsMatch(t, expectedAssets, actualAssets)
+	change := getDiff(t, actualAssets, expectedAssets)
+	assert.Equal(t, len(change), 0, "There should not no difference")
 }
diff --git a/test/read_test.go b/test/read_test.go
index bb995fbb..ee908380 100644
--- a/test/read_test.go
+++ b/test/read_test.go
@@ -9,7 +9,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
 	"github.com/GoogleCloudPlatform/terraform-validator/tfgcv"
@@ -141,7 +141,9 @@ func TestReadPlannedAssetsCoverage(t *testing.T) {
 
 			expectedAssets := normalizeAssets(t, want, true)
 			actualAssets := normalizeAssets(t, got, true)
-			require.ElementsMatch(t, actualAssets, expectedAssets)
+
+			change := getDiff(t, actualAssets, expectedAssets)
+			assert.Equal(t, len(change), 0, "There should not no difference")
 		})
 	}
 }
diff --git a/test/utils_test.go b/test/utils_test.go
index a47c2acd..2a868a9d 100644
--- a/test/utils_test.go
+++ b/test/utils_test.go
@@ -14,13 +14,15 @@ import (
 	"testing"
 
 	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
-	"github.com/stretchr/testify/require"
+	"github.com/r3labs/diff/v2"
+	"github.com/stretchr/testify/assert"
 )
 
 func defaultCompareConverterOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
 	expectedAssets := normalizeAssets(t, expected, offline)
 	actualAssets := normalizeAssets(t, actual, offline)
-	require.ElementsMatch(t, expectedAssets, actualAssets)
+	change := getDiff(t, actualAssets, expectedAssets)
+	assert.Equal(t, len(change), 0, "There should not no difference")
 }
 
 func testConvertCommand(t *testing.T, dir, name string, offline bool, compare compareConvertOutputFunc) {
@@ -232,3 +234,19 @@ func getTestPrefix() string {
 
 `, data.Provider["version"], credentials)
 }
+
+func getDiff(t *testing.T, actualAssets, expectedAssets []google.Asset) diff.Changelog {
+	d, err := diff.NewDiffer(diff.SliceOrdering(false))
+	if err != nil {
+		panic(err)
+	}
+	changes, _ := d.Diff(actualAssets, expectedAssets)
+	if len(changes) == 0 {
+		return changes
+	}
+	for _, i := range changes {
+		t.Log("[Error] The folllowing change has occoured")
+		t.Logf("        Element %v has been %sd from `%v` to  `%v`\n", i.Path, i.Type, i.From, i.To)
+	}
+	return changes
+}