Repository: GoogleContainerTools/kpt
Pull request #4018 (2023-08-08 09:17:35)

Move resource version annotation to a different file
Moving resource version annotation to a new file. The newly added file can be used for storing all the metadata going forward. It is added and removed from the client side and will not be present in porch.

Please refer this for more info: https://github.com/GoogleContainerTools/kpt/issues/4017

johnbelamaric: The commit hash will change every run, so you probably need to tweak the test code to strip those out before doing deltas. Timestamp too. I had to do that with the prior version.

You may also want to not store all the annotations (in the actual code, not the test code), but I didn't mention it before because it's harmless that they are there; they are ignored by your `push` code, and the file is hidden.
bharat-m1: > The commit hash will change every run, so you probably need to tweak the test code to strip those out before doing deltas. Timestamp too. I had to do that with the prior version.
> 
> You may also want to not store all the annotations (in the actual code, not the test code), but I didn't mention it before because it's harmless that they are there; they are ignored by your `push` code, and the file is hidden.

Additional things I can find are creationTimestamp & uid Instead of removing specific annotations, I hope we can dump them only. As we have made it a hidden file it should not affect the user too.
Let me know if that is fine or we want to remove some of them.

Patch:
diff --git a/commands/alpha/rpkg/pull/command.go b/commands/alpha/rpkg/pull/command.go
index e022f80e..3fc75718 100644
--- a/commands/alpha/rpkg/pull/command.go
+++ b/commands/alpha/rpkg/pull/command.go
@@ -113,7 +113,7 @@ func (r *runner) runE(_ *cobra.Command, args []string) error {
 		return errors.E(op, err)
 	}
 
-	if err := util.AddResourceVersionAnnotation(&resources); err != nil {
+	if err := util.AddRevisionMetadata(&resources); err != nil {
 		return errors.E(op, err)
 	}
 
@@ -200,7 +200,7 @@ func createScheme() (*runtime.Scheme, error) {
 	return scheme, nil
 }
 
-var matchResourceContents = append(kio.MatchAll, kptfilev1.KptFileName)
+var matchResourceContents = append(kio.MatchAll, kptfilev1.KptFileName, kptfilev1.RevisionMetaDataFileName)
 
 func includeFile(path string) bool {
 	for _, m := range matchResourceContents {
diff --git a/commands/alpha/rpkg/push/command.go b/commands/alpha/rpkg/push/command.go
index e96f7132..13b32231 100644
--- a/commands/alpha/rpkg/push/command.go
+++ b/commands/alpha/rpkg/push/command.go
@@ -134,12 +134,12 @@ func (r *runner) runE(cmd *cobra.Command, args []string) error {
 		},
 	}
 
-	rv, err := util.GetResourceVersionAnnotation(&pkgResources)
+	rv, err := util.GetResourceVersion(&pkgResources)
 	if err != nil {
 		return errors.E(op, err)
 	}
 	pkgResources.ResourceVersion = rv
-	if err = util.RemoveResourceVersionAnnotation(&pkgResources); err != nil {
+	if err = util.RemoveRevisionMetadata(&pkgResources); err != nil {
 		return errors.E(op, err)
 	}
 
diff --git a/commands/alpha/rpkg/util/common.go b/commands/alpha/rpkg/util/common.go
index 7dafe979..5a2048ff 100644
--- a/commands/alpha/rpkg/util/common.go
+++ b/commands/alpha/rpkg/util/common.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	fnsdk "github.com/GoogleContainerTools/kpt-functions-sdk/go/fn"
+	kptfilev1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1"
 	api "github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -67,43 +68,28 @@ func GetResourceFileKubeObject(prr *api.PackageRevisionResources, file, kind, na
 	return ko, nil
 }
 
-func GetResourceVersionAnnotation(prr *api.PackageRevisionResources) (string, error) {
-	ko, err := GetResourceFileKubeObject(prr, "Kptfile", "Kptfile", "")
-
+func GetResourceVersion(prr *api.PackageRevisionResources) (string, error) {
+	ko, err := GetResourceFileKubeObject(prr, kptfilev1.RevisionMetaDataFileName, kptfilev1.RevisionMetaDataKind, "")
 	if err != nil {
 		return "", err
 	}
-	annotations := ko.GetAnnotations()
-	rv, ok := annotations[ResourceVersionAnnotation]
-	if !ok {
-		rv = ""
-	}
+	rv, _, _ := ko.NestedString("metadata", "resourceVersion")
 	return rv, nil
 }
 
-func AddResourceVersionAnnotation(prr *api.PackageRevisionResources) error {
-	ko, err := GetResourceFileKubeObject(prr, "Kptfile", "Kptfile", "")
-	if err != nil {
-		return err
+func AddRevisionMetadata(prr *api.PackageRevisionResources) error {
+	kptMetaDataKo := fnsdk.NewEmptyKubeObject()
+	kptMetaDataKo.SetAPIVersion(prr.APIVersion)
+	kptMetaDataKo.SetKind(kptfilev1.RevisionMetaDataKind)
+	if err := kptMetaDataKo.SetNestedField(prr.GetObjectMeta(), "metadata"); err != nil {
+		return fmt.Errorf("cannot set metadata: %v", err)
 	}
-
-	ko.SetAnnotation(ResourceVersionAnnotation, prr.GetResourceVersion())
-	prr.Spec.Resources["Kptfile"] = ko.String()
+	prr.Spec.Resources[kptfilev1.RevisionMetaDataFileName] = kptMetaDataKo.String()
 
 	return nil
 }
 
-func RemoveResourceVersionAnnotation(prr *api.PackageRevisionResources) error {
-	ko, err := GetResourceFileKubeObject(prr, "Kptfile", "Kptfile", "")
-	if err != nil {
-		return err
-	}
-
-	_, err = ko.RemoveNestedField("metadata", "annotations", ResourceVersionAnnotation)
-	if err != nil {
-		return err
-	}
-	prr.Spec.Resources["Kptfile"] = ko.String()
-
+func RemoveRevisionMetadata(prr *api.PackageRevisionResources) error {
+	delete(prr.Spec.Resources, kptfilev1.RevisionMetaDataFileName)
 	return nil
 }
diff --git a/pkg/api/kptfile/v1/types.go b/pkg/api/kptfile/v1/types.go
index eed630e8..1f9659a8 100644
--- a/pkg/api/kptfile/v1/types.go
+++ b/pkg/api/kptfile/v1/types.go
@@ -29,6 +29,10 @@ import (
 const (
 	KptFileName = "Kptfile"
 
+	RevisionMetaDataFileName = ".KptRevisionMetadata"
+
+	RevisionMetaDataKind = "KptRevisionMetadata"
+
 	// Deprecated: prefer KptFileGVK
 	KptFileKind = "Kptfile"
 


Test patch:
diff --git a/commands/alpha/rpkg/pull/command_test.go b/commands/alpha/rpkg/pull/command_test.go
index 2adc9b0b..ee8d60fb 100644
--- a/commands/alpha/rpkg/pull/command_test.go
+++ b/commands/alpha/rpkg/pull/command_test.go
@@ -68,13 +68,24 @@ data:
 apiVersion: config.kubernetes.io/v1
 kind: ResourceList
 items:
+- apiVersion: porch.kpt.dev/v1alpha1
+  kind: KptRevisionMetadata
+  metadata:
+    name: repo-fjdos9u2nfe2f32
+    namespace: ns
+    creationTimestamp: null
+    resourceVersion: "999"
+    annotations:
+      config.kubernetes.io/index: '0'
+      internal.config.kubernetes.io/index: '0'
+      internal.config.kubernetes.io/path: '.KptRevisionMetadata'
+      config.kubernetes.io/path: '.KptRevisionMetadata'
 - apiVersion: kpt.dev/v1
   kind: Kptfile
   metadata:
     name: bar
     annotations:
       config.kubernetes.io/local-config: "true"
-      internal.kpt.dev/resource-version: "999"
       config.kubernetes.io/index: '0'
       internal.config.kubernetes.io/index: '0'
       internal.config.kubernetes.io/path: 'Kptfile'
@@ -121,13 +132,24 @@ data:
 apiVersion: config.kubernetes.io/v1
 kind: ResourceList
 items:
+- apiVersion: porch.kpt.dev/v1alpha1
+  kind: KptRevisionMetadata
+  metadata:
+    name: repo-fjdos9u2nfe2f32
+    namespace: ns
+    creationTimestamp: null
+    resourceVersion: "999"
+    annotations:
+      config.kubernetes.io/index: '0'
+      internal.config.kubernetes.io/index: '0'
+      internal.config.kubernetes.io/path: '.KptRevisionMetadata'
+      config.kubernetes.io/path: '.KptRevisionMetadata'
 - apiVersion: kpt.dev/v1
   kind: Kptfile
   metadata:
     name: bar
     annotations:
       config.kubernetes.io/local-config: "true"
-      internal.kpt.dev/resource-version: "999"
       config.kubernetes.io/index: '0'
       internal.config.kubernetes.io/index: '0'
       internal.config.kubernetes.io/path: 'Kptfile'
diff --git a/e2e/porch_test.go b/e2e/porch_test.go
index 2f9b277e..2e7e7ee3 100644
--- a/e2e/porch_test.go
+++ b/e2e/porch_test.go
@@ -158,13 +158,14 @@ func reorderYamlStdout(t *testing.T, buf *bytes.Buffer) {
 		return
 	}
 
-	// strip out the internal.kpt.dev/resource-version
-	// annotation, because that will change with every run
+	// strip out the resourceVersion:, creationTimestamp:
+	// because that will change with every run
 	scanner := bufio.NewScanner(buf)
 	var newBuf bytes.Buffer
 	for scanner.Scan() {
 		line := scanner.Text()
-		if !strings.Contains(line, "internal.kpt.dev/resource-version:") {
+		if !strings.Contains(line, "resourceVersion:") &&
+			!strings.Contains(line, "creationTimestamp:") {
 			newBuf.Write([]byte(line))
 			newBuf.Write([]byte("\n"))
 		}
diff --git a/e2e/testdata/porch/rpkg-clone/config.yaml b/e2e/testdata/porch/rpkg-clone/config.yaml
index 05ba6bf2..ef6bbaa9 100644
--- a/e2e/testdata/porch/rpkg-clone/config.yaml
+++ b/e2e/testdata/porch/rpkg-clone/config.yaml
@@ -61,6 +61,17 @@ commands:
     stdout: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-3465eed5831e5c372243d048631c8ef1666b47d6
+          namespace: rpkg-clone
+          uid: uid:basens-clone:clone-2
       - apiVersion: kpt.dev/v1
         info:
           description: sample description
@@ -107,6 +118,17 @@ commands:
     stdout: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-b67f9ce14d378317ba83c9504eab9cc024932dd3
+          namespace: rpkg-clone
+          uid: uid:empty-clone:clone-1
       - apiVersion: kpt.dev/v1
         info:
           description: Empty Blueprint
@@ -114,6 +136,7 @@ commands:
         metadata:
           annotations:
             config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: Kptfile
             internal.config.kubernetes.io/index: "0"
             internal.config.kubernetes.io/path: Kptfile
             internal.kpt.dev/upstream-identifier: kpt.dev|Kptfile|default|empty-clone
diff --git a/e2e/testdata/porch/rpkg-copy/config.yaml b/e2e/testdata/porch/rpkg-copy/config.yaml
index c0960ba7..eb5965ef 100644
--- a/e2e/testdata/porch/rpkg-copy/config.yaml
+++ b/e2e/testdata/porch/rpkg-copy/config.yaml
@@ -49,6 +49,17 @@ commands:
     stdout: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-a29df72d1135fd010ea49f4d4877001dee423be6
+          namespace: rpkg-copy
+          uid: uid:basens-edit:copy-2
       - apiVersion: kpt.dev/v1
         info:
           description: sample description
diff --git a/e2e/testdata/porch/rpkg-init-deploy/config.yaml b/e2e/testdata/porch/rpkg-init-deploy/config.yaml
index b030da32..9f757e12 100644
--- a/e2e/testdata/porch/rpkg-init-deploy/config.yaml
+++ b/e2e/testdata/porch/rpkg-init-deploy/config.yaml
@@ -39,6 +39,17 @@ commands:
     stdout: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-628abd0a0903f5de6cb3604d917724f6fc1b5e08
+          namespace: rpkg-init-deploy
+          uid: uid:deploy-package:deploy
       - apiVersion: kpt.dev/v1
         info:
           description: Test Package Description
diff --git a/e2e/testdata/porch/rpkg-init/config.yaml b/e2e/testdata/porch/rpkg-init/config.yaml
index db2dd906..e0392f4a 100644
--- a/e2e/testdata/porch/rpkg-init/config.yaml
+++ b/e2e/testdata/porch/rpkg-init/config.yaml
@@ -38,6 +38,17 @@ commands:
     stdout: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-95686470a1fd3a3ba726cce4c8f449f6bbe2b02a
+          namespace: rpkg-init
+          uid: uid:init-package:init-1
       - apiVersion: kpt.dev/v1
         info:
           description: Test Package Description
diff --git a/e2e/testdata/porch/rpkg-push/config.yaml b/e2e/testdata/porch/rpkg-push/config.yaml
index 378ab6e7..999e1b6b 100644
--- a/e2e/testdata/porch/rpkg-push/config.yaml
+++ b/e2e/testdata/porch/rpkg-push/config.yaml
@@ -24,6 +24,17 @@ commands:
     stdout: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-efe3d01c68dfdcdd69114c9a7c65cce0d662a46f
+          namespace: rpkg-push
+          uid: uid:test-package:push
       - apiVersion: kpt.dev/v1
         info:
           description: sample description
@@ -59,6 +70,17 @@ commands:
     stdin: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-efe3d01c68dfdcdd69114c9a7c65cce0d662a46f
+          namespace: rpkg-push
+          uid: uid:test-package:push
       - apiVersion: kpt.dev/v1
         info:
           description: sample description
@@ -122,6 +144,17 @@ commands:
     stdout: |
       apiVersion: config.kubernetes.io/v1
       items:
+      - apiVersion: ""
+        kind: KptRevisionMetadata
+        metadata:
+          annotations:
+            config.kubernetes.io/index: "0"
+            config.kubernetes.io/path: .KptRevisionMetadata
+            internal.config.kubernetes.io/index: "0"
+            internal.config.kubernetes.io/path: .KptRevisionMetadata
+          name: git-efe3d01c68dfdcdd69114c9a7c65cce0d662a46f
+          namespace: rpkg-push
+          uid: uid:test-package:push
       - apiVersion: kpt.dev/v1
         info:
           description: Updated Test Package Description