Repository: helm/helm (go)

Issue:
index.json - necessary for large projects
There are multiple issues reported regarding performance problems when the index.yaml size starts to increase:
https://github.com/helm/helm/issues/6155
https://github.com/helm/helm/issues/9931
https://github.com/helm/helm/issues/9865 (not related to the index file being .yaml, but still related)

@mattfarina has created a benchmark showing that JSON parsing is an order of magnitude faster: https://github.com/mattfarina/yamlbench
@mattfarina also created https://github.com/mattfarina/helm-index-json to show some JSON split testing.

As more and more indices are growing in size, this problem is bound to happen to more and more users.

Would it be due time to look into placing a .json index file next to the .yaml file, so that Helm opts to use the .json file but falls back to .yaml if it does not exist?

Just the switch from YAML -> JSON should help significantly in this area. JSON is also more natural for computer-written data as opposed to YAML which might be more suited for data where users also at times edit the files manually.

Pull request:
Add support for creating repository indexes in JSON format
**What this PR does / why we need it**:

This adds support for creating an `index.yaml` in JSON format using `helm repo index [...] --json`.

Fixes #10542
Supersedes #12151

**Special notes for your reviewer**:

Given that it does not create unnecessary HTTP requests to discover `index.json` and/or `index.yaml` files hosted by a repository server, nor requires rigorous changes to the caching logic, and is fully backwards compatible due to YAML being a superset of JSON. This appears to be the least invasive way to add support for the JSON format.

Benchmark using Bitnami's infamous ["full index" archive](https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/index.yaml):

```console
$  benchstat old.txt new.txt                                  
goos: linux
goarch: amd64
pkg: helm.sh/helm/v3/pkg/repo
cpu: Intel(R) Core(TM) i7-10750H CPU @ 2.60GHz
                      │   old.txt    │               new.txt               │
                      │    sec/op    │   sec/op     vs base                │
LoadIndexFile/YAML-12     2.692 ± 1%    2.770 ± 3%        ~ (p=0.242 n=20)
LoadIndexFile/JSON-12   2318.3m ± 0%   582.6m ± 1%  -74.87% (p=0.000 n=20)
geomean                   2.498         1.270       -49.15%

                      │    old.txt    │               new.txt                │
                      │     B/op      │     B/op      vs base                │
LoadIndexFile/YAML-12    836.0Mi ± 0%   836.0Mi ± 0%        ~ (p=0.355 n=20)
LoadIndexFile/JSON-12   671.99Mi ± 0%   93.84Mi ± 0%  -86.03% (p=0.000 n=20)
geomean                  749.5Mi        280.1Mi       -62.63%

                      │   old.txt    │               new.txt               │
                      │  allocs/op   │  allocs/op   vs base                │
LoadIndexFile/YAML-12    12.03M ± 0%   12.03M ± 0%        ~ (p=0.683 n=20)
LoadIndexFile/JSON-12   10.015M ± 0%   1.078M ± 0%  -89.23% (p=0.000 n=20)
geomean                  10.98M        3.601M       -67.19%
```

**If applicable**:
- [ ] this PR contains documentation
- [x] this PR contains unit tests
- [x] this PR has been tested for backwards compatibility

Patch:
diff --git a/cmd/helm/repo_index.go b/cmd/helm/repo_index.go
index 917acd442..3960380d1 100644
--- a/cmd/helm/repo_index.go
+++ b/cmd/helm/repo_index.go
@@ -43,6 +43,7 @@ type repoIndexOptions struct {
 	dir   string
 	url   string
 	merge string
+	json  bool
 }
 
 func newRepoIndexCmd(out io.Writer) *cobra.Command {
@@ -70,6 +71,7 @@ func newRepoIndexCmd(out io.Writer) *cobra.Command {
 	f := cmd.Flags()
 	f.StringVar(&o.url, "url", "", "url of chart repository")
 	f.StringVar(&o.merge, "merge", "", "merge the generated index into the given index")
+	f.BoolVar(&o.json, "json", false, "output in JSON format")
 
 	return cmd
 }
@@ -80,10 +82,10 @@ func (i *repoIndexOptions) run(out io.Writer) error {
 		return err
 	}
 
-	return index(path, i.url, i.merge)
+	return index(path, i.url, i.merge, i.json)
 }
 
-func index(dir, url, mergeTo string) error {
+func index(dir, url, mergeTo string, json bool) error {
 	out := filepath.Join(dir, "index.yaml")
 
 	i, err := repo.IndexDirectory(dir, url)
@@ -95,7 +97,7 @@ func index(dir, url, mergeTo string) error {
 		var i2 *repo.IndexFile
 		if _, err := os.Stat(mergeTo); os.IsNotExist(err) {
 			i2 = repo.NewIndexFile()
-			i2.WriteFile(mergeTo, 0644)
+			writeIndexFile(i2, mergeTo, json)
 		} else {
 			i2, err = repo.LoadIndexFile(mergeTo)
 			if err != nil {
@@ -105,5 +107,12 @@ func index(dir, url, mergeTo string) error {
 		i.Merge(i2)
 	}
 	i.SortEntries()
+	return writeIndexFile(i, out, json)
+}
+
+func writeIndexFile(i *repo.IndexFile, out string, json bool) error {
+	if json {
+		return i.WriteJSONFile(out, 0644)
+	}
 	return i.WriteFile(out, 0644)
 }
diff --git a/pkg/repo/index.go b/pkg/repo/index.go
index ba2e365c8..8a23ba060 100644
--- a/pkg/repo/index.go
+++ b/pkg/repo/index.go
@@ -18,6 +18,7 @@ package repo
 
 import (
 	"bytes"
+	"encoding/json"
 	"log"
 	"os"
 	"path"
@@ -232,6 +233,18 @@ func (i IndexFile) WriteFile(dest string, mode os.FileMode) error {
 	return fileutil.AtomicWriteFile(dest, bytes.NewReader(b), mode)
 }
 
+// WriteJSONFile writes an index file in JSON format to the given destination
+// path.
+//
+// The mode on the file is set to 'mode'.
+func (i IndexFile) WriteJSONFile(dest string, mode os.FileMode) error {
+	b, err := json.MarshalIndent(i, "", "  ")
+	if err != nil {
+		return err
+	}
+	return fileutil.AtomicWriteFile(dest, bytes.NewReader(b), mode)
+}
+
 // Merge merges the given index file into this index.
 //
 // This merges by name and version.
@@ -336,7 +349,7 @@ func loadIndex(data []byte, source string) (*IndexFile, error) {
 		return i, ErrEmptyIndexYaml
 	}
 
-	if err := yaml.UnmarshalStrict(data, i); err != nil {
+	if err := jsonOrYamlUnmarshal(data, i); err != nil {
 		return i, err
 	}
 
@@ -361,3 +374,17 @@ func loadIndex(data []byte, source string) (*IndexFile, error) {
 	}
 	return i, nil
 }
+
+// jsonOrYamlUnmarshal unmarshals the given byte slice containing JSON or YAML
+// into the provided interface.
+//
+// It automatically detects whether the data is in JSON or YAML format by
+// checking its validity as JSON. If the data is valid JSON, it will use the
+// `encoding/json` package to unmarshal it. Otherwise, it will use the
+// `sigs.k8s.io/yaml` package to unmarshal the YAML data.
+func jsonOrYamlUnmarshal(b []byte, i interface{}) error {
+	if json.Valid(b) {
+		return json.Unmarshal(b, i)
+	}
+	return yaml.UnmarshalStrict(b, i)
+}


Test patch:
diff --git a/cmd/helm/repo_index_test.go b/cmd/helm/repo_index_test.go
index ae3390154..9ba3595bf 100644
--- a/cmd/helm/repo_index_test.go
+++ b/cmd/helm/repo_index_test.go
@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"io"
 	"os"
 	"path/filepath"
@@ -68,6 +69,28 @@ func TestRepoIndexCmd(t *testing.T) {
 		t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version)
 	}
 
+	b, err := os.ReadFile(destIndex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if json.Valid(b) {
+		t.Error("did not expect index file to be valid json")
+	}
+
+	// Test with `--json`
+
+	c.ParseFlags([]string{"--json", "true"})
+	if err := c.RunE(c, []string{dir}); err != nil {
+		t.Error(err)
+	}
+
+	if b, err = os.ReadFile(destIndex); err != nil {
+		t.Fatal(err)
+	}
+	if !json.Valid(b) {
+		t.Error("index file is not valid json")
+	}
+
 	// Test with `--merge`
 
 	// Remove first two charts.
diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go
index bbc48c97e..efb50ba6a 100644
--- a/pkg/repo/index_test.go
+++ b/pkg/repo/index_test.go
@@ -19,6 +19,7 @@ package repo
 import (
 	"bufio"
 	"bytes"
+	"encoding/json"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -37,6 +38,7 @@ const (
 	annotationstestfile = "testdata/local-index-annotations.yaml"
 	chartmuseumtestfile = "testdata/chartmuseum-index.yaml"
 	unorderedTestfile   = "testdata/local-index-unordered.yaml"
+	jsonTestfile        = "testdata/local-index.json"
 	testRepo            = "test-repo"
 	indexWithDuplicates = `
 apiVersion: v1
@@ -145,6 +147,10 @@ func TestLoadIndex(t *testing.T) {
 			Name:     "chartmuseum index file",
 			Filename: chartmuseumtestfile,
 		},
+		{
+			Name:     "JSON index file",
+			Filename: jsonTestfile,
+		},
 	}
 
 	for _, tc := range tests {
@@ -548,6 +554,27 @@ func TestIndexWrite(t *testing.T) {
 	}
 }
 
+func TestIndexJSONWrite(t *testing.T) {
+	i := NewIndexFile()
+	if err := i.MustAdd(&chart.Metadata{APIVersion: "v2", Name: "clipper", Version: "0.1.0"}, "clipper-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890"); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	dir := t.TempDir()
+	testpath := filepath.Join(dir, "test")
+	i.WriteJSONFile(testpath, 0600)
+
+	got, err := os.ReadFile(testpath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !json.Valid(got) {
+		t.Fatal("Index files doesn't contain valid JSON")
+	}
+	if !strings.Contains(string(got), "clipper-0.1.0.tgz") {
+		t.Fatal("Index files doesn't contain expected content")
+	}
+}
+
 func TestAddFileIndexEntriesNil(t *testing.T) {
 	i := NewIndexFile()
 	i.APIVersion = chart.APIVersionV1
diff --git a/pkg/repo/testdata/local-index.json b/pkg/repo/testdata/local-index.json
new file mode 100644
index 000000000..25296d5ca
--- /dev/null
+++ b/pkg/repo/testdata/local-index.json
@@ -0,0 +1,53 @@
+{
+  "apiVersion": "v1",
+  "entries": {
+    "nginx": [
+      {
+        "urls": ["https://charts.helm.sh/stable/nginx-0.2.0.tgz"],
+        "name": "nginx",
+        "description": "string",
+        "version": "0.2.0",
+        "home": "https://github.com/something/else",
+        "digest": "sha256:1234567890abcdef",
+        "keywords": ["popular", "web server", "proxy"],
+        "apiVersion": "v2"
+      },
+      {
+        "urls": ["https://charts.helm.sh/stable/nginx-0.1.0.tgz"],
+        "name": "nginx",
+        "description": "string",
+        "version": "0.1.0",
+        "home": "https://github.com/something",
+        "digest": "sha256:1234567890abcdef",
+        "keywords": ["popular", "web server", "proxy"],
+        "apiVersion": "v2"
+      }
+    ],
+    "alpine": [
+      {
+        "urls": [
+          "https://charts.helm.sh/stable/alpine-1.0.0.tgz",
+          "http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz"
+        ],
+        "name": "alpine",
+        "description": "string",
+        "version": "1.0.0",
+        "home": "https://github.com/something",
+        "keywords": ["linux", "alpine", "small", "sumtin"],
+        "digest": "sha256:1234567890abcdef",
+        "apiVersion": "v2"
+      }
+    ],
+    "chartWithNoURL": [
+      {
+        "name": "chartWithNoURL",
+        "description": "string",
+        "version": "1.0.0",
+        "home": "https://github.com/something",
+        "keywords": ["small", "sumtin"],
+        "digest": "sha256:1234567890abcdef",
+        "apiVersion": "v2"
+      }
+    ]
+  }
+}