Repository: replicatedhq/kots
Pull request #31 (2019-08-31 00:09:50)

Write bundle to admin console


Patch:
diff --git a/go.sum b/go.sum
index 477883ebf..393a42bfa 100644
--- a/go.sum
+++ b/go.sum
@@ -645,12 +645,14 @@ github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/svanharmelen/jsonapi v0.0.0-20180618144545-0c0828c3f16d/go.mod h1:BSTlc8jOjh0niykqEGVXOLXdi9o0r0kR8tCYiMvjFgw=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/terraform-providers/terraform-provider-openstack v1.15.0/go.mod h1:2aQ6n/BtChAl1y2S60vebhyJyZXBsuAI5G4+lHrT1Ew=
diff --git a/pkg/base/add_bundle_part.go b/pkg/base/add_bundle_part.go
new file mode 100644
index 000000000..b77286ff0
--- /dev/null
+++ b/pkg/base/add_bundle_part.go
@@ -0,0 +1,34 @@
+package base
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/kots/pkg/k8sutil"
+)
+
+func AddBundlePart(baseDir string, filename string, content []byte) error {
+	_, err := os.Stat(path.Join(baseDir, "admin-console", filename))
+	if err == nil {
+		return errors.New("base bundle file already exists")
+	}
+
+	if err := ioutil.WriteFile(path.Join(baseDir, "admin-console", filename), content, 0644); err != nil {
+		return errors.Wrap(err, "failed to write file")
+	}
+
+	k, err := k8sutil.ReadKustomizationFromFile(path.Join(baseDir, "kustomization.yaml"))
+	if err != nil {
+		return errors.Wrap(err, "failed to read kustomization file")
+	}
+
+	k.Resources = append(k.Resources, path.Join("admin-console", filename))
+
+	if err := k8sutil.WriteKustomizationToFile(k, path.Join(baseDir, "kustomization.yaml")); err != nil {
+		return errors.Wrap(err, "failed to write kustomiation file")
+	}
+
+	return nil
+}
diff --git a/pkg/k8sutil/kustomization.go b/pkg/k8sutil/kustomization.go
index b9104c25a..60e9859df 100644
--- a/pkg/k8sutil/kustomization.go
+++ b/pkg/k8sutil/kustomization.go
@@ -8,6 +8,20 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+func ReadKustomizationFromFile(file string) (*kustomizetypes.Kustomization, error) {
+	b, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read kustomization file")
+	}
+
+	k := kustomizetypes.Kustomization{}
+	if err := yaml.Unmarshal(b, &k); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal kustomization")
+	}
+
+	return &k, nil
+}
+
 func WriteKustomizationToFile(kustomization *kustomizetypes.Kustomization, file string) error {
 	b, err := yaml.Marshal(kustomization)
 	if err != nil {
diff --git a/pkg/pull/archive.go b/pkg/pull/archive.go
new file mode 100644
index 000000000..6a81b40b3
--- /dev/null
+++ b/pkg/pull/archive.go
@@ -0,0 +1,97 @@
+package pull
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+
+	"github.com/mholt/archiver"
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/kots/pkg/base"
+	"github.com/replicatedhq/kots/pkg/upstream"
+	"github.com/replicatedhq/kots/pkg/util"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+func writeArchiveAsConfigMap(pullOptions PullOptions, u *upstream.Upstream, baseDir string) error {
+	// Package this app into a bundle so that the Admin Console can write it as the first version...
+	tarGz := archiver.TarGz{
+		Tar: &archiver.Tar{
+			ImplicitTopLevelFolder: true,
+		},
+	}
+
+	paths := []string{
+		path.Join(pullOptions.RootDir, u.Name, "upstream"),
+		path.Join(pullOptions.RootDir, u.Name, "base"),
+		path.Join(pullOptions.RootDir, u.Name, "overlays"),
+	}
+
+	tempDir, err := ioutil.TempDir("", "kots")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(tempDir)
+
+	if err := tarGz.Archive(paths, path.Join(tempDir, "kots-uploadable-archive.tar.gz")); err != nil {
+		return errors.Wrap(err, "failed to create tar gz")
+	}
+
+	archive, err := ioutil.ReadFile(path.Join(tempDir, "kots-uploadable-archive.tar.gz"))
+	if err != nil {
+		return errors.Wrap(err, "failed to read temp file")
+	}
+
+	encoded := base64.StdEncoding.EncodeToString(archive)
+
+	// well.
+	//
+	// let's write this encoded value to a config map with a known label
+	// so that kotsadm will upload it and ingest it as an app
+	// it's really the only way we can get the archive
+	// but etcd and config maps are limited to 1 mb
+	// so let's split it across multiple, if it's larger than 1 mb
+	encodedParts, err := util.SplitStringOnLen(encoded, 756*1000)
+	if err != nil {
+		return errors.Wrap(err, "failed to split encoded")
+	}
+
+	s := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme)
+	for i, encodedPart := range encodedParts {
+		labels := map[string]string{}
+		labels["kotsadm"] = "bundle"
+		labels["kotsadm-bundle-part"] = strconv.Itoa(i)
+
+		configMap := &corev1.ConfigMap{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: "v1",
+				Kind:       "ConfigMap",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   fmt.Sprintf("kotsadm-bundle-%d", i),
+				Labels: labels,
+			},
+			Data: map[string]string{
+				"part": encodedPart,
+			},
+		}
+
+		var b bytes.Buffer
+		if err := s.Encode(configMap, &b); err != nil {
+			return errors.Wrap(err, "failed to marshal bundle part config map")
+		}
+
+		if err := base.AddBundlePart(baseDir, fmt.Sprintf("kotsadm-bundle-%d.yaml", i), b.Bytes()); err != nil {
+			return errors.Wrap(err, "failed to write base")
+		}
+	}
+
+	return nil
+}
diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go
index fc17e585e..fd0628f37 100644
--- a/pkg/pull/pull.go
+++ b/pkg/pull/pull.go
@@ -97,10 +97,12 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) {
 		return "", errors.Wrap(err, "failed to fetch upstream")
 	}
 
+	includeAdminConsole := uri.Scheme == "replicated" && !pullOptions.ExcludeAdminConsole
+
 	writeUpstreamOptions := upstream.WriteOptions{
 		RootDir:             pullOptions.RootDir,
 		CreateAppDir:        true,
-		IncludeAdminConsole: uri.Scheme == "replicated" && !pullOptions.ExcludeAdminConsole,
+		IncludeAdminConsole: includeAdminConsole,
 		SharedPassword:      pullOptions.SharedPassword,
 	}
 	if err := u.WriteUpstream(writeUpstreamOptions); err != nil {
@@ -163,7 +165,13 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) {
 		log.FinishSpinner()
 	}
 
-	return path.Join(pullOptions.RootDir, u.Name), nil
+	if includeAdminConsole {
+		if err := writeArchiveAsConfigMap(pullOptions, u, u.GetBaseDir(writeUpstreamOptions)); err != nil {
+			return "", errors.Wrap(err, "failed to write archive as config map")
+		}
+	}
+
+	return path.Join(pullOptions.RootDir, u.Name, u.GetBaseDir(writeUpstreamOptions)), nil
 }
 
 func parseLicenseFromFile(filename string) (*kotsv1beta1.License, error) {
diff --git a/pkg/upstream/write.go b/pkg/upstream/write.go
index a5c38d545..9c3e515a7 100644
--- a/pkg/upstream/write.go
+++ b/pkg/upstream/write.go
@@ -28,8 +28,6 @@ func (u *Upstream) WriteUpstream(options WriteOptions) error {
 
 	renderDir = path.Join(renderDir, "upstream")
 
-	var previousValuesContent []byte
-
 	if options.IncludeAdminConsole {
 		adminConsoleFiles, err := generateAdminConsoleFiles(renderDir, options.SharedPassword)
 		if err != nil {
@@ -39,6 +37,7 @@ func (u *Upstream) WriteUpstream(options WriteOptions) error {
 		u.Files = append(u.Files, adminConsoleFiles...)
 	}
 
+	var previousValuesContent []byte
 	_, err := os.Stat(renderDir)
 	if err == nil {
 		// if there's already a values yaml, we need to save
diff --git a/pkg/util/util.go b/pkg/util/util.go
index e34f3d266..57d24c019 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -1,6 +1,9 @@
 package util
 
-import "net/url"
+import (
+	"bytes"
+	"net/url"
+)
 
 func IsURL(str string) bool {
 	_, err := url.ParseRequestURI(str)
@@ -28,3 +31,26 @@ func CommonSlicePrefix(first []string, second []string) []string {
 
 	return common
 }
+
+func SplitStringOnLen(str string, maxLength int) ([]string, error) {
+	if maxLength >= len(str) {
+		return []string{str}, nil
+	}
+
+	work := ""
+	result := []string{}
+
+	runes := bytes.Runes([]byte(str))
+
+	for i, r := range runes {
+		work = work + string(r)
+		if (i+1)%maxLength == 0 {
+			result = append(result, work)
+			work = ""
+		} else if i+1 == len(runes) {
+			result = append(result, work)
+		}
+	}
+
+	return result, nil
+}


Test patch:
diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go
index 0920dbdbc..a781b3d2c 100644
--- a/pkg/util/util_test.go
+++ b/pkg/util/util_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func Test_CommonSlicePrefix(t *testing.T) {
@@ -40,3 +41,42 @@ func Test_CommonSlicePrefix(t *testing.T) {
 		})
 	}
 }
+
+func Test_SplitStringOnLen(t *testing.T) {
+	tests := []struct {
+		name     string
+		in       string
+		max      int
+		expected []string
+	}{
+		{
+			name:     "single part",
+			in:       "this is a test",
+			max:      1000,
+			expected: []string{"this is a test"},
+		},
+		{
+			name:     "even parts",
+			in:       "fourfivenine",
+			max:      4,
+			expected: []string{"four", "five", "nine"},
+		},
+		{
+			name:     "too big",
+			in:       "one two six",
+			max:      7,
+			expected: []string{"one two", " six"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+
+			parts, err := SplitStringOnLen(test.in, test.max)
+			req.NoError(err)
+
+			assert.Equal(t, test.expected, parts)
+		})
+	}
+}