Repository: iter8-tools/iter8
Pull request #1230 (2022-04-25 20:06:06)

Fix more bugs in Kubernetes experimentation
1. It turns out that while specificying RBACs for create verbs, one cannot specify resource names ... please refer to the note [here](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources). This leaves us with two options ... 
  - create the experiment spec and result as part of the Helm install, and the experiment job only does update of the created secret.
  - create the result in the experiment job, but make the create RBAC expansive (i.e., create secrets, instead of creating secret with a specific name).

I chose the first option to keep the RBAC profile minimal. For starters, I don't need a create verb in the RBAC at all. I also eliminated release names from Helm-installed-resource-names. This means, the RBACs do not have to change from release-to-release, paving the way eventually for autox.


Patch:
diff --git a/.github/workflows/assets.yaml b/.github/workflows/assets.yaml
index 62b81e77..7a057238 100644
--- a/.github/workflows/assets.yaml
+++ b/.github/workflows/assets.yaml
@@ -98,18 +98,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-and-push
     steps:
-    - name: Install Iter8
-      run: |
-        tagref=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-        wget https://github.com/iter8-tools/iter8/releases/download/$tagref/iter8-linux-amd64.tar.gz
-        tar -xvf iter8-linux-amd64.tar.gz
-        mv linux-amd64/iter8 /usr/local/bin
+    - uses: iter8-tools/iter8@v0.10
     - uses: engineerd/setup-kind@v0.5.0
     - name: create app
       run: |
         kubectl create deployment httpbin --image=kennethreitz/httpbin
         kubectl expose deployment httpbin --type=ClusterIP --port=80
-        kubectl wait --for=condition=available --timeout=60s deployment/httpbin
+        kubectl wait --for=condition=available --timeout=300s deployment/httpbin
     - name: iter8 k launch
       run: |
         iter8 k launch -c load-test-http \
@@ -138,7 +133,7 @@ jobs:
       run: |
         kubectl create deploy hello --image=docker.io/grpc/java-example-hostname:latest --port=50051
         kubectl expose deploy hello --port=50051
-        kubectl wait --for=condition=available --timeout=60s deployment/hello
+        kubectl wait --for=condition=available --timeout=300s deployment/hello
     - name: iter8 k launch
       run: |
         iter8 k launch -c load-test-grpc \
diff --git a/action/launch.go b/action/launch.go
index d74e1dcc..9cd007a1 100644
--- a/action/launch.go
+++ b/action/launch.go
@@ -119,10 +119,5 @@ func (lOpts *LaunchOpts) KubeRun() error {
 	}
 	driver.UpdateChartDependencies(gOpts.chartDir(), lOpts.EnvSettings)
 
-	if lOpts.Revision > 0 { // last release found; setup upgrade
-		return lOpts.KubeDriver.Upgrade(gOpts.chartDir(), lOpts.Options, lOpts.Group, lOpts.DryRun)
-	} else {
-		// no release found; setup install
-		return lOpts.KubeDriver.Install(gOpts.chartDir(), lOpts.Options, lOpts.Group, lOpts.DryRun)
-	}
+	return lOpts.KubeDriver.Launch(gOpts.chartDir(), lOpts.Options, lOpts.Group, lOpts.DryRun)
 }
diff --git a/base/log/log.go b/base/log/log.go
index f5fce2db..9a7ce8bc 100644
--- a/base/log/log.go
+++ b/base/log/log.go
@@ -19,6 +19,8 @@ type Iter8Logger struct {
 
 // StackTrace is the trace from external components like a shell scripts run by an Iter8 task.
 type StackTrace struct {
+	// prefix is the string with which external traces are prefixed
+	prefix string
 	// Trace is the raw trace
 	Trace string
 }
@@ -42,17 +44,26 @@ func init() {
 // WithStackTrace yields a log entry with a formatted stack trace field embedded in it.
 func (l *Iter8Logger) WithStackTrace(t string) *logrus.Entry {
 	return l.WithField("stack-trace", &StackTrace{
-		Trace: t,
+		prefix: "::Trace:: ",
+		Trace:  t,
 	})
 }
 
-// String processes stack traces by prefixing each line of the trace with ::Trace::.
+// WithSpaces yields a log entry with a formatted indent embedded in it.
+func (l *Iter8Logger) WithIndentedTrace(t string) *logrus.Entry {
+	return l.WithField("indented-trace", &StackTrace{
+		prefix: "  ",
+		Trace:  t,
+	})
+}
+
+// String processes stack traces by prefixing each line of the trace with prefix.
 // This enables other tools like grep to easily filter out these traces if needed.
 func (st *StackTrace) String() string {
 	out := "below ... \n"
 	scanner := bufio.NewScanner(strings.NewReader(st.Trace))
 	for scanner.Scan() {
-		out += "::Trace:: " + scanner.Text() + "\n"
+		out += st.prefix + scanner.Text() + "\n"
 	}
 	out = strings.TrimSuffix(out, "\n")
 	return out
diff --git a/charts/iter8lib/templates/_k-job.tpl b/charts/iter8lib/templates/_k-job.tpl
index c41e2ce9..9d7e11a2 100644
--- a/charts/iter8lib/templates/_k-job.tpl
+++ b/charts/iter8lib/templates/_k-job.tpl
@@ -1,21 +1,22 @@
 {{- define "k.job" -}}
-{{- $name := printf "%v-%v" .Release.Name .Release.Revision -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ $name }}-job
+  name: {{ .Release.Name }}-{{ .Release.Revision }}-job
+  annotations:
+    iter8.tools/revision: {{ .Release.Revision }}
 spec:
   template:
     spec:
       containers:
       - name: iter8
-        image: {{  .Values.iter8lib.global.iter8Image }}
+        image: {{  .Values.iter8lib.iter8Image }}
         imagePullPolicy: Always
         command:
         - "/bin/sh"
         - "-c"
         - |
-          iter8 k run --namespace {{ .Release.Namespace }} --group {{ .Release.Name }} --revision {{ .Release.Revision }}
+          iter8 k run --namespace {{ .Release.Namespace }} --group {{ .Release.Name }}
       restartPolicy: Never
   backoffLimit: 0
 {{- end }}
diff --git a/charts/iter8lib/templates/_k-manifest.tpl b/charts/iter8lib/templates/_k-manifest.tpl
index 60348b82..27b7e2c2 100644
--- a/charts/iter8lib/templates/_k-manifest.tpl
+++ b/charts/iter8lib/templates/_k-manifest.tpl
@@ -7,6 +7,8 @@
 ---
 {{ include "k.spec.rolebinding" . }}
 ---
+{{ include "k.result.secret" . }}
+---
 {{ include "k.result.role" . }}
 ---
 {{ include "k.result.rolebinding" . }}
diff --git a/charts/iter8lib/templates/_k-result-role.tpl b/charts/iter8lib/templates/_k-result-role.tpl
index 9a792171..f6266591 100644
--- a/charts/iter8lib/templates/_k-result-role.tpl
+++ b/charts/iter8lib/templates/_k-result-role.tpl
@@ -1,12 +1,13 @@
 {{- define "k.result.role" -}}
-{{- $name := printf "%v-%v" .Release.Name .Release.Revision -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ $name }}-result-role
+  name: {{ .Release.Name }}-result-role
+  annotations:
+    iter8.tools/revision: {{ .Release.Revision }}
 rules:
 - apiGroups: [""]
+  resourceNames: ["{{ .Release.Name }}-result"]
   resources: ["secrets"]
-  resourceNames: ["{{ $name }}-result"]
-  verbs: ["create", "get", "update"]
+  verbs: ["get", "update"]
 {{- end }}
diff --git a/charts/iter8lib/templates/_k-result-rolebinding.tpl b/charts/iter8lib/templates/_k-result-rolebinding.tpl
index b9903110..0ee54661 100644
--- a/charts/iter8lib/templates/_k-result-rolebinding.tpl
+++ b/charts/iter8lib/templates/_k-result-rolebinding.tpl
@@ -1,14 +1,16 @@
 {{- define "k.result.rolebinding" -}}
-{{- $name := printf "%v-%v" .Release.Name .Release.Revision -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ $name }}-result-rolebinding
+  name: {{ .Release.Name }}-result-rolebinding
+  annotations:
+    iter8.tools/revision: {{ .Release.Revision }}
 subjects:
 - kind: ServiceAccount
   name: default
+  namespace: {{ .Release.Namespace }}
 roleRef:
   kind: Role
-  name: {{ $name }}-result-role
+  name: {{ .Release.Name }}-result-role
   apiGroup: rbac.authorization.k8s.io
 {{- end }}
diff --git a/charts/iter8lib/templates/_k-result-secret.tpl b/charts/iter8lib/templates/_k-result-secret.tpl
new file mode 100644
index 00000000..b7fc1768
--- /dev/null
+++ b/charts/iter8lib/templates/_k-result-secret.tpl
@@ -0,0 +1,14 @@
+{{- define "k.result.secret" -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-result
+  annotations:
+    iter8.tools/revision: {{ .Release.Revision }}
+stringData:
+  experiment.yaml: |
+    startTime:         {{ now }},
+    numCompletedTasks: 0,
+    failure:           false,
+    iter8Version:      {{ .Values.iter8lib.majorMinor }},
+{{- end }}
diff --git a/charts/iter8lib/templates/_k-spec-role.tpl b/charts/iter8lib/templates/_k-spec-role.tpl
index dc5519ed..30761141 100644
--- a/charts/iter8lib/templates/_k-spec-role.tpl
+++ b/charts/iter8lib/templates/_k-spec-role.tpl
@@ -1,12 +1,13 @@
 {{- define "k.spec.role" -}}
-{{- $name := printf "%v-%v" .Release.Name .Release.Revision -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ $name }}-spec-role
+  name: {{ .Release.Name }}-spec-role
+  annotations:
+    iter8.tools/revision: {{ .Release.Revision }}
 rules:
 - apiGroups: [""]
+  resourceNames: ["{{ .Release.Name }}-spec"]
   resources: ["secrets"]
-  resourceNames: ["{{ $name }}-spec"]
   verbs: ["get"]
 {{- end }}
diff --git a/charts/iter8lib/templates/_k-spec-rolebinding.tpl b/charts/iter8lib/templates/_k-spec-rolebinding.tpl
index 115bb616..ca67503d 100644
--- a/charts/iter8lib/templates/_k-spec-rolebinding.tpl
+++ b/charts/iter8lib/templates/_k-spec-rolebinding.tpl
@@ -1,14 +1,16 @@
 {{- define "k.spec.rolebinding" -}}
-{{- $name := printf "%v-%v" .Release.Name .Release.Revision -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ $name }}-spec-rolebinding
+  name: {{ .Release.Name }}-spec-rolebinding
+  annotations:
+    iter8.tools/revision: {{ .Release.Revision }}
 subjects:
 - kind: ServiceAccount
   name: default
+  namespace: {{ .Release.Namespace }}
 roleRef:
   kind: Role
-  name: {{ $name }}-spec-role
+  name: {{ .Release.Name }}-spec-role
   apiGroup: rbac.authorization.k8s.io
 {{- end }}
diff --git a/charts/iter8lib/templates/_k-spec-secret.tpl b/charts/iter8lib/templates/_k-spec-secret.tpl
index 87736920..107413c8 100644
--- a/charts/iter8lib/templates/_k-spec-secret.tpl
+++ b/charts/iter8lib/templates/_k-spec-secret.tpl
@@ -1,10 +1,11 @@
 {{- define "k.spec.secret" -}}
-{{- $name := printf "%v-%v" .Release.Name .Release.Revision -}}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ $name }}-spec
+  name: {{ .Release.Name }}-spec
+  annotations:
+    iter8.tools/revision: {{ .Release.Revision }}
 stringData:
   experiment.yaml: |
-{{ include "experiment" . | indent 4 }}
+{{- include "experiment" . | indent 4 }}
 {{- end }}
diff --git a/charts/iter8lib/values.yaml b/charts/iter8lib/values.yaml
index 2aaf5d11..f3e22ace 100644
--- a/charts/iter8lib/values.yaml
+++ b/charts/iter8lib/values.yaml
@@ -1,4 +1,5 @@
-### global values under global are available for charts that use iter8lib as a dependency
-global:
-  ### iter8Image default iter8 image used for running Kubernetes experiments
-  iter8Image: iter8/iter8:0.10
\ No newline at end of file
+### iter8Image default iter8 image used for running Kubernetes experiments
+iter8Image: iter8/iter8:0.10
+
+### majorMinor is the minor version of Iter8
+majorMinor: v0.10
\ No newline at end of file
diff --git a/cmd/k.go b/cmd/k.go
index 0a532477..a1dc1762 100644
--- a/cmd/k.go
+++ b/cmd/k.go
@@ -20,14 +20,6 @@ func addExperimentGroupFlag(cmd *cobra.Command, groupP *string, required bool) {
 	}
 }
 
-// addExperimentRevisionFlag adds the experiment revision flag and marks it as required or optional
-func addExperimentRevisionFlag(cmd *cobra.Command, revisionP *int, required bool) {
-	cmd.Flags().IntVar(revisionP, "revision", 0, "experiment revision")
-	if required {
-		cmd.MarkFlagRequired("revision")
-	}
-}
-
 func init() {
 	settings.AddFlags(kCmd.PersistentFlags())
 	// hiding these Helm flags for now
diff --git a/cmd/klog.go b/cmd/klog.go
index 16cab776..dc6786ff 100644
--- a/cmd/klog.go
+++ b/cmd/klog.go
@@ -27,7 +27,7 @@ func newKLogCmd(kd *driver.KubeDriver) *cobra.Command {
 			if lg, err := actor.KubeRun(); err != nil {
 				return err
 			} else {
-				log.Logger.WithStackTrace(lg).Info("experiment logs from Kubernetes cluster")
+				log.Logger.WithIndentedTrace(lg).Info("experiment logs from Kubernetes cluster")
 				return nil
 			}
 		},
diff --git a/cmd/krun.go b/cmd/krun.go
index 66f873db..65ab0cea 100644
--- a/cmd/krun.go
+++ b/cmd/krun.go
@@ -13,7 +13,7 @@ import (
 const kRunDesc = `
 Run a Kubernetes experiment. This command reads an experiment specified in a secret and writes the result to another secret.
 
-	$ iter8 k run --namespace {{ .Experiment.Namespace }} --group {{ .Experiment.group }} --revision {{ .Experiment.Revision }}
+	$ iter8 k run --namespace {{ .Experiment.Namespace }} --group {{ .Experiment.group }}
 
 This command is intended for use within the Iter8 Docker image that is used to execute Kubernetes experiments.
 `
@@ -33,7 +33,6 @@ func newKRunCmd(kd *driver.KubeDriver, out io.Writer) *cobra.Command {
 		},
 	}
 	addExperimentGroupFlag(cmd, &actor.Group, true)
-	addExperimentRevisionFlag(cmd, &actor.Revision, true)
 	actor.EnvSettings = settings
 	cmd.MarkFlagRequired("namespace")
 	return cmd
diff --git a/driver/kubedriver.go b/driver/kubedriver.go
index fb5580d3..8b252aaa 100644
--- a/driver/kubedriver.go
+++ b/driver/kubedriver.go
@@ -59,8 +59,8 @@ type KubeDriver struct {
 	*action.Configuration
 	// Group is the experiment group
 	Group string
-	// Revision is the revision of the experiment
-	Revision int
+	// revision is the revision of the experiment
+	revision int
 }
 
 // NewKubeDriver creates and returns a new KubeDriver
@@ -114,9 +114,9 @@ func (kd *KubeDriver) initHelm() error {
 // initRevision initializes the latest revision
 func (driver *KubeDriver) initRevision() error {
 	// update revision to latest, if none is specified
-	if driver.Revision <= 0 {
+	if driver.revision <= 0 {
 		if rel, err := driver.getLastRelease(); err == nil && rel != nil {
-			driver.Revision = rel.Version
+			driver.revision = rel.Version
 		} else {
 			return err
 		}
@@ -158,17 +158,17 @@ func (driver *KubeDriver) getLastRelease() (*release.Release, error) {
 
 // getSpecSecretName yields the name of the experiment spec secret
 func (driver *KubeDriver) getSpecSecretName() string {
-	return fmt.Sprintf("%v-%v-spec", driver.Group, driver.Revision)
+	return fmt.Sprintf("%v-spec", driver.Group)
 }
 
 // getResultSecretName yields the name of the experiment result secret
 func (driver *KubeDriver) getResultSecretName() string {
-	return fmt.Sprintf("%v-%v-result", driver.Group, driver.Revision)
+	return fmt.Sprintf("%v-result", driver.Group)
 }
 
 // getExperimentJobName yields the name of the experiment job
 func (driver *KubeDriver) getExperimentJobName() string {
-	return fmt.Sprintf("%v-%v-job", driver.Group, driver.Revision)
+	return fmt.Sprintf("%v-%v-job", driver.Group, driver.revision)
 }
 
 // getSecretWithRetry attempts to get a Kubernetes secret with retries
@@ -278,37 +278,6 @@ func (driver *KubeDriver) formResultSecret(r *base.ExperimentResult) (*corev1.Se
 	return &sec, nil
 }
 
-// createExperimentResultSecret creates the experiment result secret
-func (driver *KubeDriver) createExperimentResultSecret(r *base.ExperimentResult) error {
-	if sec, err := driver.formResultSecret(r); err == nil {
-		err1 := retry.OnError(
-			wait.Backoff{
-				Steps:    int(secretTimeout / retryInterval),
-				Cap:      secretTimeout,
-				Duration: retryInterval,
-				Factor:   1.0,
-				Jitter:   0.1,
-			},
-			func(err2 error) bool { // retry on specific failures
-				return kerrors.ReasonForError(err2) == metav1.StatusReasonForbidden
-			},
-			func() error {
-				secretsClient := driver.Clientset.CoreV1().Secrets(driver.Namespace())
-				_, err3 := secretsClient.Create(context.Background(), sec, metav1.CreateOptions{})
-				return err3
-			},
-		)
-		if err1 != nil {
-			err4 := fmt.Errorf("unable to create secret %v", sec.Name)
-			log.Logger.WithStackTrace(err1.Error()).Error(err4)
-			return err4
-		}
-		return nil
-	} else {
-		return err
-	}
-}
-
 // updateExperimentResultSecret updates the experiment result secret
 // as opposed to patch, update is an atomic operation
 func (driver *KubeDriver) updateExperimentResultSecret(r *base.ExperimentResult) error {
@@ -330,16 +299,6 @@ func (driver *KubeDriver) updateExperimentResultSecret(r *base.ExperimentResult)
 
 // WriteResult writes results for a Kubernetes experiment
 func (driver *KubeDriver) WriteResult(r *base.ExperimentResult) error {
-	// create result secret if need be
-	log.Logger.Debug("Write result called ... ")
-	if sec, _ := driver.getExperimentResultSecret(); sec == nil {
-		log.Logger.Info("creating experiment result secret")
-		if err := driver.createExperimentResultSecret(r); err != nil {
-			return err
-		}
-	} else {
-		log.Logger.Debug("Secret exists ... ", sec.Name)
-	}
 	if err := driver.updateExperimentResultSecret(r); err != nil {
 		return err
 	}
@@ -388,7 +347,7 @@ func writeManifest(rel *release.Release) error {
 // Credit: the logic for this function is sourced from Helm
 // https://github.com/helm/helm/blob/8ab18f7567cedffdfa5ba4d7f6abfb58efc313f8/cmd/helm/upgrade.go#L69
 // Upgrade a Kubernetes experiment to the next release
-func (driver *KubeDriver) Upgrade(chartDir string, valueOpts values.Options, group string, dry bool) error {
+func (driver *KubeDriver) upgrade(chartDir string, valueOpts values.Options, group string, dry bool) error {
 	client := action.NewUpgrade(driver.Configuration)
 	client.Namespace = driver.Namespace()
 	client.DryRun = dry
@@ -423,7 +382,7 @@ func (driver *KubeDriver) Upgrade(chartDir string, valueOpts values.Options, gro
 	}
 
 	// upgrading revision info
-	driver.Revision = rel.Version
+	driver.revision = rel.Version
 
 	// write manifest if dry
 	if dry {
@@ -439,10 +398,10 @@ func (driver *KubeDriver) Upgrade(chartDir string, valueOpts values.Options, gro
 	return nil
 }
 
-// Install a Kubernetes experiment
+// install a Kubernetes experiment
 // Credit: the logic for this function is sourced from Helm
 // https://github.com/helm/helm/blob/8ab18f7567cedffdfa5ba4d7f6abfb58efc313f8/cmd/helm/install.go#L177
-func (driver *KubeDriver) Install(chartDir string, valueOpts values.Options, group string, dry bool) error {
+func (driver *KubeDriver) install(chartDir string, valueOpts values.Options, group string, dry bool) error {
 	client := action.NewInstall(driver.Configuration)
 	client.Namespace = driver.Namespace()
 	client.DryRun = dry
@@ -478,7 +437,7 @@ func (driver *KubeDriver) Install(chartDir string, valueOpts values.Options, gro
 	}
 
 	// upgrading revision info
-	driver.Revision = rel.Version
+	driver.revision = rel.Version
 
 	// write manifest if dry
 	if dry {
@@ -494,6 +453,15 @@ func (driver *KubeDriver) Install(chartDir string, valueOpts values.Options, gro
 	return nil
 }
 
+// Launch a Kubernetes experiment
+func (driver *KubeDriver) Launch(chartDir string, valueOpts values.Options, group string, dry bool) error {
+	if driver.revision <= 0 {
+		return driver.install(chartDir, valueOpts, group, dry)
+	} else {
+		return driver.upgrade(chartDir, valueOpts, group, dry)
+	}
+}
+
 // Delete a Kubernetes experiment group
 func (driver *KubeDriver) Delete() error {
 	client := action.NewUninstall(driver.Configuration)


Test patch:
diff --git a/action/assert_test.go b/action/assert_test.go
index 6f5c7968..b1d7f60b 100644
--- a/action/assert_test.go
+++ b/action/assert_test.go
@@ -40,13 +40,12 @@ func TestLocalAssertFailing(t *testing.T) {
 func TestKubeAssert(t *testing.T) {
 	// fix aOpts
 	aOpts := NewAssertOpts(driver.NewFakeKubeDriver(cli.New()))
-	aOpts.Revision = 1
 	aOpts.Conditions = []string{Completed, NoFailure, SLOs}
 
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../testdata/assertinputs", "experiment.yaml"))
 	aOpts.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
@@ -55,7 +54,7 @@ func TestKubeAssert(t *testing.T) {
 	byteArray, _ = ioutil.ReadFile(base.CompletePath("../testdata/assertinputs", "result.yaml"))
 	aOpts.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-result",
+			Name:      "default-result",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"result.yaml": string(byteArray)},
diff --git a/action/launch_test.go b/action/launch_test.go
index 5d1f457e..5c0cb443 100644
--- a/action/launch_test.go
+++ b/action/launch_test.go
@@ -10,7 +10,7 @@ import (
 	"helm.sh/helm/v3/pkg/cli"
 )
 
-func TestLocalLaunch(t *testing.T) {
+func TestLocalLaunchNoDownload(t *testing.T) {
 	base.SetupWithMock(t)
 
 	// fix lOpts
@@ -25,6 +25,23 @@ func TestLocalLaunch(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLocalLaunch(t *testing.T) {
+	base.SetupWithMock(t)
+
+	// fix lOpts
+	lOpts := NewLaunchOpts(driver.NewFakeKubeDriver(cli.New()))
+	lOpts.ChartName = "load-test-http"
+	lOpts.Values = []string{"url=https://httpbin.org/get", "duration=2s"}
+	// fixing git ref forever
+	lOpts.RemoteFolderURL = defaultIter8Repo + "?ref=v0.10.6" + "//" + chartsFolderName
+
+	os.Chdir(t.TempDir())
+	err := lOpts.LocalRun()
+	assert.NoError(t, err)
+
+	assert.DirExists(t, chartsFolderName)
+}
+
 func TestKubeLaunch(t *testing.T) {
 	var err error
 
diff --git a/action/report_test.go b/action/report_test.go
index 666c59ca..9019c443 100644
--- a/action/report_test.go
+++ b/action/report_test.go
@@ -37,12 +37,11 @@ func TestKubeReportText(t *testing.T) {
 	base.SetupWithMock(t)
 	// fix rOpts
 	rOpts := NewReportOpts(driver.NewFakeKubeDriver(cli.New()))
-	rOpts.Revision = 1
 
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../testdata/assertinputs", "experiment.yaml"))
 	rOpts.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
@@ -51,7 +50,7 @@ func TestKubeReportText(t *testing.T) {
 	byteArray, _ = ioutil.ReadFile(base.CompletePath("../testdata/assertinputs", "result.yaml"))
 	rOpts.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-result",
+			Name:      "default-result",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"result.yaml": string(byteArray)},
diff --git a/action/run_test.go b/action/run_test.go
index 0a0f757f..dead92ba 100644
--- a/action/run_test.go
+++ b/action/run_test.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"io/ioutil"
 	"testing"
+	"time"
 
-	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
 
 	"github.com/iter8-tools/iter8/base"
 	"github.com/iter8-tools/iter8/base/log"
@@ -29,22 +30,30 @@ func TestKubeRun(t *testing.T) {
 	base.SetupWithMock(t)
 	// fix rOpts
 	rOpts := NewRunOpts(driver.NewFakeKubeDriver(cli.New()))
-	rOpts.Revision = 1
 
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../testdata", "experiment.yaml"))
 	rOpts.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
 	}, metav1.CreateOptions{})
-	rOpts.Clientset.BatchV1().Jobs("default").Create(context.TODO(), &batchv1.Job{
+
+	resultBytes, _ := yaml.Marshal(base.ExperimentResult{
+		StartTime:         time.Now(),
+		NumCompletedTasks: 0,
+		Failure:           false,
+		Iter8Version:      base.MajorMinor,
+	})
+	rOpts.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-job",
+			Name:      "default-result",
 			Namespace: "default",
 		},
+		StringData: map[string]string{"result.yaml": string(resultBytes)},
 	}, metav1.CreateOptions{})
+
 	err := rOpts.KubeRun()
 	assert.NoError(t, err)
 
diff --git a/base/log/log_test.go b/base/log/log_test.go
index a7b92fe0..8a044685 100644
--- a/base/log/log_test.go
+++ b/base/log/log_test.go
@@ -8,9 +8,14 @@ import (
 )
 
 func TestStackTrace(t *testing.T) {
+	Logger.WithIndentedTrace("hello there")
+	Logger.WithStackTrace("hello there")
+
 	st := StackTrace{
-		Trace: fmt.Sprintln("a") + fmt.Sprintln("b"),
+		prefix: "::Trace:: ",
+		Trace:  fmt.Sprintln("a") + fmt.Sprintln("b"),
 	}
 	assert.Contains(t, st.String(), "::Trace:: a")
 	assert.Contains(t, st.String(), "::Trace:: b")
+
 }
diff --git a/cmd/kassert_test.go b/cmd/kassert_test.go
index 7aac649a..20d98a69 100644
--- a/cmd/kassert_test.go
+++ b/cmd/kassert_test.go
@@ -5,11 +5,13 @@ import (
 	"fmt"
 	"io/ioutil"
 	"testing"
+	"time"
 
 	id "github.com/iter8-tools/iter8/driver"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
 
 	"github.com/iter8-tools/iter8/base"
 )
@@ -25,7 +27,7 @@ func TestKAssert(t *testing.T) {
 		// k run
 		{
 			name: "k run",
-			cmd:  "k run -g default --revision 1 --namespace default",
+			cmd:  "k run -g default --namespace default",
 		},
 		// k assert
 		{
@@ -42,12 +44,26 @@ func TestKAssert(t *testing.T) {
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../testdata", "experiment.yaml"))
 	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
 	}, metav1.CreateOptions{})
 
+	resultBytes, _ := yaml.Marshal(base.ExperimentResult{
+		StartTime:         time.Now(),
+		NumCompletedTasks: 0,
+		Failure:           false,
+		Iter8Version:      base.MajorMinor,
+	})
+	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "default-result",
+			Namespace: "default",
+		},
+		StringData: map[string]string{"result.yaml": string(resultBytes)},
+	}, metav1.CreateOptions{})
+
 	kd.Clientset.BatchV1().Jobs("default").Create(context.TODO(), &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "default-1-job",
diff --git a/cmd/klog_test.go b/cmd/klog_test.go
index 8047e968..d5e2efa8 100644
--- a/cmd/klog_test.go
+++ b/cmd/klog_test.go
@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
@@ -14,6 +15,12 @@ import (
 
 func TestKLog(t *testing.T) {
 	tests := []cmdTestCase{
+		// k launch
+		{
+			name:   "k launch",
+			cmd:    fmt.Sprintf("k launch -c load-test-http --chartsParentDir %v --noDownload --set url=https://httpbin.org/get --set duration=2s", base.CompletePath("../", "")),
+			golden: base.CompletePath("../testdata", "output/klaunch.txt"),
+		},
 		// k assert
 		{
 			name:   "k log",
@@ -25,7 +32,6 @@ func TestKLog(t *testing.T) {
 	// mock the environment
 	// fake kube cluster
 	*kd = *id.NewFakeKubeDriver(settings)
-	kd.Revision = 1
 	kd.Clientset.CoreV1().Pods("default").Create(context.TODO(), &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "default-1-job-8218s",
diff --git a/cmd/kreport_test.go b/cmd/kreport_test.go
index 8773fe8f..96f94a70 100644
--- a/cmd/kreport_test.go
+++ b/cmd/kreport_test.go
@@ -26,11 +26,10 @@ func TestKReport(t *testing.T) {
 	// mock the environment
 	// fake kube cluster
 	*kd = *id.NewFakeKubeDriver(settings)
-	kd.Revision = 1
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../testdata/assertinputs", "experiment.yaml"))
 	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
@@ -39,7 +38,7 @@ func TestKReport(t *testing.T) {
 	byteArray, _ = ioutil.ReadFile(base.CompletePath("../testdata/assertinputs", "result.yaml"))
 	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-result",
+			Name:      "default-result",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"result.yaml": string(byteArray)},
diff --git a/cmd/krun_test.go b/cmd/krun_test.go
index df434264..b262cabc 100644
--- a/cmd/krun_test.go
+++ b/cmd/krun_test.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"io/ioutil"
 	"testing"
+	"time"
 
-	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
 
 	id "github.com/iter8-tools/iter8/driver"
 
@@ -19,7 +20,7 @@ func TestKRun(t *testing.T) {
 		// k report
 		{
 			name:   "k run",
-			cmd:    "k run -g default --revision 1 --namespace default",
+			cmd:    "k run -g default --namespace default",
 			golden: base.CompletePath("../testdata", "output/krun.txt"),
 		},
 	}
@@ -28,21 +29,27 @@ func TestKRun(t *testing.T) {
 	base.SetupWithMock(t)
 	// fake kube cluster
 	*kd = *id.NewFakeKubeDriver(settings)
-	kd.Revision = 1
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../testdata", "experiment.yaml"))
 	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
 	}, metav1.CreateOptions{})
 
-	kd.Clientset.BatchV1().Jobs("default").Create(context.TODO(), &batchv1.Job{
+	resultBytes, _ := yaml.Marshal(base.ExperimentResult{
+		StartTime:         time.Now(),
+		NumCompletedTasks: 0,
+		Failure:           false,
+		Iter8Version:      base.MajorMinor,
+	})
+	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-job",
+			Name:      "default-result",
 			Namespace: "default",
 		},
+		StringData: map[string]string{"result.yaml": string(resultBytes)},
 	}, metav1.CreateOptions{})
 
 	runTestActionCmd(t, tests)
diff --git a/driver/kubedriver_test.go b/driver/kubedriver_test.go
index 436ba2d2..1510a545 100644
--- a/driver/kubedriver_test.go
+++ b/driver/kubedriver_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"io/ioutil"
 	"testing"
+	"time"
 
 	"github.com/iter8-tools/iter8/base"
 	"github.com/stretchr/testify/assert"
@@ -12,6 +13,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
 )
 
 func TestKOps(t *testing.T) {
@@ -23,7 +25,7 @@ func TestKOps(t *testing.T) {
 	assert.NoError(t, err)
 
 	// install
-	err = kd.Install(base.CompletePath("../", "charts/load-test-http"), values.Options{
+	err = kd.install(base.CompletePath("../", "charts/load-test-http"), values.Options{
 		Values: []string{"url=https://httpbin.org/get"},
 	}, kd.Group, false)
 	assert.NoError(t, err)
@@ -32,13 +34,13 @@ func TestKOps(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotNil(t, rel)
 	assert.Equal(t, 1, rel.Version)
-	assert.Equal(t, 1, kd.Revision)
+	assert.Equal(t, 1, kd.revision)
 
 	err = kd.Init()
 	assert.NoError(t, err)
 
 	// upgrade
-	err = kd.Upgrade(base.CompletePath("../", "charts/load-test-http"), values.Options{
+	err = kd.upgrade(base.CompletePath("../", "charts/load-test-http"), values.Options{
 		Values: []string{"url=https://httpbin.org/get"},
 	}, kd.Group, false)
 	assert.NoError(t, err)
@@ -46,7 +48,7 @@ func TestKOps(t *testing.T) {
 	rel, err = kd.Releases.Last(kd.Group)
 	assert.NotNil(t, rel)
 	assert.Equal(t, 2, rel.Version)
-	assert.Equal(t, 2, kd.Revision)
+	assert.Equal(t, 2, kd.revision)
 	assert.NoError(t, err)
 
 	err = kd.Init()
@@ -65,16 +67,31 @@ func TestKubeRun(t *testing.T) {
 	base.SetupWithMock(t)
 
 	kd := NewFakeKubeDriver(cli.New())
-	kd.Revision = 1
+	kd.revision = 1
 
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../", "testdata/drivertests/experiment.yaml"))
 	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
 	}, metav1.CreateOptions{})
+
+	resultBytes, _ := yaml.Marshal(base.ExperimentResult{
+		StartTime:         time.Now(),
+		NumCompletedTasks: 0,
+		Failure:           false,
+		Iter8Version:      base.MajorMinor,
+	})
+	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "default-result",
+			Namespace: "default",
+		},
+		StringData: map[string]string{"result.yaml": string(resultBytes)},
+	}, metav1.CreateOptions{})
+
 	kd.Clientset.BatchV1().Jobs("default").Create(context.TODO(), &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "default-1-job",
@@ -95,12 +112,12 @@ func TestLogs(t *testing.T) {
 	base.SetupWithMock(t)
 
 	kd := NewFakeKubeDriver(cli.New())
-	kd.Revision = 1
+	kd.revision = 1
 
 	byteArray, _ := ioutil.ReadFile(base.CompletePath("../", "testdata/drivertests/experiment.yaml"))
 	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "default-1-spec",
+			Name:      "default-spec",
 			Namespace: "default",
 		},
 		StringData: map[string]string{"experiment.yaml": string(byteArray)},
diff --git a/testdata/output/klog.txt b/testdata/output/klog.txt
index d941caa7..15a87907 100644
--- a/testdata/output/klog.txt
+++ b/testdata/output/klog.txt
@@ -1,2 +1,2 @@
-time=1977-09-02 22:04:05 level=info msg=experiment logs from Kubernetes cluster stack-trace=below ... 
-::Trace:: fake logs
+time=1977-09-02 22:04:05 level=info msg=experiment logs from Kubernetes cluster indented-trace=below ... 
+  fake logs
diff --git a/testdata/output/krun.txt b/testdata/output/krun.txt
index 9e4a89ba..f7e9ccb4 100644
--- a/testdata/output/krun.txt
+++ b/testdata/output/krun.txt
@@ -1,6 +1,3 @@
-time=1977-09-02 22:04:05 level=error msg=unable to get secret default-1-result stack-trace=below ... 
-::Trace:: secrets "default-1-result" not found
-time=1977-09-02 22:04:05 level=info msg=creating experiment result secret
 time=1977-09-02 22:04:05 level=info msg=task 1: gen-load-and-collect-metrics-http : started
 time=1977-09-02 22:04:05 level=info msg=task 1: gen-load-and-collect-metrics-http : completed
 time=1977-09-02 22:04:05 level=info msg=task 2: assess-app-versions : started