Repository: scylladb/scylla-operator
Pull request #553 (2021-04-19 14:38:25)

e2e: Dump test namespace content
**Description of your changes:**
When something goes wrong in the CI we need to dump information for debugging, like for
https://github.com/scylladb/scylla-operator/runs/2376373176?check_suite_focus=true
Given CI has limited resources, we can't use `--delete-namespace-policy=OnSuccess` and `hack/ci-gather-artifacts.sh` as someone running a real cluster or individual test would. Leaving the namespaces around might lead to resource exhaustion and failing future tests as a result.
This PR makes sure we collect testing namespace dump before deleting it.




Patch:
diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
index f9d56999a..c8b01e4cb 100644
--- a/.github/workflows/go.yaml
+++ b/.github/workflows/go.yaml
@@ -32,7 +32,7 @@ env:
   GOPATH: ${{ github.workspace }}/go
   git_repo_path: ${{ github.workspace }}/go/src/github.com/scylladb/scylla-operator
   image_repo_ref: docker.io/scylladb/scylla-operator
-  retention_days: 3
+  retention_days: 90
 
 defaults:
   run:
@@ -170,6 +170,13 @@ jobs:
       with:
         path: ${{ env.git_repo_path }}
         fetch-depth: 0  # also fetch tags
+    - name: Create artifacts dir
+      env:
+        ARTIFACTS_DIR: ${{ runner.temp }}/e2e-artifacts
+      run: |
+        set -x
+        mkdir "${ARTIFACTS_DIR}"
+        echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" | tee -a ${GITHUB_ENV}
     - uses: actions/download-artifact@v2
       with:
         name: operatorimage.tar.lz4
@@ -192,8 +199,6 @@ jobs:
         set -x
         go install github.com/mikefarah/yq/v4@v4.6.1
     - name: Deploy scylla-operator
-      env:
-        ARTIFACTS_DIR: ${{ runner.temp }}/e2e-artifacts
       run: |
         set -x
         timeout 10m ./hack/ci-deploy.sh '${{ env.image_repo_ref }}:ci'
@@ -203,15 +208,11 @@ jobs:
       run: |
         echo "SCYLLA_OPERATOR_TESTS_FLAKE_ATTEMPTS=5" | tee -a ${GITHUB_ENV}
     - name: Run e2e
-      env:
-        ARTIFACTS_DIR: ${{ runner.temp }}/e2e-artifacts
       run: |
         set -x
-        timeout 45m docker run --rm --entrypoint=/usr/bin/scylla-operator-tests -v "/root/.kube/config:/root/.kube/config" '${{ env.image_repo_ref }}:ci' run
+        timeout 45m docker run --user="$( id -u ):$( id -g )" --rm --entrypoint=/usr/bin/scylla-operator-tests -v="${HOME}/.kube/config:/kubeconfig:ro" -e='KUBECONFIG=/kubeconfig' -v="${ARTIFACTS_DIR}:${ARTIFACTS_DIR}:rw" '${{ env.image_repo_ref }}:ci' run --artifacts-dir="${ARTIFACTS_DIR}"
     - name: Dump cluster state
       if: ${{ always() }}
-      env:
-        ARTIFACTS_DIR: ${{ runner.temp }}/e2e-artifacts
       working-directory: ${{ runner.temp }}
       run: timeout 10m ${{ env.git_repo_path }}/hack/ci-gather-artifacts.sh
     - name: Get machine logs and info
@@ -227,7 +228,9 @@ jobs:
     - name: Compress artifacts
       if: ${{ always() }}
       working-directory: ${{ runner.temp }}
-      run: tar -c --use-compress-program=lz4 -f ./e2e-artifacts.tar.lz4 "e2e-artifacts/"
+      run: |
+        set -x
+        tar -c --use-compress-program=lz4 -f ./e2e-artifacts.tar.lz4 "e2e-artifacts/"
     - name: Upload artifacts
       if: ${{ always() }}
       uses: actions/upload-artifact@v2


Test patch:
diff --git a/pkg/cmd/tests/options.go b/pkg/cmd/tests/options.go
index 7524bfa7d..604a0a685 100644
--- a/pkg/cmd/tests/options.go
+++ b/pkg/cmd/tests/options.go
@@ -23,7 +23,7 @@ func NewTestFrameworkOptions() TestFrameworkOptions {
 }
 
 func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
-	cmd.PersistentFlags().StringVarP(&o.ArtifactsDir, "atrtifacts-dir", "", o.ArtifactsDir, "A directory for storing test artifacts. No data is collected until set.")
+	cmd.PersistentFlags().StringVarP(&o.ArtifactsDir, "artifacts-dir", "", o.ArtifactsDir, "A directory for storing test artifacts. No data is collected until set.")
 	cmd.PersistentFlags().StringVarP(&o.DeleteTestingNSPolicyUntyped, "delete-namespace-policy", "", o.DeleteTestingNSPolicyUntyped, fmt.Sprintf("Namespace deletion policy. Allowed values are [%s]", strings.Join(
 		[]string{
 			string(framework.DeleteTestingNSPolicyAlways),
diff --git a/test/e2e/framework/dump.go b/test/e2e/framework/dump.go
new file mode 100644
index 000000000..451148be5
--- /dev/null
+++ b/test/e2e/framework/dump.go
@@ -0,0 +1,222 @@
+package framework
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/dynamic"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/yaml"
+)
+
+func retrieveContainerLogs(ctx context.Context, podClient corev1client.PodInterface, destinationPath string, podName string, logOptions *corev1.PodLogOptions) error {
+	dest, err := os.OpenFile(destinationPath, os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := dest.Close()
+		if err != nil {
+			klog.ErrorS(err, "can't close file", "Path", destinationPath)
+		}
+	}()
+
+	logsReq := podClient.GetLogs(podName, logOptions)
+	readCloser, err := logsReq.Stream(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := readCloser.Close()
+		if err != nil {
+			klog.ErrorS(err, "can't close log stream", "Path", destinationPath, "Pod", podName, "Container", logOptions.Container)
+		}
+	}()
+
+	_, err = io.Copy(dest, readCloser)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func dumpPodLogs(ctx context.Context, coreClient corev1client.CoreV1Interface, pod *corev1.Pod, resourceDir string, namespace string) error {
+	podDir := filepath.Join(resourceDir, pod.Name)
+	err := os.Mkdir(podDir, 0777)
+	if err != nil {
+		return err
+	}
+
+	for _, c := range pod.Spec.Containers {
+		logOptions := &corev1.PodLogOptions{
+			Container:  c.Name,
+			Follow:     false,
+			Timestamps: true,
+		}
+
+		// Retrieve current logs.
+		logOptions.Previous = false
+		err = retrieveContainerLogs(ctx, coreClient.Pods(namespace), filepath.Join(podDir, fmt.Sprintf("%s.current", c.Name)), pod.Name, logOptions)
+		if err != nil {
+			return err
+		}
+
+		hasPrevious := false
+		for _, cs := range pod.Status.ContainerStatuses {
+			if cs.Name != c.Name {
+				continue
+			}
+
+			if cs.RestartCount > 0 {
+				hasPrevious = true
+			}
+		}
+		if hasPrevious {
+			logOptions.Previous = true
+			err = retrieveContainerLogs(ctx, coreClient.Pods(namespace), filepath.Join(podDir, fmt.Sprintf("%s.previous", c.Name)), pod.Name, logOptions)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func DumpResource(ctx context.Context, dynamicClient dynamic.Interface, coreClient corev1client.CoreV1Interface, gvr schema.GroupVersionResource, groupDir, namespace string) error {
+	list, err := dynamicClient.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	if len(list.Items) == 0 {
+		return nil
+	}
+
+	for _, obj := range list.Items {
+		obj.SetManagedFields(nil)
+	}
+
+	resourceDir := filepath.Join(groupDir, gvr.Resource)
+	err = os.Mkdir(resourceDir, 0777)
+	if err != nil {
+		return err
+	}
+
+	listData, err := yaml.Marshal(list.Items)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(filepath.Join(groupDir, fmt.Sprintf("%s.yaml", gvr.Resource)), listData, 0666)
+	if err != nil {
+		return err
+	}
+
+	for _, obj := range list.Items {
+		data, err := yaml.Marshal(obj)
+		if err != nil {
+			return err
+		}
+
+		err = ioutil.WriteFile(filepath.Join(resourceDir, fmt.Sprintf("%s.yaml", obj.GetName())), data, 0666)
+		if err != nil {
+			return err
+		}
+
+		switch gvr.String() {
+		case corev1.SchemeGroupVersion.WithResource("pods").String():
+			pod := &corev1.Pod{}
+			err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, pod)
+			if err != nil {
+				return err
+			}
+
+			err = dumpPodLogs(ctx, coreClient, pod, resourceDir, namespace)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func getContainedListableResources(apiResources []metav1.APIResource) []string {
+	var resources []string
+
+	for _, apiResource := range apiResources {
+		// Skip resources in different GV
+		if len(apiResource.Group) != 0 || len(apiResource.Version) != 0 {
+			continue
+		}
+
+		supportsList := false
+		for _, verb := range apiResource.Verbs {
+			if verb == "list" {
+				supportsList = true
+			}
+		}
+
+		if !supportsList {
+			continue
+		}
+
+		resources = append(resources, apiResource.Name)
+	}
+
+	return resources
+}
+
+func DumpNamespace(ctx context.Context, discoveryClient discovery.DiscoveryInterface, dynamicClient dynamic.Interface, podClient corev1client.CoreV1Interface, artifactsDir, namespace string) error {
+	namespaceDir := path.Join(artifactsDir, namespace)
+	err := os.Mkdir(namespaceDir, 0777)
+	if err != nil {
+		return err
+	}
+
+	resourceLists, err := discoveryClient.ServerPreferredNamespacedResources()
+	if err != nil {
+		return err
+	}
+
+	for _, list := range resourceLists {
+		gv, err := schema.ParseGroupVersion(list.GroupVersion)
+		if err != nil {
+			return err
+		}
+
+		var groupDir string
+		if len(gv.Group) == 0 {
+			groupDir = path.Join(namespaceDir, fmt.Sprintf("core_%s", gv.Version))
+		} else {
+			groupDir = path.Join(namespaceDir, fmt.Sprintf("%s_%s", gv.Group, gv.Version))
+		}
+
+		err = os.Mkdir(groupDir, 0777)
+		if err != nil {
+			return err
+		}
+		resources := getContainedListableResources(list.APIResources)
+		for _, r := range resources {
+			gvr := gv.WithResource(r)
+			err = DumpResource(ctx, dynamicClient, podClient, gvr, groupDir, namespace)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
index 92f48084e..a71f76d74 100644
--- a/test/e2e/framework/framework.go
+++ b/test/e2e/framework/framework.go
@@ -5,6 +5,8 @@ package framework
 import (
 	"context"
 	"fmt"
+	"os"
+	"path"
 	"time"
 
 	g "github.com/onsi/ginkgo"
@@ -280,4 +282,21 @@ func (f *Framework) afterEach() {
 		By(fmt.Sprintf("Collecting events from namespace %q.", f.namespace.Name))
 		DumpEventsInNamespace(ctx, f.KubeAdminClient(), f.namespace.Name)
 	}
+
+	// CI can't keep namespaces alive because it could get out of resources for the other tests
+	// so we need to collect the namespaced dump before destroying the namespace.
+	// Collecting artifacts even for successful runs helps to verify if it went
+	// as expected and the amount of data is bearable.
+	if len(TestContext.ArtifactsDir) != 0 {
+		By(fmt.Sprintf("Collecting dumps from namespace %q.", f.namespace.Name))
+
+		d := path.Join(TestContext.ArtifactsDir, "e2e-namespaces")
+		err := os.Mkdir(d, 0777)
+		if err != nil && !os.IsExist(err) {
+			o.Expect(err).NotTo(o.HaveOccurred())
+		}
+
+		err = DumpNamespace(ctx, f.KubeClient().Discovery(), f.DynamicAdminClient(), f.KubeClient().CoreV1(), d, f.Namespace())
+		o.Expect(err).NotTo(o.HaveOccurred())
+	}
 }