Repository: kumahq/kuma
Pull request #6189 (2023-03-02 10:10:11)

feat(e2e/universal): display full logs after fail
Logs which were for convinience saved under /tmp/e2e, now are wrapped by directories which names constists of suite start time.

Each universal component logs are also placed under directory with the name of the test (which is easier to find when needed).

When test fails, the logs are written to GinkgoWriter, so in CI we'll have access to them stright away.

### Checklist prior to review

<!--
Each of these sections need to be filled by the author when opening the PR.

If something doesn't apply please check the box and add a justification after the `--`
-->

- [ ] [Link to relevant issue][1] as well as docs and UI issues --
- [ ] This will not break child repos: it doesn't hardcode values (.e.g "kumahq" as a image registry) and it will work on Windows, system specific functions like `syscall.Mkfifo` have equivalent implementation on the other OS --
- [ ] Tests (Unit test, E2E tests, manual test on universal and k8s) --
- [ ] Do you need to update [`UPGRADE.md`](../blob/master/UPGRADE.md)? --
- [ ] Does it need to be backported according to the [backporting policy](../blob/master/CONTRIBUTING.md#backporting)? --
- [ ] Do you need to explicitly set a [`> Changelog:` entry here](../blob/master/CONTRIBUTING.md#submitting-a-patch) or add a `ci/` label to run fewer/more tests?

[1]: https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword



Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 38b3769f7..ec358b9d6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -142,7 +142,6 @@ jobs:
     - checkout
     - restore_cache:
         keys:
-        # prefer the exact match
         - golang_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
     - run:
         name: "Run code generators (go generate, protoc, ...) and code checks (go fmt, go vet, ...)"
@@ -180,7 +179,6 @@ jobs:
         go_os: linux
     - restore_cache:
         keys:
-        # prefer the exact match
         - vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
     - run:
         name: "Run tests"
@@ -289,7 +287,6 @@ jobs:
           go_arch: << parameters.arch >>
       - restore_cache:
           keys:
-            # prefer the exact match
             - vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
       # Mount files from the upstream jobs
       - attach_workspace:
@@ -367,7 +364,6 @@ jobs:
         go_arch: <<parameters.arch>>
     - restore_cache:
         keys:
-        # prefer the exact match
         - golang_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
     - run:
         name: Build all Kuma binaries (such as, kumactl, kuma-cp, kuma-dp)
diff --git a/go.mod b/go.mod
index 5d055fd8d..8b958e3dc 100644
--- a/go.mod
+++ b/go.mod
@@ -28,6 +28,7 @@ require (
 	github.com/gruntwork-io/terratest v0.41.11
 	github.com/hoisie/mustache v0.0.0-20160804235033-6375acf62c69
 	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/kennygrant/sanitize v1.2.4
 	github.com/kumahq/protoc-gen-kumadoc v0.3.1
 	github.com/lib/pq v1.10.7
 	github.com/miekg/dns v1.1.51
diff --git a/go.sum b/go.sum
index dc6f23a86..509aa30b7 100644
--- a/go.sum
+++ b/go.sum
@@ -885,6 +885,8 @@ github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0Lh
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
+github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
diff --git a/mk/dependencies/deps.lock b/mk/dependencies/deps.lock
index 47e2cf9e6..a5e11440d 100644
--- a/mk/dependencies/deps.lock
+++ b/mk/dependencies/deps.lock
@@ -1,1 +1,1 @@
-8e11a83e8c73845f5ded18962a47f8757468f063
+cc8405c4c28627c618c51a20106e02be83077ce8
diff --git a/mk/dev.mk b/mk/dev.mk
index 2db4fd926..77754896f 100644
--- a/mk/dev.mk
+++ b/mk/dev.mk
@@ -7,7 +7,7 @@ PROJECT_NAME ?= kuma
 
 CI_TOOLS_DIR ?= ${HOME}/.kuma-dev/${PROJECT_NAME}-${CI_TOOLS_VERSION}
 ifdef XDG_DATA_HOME
-	CI_TOOLS_DIR := ${XDG_DATA_HOME}/kuma-dev
+	CI_TOOLS_DIR := ${XDG_DATA_HOME}/kuma-dev/${PROJECT_NAME}-${CI_TOOLS_VERSION}
 endif
 CI_TOOLS_BIN_DIR=$(CI_TOOLS_DIR)/bin
 
diff --git a/tools/releases/dockerfiles/Dockerfile.base b/tools/releases/dockerfiles/Dockerfile.base
index 9b51d83e1..678e1cf4d 100644
--- a/tools/releases/dockerfiles/Dockerfile.base
+++ b/tools/releases/dockerfiles/Dockerfile.base
@@ -4,3 +4,5 @@ COPY /tools/releases/templates/LICENSE \
     /tools/releases/templates/README \
     /tools/releases/templates/NOTICE \
     /kuma/
+
+SHELL ["/busybox/busybox", "sh", "-c"]
diff --git a/tools/releases/dockerfiles/Dockerfile.base-root b/tools/releases/dockerfiles/Dockerfile.base-root
index 6780f0e6f..730a07218 100644
--- a/tools/releases/dockerfiles/Dockerfile.base-root
+++ b/tools/releases/dockerfiles/Dockerfile.base-root
@@ -5,3 +5,5 @@ COPY /tools/releases/templates/LICENSE \
     /tools/releases/templates/README \
     /tools/releases/templates/NOTICE \
     /kuma/
+
+SHELL ["/busybox/busybox", "sh", "-c"]
diff --git a/tools/releases/dockerfiles/Dockerfile.static b/tools/releases/dockerfiles/Dockerfile.static
index 7d9a3985f..dcf813ee1 100644
--- a/tools/releases/dockerfiles/Dockerfile.static
+++ b/tools/releases/dockerfiles/Dockerfile.static
@@ -4,3 +4,5 @@ COPY /tools/releases/templates/LICENSE \
     /tools/releases/templates/README \
     /tools/releases/templates/NOTICE \
     /kuma/
+
+SHELL ["/busybox/busybox", "sh", "-c"]


Test patch:
diff --git a/test/e2e_env/universal/universal_suite_test.go b/test/e2e_env/universal/universal_suite_test.go
index ba77405b5..257365df9 100644
--- a/test/e2e_env/universal/universal_suite_test.go
+++ b/test/e2e_env/universal/universal_suite_test.go
@@ -45,7 +45,12 @@ func TestE2E(t *testing.T) {
 	test.RunE2ESpecs(t, "E2E Universal Suite")
 }
 
-var _ = SynchronizedBeforeSuite(universal.SetupAndGetState, universal.RestoreState)
+var (
+	_ = SynchronizedBeforeSuite(universal.SetupAndGetState, universal.RestoreState)
+	_ = BeforeEach(universal.RememberSpecID)
+	_ = AfterSuite(universal.WriteLogsIfFailed)
+	_ = AfterEach(universal.WriteLogsIfFailed)
+)
 
 var (
 	_ = Describe("User Auth", auth.UserAuth)
diff --git a/test/framework/config.go b/test/framework/config.go
index c542ce95b..f1e909739 100644
--- a/test/framework/config.go
+++ b/test/framework/config.go
@@ -3,6 +3,7 @@ package framework
 import (
 	"fmt"
 	"os"
+	"path"
 	"runtime"
 	"time"
 
@@ -51,6 +52,7 @@ type E2eConfig struct {
 	ZoneIngressApp                string            `json:"zoneIngressApp,omitempty" envconfig:"KUMA_ZONE_INGRESS_APP"`
 	Arch                          string            `json:"arch,omitempty" envconfig:"ARCH"`
 	KumaCpConfig                  KumaCpConfig      `json:"kumaCpConfig,omitempty" envconfig:"KUMA_CP_CONFIG"`
+	UniversalE2ELogsPath          string            `json:"universalE2ELogsPath,omitempty" envconfig:"UNIVERSAL_E2E_LOGS_PATH"`
 
 	SuiteConfig SuiteConfig `json:"suites,omitempty"`
 }
@@ -246,8 +248,9 @@ var defaultConf = E2eConfig{
 			},
 		},
 	},
-	ZoneEgressApp:  "kuma-egress",
-	ZoneIngressApp: "kuma-ingress",
+	ZoneEgressApp:        "kuma-egress",
+	ZoneIngressApp:       "kuma-ingress",
+	UniversalE2ELogsPath: path.Join(os.TempDir(), "e2e"),
 }
 
 func init() {
diff --git a/test/framework/deployments/externalservice/deployment.go b/test/framework/deployments/externalservice/deployment.go
index bf47a4609..31588b150 100644
--- a/test/framework/deployments/externalservice/deployment.go
+++ b/test/framework/deployments/externalservice/deployment.go
@@ -35,19 +35,17 @@ func Install(name string, commands ...Command) framework.InstallFunc {
 		if len(commands) < 1 {
 			return errors.New("command list can't be empty")
 		}
-		switch cluster.(type) {
+		switch c := cluster.(type) {
 		case *framework.K8sCluster:
 			deployment = &k8SDeployment{
 				name: name,
 				cmd:  commands[0],
 			}
 		case *framework.UniversalCluster:
-			deployment = &UniversalDeployment{
-				name:     name,
-				commands: commands,
-				ports:    map[uint32]uint32{},
-				verbose:  cluster.Verbose(),
-			}
+			deployment = NewUniversalDeployment(c.LogsPath()).
+				WithName(name).
+				WithCommands(commands...).
+				WithVerbose(cluster.Verbose())
 		default:
 			return errors.New("invalid cluster")
 		}
diff --git a/test/framework/deployments/externalservice/universal.go b/test/framework/deployments/externalservice/universal.go
index cbb030a41..7ba56a9a6 100644
--- a/test/framework/deployments/externalservice/universal.go
+++ b/test/framework/deployments/externalservice/universal.go
@@ -16,6 +16,7 @@ import (
 )
 
 type UniversalDeployment struct {
+	logsPath  string
 	container string
 	ip        string
 	ports     map[uint32]uint32
@@ -28,6 +29,31 @@ type UniversalDeployment struct {
 
 var _ Deployment = &UniversalDeployment{}
 
+func NewUniversalDeployment(logsPath string) *UniversalDeployment {
+	return &UniversalDeployment{
+		logsPath: logsPath,
+		ports:    map[uint32]uint32{},
+	}
+}
+
+func (u *UniversalDeployment) WithName(name string) *UniversalDeployment {
+	u.name = name
+
+	return u
+}
+
+func (u *UniversalDeployment) WithCommands(commands ...Command) *UniversalDeployment {
+	u.commands = commands
+
+	return u
+}
+
+func (u *UniversalDeployment) WithVerbose(verbose bool) *UniversalDeployment {
+	u.verbose = verbose
+
+	return u
+}
+
 var UniversalEchoServer = func(port int, tls bool) Command {
 	args := []string{
 		"test-server", "echo",
@@ -103,19 +129,19 @@ func (u *UniversalDeployment) Deploy(cluster framework.Cluster) error {
 		return err
 	}
 
-	err = ssh.NewApp(u.name, u.verbose, port, nil, []string{"printf ", "--", "\"" + cert + "\"", ">", "/server-cert.pem"}).Run()
+	err = ssh.NewApp(u.name, u.logsPath, u.verbose, port, nil, []string{"printf ", "--", "\"" + cert + "\"", ">", "/server-cert.pem"}).Run()
 	if err != nil {
 		panic(err)
 	}
 
-	err = ssh.NewApp(u.name, u.verbose, port, nil, []string{"printf ", "--", "\"" + key + "\"", ">", "/server-key.pem"}).Run()
+	err = ssh.NewApp(u.name, u.logsPath, u.verbose, port, nil, []string{"printf ", "--", "\"" + key + "\"", ">", "/server-key.pem"}).Run()
 	if err != nil {
 		panic(err)
 	}
 
 	u.cert = cert
 	for _, arg := range u.commands {
-		u.app = ssh.NewApp(u.name, u.verbose, port, nil, arg)
+		u.app = ssh.NewApp(u.name, u.logsPath, u.verbose, port, nil, arg)
 		err = u.app.Start()
 		if err != nil {
 			return err
@@ -127,7 +153,7 @@ func (u *UniversalDeployment) Deploy(cluster framework.Cluster) error {
 
 func (u *UniversalDeployment) Exec(cmd ...string) (string, string, error) {
 	port := strconv.Itoa(int(u.ports[22]))
-	sshApp := ssh.NewApp(u.name, u.verbose, port, nil, cmd)
+	sshApp := ssh.NewApp(u.name, u.logsPath, u.verbose, port, nil, cmd)
 	err := sshApp.Run()
 	return sshApp.Out(), sshApp.Err(), err
 }
diff --git a/test/framework/envoy_admin/tunnel/universal.go b/test/framework/envoy_admin/tunnel/universal.go
index 99cc4ed34..b2797ba4a 100644
--- a/test/framework/envoy_admin/tunnel/universal.go
+++ b/test/framework/envoy_admin/tunnel/universal.go
@@ -15,18 +15,20 @@ import (
 type UniversalTunnel struct {
 	t testing.TestingT
 
-	port string
+	logsPath string
+	port     string
 
 	verbose bool
 }
 
 var _ envoy_admin.Tunnel = &UniversalTunnel{}
 
-func NewUniversalEnvoyAdminTunnel(t testing.TestingT, port string, verbose bool) (envoy_admin.Tunnel, error) {
+func NewUniversalEnvoyAdminTunnel(t testing.TestingT, logsPath string, port string, verbose bool) (envoy_admin.Tunnel, error) {
 	return &UniversalTunnel{
-		t:       t,
-		port:    port,
-		verbose: verbose,
+		t:        t,
+		logsPath: logsPath,
+		port:     port,
+		verbose:  verbose,
 	}, nil
 }
 
@@ -37,7 +39,7 @@ func (t *UniversalTunnel) GetStats(name string) (*stats.Stats, error) {
 		"curl", "--silent", "--max-time", "3", "--fail", url,
 	}
 
-	app := ssh.NewApp("tunnel", t.verbose, t.port, nil, sshArgs)
+	app := ssh.NewApp("tunnel", t.logsPath, t.verbose, t.port, nil, sshArgs)
 
 	if err := app.Run(); err != nil {
 		return nil, err
@@ -61,7 +63,7 @@ func (t *UniversalTunnel) ResetCounters() error {
 		"'http://localhost:9901/reset_counters'",
 	}
 
-	app := ssh.NewApp("tunnel", t.verbose, t.port, nil, sshArgs)
+	app := ssh.NewApp("tunnel", t.logsPath, t.verbose, t.port, nil, sshArgs)
 
 	if err := app.Run(); err != nil {
 		return err
diff --git a/test/framework/envs/universal/env.go b/test/framework/envs/universal/env.go
index 7257961f6..8f165a751 100644
--- a/test/framework/envs/universal/env.go
+++ b/test/framework/envs/universal/env.go
@@ -2,11 +2,15 @@ package universal
 
 import (
 	"encoding/json"
+	"os"
+	"path"
 
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/kumahq/kuma/pkg/config/core"
 	"github.com/kumahq/kuma/test/framework"
+	"github.com/kumahq/kuma/test/framework/universal_logs"
 )
 
 var Cluster *framework.UniversalCluster
@@ -53,3 +57,42 @@ func RestoreState(bytes []byte) {
 	Expect(Cluster.AddNetworking(state.ZoneEgress, framework.Config.ZoneEgressApp)).To(Succeed())
 	Cluster.SetCp(cp)
 }
+
+// WriteLogsIfFailed will read files used to put logs from universal tests
+func WriteLogsIfFailed(ctx SpecContext) {
+	if ctx.SpecReport().Failed() {
+		logsPath := universal_logs.GetPath(
+			framework.Config.UniversalE2ELogsPath,
+			ctx.SpecReport().FullText(),
+		)
+		spacer := "------------------------------\n"
+
+		dir, err := os.ReadDir(logsPath)
+		if err != nil {
+			GinkgoWriter.Printf("Attaching component logs failed: %s", err)
+			return
+		}
+
+		for _, file := range dir {
+			if !file.IsDir() {
+				logFilePath := path.Join(logsPath, file.Name())
+
+				bs, err := os.ReadFile(logFilePath)
+				if err != nil {
+					GinkgoWriter.Printf("Attaching component logs for %s failed: %s", logFilePath, err)
+					continue
+				}
+
+				GinkgoWriter.Printf("%sLogs from: %s\n%s", spacer, logFilePath, spacer)
+				Expect(GinkgoWriter.Write(bs)).Error().ToNot(HaveOccurred())
+			}
+		}
+	}
+}
+
+func RememberSpecID(ctx SpecContext) {
+	universal_logs.GenAndSavePath(
+		framework.Config.UniversalE2ELogsPath,
+		ctx.SpecReport().FullText(),
+	)
+}
diff --git a/test/framework/setup.go b/test/framework/setup.go
index 163638cc8..72950d9fb 100644
--- a/test/framework/setup.go
+++ b/test/framework/setup.go
@@ -299,6 +299,7 @@ func zoneRelatedResource(
 			cluster.GetTesting(),
 			uniCluster.name,
 			dpName,
+			uniCluster.logsPath,
 			"",
 			appType,
 			Config.IPV6,
diff --git a/test/framework/ssh/ssh.go b/test/framework/ssh/ssh.go
index ee7ac4532..ec94ffa93 100644
--- a/test/framework/ssh/ssh.go
+++ b/test/framework/ssh/ssh.go
@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path"
 	"syscall"
 
 	"github.com/gruntwork-io/terratest/modules/logger"
@@ -29,7 +30,7 @@ type App struct {
 	logFile *os.File
 }
 
-func NewApp(appName string, verbose bool, port string, envMap map[string]string, args []string) *App {
+func NewApp(appName string, logsPath string, verbose bool, port string, envMap map[string]string, args []string) *App {
 	app := &App{
 		port: port,
 	}
@@ -40,16 +41,15 @@ func NewApp(appName string, verbose bool, port string, envMap map[string]string,
 		"root@localhost", "-p", port,
 	}
 	for k, v := range envMap {
-		sshArgs = append(sshArgs, fmt.Sprintf("%s='%s'", k, utils.ShellEscape(v)))
+		sshArgs = append(sshArgs, fmt.Sprintf("%s=%s", k, utils.ShellEscape(v)))
 	}
 	sshArgs = append(sshArgs, args...)
 	app.cmd = exec.Command("ssh", sshArgs...)
-	logDirName := "/tmp/e2e"
-	err := os.MkdirAll(logDirName, os.ModePerm)
+	err := os.MkdirAll(logsPath, os.ModePerm)
 	if err != nil {
 		panic(errors.Wrap(err, "could not create /tmp/e2e"))
 	}
-	logFileName := logDirName + "/" + appName
+	logFileName := path.Join(logsPath, appName)
 	app.logFile, err = os.OpenFile(logFileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0o660)
 	if err != nil {
 		panic(errors.Wrap(err, "could not create "+logFileName))
diff --git a/test/framework/universal_app.go b/test/framework/universal_app.go
index e280263f0..d96b46cec 100644
--- a/test/framework/universal_app.go
+++ b/test/framework/universal_app.go
@@ -185,6 +185,7 @@ networking:
 
 type UniversalApp struct {
 	t             testing.TestingT
+	logsPath      string
 	mainApp       *ssh.App
 	mainAppEnv    map[string]string
 	mainAppArgs   []string
@@ -197,9 +198,10 @@ type UniversalApp struct {
 	mesh          string
 }
 
-func NewUniversalApp(t testing.TestingT, clusterName, dpName, mesh string, mode AppMode, isipv6, verbose bool, caps []string, volumes []string, containerName string) (*UniversalApp, error) {
+func NewUniversalApp(t testing.TestingT, clusterName, dpName, logsPath, mesh string, mode AppMode, isipv6, verbose bool, caps, volumes []string, containerName string) (*UniversalApp, error) {
 	app := &UniversalApp{
 		t:             t,
+		logsPath:      logsPath,
 		ports:         map[string]string{},
 		verbose:       verbose,
 		mesh:          mesh,
@@ -316,7 +318,7 @@ func (s *UniversalApp) GetContainerName() string {
 
 func (s *UniversalApp) GetEnvoyAdminTunnel() (envoy_admin.Tunnel, error) {
 	t, err := tunnel.NewUniversalEnvoyAdminTunnel(
-		s.t, s.ports["22"], s.verbose,
+		s.t, s.logsPath, s.ports["22"], s.verbose,
 	)
 	if err != nil {
 		return nil, err
@@ -361,12 +363,12 @@ func (s *UniversalApp) ReStart() error {
 func (s *UniversalApp) CreateMainApp(env map[string]string, args []string) {
 	s.mainAppEnv = env
 	s.mainAppArgs = args
-	s.mainApp = ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], env, args)
+	s.mainApp = ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], env, args)
 }
 
 func (s *UniversalApp) OverrideDpVersion(version string) error {
 	// It is important to store installation package in /tmp/kuma/, not /tmp/ otherwise root was taking over /tmp/ and Kuma DP could not store /tmp files
-	err := ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{
+	err := ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{
 		"wget",
 		fmt.Sprintf("https://download.konghq.com/mesh-alpine/kuma-%s-ubuntu-amd64.tar.gz", version),
 		"-O",
@@ -376,7 +378,7 @@ func (s *UniversalApp) OverrideDpVersion(version string) error {
 		return err
 	}
 
-	err = ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{
+	err = ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{
 		"mkdir",
 		"-p",
 		"/tmp/kuma/",
@@ -385,7 +387,7 @@ func (s *UniversalApp) OverrideDpVersion(version string) error {
 		return err
 	}
 
-	err = ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{
+	err = ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{
 		"tar",
 		"xvzf",
 		fmt.Sprintf("/tmp/kuma-%s-ubuntu-amd64.tar.gz", version),
@@ -396,7 +398,7 @@ func (s *UniversalApp) OverrideDpVersion(version string) error {
 		return err
 	}
 
-	err = ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{
+	err = ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{
 		"cp",
 		fmt.Sprintf("/tmp/kuma/kuma-%s/bin/kuma-dp", version),
 		"/usr/bin/kuma-dp",
@@ -405,7 +407,7 @@ func (s *UniversalApp) OverrideDpVersion(version string) error {
 		return err
 	}
 
-	err = ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{
+	err = ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{
 		"cp",
 		fmt.Sprintf("/tmp/kuma/kuma-%s/bin/envoy", version),
 		"/usr/local/bin/envoy",
@@ -424,7 +426,7 @@ func (s *UniversalApp) CreateDP(
 	concurrency int,
 ) {
 	// create the token file on the app container
-	err := ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{"printf ", "\"" + token + "\"", ">", "/kuma/token-" + name}).Run()
+	err := ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{"printf ", "\"" + token + "\"", ">", "/kuma/token-" + name}).Run()
 	if err != nil {
 		panic(err)
 	}
@@ -439,7 +441,7 @@ func (s *UniversalApp) CreateDP(
 	}
 
 	if dpyaml != "" {
-		err = ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{"printf ", "\"" + dpyaml + "\"", ">", "/kuma/dpyaml-" + name}).Run()
+		err = ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{"printf ", "\"" + dpyaml + "\"", ">", "/kuma/dpyaml-" + name}).Run()
 		if err != nil {
 			panic(err)
 		}
@@ -465,7 +467,7 @@ func (s *UniversalApp) CreateDP(
 		args = append(args, "--proxy-type", proxyType)
 	}
 
-	s.dpApp = ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, args)
+	s.dpApp = ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, args)
 }
 
 func (s *UniversalApp) setupTransparent(cpIp string, builtindns bool, transparentProxyV1 bool) {
@@ -492,7 +494,7 @@ func (s *UniversalApp) setupTransparent(cpIp string, builtindns bool, transparen
 		)
 	}
 
-	app := ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, args)
+	app := ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, args)
 	err := app.Run()
 	if err != nil {
 		panic(fmt.Sprintf("err: %s\nstderr :%s\nstdout %s", err.Error(), app.Err(), app.Out()))
@@ -500,7 +502,7 @@ func (s *UniversalApp) setupTransparent(cpIp string, builtindns bool, transparen
 }
 
 func (s *UniversalApp) getIP(isipv6 bool) (string, error) {
-	cmd := ssh.NewApp(s.containerName, s.verbose, s.ports[sshPort], nil, []string{"getent", "ahosts", s.container[:12]})
+	cmd := ssh.NewApp(s.containerName, s.logsPath, s.verbose, s.ports[sshPort], nil, []string{"getent", "ahosts", s.container[:12]})
 	err := cmd.Run()
 	if err != nil {
 		return "invalid", errors.Wrapf(err, "getent failed with %s", cmd.Err())
diff --git a/test/framework/universal_cluster.go b/test/framework/universal_cluster.go
index 56bc1b2d8..1b48edf80 100644
--- a/test/framework/universal_cluster.go
+++ b/test/framework/universal_cluster.go
@@ -18,6 +18,7 @@ import (
 	"github.com/kumahq/kuma/test/framework/envoy_admin"
 	"github.com/kumahq/kuma/test/framework/envoy_admin/tunnel"
 	"github.com/kumahq/kuma/test/framework/ssh"
+	"github.com/kumahq/kuma/test/framework/universal_logs"
 )
 
 type UniversalNetworking struct {
@@ -38,6 +39,7 @@ type UniversalNetworkingState struct {
 
 type UniversalCluster struct {
 	t              testing.TestingT
+	logsPath       string
 	name           string
 	controlplane   *UniversalControlPlane
 	apps           map[string]*UniversalApp
@@ -56,6 +58,7 @@ var _ Cluster = &UniversalCluster{}
 func NewUniversalCluster(t *TestingT, name string, verbose bool) *UniversalCluster {
 	return &UniversalCluster{
 		t:              t,
+		logsPath:       universal_logs.GetPath(Config.UniversalE2ELogsPath, t.Name()),
 		name:           name,
 		apps:           map[string]*UniversalApp{},
 		verbose:        verbose,
@@ -67,6 +70,10 @@ func NewUniversalCluster(t *TestingT, name string, verbose bool) *UniversalClust
 	}
 }
 
+func (c *UniversalCluster) LogsPath() string {
+	return c.logsPath
+}
+
 func (c *UniversalCluster) WithTimeout(timeout time.Duration) Cluster {
 	c.defaultTimeout = timeout
 
@@ -153,7 +160,7 @@ func (c *UniversalCluster) DeployKuma(mode core.CpMode, opt ...KumaDeploymentOpt
 		env["KUMA_MULTIZONE_ZONE_NAME"] = c.name
 	}
 
-	app, err := NewUniversalApp(c.t, c.name, AppModeCP, "", AppModeCP, c.opts.isipv6, true, []string{}, dockerVolumes, "")
+	app, err := NewUniversalApp(c.t, c.name, AppModeCP, c.logsPath, "", AppModeCP, c.opts.isipv6, true, []string{}, dockerVolumes, "")
 	if err != nil {
 		return err
 	}
@@ -290,7 +297,7 @@ func (c *UniversalCluster) DeployApp(opt ...AppDeploymentOption) error {
 
 	Logf("IPV6 is %v", opts.isipv6)
 
-	app, err := NewUniversalApp(c.t, c.name, opts.name, opts.mesh, AppMode(appname), opts.isipv6, *opts.verbose, caps, opts.dockerVolumes, opts.dockerContainerName)
+	app, err := NewUniversalApp(c.t, c.name, opts.name, c.logsPath, opts.mesh, AppMode(appname), opts.isipv6, *opts.verbose, caps, opts.dockerVolumes, opts.dockerContainerName)
 	if err != nil {
 		return err
 	}
@@ -373,7 +380,7 @@ func runPostgresMigration(kumaCP *UniversalApp, envVars map[string]string) error
 		return errors.New("missing public port: 22")
 	}
 
-	app := ssh.NewApp(kumaCP.containerName, kumaCP.verbose, sshPort, envVars, args)
+	app := ssh.NewApp(kumaCP.containerName, kumaCP.logsPath, kumaCP.verbose, sshPort, envVars, args)
 	if err := app.Run(); err != nil {
 		return errors.Errorf("db migration err: %s\nstderr :%s\nstdout %s", err.Error(), app.Err(), app.Out())
 	}
@@ -417,7 +424,7 @@ func (c *UniversalCluster) Exec(namespace, podName, appname string, cmd ...strin
 	if !ok {
 		return "", "", errors.Errorf("App %s not found", appname)
 	}
-	sshApp := ssh.NewApp(app.containerName, c.verbose, app.ports[sshPort], nil, cmd)
+	sshApp := ssh.NewApp(app.containerName, c.logsPath, c.verbose, app.ports[sshPort], nil, cmd)
 	err := sshApp.Run()
 	return sshApp.Out(), sshApp.Err(), err
 }
@@ -495,7 +502,7 @@ func (c *UniversalCluster) addIngressEnvoyTunnel() error {
 }
 
 func (c *UniversalCluster) createEnvoyTunnel(name string) error {
-	t, err := tunnel.NewUniversalEnvoyAdminTunnel(c.t, c.networking[name].ApiServerPort, c.verbose)
+	t, err := tunnel.NewUniversalEnvoyAdminTunnel(c.t, c.logsPath, c.networking[name].ApiServerPort, c.verbose)
 	if err != nil {
 		return err
 	}
diff --git a/test/framework/universal_controlplane.go b/test/framework/universal_controlplane.go
index b221798dd..c16773c2d 100644
--- a/test/framework/universal_controlplane.go
+++ b/test/framework/universal_controlplane.go
@@ -11,10 +11,12 @@ import (
 	"github.com/kumahq/kuma/pkg/config/core"
 	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
 	"github.com/kumahq/kuma/test/framework/ssh"
+	"github.com/kumahq/kuma/test/framework/universal_logs"
 )
 
 type UniversalControlPlane struct {
 	t            testing.TestingT
+	logsPath     string
 	mode         core.CpMode
 	name         string
 	kumactl      *KumactlOptions
@@ -27,6 +29,7 @@ func NewUniversalControlPlane(t testing.TestingT, mode core.CpMode, clusterName
 	kumactl := NewKumactlOptions(t, name, verbose)
 	ucp := &UniversalControlPlane{
 		t:            t,
+		logsPath:     universal_logs.GetPath(Config.UniversalE2ELogsPath, t.Name()),
 		mode:         mode,
 		name:         name,
 		kumactl:      kumactl,
@@ -79,7 +82,7 @@ func (c *UniversalControlPlane) GetAPIServerAddress() string {
 
 func (c *UniversalControlPlane) GetMetrics() (string, error) {
 	return retry.DoWithRetryE(c.t, "fetching CP metrics", DefaultRetries, DefaultTimeout, func() (string, error) {
-		sshApp := ssh.NewApp(c.name, c.verbose, c.cpNetworking.SshPort, nil, []string{
+		sshApp := ssh.NewApp(c.name, c.logsPath, c.verbose, c.cpNetworking.SshPort, nil, []string{
 			"curl",
 			"--fail", "--show-error",
 			"http://localhost:5680/metrics",
@@ -108,6 +111,7 @@ func (c *UniversalControlPlane) generateToken(
 		func() (string, error) {
 			sshApp := ssh.NewApp(
 				c.name,
+				c.logsPath,
 				c.verbose,
 				c.cpNetworking.SshPort,
 				nil,
@@ -141,6 +145,7 @@ func (c *UniversalControlPlane) retrieveAdminToken() (string, error) {
 		func() (string, error) {
 			sshApp := ssh.NewApp(
 				c.name,
+				c.logsPath,
 				c.verbose, c.cpNetworking.SshPort, nil, []string{
 					"curl", "--fail", "--show-error",
 					"http://localhost:5681/global-secrets/admin-user-token",
diff --git a/test/framework/universal_logs/universal_logs.go b/test/framework/universal_logs/universal_logs.go
new file mode 100644
index 000000000..d8d55f958
--- /dev/null
+++ b/test/framework/universal_logs/universal_logs.go
@@ -0,0 +1,43 @@
+package universal_logs
+
+import (
+	"path"
+	"sync"
+	"time"
+
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/kennygrant/sanitize"
+	"k8s.io/utils/strings"
+)
+
+var (
+	timePrefix = time.Now().Local().Format("060102_150405")
+	paths      = map[string]string{}
+	mutex      sync.RWMutex
+)
+
+func GenAndSavePath(logsPath, specName string) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	// Let's be sure we won't exceed max filename length
+	fileName := strings.ShortenString(sanitize.Name(specName), 243)
+
+	paths[specName] = path.Join(
+		logsPath,
+		timePrefix,
+		fileName+"-"+random.UniqueId(),
+	)
+}
+
+func GetPath(logsPath, specName string) string {
+	mutex.RLock()
+	defer mutex.RUnlock()
+
+	id, ok := paths[specName]
+	if !ok {
+		return path.Join(logsPath, timePrefix)
+	}
+
+	return id
+}