Test in Go: the order of Cleanup is not what you think
Tags: go, testing, testify, cleanup
Published: 2024-02-12 03:04:47

The framework `[testify](https://github.com/stretchr/testify/tree/v1.8.4)` might be the most popular. Surprisingly, I find the documentation is not clear enough to answer my questions and I can't find answers anywhere. Therefore, I have to write experiments to verify my guesses.

One of my questions is: when will the registered `Cleanup` functions run?

The summary of my experiment shows that the order of `Cleanup` functions is running as we normally expected except for those from **subtests**. The exception could lead severe problems.

# The experiment

The idea is simple: register `Cleanup` functions from all setup/teardown methods and use the log to infer the execution order.

Let's have a peek at the result first.

![https://cdn.hashnode.com/res/hashnode/image/upload/v1707705052178/6c2c3f7b-c487-457e-bc23-4871e7f5df34.png](https://cdn.hashnode.com/res/hashnode/image/upload/v1707705052178/6c2c3f7b-c487-457e-bc23-4871e7f5df34.png)

We can see that the `Cleanup` functions registered at `TearDownSubTest` and `SetupSubTest` are called after the subtests are finished. However, it's **NOT** guaranteed that they will be called immediately after a subtest is finished.

There are two scenarios:

- For nested subtests, the `Cleanup` functions registered at `TearDownSubTest` and `SetupSubTest` are called once their parents are finished but not cleaned.
- For direct subtests whose parents are NOT subtests, their `Cleanup` functions registered at `TearDownSubTest` and `SetupSubTest` are called once the parents' `AfterTest` and `TearDownTest` are finished.

The impact is:

- Nested subtests could pollute each other because some `Cleanup` functions are running after all sibling subtests.
- Children subtest can even pollute their parents.

The source code can be found here: [https://github.com/xuanyuwang/testify-examples/blob/main/cleanup-order/cleanup_order_test.go](https://github.com/xuanyuwang/testify-examples/blob/main/cleanup-order/cleanup_order_test.go)

```go
package cleanuporder_test

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/suite"
)

var tearDownSubTest []string

type TestCleanupOrder struct {
	suite.Suite
}

func (s *TestCleanupOrder) SetupSuite() {
	fmt.Println("Run Interface SetupAllSuite: SetupSuite")
	s.T().Cleanup(func() {
		fmt.Println("Cleanup Interface SetupAllSuite: SetupSuite")
	})
}

func (s *TestCleanupOrder) SetupTest() {
	fmt.Println("Run Interface SetupTestSuite: SetupTest")
	s.T().Cleanup(func() {
		fmt.Println("Cleanup Interface SetupTestSuite: SetupTest")
	})
}

func (s *TestCleanupOrder) TeatDownSuite() {
	fmt.Println("Run Interface TearDownAllSuite: TearDownSuite")
	s.T().Cleanup(func() {
		fmt.Println("Cleanup Interface TearDownAllSuite: TearDownSuite")
	})
}

func (s *TestCleanupOrder) TearDownTest() {
	fmt.Println("Run Interface TearDownTestSuite: TearDownTest")
	s.T().Cleanup(func() {
		fmt.Println("Cleanup Interface TearDownTestSuite: TearDownTest")
	})
}

func (s *TestCleanupOrder) BeforeTest(suiteName, testName string) {
	fmt.Printf("Run Interface BeforeTest: BeforeTest for suite %s - test %s\n", suiteName, testName)
	s.T().Cleanup(func() {
		fmt.Printf("Cleanup Interface BeforeTest: BeforeTest for suite %s - test %s\n", suiteName, testName)
	})
}

func (s *TestCleanupOrder) AfterTest(suiteName, testName string) {
	fmt.Printf("Run Interface AfterTest: AfterTest for suite %s - test %s\n", suiteName, testName)
	s.T().Cleanup(func() {
		fmt.Printf("Cleanup Interface AfterTest: AfterTest for suite %s - test %s\n", suiteName, testName)
	})
}

func (s *TestCleanupOrder) HandleStats(suiteName string, stats *suite.SuiteInformation) {
	fmt.Printf("Run Interface WithStats: HandleStats for suite %s\n", suiteName)
	s.T().Cleanup(func() {
		fmt.Printf("Cleanup Interface WithStats: HandleStats for suite %s\n", suiteName)
	})
}

func (s *TestCleanupOrder) SetupSubTest() {
	fmt.Println("Run Interface SetupSubTest: SetupSubTest")
	s.T().Cleanup(func() {
		fmt.Println("Cleanup Interface SetupSubTest: SetupSubTest")
	})
}

func (s *TestCleanupOrder) TearDownSubTest() {
	fmt.Println("Run Interface TearDownSubTest: TearDownSubTest")
	s.T().Cleanup(func() {
		fmt.Println("Cleanup Interface TearDownSubTest: TearDownSubTest")
		fmt.Printf("\tTear down sub test %v\n", tearDownSubTest)
	})
}

// Test case
func (s *TestCleanupOrder) TestCaseA() {
	fmt.Println("Run TestCase A")
	s.T().Cleanup(func() {
		fmt.Println("Cleanup TestCase A")
	})

	s.Run("Sub-TestCase A", func() {
		fmt.Println("Run Sub-TestCase A")
		s.T().Cleanup(func() {
			fmt.Println("Cleanup Sub-TestCase A")
		})
		tearDownSubTest = append(tearDownSubTest, "Sub-TestCase A")

		s.Run("Sub-Sub-TestCase A", func() {
			s.T().Cleanup(func() {
				fmt.Println("Cleanup Sub-Sub-TestCase A")
			})
			fmt.Println("Run Sub-Sub-TestCase A")
			tearDownSubTest = append(tearDownSubTest, "Sub-Sub-TestCase A")
			fmt.Println("Finish Sub-Sub-TestCase A")
		})
		s.Run("Sub-Sub-TestCase B", func() {
			s.T().Cleanup(func() {
				fmt.Println("Cleanup Sub-Sub-TestCase B")
			})
			fmt.Println("Run Sub-Sub-TestCase B")
			tearDownSubTest = append(tearDownSubTest, "Sub-Sub-TestCase B")
			fmt.Println("Finish Sub-Sub-TestCase B")
		})

		fmt.Println("Finish Sub-TestCase A")
	})
	fmt.Println("Finish TestCase A")
}

// Entry point of the whole test suite
func TestCleanupOrderSuite(t *testing.T) {
	suite.Run(t, &TestCleanupOrder{})
}
```