func setupTests(suite *AppTestSuite) {
	for _, test := range tests {
		test.Setup(suite, suite.T())
	}
}
run setup on all tests.
package node
import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sync"
	"testing"
	"time"
	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
	"github.com/spacemeshos/post/initialization"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"github.com/spacemeshos/go-spacemesh/activation"
	apicfg "github.com/spacemeshos/go-spacemesh/api/config"
	"github.com/spacemeshos/go-spacemesh/common/types"
	"github.com/spacemeshos/go-spacemesh/common/util"
	"github.com/spacemeshos/go-spacemesh/config"
	"github.com/spacemeshos/go-spacemesh/eligibility"
	"github.com/spacemeshos/go-spacemesh/log"
	"github.com/spacemeshos/go-spacemesh/log/logtest"
	"github.com/spacemeshos/go-spacemesh/p2p"
	"github.com/spacemeshos/go-spacemesh/signing"
	"github.com/spacemeshos/go-spacemesh/timesync"
	"github.com/spacemeshos/go-spacemesh/tortoisebeacon"
)
type AppTestSuite struct {
	suite.Suite
	apps        []*App
	killedApps  []*App
	killEpoch   types.EpochID
	log         log.Log
	poetCleanup func(cleanup bool) error
}
func (suite *AppTestSuite) SetupTest() {
	suite.apps = make([]*App, 0, 0)
	suite.log = logtest.New(suite.T())
	suite.poetCleanup = func(bool) error { return nil }
}
func (suite *AppTestSuite) TearDownTest() {
	if matches, err := filepath.Glob("*.bin"); err != nil {
		suite.log.With().Error("error while finding poet bin files", log.Err(err))
	} else {
		for _, f := range matches {
			if err = os.Remove(f); err != nil {
				suite.log.With().Error("error while cleaning up poet bin files", log.Err(err))
			}
		}
	}
}
func Test_PoETHarnessSanity(t *testing.T) {
	h, err := activation.NewHTTPPoetHarness(true)
	require.NoError(t, err)
	require.NotNil(t, h)
}
func (suite *AppTestSuite) initMultipleInstances(ctx context.Context, cfg *config.Config, rolacle *eligibility.FixedRolacle, numOfInstances int, genesisTime string,
	poetClient *activation.HTTPPoetClient, clock TickProvider, mesh mocknet.Mocknet) (firstDir string) {
	name := 'a'
	for i, h := range mesh.Hosts() {
		dbStorepath := suite.T().TempDir()
		if i == 0 {
			firstDir = dbStorepath
		}
		edSgn := signing.NewEdSigner()
		wrapped, err := p2p.Upgrade(h, p2p.WithContext(ctx))
		suite.Require().NoError(err)
		suite.T().Cleanup(func() { _ = wrapped.Stop() })
		smApp, err := InitSingleInstance(logtest.New(suite.T()), *cfg, i, genesisTime, dbStorepath, rolacle, poetClient, clock, wrapped, edSgn)
		suite.NoError(err)
		suite.apps = append(suite.apps, smApp)
		name++
	}
	return
}
func (suite *AppTestSuite) ClosePoet() {
	if err := suite.poetCleanup(true); err != nil {
		suite.log.With().Error("error while cleaning up poet", log.Err(err))
	}
}
var tests = []TestScenario{
	txWithRunningNonceGenerator([]int{}),
	sameRootTester([]int{0}),
	reachedEpochTester([]int{}),
	txWithUnorderedNonceGenerator([]int{1}),
	healingTester([]int{}),  
}
type sharedClock struct {
	logger log.Log
	*timesync.TimeClock
}
func (clock sharedClock) Close() {
	clock.logger.Info("simulating clock close")
}
func (clock sharedClock) RealClose() {
	clock.TimeClock.Close()
}
func (suite *AppTestSuite) TestMultipleNodes() {
	const (
		numberOfEpochs = 5  
		numOfInstances = 5
	)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	mesh, err := mocknet.FullMeshLinked(ctx, numOfInstances)
	suite.Require().NoError(err)
	cfg := getTestDefaultConfig()
	types.SetLayersPerEpoch(cfg.LayersPerEpoch)
	lg := logtest.New(suite.T())
	genesisTime := time.Now().Add(20 * time.Second).Format(time.RFC3339)
	poetHarness, err := activation.NewHTTPPoetHarness(false)
	suite.NoError(err, "failed creating poet client harness: %v", err)
	suite.poetCleanup = poetHarness.Teardown
	errorRegexp, err := regexp.Compile(`\bERROR\b`)
	require.NoError(suite.T(), err)
	failChan := make(chan struct{})
	go func() {
		poetLog := lg.WithName("poet")
		scanner := bufio.NewScanner(io.MultiReader(poetHarness.Stdout, poetHarness.Stderr))
		for scanner.Scan() {
			line := scanner.Text()
			matched := errorRegexp.MatchString(line)
			if matched {
				suite.T().Errorf("got error from poet: %s", line)
				close(failChan)
			}
			poetLog.Debug(line)
		}
	}()
	rolacle := eligibility.New(lg)
	gTime, err := time.Parse(time.RFC3339, genesisTime)
	if err != nil {
		suite.log.With().Error("cannot parse genesis time", log.Err(err))
	}
	ld := 20 * time.Second
	clock := sharedClock{suite.log, timesync.NewClock(timesync.RealClock{}, ld, gTime, logtest.New(suite.T()))}
	firstDir := suite.initMultipleInstances(ctx, cfg, rolacle, numOfInstances, genesisTime, poetHarness.HTTPPoetClient, clock, mesh)
	suite.Require().NoError(mesh.ConnectAllButSelf())
	var oldRoot types.Hash32
	var edSgn *signing.EdSigner
	func() {
		defer func() {
			GracefulShutdown(suite.apps)
		}()
		defer suite.ClosePoet()
		defer clock.RealClose()
		for _, a := range suite.apps {
			suite.NoError(a.startServices(context.TODO()))
		}
		ActivateGrpcServer(suite.apps[0])
		if err := poetHarness.Start(context.TODO(), []string{fmt.Sprintf("127.0.0.1:%d", suite.apps[0].grpcAPIService.Port)}); err != nil {
			suite.T().Fatalf("failed to start poet server: %v", err)
		}
		timeout := time.After(10 * time.Minute)
		doneChan := make(chan struct{})
		go func() {
			setupTests(suite)
			close(doneChan)
		}()
	loopSetup:
		for {
			select {
			case <-doneChan:
				break loopSetup
			case <-timeout:
				suite.T().Fatal("timed out")
			case <-failChan:
				suite.T().Fatal("error from poet harness")
			}
		}
		finished := map[int]bool{}
	loop:
		for {
			select {
			case <-timeout:
				suite.T().Fatal("timed out")
			case <-failChan:
				suite.T().Fatal("error from poet harness")
			default:
				if runTests(suite, finished) {
					break loop
				}
				time.Sleep(20 * time.Second)
			}
		}
		suite.validateBlocksAndATXs(types.NewLayerID(numberOfEpochs * suite.apps[0].Config.LayersPerEpoch).Sub(1))
		oldRoot = suite.apps[0].state.GetStateRoot()
		edSgn = suite.apps[0].edSgn
	}()
	host, err := mesh.GenPeer()
	suite.Require().NoError(err)
	wrapped, err := p2p.Upgrade(host, p2p.WithContext(ctx))
	defer wrapped.Stop()
	suite.Require().NoError(err)
	smApp, err := InitSingleInstance(lg, *cfg, 0, genesisTime, firstDir, rolacle, poetHarness.HTTPPoetClient, clock, wrapped, edSgn)
	suite.NoError(err)
	suite.validateReloadStateRoot(smApp, oldRoot)
}
type (
	ScenarioSetup        func(*AppTestSuite, *testing.T)
	ScenarioTestCriteria func(*AppTestSuite, *testing.T) bool
	TestScenario         struct {
		Setup        ScenarioSetup
		Criteria     ScenarioTestCriteria
		Dependencies []int
	}
)
func txWithUnorderedNonceGenerator(dependencies []int) TestScenario {
	acc1Signer, err := signing.NewEdSignerFromBuffer(util.FromHex(apicfg.Account2Private))
	if err != nil {
		log.With().Panic("could not build ed signer", log.Err(err))
	}
	addr := types.Address{}
	addr.SetBytes(acc1Signer.PublicKey().Bytes())
	dst := types.BytesToAddress([]byte{0x09})
	txsSent := 25
	setup := func(suite *AppTestSuite, t *testing.T) {
		for i := 0; i < txsSent; i++ {
			tx, err := types.NewSignedTx(uint64(txsSent-i), dst, 10, 1, 1, acc1Signer)
			if err != nil {
				suite.log.With().Panic("panicked creating signed tx", log.Err(err))
			}
			txbytes, _ := types.InterfaceToBytes(tx)
			pbMsg := &pb.SubmitTransactionRequest{Transaction: txbytes}
			_, err = suite.apps[0].txService.SubmitTransaction(context.TODO(), pbMsg)
			suite.Error(err)
			suite.log.With().Info("got expected error submitting tx with out of order nonce", log.Err(err))
		}
	}
	test := func(suite *AppTestSuite, t *testing.T) bool {
		ok := true
		for _, app := range suite.apps {
			suite.log.With().Info("zero acc current balance",
				app.nodeID,
				log.FieldNamed("sender", addr),
				log.FieldNamed("dest", dst),
				log.Uint64("dest_balance", app.state.GetBalance(dst)),
				log.Uint64("sender_nonce", app.state.GetNonce(addr)),
				log.Uint64("sender_balance", app.state.GetBalance(addr)))
			ok = ok && 0 == app.state.GetBalance(dst) && app.state.GetNonce(addr) == 0
		}
		if ok {
			suite.log.Info("zero addresses ok")
		}
		return ok
	}
	return TestScenario{setup, test, dependencies}
}
func txWithRunningNonceGenerator(dependencies []int) TestScenario {
	acc1Signer, err := signing.NewEdSignerFromBuffer(util.FromHex(apicfg.Account1Private))
	if err != nil {
		log.With().Panic("could not build ed signer", log.Err(err))
	}
	addr := types.Address{}
	addr.SetBytes(acc1Signer.PublicKey().Bytes())
	dst := types.BytesToAddress([]byte{0x02})
	txsSent := 25
	setup := func(suite *AppTestSuite, t *testing.T) {
		accountRequest := &pb.AccountRequest{AccountId: &pb.AccountId{Address: addr.Bytes()}}
		getNonce := func() int {
			accountResponse, err := suite.apps[0].globalstateSvc.Account(context.TODO(), accountRequest)
			assert.NoError(suite.T(), err)
			return int(accountResponse.AccountWrapper.StateProjected.Counter)
		}
		for i := 0; i < txsSent; i++ {
			actNonce := getNonce()
			suite.log.Info("waiting for nonce: %d, current projected nonce: %d", i, actNonce)
			for i != actNonce {
				time.Sleep(250 * time.Millisecond)
				actNonce = getNonce()
			}
			tx, err := types.NewSignedTx(uint64(i), dst, 10, 1, 1, acc1Signer)
			suite.NoError(err, "failed to create signed tx: %s", err)
			txbytes, _ := types.InterfaceToBytes(tx)
			pbMsg := &pb.SubmitTransactionRequest{Transaction: txbytes}
			_, err = suite.apps[0].txService.SubmitTransaction(context.TODO(), pbMsg)
			suite.NoError(err, "error submitting transaction")
		}
	}
	test := func(suite *AppTestSuite, t *testing.T) bool {
		ok := true
		for _, app := range suite.apps {
			suite.log.With().Info("valid tx recipient acc current balance",
				app.nodeID,
				log.FieldNamed("sender", addr),
				log.FieldNamed("dest", dst),
				log.Uint64("dest_balance", app.state.GetBalance(dst)),
				log.Uint64("sender_nonce", app.state.GetNonce(addr)),
				log.Uint64("sender_balance", app.state.GetBalance(addr)))
			ok = ok && app.state.GetBalance(dst) >= 250 && app.state.GetNonce(addr) == uint64(txsSent)
		}
		if ok {
			suite.log.Info("addresses ok")
		}
		return ok
	}
	return TestScenario{setup, test, dependencies}
}
func reachedEpochTester(dependencies []int) TestScenario {
	const numberOfEpochs = 5  
	setup := func(*AppTestSuite, *testing.T) {}
	test := func(suite *AppTestSuite, t *testing.T) bool {
		expectedTotalWeight := configuredTotalWeight(suite.apps)
		for _, app := range suite.apps {
			if app.mesh.LatestLayer().Before(types.NewLayerID(numberOfEpochs * uint32(app.Config.LayersPerEpoch))) {
				return false
			}
			suite.validateLastATXTotalWeight(app, numberOfEpochs, expectedTotalWeight)
		}
		suite.log.Info("epoch ok")
		suite.healingWeakcoinTester()
		return true
	}
	return TestScenario{setup, test, dependencies}
}
func (suite *AppTestSuite) healingWeakcoinTester() {
	globalLayer := suite.apps[0].mesh.LatestLayer()
	globalCoin := make(map[types.LayerID]bool)
	for i, app := range suite.apps {
		lastLayer := app.mesh.LatestLayer()
		suite.Equal(globalLayer, lastLayer, "bad last layer on node %v", i)
		for layerID := types.NewLayerID(1); layerID.Before(lastLayer); layerID = layerID.Add(1) {
			coinflip, exists := app.mesh.DB.GetCoinflip(context.TODO(), layerID)
			if !exists {
				suite.Fail("no weak coin value", "node %v layer %v last layer %v", i, layerID, lastLayer)
				continue
			}
			if gc, layerExists := globalCoin[layerID]; layerExists {
				suite.Equal(gc, coinflip, "bad weak coin value on node %v layer %v last layer %v", i, layerID, lastLayer)
			} else {
				globalCoin[layerID] = coinflip
			}
		}
	}
}
func healingTester(dependencies []int) TestScenario {
	var lastVerifiedLayer types.LayerID
	var ok1, ok2 map[int]bool
	confidenceInterval := uint32(2)
	once := sync.Once{}
	setup := func(suite *AppTestSuite, t *testing.T) {}
	test := func(suite *AppTestSuite, t *testing.T) bool {
		once.Do(func() {
			firstAppToKill := len(suite.apps) - (len(suite.apps)/2 + 1)
			suite.killedApps = suite.apps[firstAppToKill:]
			GracefulShutdown(suite.apps[firstAppToKill:])
			suite.apps = suite.apps[:firstAppToKill]
			lastVerifiedLayer = suite.apps[0].mesh.LatestLayerInState()
			suite.killEpoch = lastVerifiedLayer.GetEpoch()
			ok1 = make(map[int]bool, len(suite.apps))
			ok2 = make(map[int]bool, len(suite.apps))
		})
		success := true
		for i, app := range suite.apps {
			lyrProcessed := app.mesh.ProcessedLayer()
			lyrVerified := app.mesh.LatestLayerInState()
			suite.log.With().Info("node latest layers",
				log.FieldNamed("old_verified", lastVerifiedLayer),
				log.Uint32("confidence_interval", confidenceInterval),
				log.FieldNamed("processed", lyrProcessed),
				log.FieldNamed("verified", lyrVerified),
				log.Int("app", i),
				log.FieldNamed("nodeid", app.nodeID))
			ok1[i] = ok1[i] || lyrProcessed.After(lyrVerified.Add(confidenceInterval))
			ok2[i] = ok1[i] && lyrVerified.After(lastVerifiedLayer.Add(confidenceInterval))
			ok2[i] = ok2[i] && !lyrVerified.Add(confidenceInterval).Before(lyrProcessed)
			success = success && ok2[i]
		}
		return success
	}
	return TestScenario{setup, test, dependencies}
}
func configuredTotalWeight(apps []*App) uint64 {
	expectedTotalWeight := uint64(0)
	for _, app := range apps {
		expectedTotalWeight += uint64(app.Config.SMESHING.Opts.NumUnits)
	}
	return expectedTotalWeight
}
func sameRootTester(dependencies []int) TestScenario {
	setup := func(*AppTestSuite, *testing.T) {}
	test := func(suite *AppTestSuite, t *testing.T) bool {
		stickyClientsDone := 0
		maxClientsDone := 0
		for idx, app := range suite.apps {
			clientsDone := 0
			for idx2, app2 := range suite.apps {
				if idx != idx2 {
					r1 := app.state.IntermediateRoot(false).String()
					r2 := app2.state.IntermediateRoot(false).String()
					if r1 == r2 {
						clientsDone++
						if clientsDone == len(suite.apps)-1 {
							suite.log.Info("%d roots confirmed out of %d return ok", clientsDone, len(suite.apps))
							return true
						}
					}
				}
			}
			if clientsDone > maxClientsDone {
				maxClientsDone = clientsDone
			}
		}
		if maxClientsDone != stickyClientsDone {
			stickyClientsDone = maxClientsDone
			suite.log.Info("%d roots confirmed out of %d", maxClientsDone, len(suite.apps))
		}
		return false
	}
	return TestScenario{setup, test, dependencies}
}