Repository: hashgraph/hedera-services
Issue: Unavailable channel when using SuiteRunner to run multiple suites that include load/perf test

https://github.com/hashgraph/hedera-services/blob/c3aa15f4edc8f406ada2c98dadc9d7af2ed252e0/test-clients/src/main/java/com/hedera/services/bdd/spec/HapiApiSpec.java#L231-L241
https://github.com/hashgraph/hedera-services/blob/c3aa15f4edc8f406ada2c98dadc9d7af2ed252e0/test-clients/src/main/java/com/hedera/services/bdd/spec/HapiApiSpec.java#L284-L297
With the current logic, the channels between clients and servers are closed when a spec failed or there are pending ops (in case of load/perf test). This causes the following exception when using SuiteRunner to run multiple suites that include load/perf test.
```
io.grpc.StatusRuntimeException: UNAVAILABLE: Channel shutdown invoked
	at io.grpc.stub.ClientCalls.toStatusRuntimeException(ClientCalls.java:235) ~[SuiteRunner.jar:?]
	at io.grpc.stub.ClientCalls.getUnchecked(ClientCalls.java:216) ~[SuiteRunner.jar:?]
	at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:141) ~[SuiteRunner.jar:?]
	at com.hederahashgraph.service.proto.java.FileServiceGrpc$FileServiceBlockingStub.getFileContent(FileServiceGrpc.java:638) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.spec.fees.FeesAndRatesProvider.downloadWith(FeesAndRatesProvider.java:157) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.spec.fees.FeesAndRatesProvider.lookupDownloadFee(FeesAndRatesProvider.java:150) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.spec.fees.FeesAndRatesProvider.downloadRateSet(FeesAndRatesProvider.java:120) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.spec.fees.FeesAndRatesProvider.init(FeesAndRatesProvider.java:79) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.spec.HapiApiSpec.init(HapiApiSpec.java:187) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.spec.HapiApiSpec.run(HapiApiSpec.java:168) ~[SuiteRunner.jar:?]
	at java.util.AbstractList$RandomAccessSpliterator.forEachRemaining(AbstractList.java:720) ~[?:?]
	at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:658) ~[?:?]
	at com.hedera.services.bdd.suites.HapiApiSuite.runSync(HapiApiSuite.java:246) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.suites.HapiApiSuite.runSuite(HapiApiSuite.java:138) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.suites.HapiApiSuite.runSuiteSync(HapiApiSuite.java:131) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.suites.SuiteRunner.lambda$runSuitesSync$14(SuiteRunner.java:407) ~[SuiteRunner.jar:?]
	at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:176) ~[?:?]
	at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:?]
	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) ~[?:?]
	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) ~[?:?]
	at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) ~[?:?]
	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:?]
	at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) ~[?:?]
	at com.hedera.services.bdd.suites.SuiteRunner.runSuitesSync(SuiteRunner.java:408) ~[SuiteRunner.jar:?]
	at com.hedera.services.bdd.suites.SuiteRunner.lambda$runTargetCategories$10(SuiteRunner.java:381) ~[SuiteRunner.jar:?]
	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195) [?:?]
	at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1654) [?:?]
	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) [?:?]
	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) [?:?]
	at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) [?:?]
	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) [?:?]
	at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) [?:?]
	at com.hedera.services.bdd.suites.SuiteRunner.runTargetCategories(SuiteRunner.java:381) [SuiteRunner.jar:?]
	at com.hedera.services.bdd.suites.SuiteRunner.runCategories(SuiteRunner.java:350) [SuiteRunner.jar:?]
	at com.hedera.services.bdd.suites.SuiteRunner.main(SuiteRunner.java:290) [SuiteRunner.jar:?]
```

**Solution:**
- Add correct tearDown function for `HapiApiSpec`, `HapiApiSuite` and `HapiApiClients`
- Move the logic of tearing down clients to in suite instead of in spec.

Changed files:
test-clients/src/main/java/com/hedera/services/bdd/spec/HapiApiSpec.java
test-clients/src/main/java/com/hedera/services/bdd/suites/HapiApiSuite.java
test-clients/src/main/java/com/hedera/services/bdd/spec/infrastructure/HapiApiClients.java

diff --git a/test-clients/src/main/java/com/hedera/services/bdd/spec/HapiApiSpec.java b/test-clients/src/main/java/com/hedera/services/bdd/spec/HapiApiSpec.java
index 103667656c..7e2647612f 100644
--- a/test-clients/src/main/java/com/hedera/services/bdd/spec/HapiApiSpec.java
+++ b/test-clients/src/main/java/com/hedera/services/bdd/spec/HapiApiSpec.java
@@ -202,6 +202,12 @@ public class HapiApiSpec implements Runnable {
 		return true;
 	}
 
+	private void tearDown() {
+		if (finalizingExecutor != null) {
+			finalizingExecutor.shutdown();
+		}
+	}
+
 	private void exec(List<HapiSpecOperation> ops) {
 		if (status == ERROR) {
 			log.warn("'" + name + "' failed to initialize, being skipped!");
@@ -233,12 +239,8 @@ public class HapiApiSpec implements Runnable {
 			if (finishingError.get().isPresent()) {
 				status = FAILED;
 			}
-		} else {
-			if (finalizingExecutor != null) {
-				finalizingExecutor.shutdown();
-				this.clients().closeChannels();
-			}
 		}
+		tearDown();
 		log.info(logPrefix() + "final status: " + status + "!");
 
 		if(saveContextFlag) {
@@ -283,7 +285,6 @@ public class HapiApiSpec implements Runnable {
 
 	private void finishFinalizingOps() {
 		if (pendingOps.isEmpty()) {
-			finalizingExecutor.shutdown();
 			return;
 		}
 		log.info(logPrefix() + "executed " + numLedgerOpsExecuted.get() + " ledger ops.");
@@ -292,8 +293,6 @@ public class HapiApiSpec implements Runnable {
 			startFinalizingOps();
 		}
 		finalizingFuture.join();
-		finalizingExecutor.shutdown();
-		this.clients().closeChannels();
 	}
 
 	public void offerFinisher(HapiSpecOpFinisher finisher) {
diff --git a/test-clients/src/main/java/com/hedera/services/bdd/spec/infrastructure/HapiApiClients.java b/test-clients/src/main/java/com/hedera/services/bdd/spec/infrastructure/HapiApiClients.java
index d53b5e76b1..db96944cd4 100644
--- a/test-clients/src/main/java/com/hedera/services/bdd/spec/infrastructure/HapiApiClients.java
+++ b/test-clients/src/main/java/com/hedera/services/bdd/spec/infrastructure/HapiApiClients.java
@@ -40,7 +40,6 @@ import com.hederahashgraph.service.proto.java.NetworkServiceGrpc.NetworkServiceB
 import com.hederahashgraph.service.proto.java.TokenServiceGrpc;
 import com.hederahashgraph.service.proto.java.TokenServiceGrpc.TokenServiceBlockingStub;
 import io.grpc.ManagedChannel;
-import io.grpc.ManagedChannelBuilder;
 import io.grpc.netty.GrpcSslContexts;
 import io.grpc.netty.NegotiationType;
 import io.grpc.netty.NettyChannelBuilder;
@@ -49,12 +48,10 @@ import io.netty.handler.ssl.SupportedCipherSuiteFilter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 
 import static java.util.stream.Collectors.toMap;
 
@@ -73,7 +70,7 @@ public class HapiApiClients {
 	private final List<NodeConnectInfo> nodes;
 	private final Map<AccountID, String> stubIds;
 	private final Map<AccountID, String> tlsStubIds;
-	private List<ManagedChannel> channels;
+	private static Map<String, ManagedChannel> channels = new HashMap<>();
 
 	private final ManagedChannel createNettyChannel(NodeConnectInfo node, boolean useTls) {
 		try {
@@ -100,7 +97,6 @@ public class HapiApiClients {
 						.usePlaintext()
 						.build();
 			}
-			channels.add(channel);
 			return channel;
 		} catch (Exception e) {
 			log.error("Error creating Netty channel", e);
@@ -108,8 +104,21 @@ public class HapiApiClients {
 		return null;
 	}
 
+	private void addStubs(NodeConnectInfo node, String uri, boolean useTls) {
+		if (!channels.containsKey(uri)) {
+			ManagedChannel channel = createNettyChannel(node, useTls);
+			channels.put(uri, channel);
+			scSvcStubs.put(uri, SmartContractServiceGrpc.newBlockingStub(channel));
+			consSvcStubs.put(uri, ConsensusServiceGrpc.newBlockingStub(channel));
+			fileSvcStubs.put(uri, FileServiceGrpc.newBlockingStub(channel));
+			tokenSvcStubs.put(uri, TokenServiceGrpc.newBlockingStub(channel));
+			cryptoSvcStubs.put(uri, CryptoServiceGrpc.newBlockingStub(channel));
+			freezeSvcStubs.put(uri, FreezeServiceGrpc.newBlockingStub(channel));
+			networkSvcStubs.put(uri, NetworkServiceGrpc.newBlockingStub(channel));
+		}
+	}
+
 	private HapiApiClients(List<NodeConnectInfo> nodes, AccountID defaultNode) {
-		this.channels = new ArrayList<>();
 		this.nodes = nodes;
 		stubIds = nodes
 				.stream()
@@ -119,25 +128,8 @@ public class HapiApiClients {
 				.collect(toMap(NodeConnectInfo::getAccount, NodeConnectInfo::tlsUri));
 		int before = stubCount();
 		nodes.forEach(node -> {
-			ManagedChannel channel = createNettyChannel(node, false);
-			String stubsId = node.uri();
-			scSvcStubs.computeIfAbsent(stubsId, ignore -> SmartContractServiceGrpc.newBlockingStub(channel));
-			consSvcStubs.computeIfAbsent(stubsId, ignore -> ConsensusServiceGrpc.newBlockingStub(channel));
-			fileSvcStubs.computeIfAbsent(stubsId, ignore -> FileServiceGrpc.newBlockingStub(channel));
-			tokenSvcStubs.computeIfAbsent(stubsId, ignore -> TokenServiceGrpc.newBlockingStub(channel));
-			cryptoSvcStubs.computeIfAbsent(stubsId, ignore -> CryptoServiceGrpc.newBlockingStub(channel));
-			freezeSvcStubs.computeIfAbsent(stubsId, ignore -> FreezeServiceGrpc.newBlockingStub(channel));
-			networkSvcStubs.computeIfAbsent(stubsId, ignore -> NetworkServiceGrpc.newBlockingStub(channel));
-
-			ManagedChannel tlsChannel = createNettyChannel(node, true);
-			String tlsStubsId = node.tlsUri();
-			scSvcStubs.computeIfAbsent(tlsStubsId, ignore -> SmartContractServiceGrpc.newBlockingStub(tlsChannel));
-			consSvcStubs.computeIfAbsent(tlsStubsId, ignore -> ConsensusServiceGrpc.newBlockingStub(tlsChannel));
-			fileSvcStubs.computeIfAbsent(tlsStubsId, ignore -> FileServiceGrpc.newBlockingStub(tlsChannel));
-			tokenSvcStubs.computeIfAbsent(tlsStubsId, ignore -> TokenServiceGrpc.newBlockingStub(tlsChannel));
-			cryptoSvcStubs.computeIfAbsent(tlsStubsId, ignore -> CryptoServiceGrpc.newBlockingStub(tlsChannel));
-			freezeSvcStubs.computeIfAbsent(tlsStubsId, ignore -> FreezeServiceGrpc.newBlockingStub(tlsChannel));
-			networkSvcStubs.computeIfAbsent(stubsId, ignore -> NetworkServiceGrpc.newBlockingStub(tlsChannel));
+			addStubs(node, node.uri(), false);
+			addStubs(node, node.tlsUri(), true);
 		});
 		int after = stubCount();
 		this.defaultNode = defaultNode;
@@ -207,10 +199,26 @@ public class HapiApiClients {
 	/**
 	 * Close all netty channels that are opened for clients
 	 */
-	public void closeChannels() {
+	private static void closeChannels() {
 		if (channels.isEmpty()) {
 			return;
 		}
-		channels.forEach(channel -> channel.shutdown());
+		channels.forEach((uri, channel) -> channel.shutdown());
+		channels.clear();
+	}
+
+	private static void clearStubs() {
+		scSvcStubs.clear();
+		consSvcStubs.clear();
+		fileSvcStubs.clear();
+		tokenSvcStubs.clear();
+		cryptoSvcStubs.clear();
+		freezeSvcStubs.clear();
+		networkSvcStubs.clear();
+	}
+
+	public static void tearDown() {
+		closeChannels();
+		clearStubs();
 	}
 }
diff --git a/test-clients/src/main/java/com/hedera/services/bdd/suites/HapiApiSuite.java b/test-clients/src/main/java/com/hedera/services/bdd/suites/HapiApiSuite.java
index 62682887d2..7306af93bd 100644
--- a/test-clients/src/main/java/com/hedera/services/bdd/suites/HapiApiSuite.java
+++ b/test-clients/src/main/java/com/hedera/services/bdd/suites/HapiApiSuite.java
@@ -24,6 +24,7 @@ import com.google.common.math.Stats;
 import com.hedera.services.bdd.spec.HapiApiSpec;
 import com.hedera.services.bdd.spec.HapiSpecOperation;
 import com.hedera.services.bdd.spec.HapiSpecSetup;
+import com.hedera.services.bdd.spec.infrastructure.HapiApiClients;
 import com.hedera.services.bdd.spec.queries.HapiQueryOp;
 import com.hedera.services.bdd.spec.stats.HapiStats;
 import com.hedera.services.bdd.spec.stats.OpObs;
@@ -42,7 +43,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.Random;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -138,6 +138,7 @@ public abstract class HapiApiSuite {
 		runner.accept(specs);
 		finalSpecs = specs;
 		summarizeResults(getResultsLogger());
+		HapiApiClients.tearDown();
 		return finalOutcomeFor(finalSpecs);
 	}