Repository: hashgraph/hedera-services
Issue: Hedera services shouldn't proceed if initial total ledger balance doesn't add up 

**Summary of the defect**
When hedera services starts and the initial total ledger balance doesn't add up the expected value, the services should stop and refuse to move forward. Also this should be a severe ERROR, not just an WARN message.


**How to reproduce (if possible)**
When in CircleCi running three test jobs in parallel, this issue surfaces. Please refer to this URL for more info: https://app.circleci.com/pipelines/github/hashgraph/hedera-services/715/workflows/25649e74-eb55-45f0-a8ce-54e36f6f6bdf/jobs/9371



**Service logs (if applicable)**
Log message excerpt from CircleCi is here:
```
>>>> [CI] >> ================
 
>>>> [CI] >> === /repo/HapiApp2.0/18.237.197.98/output/hgcaa.log ===
 
>>>> [CI] >> ================
 
2020-06-08 22:15:32.908 INFO  87   ServicesState - Initializing context of Services node 0 with platform and address book...
 
2020-06-08 22:15:32.928 INFO  94   ServicesState - ...done, context is set for Services node 0!
 
2020-06-08 22:15:32.936 INFO  321  ServicesMain - Using context to initialize HederaNode#0...
 
2020-06-08 22:15:33.037 INFO  171  ServicesMain - Property sources are available.
 
2020-06-08 22:15:33.044 INFO  173  ServicesMain - Platform is configured.
 
2020-06-08 22:15:33.044 INFO  175  ServicesMain - Migrations complete.
 
2020-06-08 22:15:33.060 WARN  307  ServicesMain - Unexpected total balance in ledger, nodeId=0!
 
2020-06-08 22:15:33.078 INFO  177  ServicesMain - Ledger state ok.
 
2020-06-08 22:15:33.151 INFO  52   RecordCache - Constructing the RecordCache with TTL 180s
 
2020-06-08 22:15:33.190 INFO  224  HfsSystemFilesManager - --- Bootstrapping network properties from 'data/config/application.properties' as below ---
 
2020-06-08 22:15:33.193 INFO  228  HfsSystemFilesManager - NETTY_CHANNEL_TYPE = EPOLL
 
2020-06-08 22:15:33.194 INFO  228  HfsSystemFilesManager - accountBalanceExportDir = data/accountBalances/
 
2020-06-08 22:15:33.195 INFO  228  HfsSystemFilesManager - accountBalanceExportEnabled = true
 
2020-06-08 22:15:33.195 INFO  228  HfsSystemFilesManager - accountBalanceExportPeriodMinutes = 10
...
```
**Environment:**
- OS: [e.g. Ubuntu 18.04]
- Java: [e.g. OpenJDK 12.0.2]
- Hedera Services Version: [0.5.0]
- HAPI Version: []

**Additional Context**
Add any other context about the problem here. Attach any logs here, if applicable.




Changed files:
hedera-node/src/main/java/com/hedera/services/ServicesMain.java
hedera-node/src/main/java/com/hedera/services/legacy/handler/TransactionHandler.java
hedera-node/src/test/java/com/hedera/services/ServicesMainTest.java
hedera-node/src/test/java/com/hedera/services/legacy/unit/CrptDelAcctValtionAndStartupBalCheckTest.java

diff --git a/hedera-node/src/main/java/com/hedera/services/ServicesMain.java b/hedera-node/src/main/java/com/hedera/services/ServicesMain.java
index f26bbe8e1c..ef2ad4c73d 100644
--- a/hedera-node/src/main/java/com/hedera/services/ServicesMain.java
+++ b/hedera-node/src/main/java/com/hedera/services/ServicesMain.java
@@ -279,7 +279,8 @@ public class ServicesMain implements SwirldMain {
 	private void validateLedgerState() {
 		ctx.ledgerValidator().assertIdsAreValid(ctx.accounts());
 		if (!ctx.ledgerValidator().hasExpectedTotalBalance(ctx.accounts())) {
-			log.warn("Unexpected total balance in ledger, nodeId={}!", ctx.id());
+			log.error("Unexpected total balance in ledger, nodeId={}!", ctx.id());
+			systemExits.fail(1);
 		}
 		if (ctx.nodeAccount() == null) {
 			throwIseOrLogError(new IllegalStateException("Unknown ledger account!"));
diff --git a/hedera-node/src/main/java/com/hedera/services/legacy/handler/TransactionHandler.java b/hedera-node/src/main/java/com/hedera/services/legacy/handler/TransactionHandler.java
index a2e6c7a299..f2e894cb14 100644
--- a/hedera-node/src/main/java/com/hedera/services/legacy/handler/TransactionHandler.java
+++ b/hedera-node/src/main/java/com/hedera/services/legacy/handler/TransactionHandler.java
@@ -615,7 +615,7 @@ public class TransactionHandler {
   }
 
   /**
-   * Validates Account IDs and Total Balance in Account Map on Start Up .
+   * Validates Account IDs and Total Balance in Account Map on Start Up .  
    * If it finds any invalid Account ID  it stops checking further and returns Invalid Account ID response code.
    * If all the Account IDs are valid, it checks the total balance. If its not equal to expected balance, it
    * returns Invalid Balance response code.
@@ -639,7 +639,7 @@ public class TransactionHandler {
         break;
       }
     }
-    if(response == OK) {
+    if(response == OK && !accountMap.isEmpty()) {
     	if(totalBalance != PropertiesLoader.getInitialGenesisCoins()) {
     		response = ResponseCodeEnum.TOTAL_LEDGER_BALANCE_INVALID;
     	}
diff --git a/hedera-node/src/test/java/com/hedera/services/ServicesMainTest.java b/hedera-node/src/test/java/com/hedera/services/ServicesMainTest.java
index 2ff7abbe09..3254562124 100644
--- a/hedera-node/src/test/java/com/hedera/services/ServicesMainTest.java
+++ b/hedera-node/src/test/java/com/hedera/services/ServicesMainTest.java
@@ -67,6 +67,7 @@ import java.time.Instant;
 import static com.hedera.services.context.SingletonContextsManager.CONTEXTS;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.BDDMockito.any;
 import static org.mockito.BDDMockito.given;
 import static org.mockito.BDDMockito.inOrder;
@@ -76,6 +77,7 @@ import static org.mockito.BDDMockito.never;
 import static org.mockito.BDDMockito.verify;
 import static org.mockito.BDDMockito.verifyNoInteractions;
 import static org.mockito.BDDMockito.willThrow;
+import static org.mockito.Mockito.atLeastOnce;
 
 @RunWith(JUnitPlatform.class)
 public class ServicesMainTest {
@@ -184,9 +186,48 @@ public class ServicesMainTest {
 		ServicesMain.log = LogManager.getLogger(ServicesMain.class);
 	}
 
+	@Test
+	public void shouldAlwaysFailFastOnUnexpectedInitialLedgerBalance() {
+		given(properties.getBooleanProperty("hedera.exitOnNodeStartupFailure")).willReturn(anyBoolean());
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(false);
+
+		// when:
+		subject.init(null, new NodeId(false, NODE_ID));
+
+		// then:
+		verify(systemExits,atLeastOnce()).fail(1);
+	}
+
+	@Test
+	public void shouldNotFailOnExpectedInitialLedgerBalance() {
+		given(properties.getBooleanProperty("hedera.exitOnNodeStartupFailure")).willReturn(true);
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
+
+		// when:
+		subject.init(null, new NodeId(false, NODE_ID));
+
+		// then:
+		verify(systemExits,never()).fail(1);
+	}
+	
+	@Test
+	public void shouldFailFastOnMissingNodeAccountIdIfSkippingNotExits() {
+		given(properties.getBooleanProperty("hedera.exitOnNodeStartupFailure")).willReturn(true);
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
+		given(ctx.nodeAccount()).willReturn(null);
+
+		// when:
+		subject.init(null, new NodeId(false, NODE_ID));
+
+		// then:
+		verify(systemExits, atLeastOnce()).fail(1);
+	}
+
+
 	@Test
 	public void doesntFailFastOnMissingNodeAccountIdIfSkippingExits() {
 		given(properties.getBooleanProperty("hedera.exitOnNodeStartupFailure")).willReturn(false);
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
 		given(ctx.nodeAccount()).willReturn(null);
 
 		// when:
@@ -196,10 +237,13 @@ public class ServicesMainTest {
 		verify(systemExits, never()).fail(1);
 	}
 
+
+
 	@Test
 	public void failsFastOnNonUtf8DefaultCharset() {
 		// setup:
 		subject.defaultCharset = () -> StandardCharsets.US_ASCII;
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
 
 		// when:
 		subject.init(null, new NodeId(false, NODE_ID));
@@ -216,13 +260,14 @@ public class ServicesMainTest {
 		subject.init(null, new NodeId(false, NODE_ID));
 
 		// then:
-		verify(systemExits).fail(1);
+		verify(systemExits, atLeastOnce()).fail(1);
 	}
 
 	@Test
 	public void exitsOnApplicationPropertiesLoading() {
 		willThrow(IllegalStateException.class)
 				.given(systemFilesManager).loadApplicationProperties();
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
 
 		// when:
 		subject.init(null, new NodeId(false, NODE_ID));
@@ -235,6 +280,7 @@ public class ServicesMainTest {
 	public void exitsOnAddressBookCreationFailure() {
 		willThrow(IllegalStateException.class)
 				.given(systemFilesManager).createAddressBookIfMissing();
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
 
 		// when:
 		subject.init(null, new NodeId(false, NODE_ID));
@@ -247,6 +293,7 @@ public class ServicesMainTest {
 	public void exitsOnCreationFailure() throws Exception {
 		given(properties.getBooleanProperty("hedera.createSystemAccountsOnStartup")).willReturn(true);
 		given(properties.getBooleanProperty("hedera.exitOnNodeStartupFailure")).willReturn(true);
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
 
 		willThrow(Exception.class)
 				.given(systemAccountsCreator).createSystemAccounts(any(), any());
@@ -428,6 +475,7 @@ public class ServicesMainTest {
 	public void rethrowsAccountsCreationFailureAsIse() {
 		given(properties.getBooleanProperty("hedera.createSystemAccountsOnStartup")).willReturn(true);
 		given(ctx.systemAccountsCreator()).willReturn(null);
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
 
 		// when:
 		subject.init(null, new NodeId(false, NODE_ID));
@@ -453,6 +501,7 @@ public class ServicesMainTest {
 		given(properties.getStringProperty("hedera.accountsExportPath")).willReturn(PATH);
 		given(properties.getBooleanProperty("hedera.exportAccountsOnStartup")).willReturn(true);
 		given(ctx.accountsExporter()).willReturn(null);
+		given(ledgerValidator.hasExpectedTotalBalance(accounts)).willReturn(true);
 
 		// when:
 		subject.init(null, new NodeId(false, NODE_ID));
diff --git a/hedera-node/src/test/java/com/hedera/services/legacy/unit/CrptDelAcctValtionAndStartupBalCheckTest.java b/hedera-node/src/test/java/com/hedera/services/legacy/unit/CrptDelAcctValtionAndStartupBalCheckTest.java
index b268e9bf46..7400c9ec25 100644
--- a/hedera-node/src/test/java/com/hedera/services/legacy/unit/CrptDelAcctValtionAndStartupBalCheckTest.java
+++ b/hedera-node/src/test/java/com/hedera/services/legacy/unit/CrptDelAcctValtionAndStartupBalCheckTest.java
@@ -56,6 +56,8 @@ import com.hedera.services.legacy.core.jproto.JKey;
 import net.i2p.crypto.eddsa.EdDSAPrivateKey;
 import net.i2p.crypto.eddsa.EdDSAPublicKey;
 import net.i2p.crypto.eddsa.KeyPairGenerator;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.platform.runner.JUnitPlatform;
 
 public class CrptDelAcctValtionAndStartupBalCheckTest {
 
@@ -102,27 +104,78 @@ public class CrptDelAcctValtionAndStartupBalCheckTest {
 		keys = Key.newBuilder().setKeyList(KeyList.newBuilder().addAllKeys(keyListp).build()).build();
 	}
 
-	    @Test
-		public void testAccountMapBalanceForStartup() throws Exception {
-			long account1Balance = 100000l;
-			long account2Balance = 200000l;
-			// Total Balance is less than 50B
-			account1ID = RequestBuilder.getAccountIdBuild(1022l, 0l, 0l);
-			account2ID = RequestBuilder.getAccountIdBuild(1023l, 0l, 0l);
-			createAccount(account1ID, account1Balance, keys);
-			createAccount(account2ID, account2Balance, keys);
-			ResponseCodeEnum response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
-			Assert.assertEquals(ResponseCodeEnum.TOTAL_LEDGER_BALANCE_INVALID, response);
+	@Test
+	public void testAccountMapBalanceForStartup() throws Exception {
+		long account1Balance = 100000l;
+		long account2Balance = 200000l;
+		// Total Balance is less than 50B
+		account1ID = RequestBuilder.getAccountIdBuild(1022l, 0l, 0l);
+		account2ID = RequestBuilder.getAccountIdBuild(1023l, 0l, 0l);
+		createAccount(account1ID, account1Balance, keys);
+		createAccount(account2ID, account2Balance, keys);
+		ResponseCodeEnum response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
+		Assert.assertEquals(ResponseCodeEnum.TOTAL_LEDGER_BALANCE_INVALID, response);
+
+		// Total balance is 50B
+		account3ID = RequestBuilder.getAccountIdBuild(1024l, 0l, 0l);
+		long account3Balance = 5000000000000000000l - (account2Balance + account1Balance);
+		createAccount(account3ID, account3Balance, keys);
+		response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
+		Assert.assertEquals(ResponseCodeEnum.OK, response);
+	}
 
-			// Total balance is 50B
-			account3ID = RequestBuilder.getAccountIdBuild(1024l, 0l, 0l);
-			long account3Balance = 5000000000000000000l - (account2Balance + account1Balance);
-			createAccount(account3ID, account3Balance, keys);
-			response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
-			Assert.assertEquals(ResponseCodeEnum.OK, response);
+	@Test
+	public void testInvalidAccountIdWithZeroAcctNumForStartup() throws Exception {
+		long account1Balance = 100000l;
+		// account id as 0.0.0
+		account1ID = RequestBuilder.getAccountIdBuild(0l, 0l, 0l);
+		createAccount(account1ID, account1Balance, keys);
+		ResponseCodeEnum response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
+		Assert.assertEquals(ResponseCodeEnum.INVALID_ACCOUNT_ID, response);
+	}
 
+	@Test
+	public void testInvalidAccountIdWithTooLargeAcctNumForStartup2() throws Exception {
+		long account1Balance = 100000l;
+		// account num as larger than allowef
+		account1ID = RequestBuilder.getAccountIdBuild(PropertiesLoader.getConfigAccountNum() + 1, 0l, 0l);
+		createAccount(account1ID, account1Balance, keys);
+		ResponseCodeEnum response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
+		Assert.assertEquals(ResponseCodeEnum.INVALID_ACCOUNT_ID, response);
+	}
 
-		}
+	@Test
+	public void testInvalidAccountIdDueToInvalidRealmForStartup() throws Exception {
+		fcMap.clear();
+		long account1Balance = 100000l;
+		// account id has bad realm value
+		account1ID = RequestBuilder.getAccountIdBuild(1021l, -1l, 0l);
+		createAccount(account1ID, account1Balance, keys);
+
+		ResponseCodeEnum response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
+		Assert.assertEquals(ResponseCodeEnum.INVALID_ACCOUNT_ID, response);
+	}
+
+	@Test
+	public void testInvalidAccountIdDueToInvalidShardForStartup() throws Exception {
+		fcMap.clear();
+		long account1Balance = 100000l;
+		// account id has bad shard value
+		account1ID = RequestBuilder.getAccountIdBuild(1022l, 0l, 100l);
+		createAccount(account1ID, account1Balance, keys);
+
+		ResponseCodeEnum response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
+		Assert.assertEquals(ResponseCodeEnum.INVALID_ACCOUNT_ID, response);
+	}
+
+
+	@Test
+	public void testEmptyAccountMapBalanceForStartup() throws Exception {
+		fcMap.clear();
+		ResponseCodeEnum response = TransactionHandler.validateAccountIDAndTotalBalInMap(fcMap);
+		Assert.assertEquals(ResponseCodeEnum.OK, response);
+
+	}
 
 	private static Key PrivateKeyToKey(PrivateKey privateKey) {
 		byte[] pubKey = ((EdDSAPrivateKey) privateKey).getAbyte();
@@ -134,13 +187,14 @@ public class CrptDelAcctValtionAndStartupBalCheckTest {
 		byte[] pubKey = ((EdDSAPublicKey) pair.getPublic()).getAbyte();
 		String pubKeyHex = MiscUtils.commonsBytesToHex(pubKey);
 		pubKey2privKeyMap.put(pubKeyHex, pair.getPrivate());
-	}	
+	}
 
 
 	private void createAccount(AccountID payerAccount, long balance, Key key) throws Exception {
 		MerkleEntityId mk = new MerkleEntityId();
 		mk.setNum(payerAccount.getAccountNum());
-		mk.setRealm(0);
+		mk.setRealm(payerAccount.getRealmNum());
+		mk.setShard(payerAccount.getShardNum());
 		MerkleAccount mv = new MerkleAccount();
 		mv.setBalance(balance);
 		JKey jkey = JKey.mapKey(key);