Repository: hashgraph/hedera-local-node (ts)

Issue:
Intermittent failure on start of AccountCreationState
### Description

If the `AccountCreationState` begins, but the SDK is not yet connected to a healthy node, the local node would throw this error when starting up. We need to implement some sort of retry mechanism in case this issue happens.

### Steps to reproduce

Run the local node multiple times until you see the error.

### Additional context

```bash
[Hedera-Local-Node] INFO (StartState) ⏳ Starting Hedera Local Node...
[Hedera-Local-Node] INFO (StartState) ⏳ Detecting network...
[Hedera-Local-Node] INFO (StartState) [✔︎] Hedera Local Node successfully started!
[Hedera-Local-Node] INFO (NetworkPrepState) ⏳ Starting Network Preparation State...
[Hedera-Local-Node] INFO (NetworkPrepState) [✔︎] Imported fees successfully!
[Hedera-Local-Node] INFO (NetworkPrepState) [✔︎] Topic was created!
[Hedera-Local-Node] INFO (AccountCreationState) ⏳ Starting Account Creation state in synchronous mode ...
[Hedera-Local-Node] ERROR (StateController) [✘] Trying to transition to next state was not possible. Error is: Error: failed to find a healthy working node
```

### Hedera network

_No response_

### Version

v2.29.0

### Operating system

None

Pull request description:
fix: Intermittent failure on start of `AccountCreationState`
**Description**:

If the AccountCreationState begins, but the SDK is not yet connected to a healthy node, the local node would throw this error when starting up. We need to implement some sort of retry mechanism in case this issue happens.

This PR implements a retry mechanism to handle the scenario where the SDK is not connected to a healthy node at the start of the `AccountCreationState`. The retry mechanism will attempt to reconnect to a healthy node a specified number of times before failing.

**Changes:**
1. **Retry Mechanism:**
   - Added a new class `RetryUtils.ts` with a `retryTask` function to handle retry logic.
   - Configured the retry mechanism to execute the given task a specified number of times with a delay between attempts.

2. **Integration with `AccountCreationState`:**
   - Integrated the retry mechanism into the `AccountCreationState` to ensure it retries each call to the SDK for account creations

**Related issue(s)**:

Fixes #733

**Notes for reviewer**:

- The retry mechanism parameters (number of retries and delay) can be configured as needed.
- This change improves the robustness of the startup process by handling transient connection issues gracefully.

**Checklist**

- [x] Documented (Code comments, README, etc.)
- [x] Tested (unit, integration, etc.)
  - [x] Added unit tests in `RetryUtils.spec.ts` to verify the retry logic.
  - [x] Updated `AccountCreationState.spec.ts` to test the retry mechanism integration.

Patch:
diff --git a/src/constants.ts b/src/constants.ts
index 5d29eff..9740cfc 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -36,6 +36,10 @@ export const CONTAINERS = [
     },
 ];
 
+export const SDK_ERRORS = {
+    FAILED_TO_FIND_A_HEALTHY_NODE: "failed to find a healthy working node",
+}
+
 export const NETWORK_PREFIX = 'hedera-';
 
 export const CONSENSUS_NODE_LABEL = "network-node";
@@ -76,7 +80,7 @@ export const TRACE_COLOR = '\x1b[37m'
 // check success
 export const CHECK_SUCCESS = `${INFO_COLOR}[✔︎]${COLOR_RESET}`
 // check warn
-export const CHEKC_WARN = `${WARNING_COLOR}[!]${COLOR_RESET}`
+export const CHECK_WARN = `${WARNING_COLOR}[!]${COLOR_RESET}`
 // check fail
 export const CHECK_FAIL = `${ERROR_COLOR}[✘]${COLOR_RESET}`
 // waiting
diff --git a/src/services/LoggerService.ts b/src/services/LoggerService.ts
index 65f3b5b..aa6d5b4 100644
--- a/src/services/LoggerService.ts
+++ b/src/services/LoggerService.ts
@@ -22,7 +22,7 @@ import { log, screen, Widgets } from 'blessed';
 import terminal from 'blessed-terminal';
 import {
     CHECK_FAIL,
-    CHEKC_WARN,
+    CHECK_WARN,
     COLOR_DIM,
     COLOR_RESET,
     CONSENSUS_NODE_LABEL,
@@ -33,7 +33,7 @@ import {
     MIRROR_NODE_LABEL,
     RELAY_LABEL,
     TRACE_COLOR,
-    WARNING_COLOR
+    WARNING_COLOR,
 } from '../constants';
 import { AccountCreationState } from '../state/AccountCreationState';
 import { VerboseLevel } from '../types/VerboseLevel';
@@ -233,7 +233,7 @@ export class LoggerService implements IService{
         if (this.verboseLevel < VerboseLevel.WARNING) {
             return;
         }
-        const msgToLog = LoggerService.messageCompute(`${CHEKC_WARN} ${msg}`, module, VerboseLevel.WARNING);
+        const msgToLog = LoggerService.messageCompute(`${CHECK_WARN} ${msg}`, module, VerboseLevel.WARNING);
         this.writeToLog(msgToLog, module);
     }
 
diff --git a/src/state/AccountCreationState.ts b/src/state/AccountCreationState.ts
index 43925f4..09b2cba 100644
--- a/src/state/AccountCreationState.ts
+++ b/src/state/AccountCreationState.ts
@@ -33,11 +33,18 @@ import { ClientService } from '../services/ClientService';
 import {
     privateKeysAliasECDSA,
     privateKeysECDSA,
-    privateKeysED25519
+    privateKeysED25519,
 } from '../configuration/accountConfiguration.json';
-import { ACCOUNT_CREATION_STATE_INIT_MESSAGE, CHECK_SUCCESS, EVM_ADDRESSES_BLOCKLIST_FILE_RELATIVE_PATH, LOADING } from '../constants';
+import {
+    ACCOUNT_CREATION_STATE_INIT_MESSAGE,
+    CHECK_SUCCESS,
+    EVM_ADDRESSES_BLOCKLIST_FILE_RELATIVE_PATH,
+    LOADING,
+    SDK_ERRORS,
+} from '../constants';
 import local from '../configuration/local.json';
 import { AccountUtils } from '../utils/AccountUtils';
+import { RetryUtils } from '../utils/RetryUtils';
 
 /**
  * Represents the state of account creation.
@@ -256,10 +263,14 @@ export class AccountCreationState implements IState {
         accountData.forEach((account) => {
             const { privateKey, balance } = account;
             const client = this.clientService.getClient();
-
-            const createAccountPromise: Promise<Account> = AccountUtils
-                .createAccount(privateKey.publicKey, balance, client)
-                .then((accountInfo) => {
+            const publicKey = privateKey.publicKey;
+
+            const createAccountPromise: Promise<Account> = RetryUtils.retryTask(
+                () => AccountUtils.createAccount(publicKey, balance, client),
+                {
+                    shouldRetry: error => this.shouldRetry(error),
+                    doOnRetry: error => this.doOnRetry(error)
+                }).then((accountInfo) => {
                     const address = accountInfo.accountId.toSolidityAddress();
                     return {
                         accountId: accountInfo.accountId.toString(),
@@ -305,9 +316,12 @@ export class AccountCreationState implements IState {
             const client = this.clientService.getClient();
             const aliasAccountId = privateKey.publicKey.toAccountId(0, 0);
 
-            const createAccountPromise: Promise<Account> = AccountUtils
-                .createAliasedAccount(aliasAccountId, balance, client)
-                .then((accountInfo) => {
+            const createAccountPromise: Promise<Account> = RetryUtils.retryTask(
+                () => AccountUtils.createAliasedAccount(aliasAccountId, balance, client),
+                {
+                    shouldRetry: error => this.shouldRetry(error),
+                    doOnRetry: error => this.doOnRetry(error)
+                }).then((accountInfo) => {
                     const address = privateKey.publicKey.toEvmAddress();
                     return {
                         accountId: accountInfo.accountId.toString(),
@@ -479,4 +493,12 @@ export class AccountCreationState implements IState {
             this.stateName
         );
     }
+
+    private shouldRetry = (error: unknown): boolean => {
+        return error?.toString().includes(SDK_ERRORS.FAILED_TO_FIND_A_HEALTHY_NODE) ?? false;
+    }
+
+    private doOnRetry = (error: unknown): void => {
+        this.logger.warn(`Error occurred during task execution: "${error?.toString()}"`, this.stateName);
+    }
 }
diff --git a/src/utils/RetryUtils.ts b/src/utils/RetryUtils.ts
new file mode 100644
index 0000000..336881f
--- /dev/null
+++ b/src/utils/RetryUtils.ts
@@ -0,0 +1,75 @@
+/*-
+ *
+ * Hedera Local Node
+ *
+ * Copyright (C) 2023-2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+export class RetryUtils {
+
+  /**
+   * Retries a task up to a specified number of times.
+   *
+   * @param {() => Promise<T>} task - The task to retry.
+   * @param {Object} options - The options for the retry.
+   * @param [options.maxAttempts=3] - The maximum number of attempts.
+   * @param [options.backOff=500] - The time to wait between attempts in milliseconds.
+   * @param [options.doOnRetry=(error, attempt) => {}] - The function to call on each retry.
+   * @returns {Promise<T>} - A promise that resolves with the result of the task.
+   * @template T
+   */
+  public static async retryTask<T>(
+    task: () => Promise<T>,
+    {
+      maxRetries = 3,
+      backOff = 500,
+      shouldRetry = (_error: unknown) => true,
+      doOnRetry = (_error: unknown) => {},
+    }: {
+      maxRetries?: number;
+      backOff?: number;
+      shouldRetry?: (error: unknown) => boolean;
+      doOnRetry?: (error: unknown) => void;
+    } = {}
+  ): Promise<T> {
+    let retries = 0;
+    while (retries < maxRetries) {
+      try {
+        // eslint-disable-next-line no-await-in-loop
+        return await task();
+      } catch (error) {
+        if (!shouldRetry(error) || retries === maxRetries - 1) {
+          throw error;
+        }
+        doOnRetry(error);
+        await this.delay(backOff);
+      }
+      retries += 1;
+    }
+    /* istanbul ignore next */
+    throw new Error('Unreachable code');
+  }
+
+  /**
+   * Delays the execution of the task.
+   * @param ms The time to wait in milliseconds.
+   * @returns {Promise<void>} A promise that resolves after the delay.
+   * @private
+   */
+  private static delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}


Test patch:
diff --git a/test/unit/states/AccountCreationState.spec.ts b/test/unit/states/AccountCreationState.spec.ts
index 4d5c35e..3ef1a01 100644
--- a/test/unit/states/AccountCreationState.spec.ts
+++ b/test/unit/states/AccountCreationState.spec.ts
@@ -28,7 +28,8 @@ import { getTestBed } from '../testBed';
 import { SinonSandbox, SinonSpy, SinonStub, SinonStubbedInstance } from 'sinon';
 import {
   ACCOUNT_CREATION_STARTING_ASYNCHRONOUS_BLOCKLIST_MESSAGE,
-  ACCOUNT_CREATION_STATE_INIT_MESSAGE
+  ACCOUNT_CREATION_STATE_INIT_MESSAGE,
+  SDK_ERRORS,
 } from '../../../src/constants';
 import { IOBserver } from '../../../src/controller/IObserver';
 import { AccountUtils } from '../../../src/utils/AccountUtils';
@@ -66,6 +67,11 @@ describe('AccountCreationState', () => {
     testSandbox.resetHistory();
   });
 
+  afterEach(() => {
+    loggerService.info.resetHistory();
+    loggerService.warn.resetHistory();
+  });
+
   it('should initialize the Init State', async () => {
     expect(accountCreationState).to.be.instanceOf(AccountCreationState);
     testSandbox.assert.calledWith(serviceLocator, LoggerService.name);
@@ -96,54 +102,40 @@ describe('AccountCreationState', () => {
     });
 
     it('should log information when starting in synchronous mode', async () => {
-      await accountCreationState.subscribe(observer);
+      accountCreationState.subscribe(observer);
       await accountCreationState.onStart();
 
-      testSandbox.assert.calledTwice(loggerService.info);
+      commonAssertsForStart();
       testSandbox.assert.calledOnce(generateECDSAStub);
       testSandbox.assert.calledOnce(generateAliasECDSAStub);
       testSandbox.assert.calledOnce(generateED25519Stub);
-      testSandbox.assert.called(observer.update);
     });
 
     it('should log information when starting in asynchronous mode', async () => {
-      cliService.getCurrentArgv.returns({
-        async: true,
-        blocklisting: false,
-        balance: 1000,
-        accounts: 5,
-        startup: false,
-      } as any);
+      stubCliOptions({ async: true });
 
-      await accountCreationState.subscribe(observer);
+      accountCreationState.subscribe(observer);
       await accountCreationState.onStart();
 
-      testSandbox.assert.callCount(loggerService.info, 4);
+      commonAssertsForStart();
       testSandbox.assert.calledOnce(generateECDSAStub);
       testSandbox.assert.calledOnce(generateAliasECDSAStub);
       testSandbox.assert.calledOnce(generateED25519Stub);
-      testSandbox.assert.called(observer.update);
     });
 
     it('should log information when blocklisting is enabled', async () => {
       const getBlocklistedAccountsCountStub = testSandbox.stub(AccountCreationState.prototype, <any>'getBlocklistedAccountsCount').returns(1);
-      cliService.getCurrentArgv.returns({
-        async: true,
-        blocklisting: true,
-        balance: 1000,
-        accounts: 5,
-        startup: false,
-      } as any);
+      stubCliOptions({ async: true, blocklisting: true });
 
-      await accountCreationState.subscribe(observer);
+      accountCreationState.subscribe(observer);
       await accountCreationState.onStart();
 
+      commonAssertsForStart();
       testSandbox.assert.calledWith(loggerService.info, ACCOUNT_CREATION_STARTING_ASYNCHRONOUS_BLOCKLIST_MESSAGE, 'AccountCreationState');
       testSandbox.assert.called(getBlocklistedAccountsCountStub);
       testSandbox.assert.calledOnce(generateECDSAStub);
       testSandbox.assert.calledOnce(generateAliasECDSAStub);
       testSandbox.assert.calledOnce(generateED25519Stub);
-      testSandbox.assert.called(observer.update);
 
       getBlocklistedAccountsCountStub.restore();
     });
@@ -160,6 +152,10 @@ describe('AccountCreationState', () => {
     const expectedAccountId: AccountId = AccountId.fromString('0.0.1234');
     expectedAccountId.evmAddress = EvmAddress.fromString('0x1234');
     const expectedBalance: Hbar = new Hbar(1000);
+    const accountInfo: AccountInfo = {
+      accountId: expectedAccountId,
+      balance: expectedBalance
+    } as AccountInfo;
 
     let createAccountStub: SinonStub;
     let createAliasAccountStub: SinonStub;
@@ -169,12 +165,23 @@ describe('AccountCreationState', () => {
     let logAliasAccountTitleStub: SinonStub;
     let logAccountDividerStub: SinonStub;
     let logAliasAccountDividerStub: SinonStub;
+    let privateKeySpy: SinonSpy;
+
+    beforeEach(() => {
+      stubCliOptions({
+        async: false,
+        blocklisting: false,
+        balance: 1000,
+        accounts: 5,
+        startup: false,
+      });
+    });
 
     beforeEach(() => {
       createAccountStub = testSandbox.stub(AccountUtils, <any>'createAccount')
-        .resolves({ accountId: expectedAccountId, balance: expectedBalance } as AccountInfo);
+        .resolves(accountInfo);
       createAliasAccountStub = testSandbox.stub(AccountUtils, <any>'createAliasedAccount')
-        .resolves({ accountId: expectedAccountId, balance: expectedBalance } as AccountInfo);
+        .resolves(accountInfo);
       logAccountStub = testSandbox.stub(AccountCreationState.prototype, <any>'logAccount').resolves();
       logAccountTitleStub = testSandbox.stub(AccountCreationState.prototype, <any>'logAccountTitle').resolves();
       logAliasAccountStub = testSandbox.stub(AccountCreationState.prototype, <any>'logAliasAccount').resolves();
@@ -192,13 +199,13 @@ describe('AccountCreationState', () => {
       logAliasAccountTitleStub.restore();
       logAccountDividerStub.restore();
       logAliasAccountDividerStub.restore();
+      privateKeySpy.restore();
     })
 
     describe('generateECDSA', () => {
       let generateAliasECDSA: SinonStub;
       let generateED25519: SinonStub;
       let generateECDSA: SinonSpy;
-      let privateKeySpy: SinonSpy;
   
       beforeEach(() => {
         generateAliasECDSA = testSandbox.stub(AccountCreationState.prototype, <any>'generateAliasECDSA').resolves();
@@ -211,59 +218,61 @@ describe('AccountCreationState', () => {
         generateAliasECDSA.restore();
         generateED25519.restore();
         generateECDSA.restore();
-        privateKeySpy.restore();
       })
   
       it('should generate ECDSA accounts synchronously and log the title and divider', async () => {
-        cliService.getCurrentArgv.returns({
-          async: false,
-          blocklisting: false,
-          balance: 1000,
-          accounts: 5,
-          startup: false,
-        } as CLIOptions);
-
-        await accountCreationState.subscribe(observer);
+        accountCreationState.subscribe(observer);
         await accountCreationState.onStart();
 
-        testSandbox.assert.callCount(loggerService.info, 8);
         testSandbox.assert.called(generateECDSA);
         testSandbox.assert.called(generateAliasECDSA);
         testSandbox.assert.called(generateED25519);
-        testSandbox.assert.called(observer.update);
 
-        testSandbox.assert.callCount(privateKeySpy, 5);
+        assertCreatedAccounts(5, 'ECDSA');
         testSandbox.assert.callCount(createAccountStub, 5);
-        testSandbox.assert.calledOnceWithExactly(logAccountTitleStub, 'ECDSA');
-        testSandbox.assert.callCount(logAccountStub, 5);
-        testSandbox.assert.calledOnce(logAccountDividerStub);
       });
 
       it('should generate ECDSA accounts аsynchronously and log the title and divider', async () => {
-        cliService.getCurrentArgv.returns({
-          async: true,
-          blocklisting: false,
-          balance: 1000,
-          accounts: 10,
-          startup: false,
-        } as any);
-
-        await accountCreationState.subscribe(observer);
+        stubCliOptions({ async: true });
+
+        accountCreationState.subscribe(observer);
         await accountCreationState.onStart();
 
-        testSandbox.assert.callCount(loggerService.info, 10);
         testSandbox.assert.called(generateECDSA);
         testSandbox.assert.called(generateAliasECDSA);
         testSandbox.assert.called(generateED25519);
-        testSandbox.assert.called(observer.update);
 
         await generateECDSA.returnValues[0];
 
-        testSandbox.assert.callCount(privateKeySpy, 10);
-        testSandbox.assert.callCount(createAccountStub, 10);
-        testSandbox.assert.calledOnceWithExactly(logAccountTitleStub, 'ECDSA');
-        testSandbox.assert.callCount(logAccountStub, 10);
-        testSandbox.assert.calledOnce(logAccountDividerStub);
+        assertCreatedAccounts(5, 'ECDSA');
+        testSandbox.assert.callCount(createAccountStub, 5);
+      });
+
+      it('should retry the creation of ECDSA accounts if there is an error', async () => {
+        const error = new Error(SDK_ERRORS.FAILED_TO_FIND_A_HEALTHY_NODE);
+        const errorCount = 2;
+        rejectWithError(createAccountStub, error, errorCount);
+
+        accountCreationState.subscribe(observer);
+        await accountCreationState.onStart();
+
+        const accounts = cliService.getCurrentArgv().accounts;
+        assertCreatedAccounts(accounts, 'ECDSA');
+        assertRetries(createAccountStub, errorCount, error);
+      });
+
+      it('should not retry the creation of ECDSA accounts if there is an error which is not retryable', async () => {
+        // Stub the createAccount method to throw an error
+        // on the first two calls and then resolve any subsequent calls
+        const error = new Error('Some other error');
+        rejectWithError(createAccountStub, error, 2);
+
+        accountCreationState.subscribe(observer);
+        await assertThrowsError(() => accountCreationState.onStart(), error);
+
+        const accounts = cliService.getCurrentArgv().accounts;
+        assertCreatedAccounts(accounts, 'ECDSA', false);
+        assertRetries(createAccountStub, 0);
       });
     });
   
@@ -271,7 +280,6 @@ describe('AccountCreationState', () => {
       let generateECDSA: SinonStub;
       let generateED25519: SinonStub;
       let generateAliasECDSA: SinonSpy;
-      let privateKeySpy: SinonSpy;
   
       beforeEach(() => {
         generateECDSA = testSandbox.stub(AccountCreationState.prototype, <any>'generateECDSA').resolves();
@@ -284,60 +292,60 @@ describe('AccountCreationState', () => {
         generateECDSA.restore();
         generateED25519.restore();
         generateAliasECDSA.restore();
-        privateKeySpy.restore();
-        loggerService.info.resetHistory()
       })
   
       it('should generate AliasECDSA accounts synchronously and log the title and divider', async () => {
-        cliService.getCurrentArgv.returns({
-          async: false,
-          blocklisting: false,
-          balance: 1000,
-          accounts: 10,
-          startup: false,
-        } as any);
-        
-        await accountCreationState.subscribe(observer);
+        accountCreationState.subscribe(observer);
         await accountCreationState.onStart();
 
-        testSandbox.assert.callCount(loggerService.info, 12);
         testSandbox.assert.called(generateAliasECDSA);
         testSandbox.assert.called(generateECDSA);
         testSandbox.assert.called(generateED25519);
-        testSandbox.assert.called(observer.update);
 
-        testSandbox.assert.callCount(privateKeySpy, 10);
-        testSandbox.assert.callCount(createAliasAccountStub, 10);
-        testSandbox.assert.calledOnce(logAliasAccountTitleStub);
-        testSandbox.assert.callCount(logAliasAccountStub, 10);
-        testSandbox.assert.calledOnce(logAliasAccountDividerStub);
+        assertCreatedAccountAliases(5);
+        testSandbox.assert.callCount(createAliasAccountStub, 5);
       });
 
       it('should generate AliasECDSA accounts asynchronously and log the title and divider', async () => {
-        cliService.getCurrentArgv.returns({
-          async: true,
-          blocklisting: false,
-          balance: 1000,
-          accounts: 6,
-          startup: false,
-        } as any);
-
-        await accountCreationState.subscribe(observer);
+        stubCliOptions({ async: true });
+
+        accountCreationState.subscribe(observer);
         await accountCreationState.onStart();
 
-        testSandbox.assert.callCount(loggerService.info, 2);
         testSandbox.assert.called(generateAliasECDSA);
         testSandbox.assert.called(generateECDSA);
         testSandbox.assert.called(generateED25519);
-        testSandbox.assert.called(observer.update);
 
         await generateAliasECDSA.returnValues[0];
 
-        testSandbox.assert.callCount(privateKeySpy, 6);
-        testSandbox.assert.callCount(createAliasAccountStub, 6);
-        testSandbox.assert.calledOnce(logAliasAccountTitleStub);
-        testSandbox.assert.callCount(logAliasAccountStub, 6);
-        testSandbox.assert.calledOnce(logAliasAccountDividerStub);
+        assertCreatedAccountAliases(5);
+        testSandbox.assert.callCount(createAliasAccountStub, 5);
+      });
+
+
+      it('should retry the creation of AliasECDSA accounts if there is an error', async () => {
+        const error = new Error(SDK_ERRORS.FAILED_TO_FIND_A_HEALTHY_NODE);
+        const errorCount = 2;
+        rejectWithError(createAliasAccountStub, error, errorCount);
+
+        accountCreationState.subscribe(observer);
+        await accountCreationState.onStart();
+
+        const accounts = cliService.getCurrentArgv().accounts;
+        assertCreatedAccountAliases(accounts);
+        assertRetries(createAliasAccountStub, errorCount, error);
+      });
+
+      it('should not retry the creation of AliasECDSA accounts if there is an error which is not retryable', async () => {
+        const error = new Error('Some other error');
+        rejectWithError(createAliasAccountStub, error, 2);
+
+        accountCreationState.subscribe(observer);
+        await assertThrowsError(() => accountCreationState.onStart(), error);
+
+        const accounts = cliService.getCurrentArgv().accounts;
+        assertCreatedAccountAliases(accounts, false);
+        assertRetries(createAliasAccountStub, 0);
       });
     });
   
@@ -345,7 +353,6 @@ describe('AccountCreationState', () => {
       let generateAliasECDSA: SinonStub;
       let generateECDSA: SinonStub;
       let generateED25519: SinonSpy;
-      let privateKeySpy: SinonSpy;
   
       beforeEach(() => {
         generateAliasECDSA = testSandbox.stub(AccountCreationState.prototype, <any>'generateAliasECDSA').resolves();
@@ -358,61 +365,145 @@ describe('AccountCreationState', () => {
         generateAliasECDSA.restore();
         generateECDSA.restore();
         generateED25519.restore();
-        privateKeySpy.restore();
-        loggerService.info.resetHistory();
       })
   
       it('should generate ED25519 accounts synchronously and log the title and divider', async () => {
-        cliService.getCurrentArgv.returns({
-          async: false,
-          blocklisting: false,
-          balance: 1000,
-          accounts: 10,
-          startup: false,
-        } as any);
-
-        await accountCreationState.subscribe(observer);
+        accountCreationState.subscribe(observer);
         await accountCreationState.onStart();
 
-        testSandbox.assert.callCount(loggerService.info, 2);
         testSandbox.assert.called(generateAliasECDSA);
         testSandbox.assert.called(generateECDSA);
         testSandbox.assert.called(generateED25519);
-        testSandbox.assert.called(observer.update);
-
-        testSandbox.assert.callCount(privateKeySpy, 10);
-        testSandbox.assert.callCount(createAccountStub, 10);
-        testSandbox.assert.calledOnceWithExactly(logAccountTitleStub, 'ED25519');
-        testSandbox.assert.callCount(logAccountStub, 10);
-        testSandbox.assert.calledOnce(logAccountDividerStub);
+        testSandbox.assert.callCount(createAccountStub, 5);
+        assertCreatedAccounts(5, 'ED25519');
       });
 
       it('should generate ED25519 accounts asynchronously and log the title and divider', async () => {
-        cliService.getCurrentArgv.returns({
-          async: true,
-          blocklisting: false,
-          balance: 1000,
-          accounts: 6,
-          startup: false,
-        } as any);
-
-        await accountCreationState.subscribe(observer);
+        stubCliOptions({ async: true });
+
+        accountCreationState.subscribe(observer);
         await accountCreationState.onStart();
 
-        testSandbox.assert.callCount(loggerService.info, 2);
         testSandbox.assert.called(generateAliasECDSA);
         testSandbox.assert.called(generateECDSA);
         testSandbox.assert.called(generateED25519);
-        testSandbox.assert.called(observer.update);
 
         await generateED25519.returnValues[0];
 
-        testSandbox.assert.callCount(privateKeySpy, 6);
-        testSandbox.assert.callCount(createAccountStub, 6);
-        testSandbox.assert.calledOnceWithExactly(logAccountTitleStub, 'ED25519');
-        testSandbox.assert.callCount(logAccountStub, 6);
-        testSandbox.assert.calledOnce(logAccountDividerStub);
+        assertCreatedAccounts(5, 'ED25519');
+        testSandbox.assert.callCount(createAccountStub, 5);
+      });
+
+      it('should retry the creation of ED25519 accounts if there is an error', async () => {
+        const error = new Error(SDK_ERRORS.FAILED_TO_FIND_A_HEALTHY_NODE);
+        const errorCount = 2;
+        rejectWithError(createAccountStub, error, errorCount);
+
+        accountCreationState.subscribe(observer);
+        await accountCreationState.onStart();
+
+        const accounts = cliService.getCurrentArgv().accounts;
+        assertCreatedAccounts(accounts, 'ED25519');
+        assertRetries(createAccountStub, errorCount, error);
+      });
+
+      it('should not retry the creation of ED25519 accounts if there is an error which is not retryable', async () => {
+        const error = new Error('Some other error');
+        rejectWithError(createAccountStub, error, 2);
+
+        accountCreationState.subscribe(observer);
+        await assertThrowsError(() => accountCreationState.onStart(), error);
+
+        const accounts = cliService.getCurrentArgv().accounts;
+        assertCreatedAccounts(accounts, 'ED25519', false);
+        assertRetries(createAccountStub, 0);
       });
     });
-  })
+
+    /**
+     * Stubs the createAccount method to throw an error on the first two calls
+     * and then resolve any subsequent calls
+     *
+     * @param stub The stub to reset and configure
+     * @param error The error to throw
+     * @param calls The number of calls to make before resolving
+     */
+    function rejectWithError(stub: SinonStub, error: Error, calls: number) {
+      stub.reset();
+      for (let i = 0; i < calls; i++) {
+        stub.onCall(i).rejects(error);
+      }
+      stub.resolves(accountInfo);
+    }
+
+    function assertRetries(stub: SinonStub, retries: number, error?: Error) {
+      const accounts = cliService.getCurrentArgv().accounts;
+
+      // Assert that the task was retried until it succeeded
+      testSandbox.assert.callCount(stub, accounts + retries);
+
+      // Assert that the error was logged on each failure
+      if (retries > 0) {
+        testSandbox.assert.callCount(loggerService.warn, 2);
+        for (const call of loggerService.warn.getCalls()) {
+          expect(call.args[0]).to.equal(`Error occurred during task execution: "${error!.toString()}"`);
+        }
+      } else {
+        testSandbox.assert.notCalled(loggerService.warn);
+      }
+    }
+
+    function assertCreatedAccounts(accounts: number, accountType: string, success: boolean = true): void {
+      commonAssertsForStart(success);
+      testSandbox.assert.callCount(privateKeySpy, accounts);
+      if (success) {
+        testSandbox.assert.calledOnceWithExactly(logAccountTitleStub, accountType);
+        testSandbox.assert.callCount(logAccountStub, accounts);
+        testSandbox.assert.calledOnce(logAccountDividerStub);
+      } else {
+        testSandbox.assert.notCalled(logAccountTitleStub);
+        testSandbox.assert.notCalled(logAccountStub);
+        testSandbox.assert.notCalled(logAccountDividerStub);
+      }
+    }
+
+    function assertCreatedAccountAliases(count: number, success: boolean = true): void {
+      commonAssertsForStart(success);
+      testSandbox.assert.callCount(privateKeySpy, count);
+      if (success) {
+        testSandbox.assert.calledOnce(logAliasAccountTitleStub);
+        testSandbox.assert.callCount(logAliasAccountStub, count);
+        testSandbox.assert.calledOnce(logAliasAccountDividerStub);
+      } else {
+        testSandbox.assert.notCalled(logAliasAccountTitleStub);
+        testSandbox.assert.notCalled(logAliasAccountStub);
+        testSandbox.assert.notCalled(logAliasAccountDividerStub);
+      }
+    }
+  });
+
+  function stubCliOptions(mockedOptions: Partial<CLIOptions>): void {
+    cliService.getCurrentArgv.returns({
+      ...cliService.getCurrentArgv(),
+      ...mockedOptions,
+    } as CLIOptions);
+  }
+
+  function commonAssertsForStart(success: boolean = true): void {
+    testSandbox.assert.called(observer.update);
+    if (success) {
+      testSandbox.assert.calledTwice(loggerService.info);
+    } else {
+      testSandbox.assert.calledOnce(loggerService.info);
+    }
+  }
+
+  async function assertThrowsError(fn: () => Promise<void>, error: Error): Promise<void> {
+    try {
+      await fn();
+      expect.fail('Expected an exception to be thrown');
+    } catch (e) {
+      expect(e).to.equal(error);
+    }
+  }
 });
diff --git a/test/unit/utils/RetryUtils.spec.ts b/test/unit/utils/RetryUtils.spec.ts
new file mode 100644
index 0000000..d65ad5a
--- /dev/null
+++ b/test/unit/utils/RetryUtils.spec.ts
@@ -0,0 +1,135 @@
+/*-
+ *
+ * Hedera Local Node
+ *
+ * Copyright (C) 2023 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { createStubInstance, SinonSpy, SinonStubbedInstance, spy, stub } from 'sinon';
+import { LoggerService } from '../../../src/services/LoggerService';
+import { RetryUtils } from '../../../src/utils/RetryUtils';
+import { expect } from 'chai';
+
+describe('RetryUtils', () => {
+  const backOff = 100;
+  let doOnRetry: SinonSpy;
+  let loggerService: SinonStubbedInstance<LoggerService>;
+
+  beforeEach(() => {
+    doOnRetry = spy((e) => loggerService.error(e.message));
+    loggerService = createStubInstance(LoggerService);
+  });
+
+  afterEach(() => {
+    doOnRetry.resetHistory();
+  });
+
+  it('should retry the task until it succeeds', async () => {
+    const error = new Error('First attempt failed');
+    const task = stub()
+      .onFirstCall().rejects(error)
+      .onSecondCall().resolves('Success');
+
+    const result = await RetryUtils.retryTask(task, { doOnRetry, backOff });
+
+    expect(result).to.equal('Success');
+    expect(task.callCount).to.equal(2);
+    expect(doOnRetry.callCount).to.equal(1);
+    expect(loggerService.error.callCount).to.equal(1);
+    expect(loggerService.error.getCall(0).args[0]).to.equal(error.message);
+  });
+
+  it('should throw an error after max retries', async () => {
+    const error = new Error('Task failed');
+    const task = stub().rejects(error);
+
+    try {
+      await RetryUtils.retryTask(task, { doOnRetry, backOff, maxRetries: 5 });
+    } catch (e) {
+      expect(e).to.equal(error);
+    }
+
+    expect(task.callCount).to.equal(5);
+    expect(doOnRetry.callCount).to.equal(4);
+  });
+
+  it('should log an error message on each failure', async () => {
+    const error = new Error('Task failed');
+    const task = stub().rejects(error);
+
+    try {
+      await RetryUtils.retryTask(task, { doOnRetry, backOff });
+    } catch (e) {
+      expect(e).to.equal(error);
+    }
+
+    expect(task.callCount).to.equal(3);
+    expect(doOnRetry.callCount).to.equal(2);
+    expect(loggerService.error.callCount).to.equal(2);
+    for (let i = 0; i < 2; i++) {
+      expect(doOnRetry.getCall(i).args[0]).to.equal(error);
+      expect(loggerService.error.getCall(i).args[0]).to.equal(error.message);
+    }
+  });
+
+  it('should succeed on the first attempt without retries', async () => {
+    const task = stub().resolves('Success');
+
+    const result = await RetryUtils.retryTask(task, { doOnRetry, backOff });
+
+    expect(result).to.equal('Success');
+    expect(task.callCount).to.equal(1);
+    expect(doOnRetry.callCount).to.equal(0);
+    expect(loggerService.error.callCount).to.equal(0);
+  });
+
+  it('should not retry if shouldRetry returns false', async () => {
+    const error = new Error('Non-retryable error');
+    const task = stub().rejects(error);
+    const shouldRetry = stub().returns(false);
+
+    try {
+      await RetryUtils.retryTask(task, { doOnRetry, backOff, shouldRetry });
+    } catch (e) {
+      expect(e).to.equal(error);
+    }
+
+    expect(task.callCount).to.equal(1);
+    expect(doOnRetry.callCount).to.equal(0);
+    expect(shouldRetry.callCount).to.equal(1);
+  });
+
+  it('should retry only for retryable errors', async () => {
+    const retryableError = new Error('Retryable error');
+    const nonRetryableError = new Error('Non-retryable error');
+    const task = stub()
+      .onFirstCall().rejects(retryableError)
+      .onSecondCall().rejects(nonRetryableError);
+    const shouldRetry = stub()
+      .onFirstCall().returns(true)
+      .onSecondCall().returns(false);
+
+    try {
+      await RetryUtils.retryTask(task, { doOnRetry, backOff, shouldRetry });
+    } catch (e) {
+      expect(e).to.equal(nonRetryableError);
+    }
+
+    expect(task.callCount).to.equal(2);
+    expect(doOnRetry.callCount).to.equal(1);
+    expect(shouldRetry.callCount).to.equal(2);
+  });
+});