Repository: atsign-foundation/at_client_sdk

Issue:
getKeys, when specifying sharedBy, results in creation of a single-use-only RemoteSecondary each time
### Describe the bug

getKeys, when specifying sharedBy, results in LocalSecondary._scan() making a new one-time-use-only connection to the cloud secondary, by creating a new RemoteSecondary object

### Steps to reproduce

It's what happens now

### Expected behavior

Should use the existing remote secondary

### Screenshots

_No response_

### Smartphones

_No response_

### Were you using an atApplication when the bug was found?

_No response_

### Additional context

See also #744 



Fix patch:
diff --git a/packages/at_client/CHANGELOG.md b/packages/at_client/CHANGELOG.md
index b16e90292..af86edd08 100644
--- a/packages/at_client/CHANGELOG.md
+++ b/packages/at_client/CHANGELOG.md
@@ -1,3 +1,6 @@
+## 3.0.42
+- fix: Improved performance of getKeys (and getAtKeys) when sharedBy is specified, by using the existing 
+RemoteSecondary connection rather than creating a new one
 ## 3.0.41
 - chore: upgrade persistence secondary to version 3.0.38 which reverts sync of signing keys and statsNotificationKey
 ## 3.0.40
diff --git a/packages/at_client/lib/src/client/at_client_impl.dart b/packages/at_client/lib/src/client/at_client_impl.dart
index 3d49053d5..2f3745e06 100644
--- a/packages/at_client/lib/src/client/at_client_impl.dart
+++ b/packages/at_client/lib/src/client/at_client_impl.dart
@@ -36,6 +36,7 @@ import 'package:at_commons/at_commons.dart';
 import 'package:at_lookup/at_lookup.dart';
 import 'package:at_persistence_secondary_server/at_persistence_secondary_server.dart';
 import 'package:at_utils/at_utils.dart';
+import 'package:meta/meta.dart';
 import 'package:path/path.dart';
 import 'package:uuid/uuid.dart';
 
@@ -46,6 +47,7 @@ class AtClientImpl implements AtClient {
   AtClientPreference? get preference => _preference;
   String? currentAtSign;
   String? _namespace;
+  SecondaryKeyStore? _localSecondaryKeyStore;
   LocalSecondary? _localSecondary;
   RemoteSecondary? _remoteSecondary;
 
@@ -57,85 +59,72 @@ class AtClientImpl implements AtClient {
   AtClientManager? _atClientManager;
 
   final _logger = AtSignLogger('AtClientImpl');
-  static final Map _atClientInstanceMap = <String, AtClient>{};
 
-  /// Returns a new instance of [AtClient]. App has to pass the current user atSign
-  /// and the client preference.
-  @Deprecated("Use AtClientManger to get instance of atClient")
-  static Future<AtClient?> getClient(String? currentAtSign) async {
-    if (_atClientInstanceMap.containsKey(currentAtSign)) {
-      return _atClientInstanceMap[currentAtSign];
-    }
-    AtSignLogger('AtClientImpl')
-        .severe('Instance of atclientimpl for $currentAtSign is not created');
-    return null;
-  }
-
-  @Deprecated("Use [create]")
-
-  /// use [create]
-  static Future<void> createClient(String currentAtSign, String? namespace,
-      AtClientPreference preferences) async {
-    currentAtSign = AtUtils.formatAtSign(currentAtSign)!;
-    if (_atClientInstanceMap.containsKey(currentAtSign)) {
-      return;
-    }
-    if (preferences.isLocalStoreRequired) {
-      var storageManager = StorageManager(preferences);
-      await storageManager.init(currentAtSign, preferences.keyStoreSecret);
-    }
-    var atClientImpl = AtClientImpl(currentAtSign, namespace, preferences);
-    await atClientImpl._init();
-    _atClientInstanceMap[currentAtSign] = atClientImpl;
-  }
-
-  @Deprecated("Use [create]")
-  AtClientImpl(
-      // ignore: no_leading_underscores_for_local_identifiers
-      String _atSign,
-      String? namespace,
-      AtClientPreference preference) {
-    currentAtSign = AtUtils.formatAtSign(_atSign);
-    _preference = preference;
-    _namespace = namespace;
-  }
+  @visibleForTesting
+  static final Map atClientInstanceMap = <String, AtClient>{};
 
   static Future<AtClient> create(
       String currentAtSign, String? namespace, AtClientPreference preferences,
-      {AtClientManager? atClientManager}) async {
+      {
+        AtClientManager? atClientManager,
+        RemoteSecondary? remoteSecondary,
+        EncryptionService? encryptionService,
+        SecondaryKeyStore? localSecondaryKeyStore
+      }) async {
     currentAtSign = AtUtils.formatAtSign(currentAtSign)!;
-    if (_atClientInstanceMap.containsKey(currentAtSign)) {
-      return _atClientInstanceMap[currentAtSign];
-    }
-    if (preferences.isLocalStoreRequired) {
-      var storageManager = StorageManager(preferences);
-      await storageManager.init(currentAtSign, preferences.keyStoreSecret);
+    if (atClientInstanceMap.containsKey(currentAtSign)) {
+      return atClientInstanceMap[currentAtSign];
     }
+
     atClientManager ??= AtClientManager.getInstance();
     var atClientImpl =
-        AtClientImpl._(currentAtSign, namespace, preferences, atClientManager);
+        AtClientImpl._(currentAtSign, namespace, preferences, atClientManager,
+          remoteSecondary: remoteSecondary,
+          encryptionService: encryptionService,
+          localSecondaryKeyStore: localSecondaryKeyStore);
+
     await atClientImpl._init();
-    _atClientInstanceMap[currentAtSign] = atClientImpl;
-    return _atClientInstanceMap[currentAtSign];
+
+    atClientInstanceMap[currentAtSign] = atClientImpl;
+    return atClientInstanceMap[currentAtSign];
   }
 
-  // ignore: no_leading_underscores_for_local_identifiers
-  AtClientImpl._(String _atSign, String? namespace,
-      AtClientPreference preference, AtClientManager atClientManager) {
-    currentAtSign = AtUtils.formatAtSign(_atSign);
+  AtClientImpl._(String atSign, String? namespace,
+      AtClientPreference preference, AtClientManager atClientManager,
+      {
+        RemoteSecondary? remoteSecondary,
+        EncryptionService? encryptionService,
+        SecondaryKeyStore? localSecondaryKeyStore
+      }) {
+    currentAtSign = AtUtils.formatAtSign(atSign);
     _preference = preference;
     _preference?.namespace ??= namespace;
     _namespace = namespace;
     _atClientManager = atClientManager;
+    _localSecondaryKeyStore = localSecondaryKeyStore;
+    if (_localSecondaryKeyStore != null && ! _preference!.isLocalStoreRequired) {
+      throw IllegalArgumentException('A SecondaryKeyStore was injected, but preference.isLocalStoreRequired is false');
+    }
+    _remoteSecondary = remoteSecondary;
+    _encryptionService = encryptionService;
   }
 
   Future<void> _init() async {
     if (_preference!.isLocalStoreRequired) {
-      _localSecondary = LocalSecondary(this);
+      if (_localSecondaryKeyStore == null) {
+        var storageManager = StorageManager(preference);
+        await storageManager.init(currentAtSign!, preference!.keyStoreSecret);
+      }
+
+      _localSecondary = LocalSecondary(this, keyStore: _localSecondaryKeyStore);
     }
-    _remoteSecondary = RemoteSecondary(currentAtSign!, _preference!,
-        privateKey: _preference!.privateKey);
-    _encryptionService = EncryptionService();
+
+    // Now using ??= because we may be injecting a RemoteSecondary
+    _remoteSecondary ??= RemoteSecondary(currentAtSign!, _preference!, privateKey: _preference!.privateKey);
+
+    // Now using ??= because we may be injecting an EncryptionService
+    _encryptionService ??= EncryptionService();
+
     _encryptionService!.remoteSecondary = _remoteSecondary;
     _encryptionService!.currentAtSign = currentAtSign;
     _encryptionService!.localSecondary = _localSecondary;
@@ -286,8 +275,7 @@ class AtClientImpl implements AtClient {
       {String? regex,
       String? sharedBy,
       String? sharedWith,
-      bool showHiddenKeys = false,
-      bool isDedicated = false}) async {
+      bool showHiddenKeys = false}) async {
     var builder = ScanVerbBuilder()
       ..sharedWith = sharedWith
       ..sharedBy = sharedBy
@@ -308,12 +296,12 @@ class AtClientImpl implements AtClient {
       {String? regex,
       String? sharedBy,
       String? sharedWith,
-      bool isDedicated = false}) async {
+      bool showHiddenKeys = false}) async {
     var getKeysResult = await getKeys(
         regex: regex,
         sharedBy: sharedBy,
         sharedWith: sharedWith,
-        isDedicated: isDedicated);
+        showHiddenKeys: showHiddenKeys);
     var result = <AtKey>[];
     if (getKeysResult.isNotEmpty) {
       for (var key in getKeysResult) {
@@ -323,7 +311,7 @@ class AtClientImpl implements AtClient {
           _logger.severe('$key is not a well-formed key');
         } on Exception catch (e) {
           _logger.severe(
-              'Exception occured: ${e.toString()}. Unable to form key $key');
+              'Exception occurred: ${e.toString()}. Unable to form key $key');
         }
       }
     }
@@ -903,4 +891,49 @@ class AtClientImpl implements AtClient {
     }
     return await getRemoteSecondary()?.executeVerb(builder);
   }
+
+  //
+  // Everything after this point has been deprecated
+  //
+
+  /// Returns a new instance of [AtClient]. App has to pass the current user atSign
+  /// and the client preference.
+  @Deprecated("Use AtClientManger to get instance of atClient")
+  static Future<AtClient?> getClient(String? currentAtSign) async {
+    if (atClientInstanceMap.containsKey(currentAtSign)) {
+      return atClientInstanceMap[currentAtSign];
+    }
+    AtSignLogger('AtClientImpl')
+        .severe('Instance of AtClientImpl for $currentAtSign has not been created');
+    return null;
+  }
+
+  @Deprecated("Use [create]")
+  /// use [create]
+  static Future<void> createClient(String currentAtSign, String? namespace,
+      AtClientPreference preferences) async {
+    currentAtSign = AtUtils.formatAtSign(currentAtSign)!;
+    if (atClientInstanceMap.containsKey(currentAtSign)) {
+      return;
+    }
+    if (preferences.isLocalStoreRequired) {
+      var storageManager = StorageManager(preferences);
+      await storageManager.init(currentAtSign, preferences.keyStoreSecret);
+    }
+    var atClientImpl = AtClientImpl(currentAtSign, namespace, preferences);
+    await atClientImpl._init();
+    atClientInstanceMap[currentAtSign] = atClientImpl;
+  }
+
+  @Deprecated("Use [create]")
+  AtClientImpl(
+      // ignore: no_leading_underscores_for_local_identifiers
+      String _atSign,
+      String? namespace,
+      AtClientPreference preference) {
+    currentAtSign = AtUtils.formatAtSign(_atSign);
+    _preference = preference;
+    _namespace = namespace;
+  }
+
 }
diff --git a/packages/at_client/lib/src/client/at_client_spec.dart b/packages/at_client/lib/src/client/at_client_spec.dart
index 2f7accb31..17f36f03c 100644
--- a/packages/at_client/lib/src/client/at_client_spec.dart
+++ b/packages/at_client/lib/src/client/at_client_spec.dart
@@ -20,7 +20,7 @@ abstract class AtClient {
   @Deprecated("Use SyncManager.sync")
   SyncManager? getSyncManager();
 
-  /// Returns a singleton instance of [RemoteSecondary] to communicate with user's secondary server.
+  /// Returns a [RemoteSecondary] to communicate with user's cloud secondary server.
   RemoteSecondary? getRemoteSecondary();
 
   LocalSecondary? getLocalSecondary();
@@ -36,7 +36,8 @@ abstract class AtClient {
   /// phone.persona@alice where 'persona' is the namespace. If you want to save by ignoring the namespace set [AtKey.metadata.namespaceAware]
   /// to false.
   /// Additional metadata can be set using [AtKey.metadata]
-  /// [isDedicated] need to be set to true to create a dedicated connection
+  ///
+  /// [isDedicated] is currently ignored and will be removed in next major version
   /// ```
   /// update:phone@alice +1 999 9999
   ///   var key = AtKey()..key='phone'
@@ -137,6 +138,8 @@ abstract class AtClient {
 
   /// Get the value of [AtKey.key] from user's cloud secondary if [AtKey.sharedBy] is set. Otherwise looks up the key from local secondary.
   /// If the key was stored with public access, set [AtKey.metadata.isPublic] to true. If the key was shared with another atSign set [AtKey.sharedWith]
+  ///
+  /// [isDedicated] is currently ignored and will be removed in next major version
   /// ```
   /// e.g alice is current atsign
   /// llookup:phone@alice
@@ -197,6 +200,7 @@ abstract class AtClient {
   /// Throws [SelfKeyNotFoundException] if the self encryption key is not found.
   ///
   /// Throws [AtClientException] if the cloud secondary is invalid or not reachable
+  ///
   Future<AtValue> get(AtKey key,
       {bool isDedicated = false, GetRequestOptions? getRequestOptions});
 
@@ -224,7 +228,8 @@ abstract class AtClient {
   /// By default namespace that is used to create the [AtClient] instance will be appended to the key. phone@alice translates to
   /// phone.persona@alice where 'persona' is the namespace. If you want to ignoring the namespace set [AtKey.metadata.namespaceAware]
   /// to false.
-  /// [isDedicated] need to be set to true to create a dedicated connection
+  ///
+  /// [isDedicated] is currently ignored and will be removed in next major version
   /// ```
   /// e.g alice is current atsign
   /// delete:phone@alice
@@ -280,7 +285,7 @@ abstract class AtClient {
   ///   getKeys(sharedBy:'@bob');
   ///```
   Future<List<AtKey>> getAtKeys(
-      {String? regex, String? sharedBy, String? sharedWith});
+      {String? regex, String? sharedBy, String? sharedWith, bool showHiddenKeys = false});
 
   /// Get all the keys stored in user's secondary in string format. If [regex] is specified only matching keys are returned.
   /// If [sharedBy] is specified, then gets the keys from [sharedBy] user shared with current atClient user.
@@ -301,7 +306,6 @@ abstract class AtClient {
       bool showHiddenKeys = false});
 
   /// Notifies the [AtKey] with the [sharedWith] user of the atsign. Optionally, operation, value and metadata can be set along with key to notify.
-  /// [isDedicated] need to be set to true to create a dedicated connection
   ///```
   ///e.g alice is the current atsign
   /// notify:update:@bob:phone@alice:+1 999 9999
diff --git a/packages/at_client/lib/src/client/local_secondary.dart b/packages/at_client/lib/src/client/local_secondary.dart
index 06c72e6f7..a4c61662d 100644
--- a/packages/at_client/lib/src/client/local_secondary.dart
+++ b/packages/at_client/lib/src/client/local_secondary.dart
@@ -22,8 +22,8 @@ class LocalSecondary implements Secondary {
   /// Local keystore used to store data for the current atSign.
   SecondaryKeyStore? keyStore;
 
-  LocalSecondary(this._atClient) {
-    keyStore = SecondaryPersistenceStoreFactory.getInstance()
+  LocalSecondary(this._atClient, {this.keyStore}) {
+    keyStore ??= SecondaryPersistenceStoreFactory.getInstance()
         .getSecondaryPersistenceStore(_atClient.getCurrentAtSign())!
         .getSecondaryKeyStore();
   }
@@ -176,9 +176,7 @@ class LocalSecondary implements Secondary {
       // shared with current atSign
       if (builder.sharedBy != null) {
         var command = builder.buildCommand();
-        return await RemoteSecondary(
-                _atClient.getCurrentAtSign()!, _atClient.getPreferences()!,
-                privateKey: _atClient.getPreferences()!.privateKey)
+        return _atClient.getRemoteSecondary()!
             .executeCommand(command, auth: true);
       }
       List<String?> keys;
diff --git a/packages/at_client/lib/src/manager/storage_manager.dart b/packages/at_client/lib/src/manager/storage_manager.dart
index 410eee536..2f011a672 100644
--- a/packages/at_client/lib/src/manager/storage_manager.dart
+++ b/packages/at_client/lib/src/manager/storage_manager.dart
@@ -3,14 +3,6 @@ import 'package:at_persistence_secondary_server/at_persistence_secondary_server.
 
 /// Manager to create local storage
 class StorageManager {
-  static final StorageManager _singleton = StorageManager._internal();
-
-  StorageManager._internal();
-
-  factory StorageManager.getInstance() {
-    return _singleton;
-  }
-
   bool isStorageInitialized = false;
 
   AtClientPreference? preferences;
diff --git a/packages/at_client/lib/src/service/sync_service.dart b/packages/at_client/lib/src/service/sync_service.dart
index 3e0c30f73..c23f85460 100644
--- a/packages/at_client/lib/src/service/sync_service.dart
+++ b/packages/at_client/lib/src/service/sync_service.dart
@@ -4,6 +4,12 @@ import 'package:at_client/src/service/sync/sync_status.dart';
 abstract class SyncService {
   /// Sync local secondary and cloud secondary.
   ///
+  /// NB: With current SyncServiceImpl this may not immediately sync. Instead, it will enqueue a request for sync to take place.
+  ///
+  /// This method will be obsolete in the forthcoming alternative implementation which takes an event-driven
+  /// streaming approach to syncing. Instead of requesting a sync, you will request that processing
+  /// of the data events (whether originating from client or server) be paused or resumed or re-initialized
+  ///
   /// If local secondary is ahead, pushes the changes to the cloud secondary.
   /// If cloud secondary is ahead, pulls the changes to the local secondary.
   ///
@@ -22,7 +28,7 @@ abstract class SyncService {
   ///   print(syncResult.lastSyncedOn);
   /// }
   /// ```
-  void sync({Function? onDone});
+  void sync({Function? onDone, Function? onError});
 
   /// Call this method to set the Global onDone callback.
   /// This method will be called when a sync is completed.
diff --git a/packages/at_client/lib/src/service/sync_service_impl.dart b/packages/at_client/lib/src/service/sync_service_impl.dart
index 7fcba9320..3c3b9b448 100644
--- a/packages/at_client/lib/src/service/sync_service_impl.dart
+++ b/packages/at_client/lib/src/service/sync_service_impl.dart
@@ -143,7 +143,7 @@ class SyncServiceImpl implements SyncService, AtSignChangeListener {
   }
 
   @visibleForTesting
-  Future<void> processSyncRequests() async {
+  Future<void> processSyncRequests({bool respectSyncRequestQueueSizeAndRequestTriggerDuration = true}) async {
     final syncProgress = SyncProgress()..syncStatus = SyncStatus.started;
     syncProgress.startedAt = DateTime.now().toUtc();
     _logger.finest('in _processSyncRequests');
@@ -160,16 +160,21 @@ class SyncServiceImpl implements SyncService, AtSignChangeListener {
       _informSyncProgress(syncProgress);
       return;
     }
-    if (_syncRequests.isEmpty ||
-        (_syncRequests.length < _syncRequestThreshold &&
-            (_syncRequests.isNotEmpty &&
-                DateTime.now()
-                        .toUtc()
-                        .difference(_syncRequests.elementAt(0).requestedOn)
-                        .inSeconds <
-                    _syncRequestTriggerInSeconds))) {
-      _logger.finest('skipping sync - queue length ${_syncRequests.length}');
-      return;
+    if (respectSyncRequestQueueSizeAndRequestTriggerDuration) {
+      if (_syncRequests.isEmpty ||
+          (_syncRequests.length < _syncRequestThreshold &&
+              (_syncRequests.isNotEmpty &&
+                  DateTime
+                      .now()
+                      .toUtc()
+                      .difference(_syncRequests
+                      .elementAt(0)
+                      .requestedOn)
+                      .inSeconds <
+                      _syncRequestTriggerInSeconds))) {
+        _logger.finest('skipping sync - queue length ${_syncRequests.length}');
+        return;
+      }
     }
     final syncRequest = _getSyncRequest();
     try {
diff --git a/packages/at_client/pubspec.yaml b/packages/at_client/pubspec.yaml
index cfcc75cf8..b51703e13 100644
--- a/packages/at_client/pubspec.yaml
+++ b/packages/at_client/pubspec.yaml
@@ -1,7 +1,7 @@
 name: at_client
 description: The at_client library is the non-platform specific Client SDK which provides the essential methods for building an app using the atProtocol.
 ## When incrementing the version, update the version in AtClientConfig file
-version: 3.0.41
+version: 3.0.42
 repository: https://github.com/atsign-foundation/at_client_sdk
 homepage: https://docs.atsign.com/
 
@@ -27,7 +27,7 @@ dependencies:
   at_lookup: ^3.0.30
   at_utf7: ^1.0.0
   at_base2e15: ^1.0.0
-  at_commons: ^3.0.26
+  at_commons: ^3.0.27
   at_utils: ^3.0.11
   meta: ^1.8.0
 


Test patch:
diff --git a/packages/at_client/test/local_secondary_test.dart b/packages/at_client/test/local_secondary_test.dart
index 6ddf3e25f..459c7b98c 100644
--- a/packages/at_client/test/local_secondary_test.dart
+++ b/packages/at_client/test/local_secondary_test.dart
@@ -9,9 +9,38 @@ import 'package:test/test.dart';
 import 'package:mocktail/mocktail.dart';
 
 class MockSecondaryKeyStore extends Mock implements SecondaryKeyStore {
+  static const String hiddenKey1 = 'public:__location.wavi@alice';
+  static const String hiddenKey2 = '_profilePic.wavi@alice';
+  static const String nonHiddenKey1 = 'public:nickname.wavi@alice';
+  static const String nonHiddenKey2 = 'some.self.key.wavi@alice';
+  static const String otherWaviHiddenKey = 'public:__location.other_wavi@alice';
+  static const String waviOtherHiddenKey = 'public:__location.wavi_other@alice';
+  static const String otherWaviOtherHiddenKey = 'public:__location.other_wavi_other@alice';
+  static const String otherWaviNonHiddenKey = 'public:nickname.other_wavi@alice';
+  static const String waviOtherNonHiddenKey = 'public:nickname.wavi_other@alice';
+  static const String otherWaviOtherNonHiddenKey = 'public:nickname.other_wavi_other@alice';
+  static const List<String> keysInKeyStore = [
+    nonHiddenKey1,
+    hiddenKey1,
+    otherWaviHiddenKey,
+    nonHiddenKey2,
+    hiddenKey2,
+    otherWaviNonHiddenKey,
+    waviOtherHiddenKey,
+    otherWaviOtherHiddenKey,
+    waviOtherNonHiddenKey,
+    otherWaviOtherNonHiddenKey
+  ];
   @override
   List<String> getKeys({String? regex}) {
-    return ['public:__location.wavi@alice', '_profilePic.wavi@alice'];
+    if (regex != null) {
+      RegExp re = RegExp(regex);
+      return keysInKeyStore.where((key) {
+        return key.contains(re);
+      }).toList();
+    } else {
+      return keysInKeyStore.toList();
+    }
   }
 }
 
@@ -19,10 +48,20 @@ class MockAtClientImpl extends Mock implements AtClientImpl {}
 
 void main() {
   var storageDir = '${Directory.current.path}/test/hive';
-  setUp(() async => await setUpFunc(storageDir));
+
+  final String atSign = '@alice';
+
   group('A group of local secondary get keys test', () {
+    setUp(() async {
+      AtClientImpl.atClientInstanceMap.remove(atSign);
+      await setupLocalStorage(storageDir, atSign);
+    });
+    tearDown(() async {
+      AtClientImpl.atClientInstanceMap.remove(atSign);
+      await tearDownLocalStorage(storageDir);
+    });
+
     test('test get private key', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -34,8 +73,8 @@ void main() {
       await localSecondary.putValue(AT_PKAM_PRIVATE_KEY, pkamPrivateKey);
       expect(await localSecondary.getPrivateKey(), pkamPrivateKey);
     });
+
     test('test get public key', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -47,8 +86,8 @@ void main() {
       await localSecondary.putValue(AT_PKAM_PUBLIC_KEY, pkamPublicKey);
       expect(await localSecondary.getPublicKey(), pkamPublicKey);
     });
+
     test('test get encryption private key', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -63,8 +102,8 @@ void main() {
       expect(
           await localSecondary.getEncryptionPrivateKey(), encryptionPrivateKey);
     });
+
     test('test get encryption public key', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -78,8 +117,8 @@ void main() {
       expect(await localSecondary.getEncryptionPublicKey(atSign),
           encryptionPublicKey);
     });
+
     test('test get self encryption key', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -91,15 +130,13 @@ void main() {
       await localSecondary.putValue(AT_ENCRYPTION_SELF_KEY, selfEncryptionKey);
       expect(await localSecondary.getEncryptionSelfKey(), selfEncryptionKey);
     });
-    try {
-      tearDown(() async => await tearDownFunc(storageDir));
-    } on Exception catch (e) {
-      print('error in tear down:${e.toString()}');
-    }
   });
+
   group('A group of local secondary execute verb tests', () {
+    setUp(() async => await setupLocalStorage(storageDir, atSign));
+    tearDown(() async => await tearDownLocalStorage(storageDir));
+
     test('test update verb builder', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -111,14 +148,14 @@ void main() {
         ..isPublic = true
         ..value = 'alice@gmail.com'
         ..atKey = 'email'
-        ..sharedBy = '@alice';
+        ..sharedBy = atSign;
       final executeResult =
           await localSecondary.executeVerb(verbBuilder, sync: false);
       expect(executeResult, isNotNull);
       expect(executeResult!.startsWith('data:'), true);
     });
+
     test('test llookup verb builder', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -139,8 +176,8 @@ void main() {
           await localSecondary.executeVerb(llookupVerbBuilder, sync: false);
       expect(llookupResult, 'data:alice@gmail.com');
     });
+
     test('test delete verb builder', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -164,8 +201,8 @@ void main() {
       expect(localSecondary.executeVerb(llookupVerbBuilder, sync: false),
           throwsA(isA<KeyNotFoundException>()));
     });
+
     test('test scan verb builder', () async {
-      final atSign = '@alice';
       final atClientManager = AtClientManager(atSign);
       final preference = AtClientPreference()
         ..syncRegex = '.wavi'
@@ -189,54 +226,227 @@ void main() {
           await localSecondary.executeVerb(scanVerbBuilder, sync: false);
       final scanJson = jsonDecode(scanResult!);
       print(scanJson);
-      expect(scanJson.contains('phone@alice'), true);
-      expect(scanJson.contains('public:email@alice'), true);
+      expect(scanJson.contains('phone$atSign'), true);
+      expect(scanJson.contains('public:email$atSign'), true);
     });
   });
-  try {
-    tearDown(() async => await tearDownFunc(storageDir));
-  } on Exception catch (e) {
-    print('error in tear down:${e.toString()}');
-  }
 
-  group('A group of tests to validate getKeys', () {
-    test('A test to validate getKeys when showHidden is set to true', () async {
-      AtClientImpl mockAtClientImpl = MockAtClientImpl();
-      SecondaryKeyStore mockSecondaryKeyStore = MockSecondaryKeyStore();
-      LocalSecondary localSecondary = LocalSecondary(mockAtClientImpl);
-      localSecondary.keyStore = mockSecondaryKeyStore;
+  group('A group of tests to validate getKeys and getAtKeys', () {
+    late SecondaryKeyStore mockSecondaryKeyStore;
+    late LocalSecondary localSecondary;
+    late AtClient atClient;
+    final String namespace = 'validate_get_keys';
+    final preference = AtClientPreference()
+      ..syncRegex = '.$namespace'
+      ..hiveStoragePath = '*&@should not be used by these tests, we will mock local storage'
+      ..isLocalStoreRequired = true;
+
+
+    setUp(() async {
+      AtClientImpl.atClientInstanceMap.remove(atSign);
+
+      mockSecondaryKeyStore = MockSecondaryKeyStore();
+      atClient = await AtClientImpl.create(atSign, namespace, preference,
+        atClientManager: AtClientManager(atSign),
+        localSecondaryKeyStore: mockSecondaryKeyStore
+      );
+      localSecondary = LocalSecondary(atClient, keyStore: mockSecondaryKeyStore);
+    });
+
+    tearDown(() async {
+      AtClientImpl.atClientInstanceMap.remove(atSign);
+    });
+
+    test('LocalSecondary scan, showHiddenKeys:true, regex:<actualDot>wavi@', () async {
       var response = await localSecondary
-          .executeVerb(ScanVerbBuilder()..showHiddenKeys = true);
-      expect(response?.contains('public:__location.wavi@alice'), true);
-      expect(response?.contains('_profilePic.wavi@alice'), true);
+          .executeVerb(ScanVerbBuilder()..showHiddenKeys = true..regex='\\.wavi@');
+      expect(response?.contains(MockSecondaryKeyStore.hiddenKey1), true);
+      expect(response?.contains(MockSecondaryKeyStore.hiddenKey2), true);
+      expect(response?.contains(MockSecondaryKeyStore.nonHiddenKey1), true);
+      expect(response?.contains(MockSecondaryKeyStore.nonHiddenKey2), true);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.waviOtherHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviOtherHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviNonHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.waviOtherNonHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey), false);
+    });
+
+    test('getKeys, showHiddenKeys:true, regex:<actualDot>wavi@', () async {
+      List<String> response = await atClient.getKeys(showHiddenKeys: true, regex: '\\.wavi@');
+      expect(response.contains(MockSecondaryKeyStore.hiddenKey1), true);
+      expect(response.contains(MockSecondaryKeyStore.hiddenKey2), true);
+      expect(response.contains(MockSecondaryKeyStore.nonHiddenKey1), true);
+      expect(response.contains(MockSecondaryKeyStore.nonHiddenKey2), true);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.waviOtherHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviOtherHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviNonHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.waviOtherNonHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey), false);
+    });
+
+    // We'll test getAtKeys (which calls getKeys, which calls LocalSecondary scan)
+    // with multiple regex variants to verify regex is being processed correctly
+    test('getAtKeys, showHiddenKeys:true, regex:<actualDot>wavi@', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: true, regex: '\\.wavi@');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), false);
+    });
+
+    test('getAtKeys, showHiddenKeys:true, regex:<regexDot>wavi@', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: true, regex: '.wavi@');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), false);
+    });
+
+    test('getAtKeys, showHiddenKeys:true, regex:<actualDot>wavi', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: true, regex: '\\.wavi');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), false);
+    });
+
+    test('getAtKeys, showHiddenKeys:true, regex:<regexDot>wavi', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: true, regex: '.wavi');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), true);
     });
 
-    test('A test to validate getKeys when showHidden is set to true', () async {
-      AtClientImpl mockAtClientImpl = MockAtClientImpl();
-      SecondaryKeyStore mockSecondaryKeyStore = MockSecondaryKeyStore();
-      LocalSecondary localSecondary = LocalSecondary(mockAtClientImpl);
-      localSecondary.keyStore = mockSecondaryKeyStore;
+    test('LocalSecondary scan, showHiddenKeys:false, regex:<actualDot>wavi@', () async {
       var response = await localSecondary
-          .executeVerb(ScanVerbBuilder()..showHiddenKeys = false);
-      expect(response?.contains('public:__location.wavi@alice'), false);
-      expect(response?.contains('_profilePic.wavi@alice'), false);
+          .executeVerb(ScanVerbBuilder()..showHiddenKeys = false..regex='\\.wavi@');
+      expect(response?.contains(MockSecondaryKeyStore.hiddenKey1), false);
+      expect(response?.contains(MockSecondaryKeyStore.hiddenKey2), false);
+      expect(response?.contains(MockSecondaryKeyStore.nonHiddenKey1), true);
+      expect(response?.contains(MockSecondaryKeyStore.nonHiddenKey2), true);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.waviOtherHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviOtherHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviNonHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.waviOtherNonHiddenKey), false);
+      expect(response?.contains(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey), false);
+    });
+
+    test('getKeys, showHiddenKeys:false, regex:<actualDot>wavi@', () async {
+      List<String> response = await atClient.getKeys(showHiddenKeys: false, regex: '\\.wavi@');
+      expect(response.contains(MockSecondaryKeyStore.hiddenKey1), false);
+      expect(response.contains(MockSecondaryKeyStore.hiddenKey2), false);
+      expect(response.contains(MockSecondaryKeyStore.nonHiddenKey1), true);
+      expect(response.contains(MockSecondaryKeyStore.nonHiddenKey2), true);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.waviOtherHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviOtherHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviNonHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.waviOtherNonHiddenKey), false);
+      expect(response.contains(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey), false);
+    });
+
+    test('getAtKeys, showHiddenKeys:false, regex:<actualDot>wavi@', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: false, regex: '\\.wavi@');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), false);
+    });
+
+    test('getAtKeys, showHiddenKeys:false, regex:<regexDot>wavi@', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: false, regex: '.wavi@');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), false);
+    });
+
+    test('getAtKeys, showHiddenKeys:false, regex:<actualDot>wavi', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: false, regex: '\\.wavi');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), false);
+    });
+
+    test('getAtKeys, showHiddenKeys:false, regex:<regexDot>wavi', () async {
+      List<AtKey> response = await atClient.getAtKeys(showHiddenKeys: false, regex:'.wavi');
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey1)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.hiddenKey2)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey1)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.nonHiddenKey2)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherHiddenKey)), false);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.waviOtherNonHiddenKey)), true);
+      expect(response.contains(AtKey.fromString(MockSecondaryKeyStore.otherWaviOtherNonHiddenKey)), true);
     });
   });
 }
 
-Future<void> setUpFunc(storageDir) async {
+Future<void> setupLocalStorage(String storageDir, String atSign) async {
   var commitLogInstance = await AtCommitLogManagerImpl.getInstance()
-      .getCommitLog('@alice', commitLogPath: storageDir);
+      .getCommitLog(atSign, commitLogPath: storageDir);
   var persistenceManager = SecondaryPersistenceStoreFactory.getInstance()
-      .getSecondaryPersistenceStore('@alice')!;
+      .getSecondaryPersistenceStore(atSign)!;
   await persistenceManager.getHivePersistenceManager()!.init(storageDir);
   persistenceManager.getSecondaryKeyStore()!.commitLog = commitLogInstance;
 }
 
-Future<void> tearDownFunc(storageDir) async {
-  print('***local sec tearDown');
-  var isExists = await Directory(storageDir).exists();
-  if (isExists) {
-    Directory(storageDir).deleteSync(recursive: true);
+Future<void> tearDownLocalStorage(storageDir) async {
+  try {
+    var isExists = await Directory(storageDir).exists();
+    if (isExists) {
+        Directory(storageDir).deleteSync(recursive: true);
+      }
+  } catch (e, st) {
+    print('local_secondary_test.dart: exception / error in tearDown: $e, $st');
   }
 }
diff --git a/packages/at_client/test/sync_service_test.dart b/packages/at_client/test/sync_service_test.dart
index 1006e9197..db3d635fa 100644
--- a/packages/at_client/test/sync_service_test.dart
+++ b/packages/at_client/test/sync_service_test.dart
@@ -192,7 +192,7 @@ void main() async {
         actualSyncException = syncResult.atClientException;
       });
 
-      await syncServiceImpl.processSyncRequests();
+      await syncServiceImpl.processSyncRequests(respectSyncRequestQueueSizeAndRequestTriggerDuration: false);
 
       while (!listener.syncComplete) {
         await Future.delayed(Duration(milliseconds: 10));