Repository: atsign-foundation/at_client_sdk

Issue:
Clients will miss pending notifications upon startup if there is a delay between NotificationService creation and client calling NotificationService.subscribe
### Describe the bug

Clients will miss pending notifications upon startup if there is a delay between Monitor creation and client calling NotificationService.subscribe



### Steps to reproduce

Introduce a delay (for example by waiting for sync to complete) before calling subscribe on NotificationService. 

### Expected behavior

Clients should not miss notifications

### Screenshots

_No response_

### Smartphones

_No response_

### Were you using an atApplication when the bug was found?

_No response_

### Additional context

Reported by a customer, encountered by them when using some sample code I had supplied which waited for an initial sync to complete before calling NotificationService.subscribe()

1. I have implemented an initial patch which minimizes the risk of this happening and customer has verified. Will raise PR for that
2. Have filed an [enhancement request](https://github.com/atsign-foundation/at_client_sdk/issues/771) to fix this properly.


Discussion: More urgent stuff came up. The patch fix is with the customer via a git branch, but needs a PR raised. Will do in PR50

Fix patch:
diff --git a/packages/at_client/analysis_options.yaml b/packages/at_client/analysis_options.yaml
index 5dbe876df..1fc92ff8f 100644
--- a/packages/at_client/analysis_options.yaml
+++ b/packages/at_client/analysis_options.yaml
@@ -10,6 +10,7 @@ linter:
     - camel_case_types
     - unnecessary_string_interpolations
     - await_only_futures
+    - unawaited_futures
 
 analyzer:
 #   exclude:
diff --git a/packages/at_client/lib/src/service/notification_service_impl.dart b/packages/at_client/lib/src/service/notification_service_impl.dart
index 1c58aeadc..487caaba8 100644
--- a/packages/at_client/lib/src/service/notification_service_impl.dart
+++ b/packages/at_client/lib/src/service/notification_service_impl.dart
@@ -34,10 +34,14 @@ class NotificationServiceImpl
   static final Map<String, NotificationService> notificationServiceMap = {};
 
   final _logger = AtSignLogger('NotificationServiceImpl');
-  /// monitorIsPaused is set to true when [stopAllSubscriptions] is called,
-  /// and to false when [start] is successful.
+
+  /// Controls whether or not the monitor is actually running.
+  /// * monitorIsPaused is initially set to true (monitor should not be running)
+  /// * it is set to false when [_startMonitor] is called (monitor should be running)
+  /// * and it is set to true when [stopAllSubscriptions] is called (monitor should not be running).
+  /// ( Note that stopAllSubscriptions also calls Monitor.stop() )
   @visibleForTesting
-  var monitorIsPaused = false;
+  var monitorIsPaused = true;
 
   late AtClient _atClient;
   Monitor? _monitor;
@@ -76,7 +80,8 @@ class NotificationServiceImpl
     }
     final notificationService =
         NotificationServiceImpl._(atClientManager, atClient, monitor: monitor);
-    await notificationService._init();
+    // We used to call _init() at this point which would start the monitor, but now we
+    // call _init() from the [subscribe] method
     notificationServiceMap[atClient.getCurrentAtSign()!] = notificationService;
     return notificationServiceMap[atClient.getCurrentAtSign()]!;
   }
@@ -100,46 +105,73 @@ class NotificationServiceImpl
         .build();
   }
 
+  /// Simple state to prevent _init() running more than once *concurrently*
+  bool _initializing = false;
+
   Future<void> _init() async {
-    _logger.finer('${_atClient.getCurrentAtSign()} notification service init');
-    await _startMonitor();
-    _logger.finer(
-        '${_atClient.getCurrentAtSign()} monitor status: ${_monitor?.getStatus()}');
-    if (_connectivityListener == null) {
-      _connectivityListener = ConnectivityListener();
-      _connectivityListener!.subscribe().listen((isConnected) {
-        if (isConnected) {
-          _logger.finer(
-              '${_atClient.getCurrentAtSign()} starting monitor through connectivity listener event');
-          _startMonitor();
-        } else {
-          _logger.finer('lost network connectivity');
-        }
-      });
+    // Note that it is safe to call _init() more than once, sequentially, because it only does two things,
+    // and both of those things are safe guarded:
+    // (1) calls _startMonitor() - which won't do anything if the monitor is already started
+    // (2) creates a connectivity listener and subscription - but only if _connectivityListener is currently null
+    if (_initializing) {
+      return;
+    }
+    try {
+      _initializing = true;
+      _logger.finer('${_atClient.getCurrentAtSign()} notification service _init()');
+      await _startMonitor();
+      _logger.finer(
+          '${_atClient.getCurrentAtSign()} monitor status: ${_monitor?.getStatus()}');
+      if (_connectivityListener == null) {
+        _connectivityListener = ConnectivityListener();
+        // Note that this subscription is cancelled by stopAllSubscriptions(), so we don't need to worry
+        // about this subscription accidentally starting the monitor after it has been intentionally stopped,
+        // for example by switching atSigns
+        _connectivityListener!.subscribe().listen((isConnected) {
+          if (isConnected) {
+            _logger.finer('${_atClient.getCurrentAtSign()} starting monitor through connectivity listener event');
+            _startMonitor();
+          } else {
+            _logger.finer('lost network connectivity');
+          }
+        });
+      }
+    } finally {
+      _initializing = false;
     }
   }
 
   Future<void> _startMonitor() async {
+    monitorIsPaused = false;
+
     if (_monitor != null && _monitor!.status == MonitorStatus.started) {
       _logger.finer(
           'monitor is already started for ${_atClient.getCurrentAtSign()}');
       return;
     }
-    await _monitor!
-        .start(lastNotificationTime: await getLastNotificationTime());
 
-    if (_monitor!.status == MonitorStatus.started) {
-      monitorIsPaused = false;
+    int? lastNotificationTime;
+    try {
+      lastNotificationTime = await getLastNotificationTime();
+    } catch (e) {
+      _logger.warning('${_atClient.getCurrentAtSign()}: startMonitor(): getLastNotificationTime() failed : $e');
+      return;
+    }
+
+    try {
+      await _monitor!.start(lastNotificationTime: lastNotificationTime);
+    } catch (e) {
+      _logger.warning('${_atClient.getCurrentAtSign()}: startMonitor(): Failed to start monitor : $e');
+      return;
     }
   }
 
-  @visibleForTesting
-
   /// Return the last received notification DateTime in epochMillis when
   /// [AtClientPreference.fetchOfflineNotifications] is set true.
   ///
   /// Returns null when the key which holds the lastNotificationReceived
   /// does not exist.
+  @visibleForTesting
   Future<int?> getLastNotificationTime() async {
     if (_atClientManager.atClient.getPreferences()!.fetchOfflineNotifications ==
         false) {
@@ -187,9 +219,11 @@ class NotificationServiceImpl
 
   @override
   void stopAllSubscriptions() {
+    _logger.finer('stopAllSubscriptions() called - setting monitorIsPaused to true');
     monitorIsPaused = true;
     _monitor?.stop();
     _connectivityListener?.unSubscribe();
+    _connectivityListener = null;
     _streamListeners.forEach((regex, streamController) {
       if (!streamController.isClosed) () => streamController.close();
     });
@@ -198,6 +232,8 @@ class NotificationServiceImpl
 
   Future<void> _internalNotificationCallback(String notificationJSON) async {
     try {
+      _logger.finest('DEBUG: $notificationJSON');
+
       final notificationParser = NotificationResponseParser();
       final atNotifications = await notificationParser
           .getAtNotifications(notificationParser.parse(notificationJSON));
@@ -263,7 +299,7 @@ class NotificationServiceImpl
     _logger.finer('monitor retry for ${_atClient.getCurrentAtSign()}');
     Future.delayed(monitorRetryInterval, () async {
       monitorRestartQueued = false;
-      if (monitorIsPaused) { // maybe it's been paused since we requested the retry
+      if (monitorIsPaused) { // maybe it's been paused during the time since the retry was requested
         _logger.warning("monitorRetry() will NOT call Monitor.start() because we've stopped all subscriptions");
       } else {
         monitorRetryCallsToMonitorStart++;
@@ -282,10 +318,8 @@ class NotificationServiceImpl
 
   @override
   Future<NotificationResult> notify(NotificationParams notificationParams,
-      {bool waitForFinalDeliveryStatus =
-          true, // this was the behaviour before introducing this parameter
-      bool checkForFinalDeliveryStatus =
-          true, // this was the behaviour before introducing this parameter
+      {bool waitForFinalDeliveryStatus = true, // this was the behaviour before introducing this parameter
+      bool checkForFinalDeliveryStatus = true, // this was the behaviour before introducing this parameter
       Function(NotificationResult)? onSuccess,
       Function(NotificationResult)? onError,
       Function(NotificationResult)? onSentToSecondary}) async {
@@ -340,8 +374,8 @@ class NotificationServiceImpl
         return notificationResult;
       } else {
         // no wait? no await
-        _waitForAndHandleFinalNotificationSendStatus(
-            notificationParams, notificationResult, onSuccess, onError);
+        unawaited(_waitForAndHandleFinalNotificationSendStatus(
+            notificationParams, notificationResult, onSuccess, onError));
         return notificationResult;
       }
     }
@@ -400,12 +434,42 @@ class NotificationServiceImpl
   @override
   Stream<AtNotification> subscribe(
       {String? regex, bool shouldDecrypt = false}) {
+    _logger.finer('subscribe(regex: $regex, shouldDecrypt: $shouldDecrypt');
     regex ??= emptyRegex;
     var notificationConfig = NotificationConfig()
       ..regex = regex
       ..shouldDecrypt = shouldDecrypt;
     var atNotificationStream = _streamListeners.putIfAbsent(
         notificationConfig, () => StreamController<AtNotification>.broadcast());
+
+    // Temporary fix for https://github.com/atsign-foundation/at_client_sdk/issues/770
+    //     (Temporary because it's a bit of a kludge, but the proper fix requires the implementation
+    //      of enhancements to the monitor verb which allow for multiple subscriptions, and changes
+    //      to this service and to the Monitor to make use of that enhancement.)
+    // Previously we were initializing the notification service
+    // before there were any 'real' subscriptions
+    // and because the notification service currently starts the monitor
+    // without any regex, the monitor immediately starts to stream all notifications
+    //
+    // As a result, if there is even a very short delay between when the notification service
+    // is created and when the app calls 'subscribe', then the app will 'miss'
+    // those notifications.
+    //
+    // So - for now, if the subscription is 'statsNotification', then we will delay
+    // initialization of the service until when the app does a real 'subscribe' call.
+    //
+    // Normally, the app code will call subscribe which will
+    // start the monitor, and SyncService will start receiving statsNotifications.
+    // However, if the app isn't explicitly calling 'sync', and hasn't called subscribe(),
+    // then there will be a delay of 30 seconds before the monitor is started, the first
+    // statsNotification message is received, and a sync request is queued.
+    // In order to compensate for that, the SyncServiceImpl itself now queues a sync request
+    // when it is initialized.
+    if (regex == 'statsNotification') {
+      Future.delayed(Duration(seconds: 30), () async => _init());
+    } else {
+      _init();
+    }
     return atNotificationStream.stream as Stream<AtNotification>;
   }
 
diff --git a/packages/at_client/lib/src/service/sync_service_impl.dart b/packages/at_client/lib/src/service/sync_service_impl.dart
index 76ac9fbd6..46bfc6fc9 100644
--- a/packages/at_client/lib/src/service/sync_service_impl.dart
+++ b/packages/at_client/lib/src/service/sync_service_impl.dart
@@ -89,6 +89,17 @@ class SyncServiceImpl implements SyncService, AtSignChangeListener {
         () async {
       try {
         await processSyncRequests();
+        // If no sync request has ever been made, let's enqueue one now.
+        // See https://github.com/atsign-foundation/at_client_sdk/issues/770
+        if (hasHadNoSyncRequests) {
+          final syncRequest = SyncRequest();
+          syncRequest.onDone = _onDone;
+          syncRequest.onError = _onError;
+          syncRequest.requestSource = SyncRequestSource.system;
+          syncRequest.requestedOn = DateTime.now().toUtc();
+          syncRequest.result = SyncResult();
+          _addSyncRequestToQueue(syncRequest);
+        }
       } on Exception catch (e, trace) {
         var cause = (e is AtException) ? e.getTraceMessage() : e.toString();
         _logger.finest(trace);
@@ -281,7 +292,13 @@ class SyncServiceImpl implements SyncService, AtSignChangeListener {
         .severe('system sync error ${syncResult.atClientException?.message}');
   }
 
+  /// We use this so that after [processSyncRequests] runs, it can enqueue a sync
+  /// request if none have yet been received. This is to address a side-effect
+  /// of the fix for https://github.com/atsign-foundation/at_client_sdk/issues/770
+  @visibleForTesting
+  bool hasHadNoSyncRequests = true;
   void _addSyncRequestToQueue(SyncRequest syncRequest) {
+    hasHadNoSyncRequests = false;
     if (_syncRequests.length == _queueSize) {
       _syncRequests.removeLast();
     }
@@ -637,7 +654,7 @@ class SyncServiceImpl implements SyncService, AtSignChangeListener {
       _logger.severe('exception in isInSync $cause');
       throw AtClientException.message(e.toString());
     } finally {
-      remoteSecondary.atLookUp.close();
+      unawaited(remoteSecondary.atLookUp.close());
     }
   }
 


Test patch:
diff --git a/packages/at_client/test/notification_service_test.dart b/packages/at_client/test/notification_service_test.dart
index 803b80568..5c9f2a94c 100644
--- a/packages/at_client/test/notification_service_test.dart
+++ b/packages/at_client/test/notification_service_test.dart
@@ -811,6 +811,10 @@ void main() {
           await NotificationServiceImpl.create(mockAtClientImpl, atClientManager: mockAtClientManager, monitor: mockMonitor)
               as NotificationServiceImpl;
 
+      // Set monitorIsPaused to false, since it isn't explicitly set to false
+      // until NotificationServiceImpl._init runs.
+      notificationServiceImpl.monitorIsPaused = false;
+
       notificationServiceImpl.monitorRetryInterval = Duration(milliseconds: 50);
       notificationServiceImpl.monitorRetry();
       expect(notificationServiceImpl.callsToMonitorRetry, 1);
@@ -829,6 +833,10 @@ void main() {
       await NotificationServiceImpl.create(mockAtClientImpl, atClientManager: mockAtClientManager, monitor: mockMonitor)
       as NotificationServiceImpl;
 
+      // Set monitorIsPaused to false, since it isn't explicitly set to false
+      // until NotificationServiceImpl._init runs.
+      notificationServiceImpl.monitorIsPaused = false;
+
       expect(notificationServiceImpl.monitorRestartQueued, false);
 
       // First call to monitorRetry should queue a call to Monitor.start and therefore return true
@@ -849,6 +857,10 @@ void main() {
       await NotificationServiceImpl.create(mockAtClientImpl, atClientManager: mockAtClientManager, monitor: mockMonitor)
       as NotificationServiceImpl;
 
+      // Set monitorIsPaused to false, since it isn't explicitly set to false
+      // until NotificationServiceImpl._init runs.
+      notificationServiceImpl.monitorIsPaused = false;
+
       expect(notificationServiceImpl.monitorIsPaused, false);
       expect(notificationServiceImpl.monitorRestartQueued, false);
 
@@ -871,6 +883,10 @@ void main() {
       await NotificationServiceImpl.create(mockAtClientImpl, atClientManager: mockAtClientManager, monitor: mockMonitor)
       as NotificationServiceImpl;
 
+      // Set monitorIsPaused to false, since it isn't explicitly set to false
+      // until NotificationServiceImpl._init runs.
+      notificationServiceImpl.monitorIsPaused = false;
+
       notificationServiceImpl.monitorRetryInterval = Duration(milliseconds: 50);
 
       expect(notificationServiceImpl.monitorIsPaused, false);
@@ -897,6 +913,10 @@ void main() {
       await NotificationServiceImpl.create(mockAtClientImpl, atClientManager: mockAtClientManager, monitor: mockMonitor)
       as NotificationServiceImpl;
 
+      // Set monitorIsPaused to false, since it isn't explicitly set to false
+      // until NotificationServiceImpl._init runs.
+      notificationServiceImpl.monitorIsPaused = false;
+
       notificationServiceImpl.monitorRetryInterval = Duration(milliseconds: 50);
 
       notificationServiceImpl.monitorRetry();