Repository: ordo-one/package-distributed-system
Pull request #104: fix(major): [sc-19639] Add possibility to remove service.
Author: login: ser-0xff
Created: 2025-03-13T13:21:02Z | Merged: 2025-03-16T16:11:15Z

## Description

Include a summary of the change and which issue is fixed. Please also include relevant motivation and context. 

## How Has This Been Tested?

Please describe the tests that you ran to verify your changes. 

## Minimal checklist:

- [ ] I have performed a self-review of my own code 
- [ ] I have added `DocC` code-level documentation for any public interfaces exported by the package
- [ ] I have added unit and/or integration tests that prove my fix is effective or that my feature works



Diff:
diff --git a/Package.swift b/Package.swift
index 54cf1c8..896296a 100644
--- a/Package.swift
+++ b/Package.swift
@@ -13,7 +13,7 @@ let externalDependencies: [String: Range<Version>] = [
 ]
 
 let internalDependencies: [String: Range<Version>] = [
-    "package-consul": .upToNextMajor(from: "7.0.0"),
+    "package-consul": .upToNextMajor(from: "9.0.0"),
     "package-lz4": .upToNextMinor(from: "1.10.0"),
 ]
 
diff --git a/Sources/DistributedSystem/DiscoveryManager.swift b/Sources/DistributedSystem/DiscoveryManager.swift
index 57691ad..97e7c46 100644
--- a/Sources/DistributedSystem/DiscoveryManager.swift
+++ b/Sources/DistributedSystem/DiscoveryManager.swift
@@ -87,6 +87,7 @@ final class DiscoveryManager {
     private var logger: Logger { loggerBox.value }
 
     private var lock = NIOLock()
+    private var stateVersion = 0 // incremented on service remove
     private var processes: [SocketAddress: ProcessInfo] = [:]
     private var discoveries: [String: DiscoveryInfo] = [:]
     private var updateHealthStatus = true
@@ -231,7 +232,7 @@ final class DiscoveryManager {
                 fatalError("Internal error: duplicated service \(serviceName)/\(serviceID)")
             }
 
-            logger.debug("addService: \(serviceName)/\(serviceID) (updateHealthStatus=\(updateHealthStatus))")
+            logger.debug("addService: '\(serviceName)'/\(serviceID) (updateHealthStatus=\(updateHealthStatus))")
             discoveryInfo.services[serviceID] = ServiceInfo(service, factory)
 
             var services = [(ConsulServiceDiscovery.Instance, DistributedSystem.ConnectionHandler)]()
@@ -251,6 +252,24 @@ final class DiscoveryManager {
         return updateHealthStatus
     }
 
+    func removeLocalService(_ serviceID: UUID) -> Int? {
+        logger.debug("removeLocalService: \(serviceID)")
+        return lock.withLock {
+            for discoveryInfo in self.discoveries.values {
+                let serviceInfo = discoveryInfo.services[serviceID]
+                if let serviceInfo {
+                    // a sanity check the service should be local
+                    if case .local = serviceInfo.address {
+                        stateVersion += 1
+                        discoveryInfo.services.removeValue(forKey: serviceID)
+                        return stateVersion
+                    }
+                }
+            }
+            return nil
+        }
+    }
+
     func setAddress(_ address: SocketAddress, for serviceName: String, _ serviceID: UUID, _ service: NodeService) -> Bool {
         let (connect, process) = lock.withLock { () -> (Bool, (channel: (UInt32, Channel), connectionHandlers: [DistributedSystem.ConnectionHandler])?) in
             guard let discoveryInfo = self.discoveries[serviceName] else {
@@ -374,9 +393,9 @@ final class DiscoveryManager {
         }
     }
 
-    func getLocalServices() -> [NodeService] {
+    func getLocalServices() -> ([NodeService], Int) {
         lock.withLock {
-            self.getLocalServicesLocked()
+            (self.getLocalServicesLocked(), stateVersion)
         }
     }
 
diff --git a/Sources/DistributedSystem/DistributedSystem.swift b/Sources/DistributedSystem/DistributedSystem.swift
index 3e11393..a1e8191 100644
--- a/Sources/DistributedSystem/DistributedSystem.swift
+++ b/Sources/DistributedSystem/DistributedSystem.swift
@@ -284,14 +284,13 @@ public class DistributedSystem: DistributedActorSystem, @unchecked Sendable {
         self.systemName = systemName
         self.addressTag = addressTag
         eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2)
-        consul = Consul()
+        consul = Consul(logLevel: logLevel)
         consulServiceDiscovery = ConsulServiceDiscovery(consul)
         discoveryManager = DiscoveryManager(loggerBox)
         syncCallManager = SyncCallManager(loggerBox)
         self.compressionMode = compressionMode
 
         loggerBox.value.logLevel = logLevel
-        consul.logLevel = logLevel
     }
 
     deinit {
@@ -730,7 +729,11 @@ public class DistributedSystem: DistributedActorSystem, @unchecked Sendable {
         return (serviceID, updateHealthStatus)
     }
 
-    func getLocalServices() -> [NodeService] {
+    func removeService(_ serviceID: UUID) -> Int? {
+        discoveryManager.removeLocalService(serviceID)
+    }
+
+    func getLocalServices() -> ([NodeService], Int) {
         discoveryManager.getLocalServices()
     }
 
diff --git a/Sources/DistributedSystem/DistributedSystemServer.swift b/Sources/DistributedSystem/DistributedSystemServer.swift
index 1466a18..b43a375 100644
--- a/Sources/DistributedSystem/DistributedSystemServer.swift
+++ b/Sources/DistributedSystem/DistributedSystemServer.swift
@@ -16,6 +16,7 @@ import Logging
 import NIOCore
 internal import NIOPosix
 import class ServiceDiscovery.CancellationToken
+import Synchronization
 
 public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
     private var localAddress: String?
@@ -27,6 +28,14 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
     var healthStatusUpdateInterval = TimeAmount.seconds(10)
     var healthStatusTTL = TimeAmount.seconds(15)
 
+    struct HealthUpdateState {
+        var version = 0
+        var pendingUpdate = false
+        var continuations = [CheckedContinuation<Void, Never>]()
+    }
+
+    let healthUpdateState = Mutex<HealthUpdateState>(.init())
+
     private static func localAddress(_ consulAddress: String) throws -> String? {
         #if os(Linux)
         let sockType = Int32(SOCK_DGRAM.rawValue)
@@ -132,6 +141,14 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         logger.info("starting server '\(systemName)' @ \(localPort) (compression mode = \(compressionMode))")
     }
 
+    private static func makeCheckID(_ serviceID: String) -> String {
+        "service:\(serviceID)"
+    }
+
+    private static func makeCheckID(_ serviceID: UUID) -> String {
+        makeCheckID(serviceID.uuidString)
+    }
+
     private func registerService(_ serviceName: String, _ serviceID: UUID, metadata: [String: String]) -> EventLoopFuture<Void> {
         // Use TTL type service health check
         // One could think we could use TCP,
@@ -139,9 +156,10 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         // Service can crash, then OS will allocate the same port for another service,
         // consul will be able to connect to that port and report that service as 'passing'
         // while in the reality the service will be down
-        let check = Check(checkID: "service:\(serviceID)",
+        let checkID = Self.makeCheckID(serviceID)
+        let check = Check(checkID: checkID,
                           deregisterCriticalServiceAfter: "\(criticalServiceDeregisterTimeout.nanoseconds / Self.nanosecondsInSecond)s",
-                          name: "service:\(serviceID)",
+                          name: checkID,
                           status: .passing,
                           ttl: "\(healthStatusTTL.nanoseconds / Self.nanosecondsInSecond)s")
         let service = Service(address: localAddress, checks: [check], id: "\(serviceID)", meta: metadata, name: serviceName, port: localPort)
@@ -156,45 +174,102 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
+        let services = {
+            while true {
+                let (services, servicesVersion) = getLocalServices()
+
+                let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                    if servicesVersion < $0.version {
+                        return $0.version
+                    } else {
+                        $0.pendingUpdate = true
+                        return nil
+                    }
+                }
+
+                if let expectedServicesVersion {
+                    logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                } else {
+                    logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+                    return services
+                }
+            }
+        }()
+
+        var futures = [EventLoopFuture<Void>]()
 
         for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+            let checkID = Self.makeCheckID(service.serviceID)
+            let promise = eventLoop.makePromise(of: Void.self)
+            let checkFuture = consul.agent.check(checkID, status: .passing)
+            checkFuture.whenComplete { result in
+                switch result {
+                case .success:
+                    promise.succeed()
+                case let .failure(error):
+                    // depending on the error we could try to re-register service
+                    if let error = error as? ConsulError,
+                       case let .httpResponseError(status, _) = error,
+                       case status = HTTPResponseStatus.notFound {
+                        if let serviceName = service.serviceName,
+                           let serviceID = UUID(uuidString: service.serviceID) {
+                            self.logger.error("check '\(checkID)' failed: \(error), trying to register service again")
+                            let serviceMeta = service.serviceMeta ?? [:]
+                            let registerFuture = self.registerService(serviceName, serviceID, metadata: serviceMeta)
+                            registerFuture.whenComplete {
+                                switch $0 {
+                                case .success:
+                                    promise.succeed()
+                                case let .failure(error):
+                                    self.logger.error("failed to re-register service \(serviceName)/\(serviceID): \(error)")
+                                    promise.fail(error)
+                                }
+                            }
+                        } else {
+                            self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                            promise.fail(error)
                         }
                     } else {
-                        self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                        self.logger.error("check '\(checkID)' failed: \(error)")
+                        promise.fail(error)
                     }
-                } else {
-                    self.logger.error("check '\(checkID)' failed: \(error)")
                 }
             }
+
+            futures.append(promise.futureResult)
         }
 
-        eventLoop.scheduleTask(in: healthStatusUpdateInterval) {
-            self.updateHealthStatus(with: eventLoop)
+        let allCompletedFuture = EventLoopFuture.whenAllComplete(futures, on: eventLoop)
+        _ = allCompletedFuture.map { _ in
+            let continuations = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return exchange(&$0.continuations, with: [])
+            }
+
+            self.logger.trace("""
+                resuming \(continuations.count) continuations,
+                schedule next health update in \(self.healthStatusUpdateInterval)"
+                """
+            )
+
+            for continuation in continuations {
+                continuation.resume()
+            }
+
+            eventLoop.scheduleTask(in: self.healthStatusUpdateInterval) {
+                self.updateHealthStatus(with: eventLoop)
+            }
         }
     }
 
@@ -203,7 +278,7 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         var metadata = metadata ?? [:]
         metadata[ServiceMetadata.systemName.rawValue] = systemName
         metadata[ServiceMetadata.processIdentifier.rawValue] = String(ProcessInfo.processInfo.processIdentifier)
@@ -224,5 +299,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        guard let servicesVersion = super.removeService(serviceID) else {
+            return false
+        }
+
+        await withCheckedContinuation { continuation in
+            let resumeContinuation = healthUpdateState.withLock {
+                $0.version = servicesVersion
+                if $0.pendingUpdate {
+                    $0.continuations.append(continuation)
+                    return false
+                } else {
+                    return true
+                }
+            }
+
+            if resumeContinuation {
+                continuation.resume()
+            } else {
+                logger.debug("waiting for health status update to finish")
+            }
+        }
+
+        let checkID = Self.makeCheckID(serviceID)
+        let future = consul.agent.deregisterCheck(checkID).flatMap {
+            self.consul.agent.deregisterServiceID(serviceID.uuidString)
+        }
+
+        do {
+            _ = try await future.get()
+            logger.info("service \(serviceID) deregistered")
+        } catch {
+            logger.error("\(error)")
+        }
+
+        return true
     }
 }
diff --git a/Tests/DistributedSystemTests/DistributedSystemTests.swift b/Tests/DistributedSystemTests/DistributedSystemTests.swift
index e172b0d..d3dee77 100644
--- a/Tests/DistributedSystemTests/DistributedSystemTests.swift
+++ b/Tests/DistributedSystemTests/DistributedSystemTests.swift
@@ -84,6 +84,27 @@ func checkCanRunTimeConsumingTest() throws {
 }
 
 final class DistributedSystemTests: XCTestCase {
+    class DummyServiceImpl: TestableService, @unchecked Sendable {
+        func openStream(byRequest request: TestMessages.OpenRequest) async {
+            fatalError("Should never be called")
+        }
+
+        func getMonster() -> Monster {
+            fatalError("Should never be called")
+        }
+
+        func doNothing() {
+            fatalError("Should never be called")
+        }
+
+        func handleMonsters(_ monsters: [Monster]) async {
+            fatalError("Should never be called")
+        }
+
+        func handleConnectionState(_ state: ConnectionState) async {
+        }
+    }
+
     class Flags {
         var serviceDeallocated = false
         var serviceConnectionClosed = false
@@ -372,27 +393,6 @@ final class DistributedSystemTests: XCTestCase {
     func testReconnectAfterServerRestart() async throws {
         try checkCanRunTimeConsumingTest()
 
-        class ServiceImpl: TestableService, @unchecked Sendable {
-            func openStream(byRequest request: TestMessages.OpenRequest) async {
-                fatalError("Should never be called")
-            }
-
-            func getMonster() -> Monster {
-                fatalError("Should never be called")
-            }
-
-            func doNothing() {
-                fatalError("Should never be called")
-            }
-
-            func handleMonsters(_ monsters: [Monster]) async {
-                fatalError("Should never be called")
-            }
-
-            func handleConnectionState(_ state: ConnectionState) async {
-            }
-        }
-
         let processInfo = ProcessInfo.processInfo
         let systemName = "\(processInfo.hostName)-ts-\(processInfo.processIdentifier)-\(#line)"
 
@@ -402,7 +402,7 @@ final class DistributedSystemTests: XCTestCase {
         let serverSystem1 = DistributedSystemServer(name: systemName)
         try await serverSystem1.start()
         try await serverSystem1.addService(ofType: TestServiceEndpoint.self, toModule: moduleID) { actorSystem in
-            try TestServiceEndpoint(ServiceImpl(), in: actorSystem)
+            try TestServiceEndpoint(DummyServiceImpl(), in: actorSystem)
         }
 
         let clientSystem = DistributedSystem(name: systemName)
@@ -431,7 +431,7 @@ final class DistributedSystemTests: XCTestCase {
         let serverSystem2 = DistributedSystemServer(name: systemName)
         try await serverSystem2.start()
         try await serverSystem2.addService(ofType: TestServiceEndpoint.self, toModule: moduleID) { actorSystem in
-            try TestServiceEndpoint(ServiceImpl(), in: actorSystem)
+            try TestServiceEndpoint(DummyServiceImpl(), in: actorSystem)
         }
 
         for await _ in stream { break }
@@ -1326,4 +1326,39 @@ final class DistributedSystemTests: XCTestCase {
 
         distributedSystem.stop()
     }
+
+    func testRemoveService() async throws {
+        let processInfo = ProcessInfo.processInfo
+        let systemName = "\(processInfo.hostName)-ts-\(processInfo.processIdentifier)-\(#line)"
+
+        let (stream, continuation) = AsyncStream<Void>.makeStream()
+
+        let moduleID = DistributedSystem.ModuleIdentifier(1)
+        let serverSystem = DistributedSystemServer(name: systemName)
+        try await serverSystem.start()
+        let serviceID = try await serverSystem.addService(ofType: TestServiceEndpoint.self, toModule: moduleID) { actorSystem in
+            continuation.yield()
+            return try TestServiceEndpoint(DummyServiceImpl(), in: actorSystem)
+        }
+
+        let clientSystem = DistributedSystem(name: systemName)
+        try clientSystem.start()
+
+        clientSystem.connectToServices(
+            TestServiceEndpoint.self,
+            withFilter: { _ in true },
+            clientFactory: { actorSystem, _ in
+                return TestClientEndpoint(Client(), in: actorSystem)
+            },
+            serviceHandler: { serviceEndpoint, _ in }
+        )
+
+        for await _ in stream { break }
+
+        // client and server connected now, removing server
+        try await serverSystem.removeService(serviceID)
+
+        clientSystem.stop()
+        serverSystem.stop()
+    }
 }


Reviews:
user: login: dimlio
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T15:36:21Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:41:25Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:41:35Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:41:43Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:41:52Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:42:03Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:42:25Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:42:35Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:42:46Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:43:02Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:43:10Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T10:43:22Z
user: login: dimlio
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T14:00:47Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-16T16:05:50Z
user: login: ser-0xff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-16T16:05:57Z

Review comments:
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: ```suggestion
                continuation.resume()
```
created_at: 2025-03-13T13:37:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -30,10 +30,10 @@ extension Channel {
     }
 }
 
-public final class Box<T> {
+public final class Box<T: ~Copyable>: @unchecked Sendable {
path: Sources/DistributedSystem/DistributedSystem.swift
user: login: dimlio
body: `Sendable` conformance doesn't look correct.
created_at: 2025-03-13T13:37:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
+            await withCheckedContinuation { continuation in
+                let resumeContinuation = healthUpdateState.withLock {
+                    $0.version = version
+                    if $0.pendingUpdate {
+                        $0.continuation = continuation
+                        return false
+                    } else {
+                        return true
+                    }
+                }
+
+                if resumeContinuation {
+                    continuation.resume(returning: ())
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: ```suggestion
                    continuation.resume()
```
created_at: 2025-03-13T13:50:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
+            await withCheckedContinuation { continuation in
+                let resumeContinuation = healthUpdateState.withLock {
+                    $0.version = version
+                    if $0.pendingUpdate {
+                        $0.continuation = continuation
+                        return false
+                    } else {
+                        return true
+                    }
+                }
+
+                if resumeContinuation {
+                    continuation.resume(returning: ())
+                } else {
+                    logger.debug("waiting for health status update finish")
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: ```suggestion
                    logger.debug("waiting for health status update to finish")
```
?
created_at: 2025-03-13T13:51:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: ```suggestion
        guard let version = super.removeService(serviceID) else { return false }
```
To avoid extra indentation ?
created_at: 2025-03-13T13:52:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1326,4 +1326,39 @@ final class DistributedSystemTests: XCTestCase {
 
         distributedSystem.stop()
     }
+
+    func testRemoveService() async throws {
+        let processInfo = ProcessInfo.processInfo
+        let systemName = "\(processInfo.hostName)-ts-\(processInfo.processIdentifier)-\(#line)"
+
+        let (stream, continuation) = AsyncStream<Void>.makeStream()
+
+        let moduleID = DistributedSystem.ModuleIdentifier(1)
+        let serverSystem = DistributedSystemServer(name: systemName)
+        try await serverSystem.start()
+        let serviceID = try await serverSystem.addService(ofType: TestServiceEndpoint.self, toModule: moduleID) { actorSystem in
+            continuation.yield()
+            return try TestServiceEndpoint(DummyServiceImpl(), in: actorSystem)
+        }
+
+        let clientSystem = DistributedSystem(name: systemName)
+        try clientSystem.start()
+
+        clientSystem.connectToServices(
+            TestServiceEndpoint.self,
+            withFilter: { _ in true },
+            clientFactory: { actorSystem, _ in
+                return TestClientEndpoint(Client(), in: actorSystem)
+            },
+            serviceHandler: { serviceEndpoint, _ in }
+        )
+
+        for await _ in stream { break }
+
+        // client and server connected now, restarting server
path: Tests/DistributedSystemTests/DistributedSystemTests.swift
user: login: dimlio
body: ```suggestion
        // client and server connected now, removing server
```
?
created_at: 2025-03-13T13:56:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
+            await withCheckedContinuation { continuation in
+                let resumeContinuation = healthUpdateState.withLock {
+                    $0.version = version
+                    if $0.pendingUpdate {
+                        $0.continuation = continuation
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: Can there already be continuation from another `removeService()` call?
created_at: 2025-03-13T14:48:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
+            }
+
+            eventLoop.scheduleTask(in: self.healthStatusUpdateInterval) {
+                self.updateHealthStatus(with: eventLoop)
+            }
+        }
+    }
+
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
-
-        for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+        while true {
+            let (services, servicesVersion) = getLocalServices()
+            let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                if servicesVersion < $0.version {
+                    return $0.version
+                } else {
+                    $0.pendingUpdate = true
+                    return nil
+                }
+            }
+
+            if let expectedServicesVersion {
+                logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                continue
+            }
+
+            logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+            let pendingRequestsBox = Box<Atomic<Int>>(.init(services.count))
+
+            for service in services {
+                let checkID = "service:\(service.serviceID)"
+                let future = consul.agent.check(checkID, status: .passing)
+                future.whenComplete { result in
+                    switch result {
+                    case .success:
+                        self.removePendingRequest(pendingRequestsBox, eventLoop)
+                    case let .failure(error):
+                        // dependning on the error we could try to re-register service
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: ```suggestion
                        // depending on the error we could try to re-register service
```
created_at: 2025-03-13T15:17:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
+            }
+
+            eventLoop.scheduleTask(in: self.healthStatusUpdateInterval) {
+                self.updateHealthStatus(with: eventLoop)
+            }
+        }
+    }
+
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
-
-        for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+        while true {
+            let (services, servicesVersion) = getLocalServices()
+            let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                if servicesVersion < $0.version {
+                    return $0.version
+                } else {
+                    $0.pendingUpdate = true
+                    return nil
+                }
+            }
+
+            if let expectedServicesVersion {
+                logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                continue
+            }
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: Maybe wrap only this part in `while true { ... }` ?
created_at: 2025-03-13T15:20:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
+            }
+
+            eventLoop.scheduleTask(in: self.healthStatusUpdateInterval) {
+                self.updateHealthStatus(with: eventLoop)
+            }
+        }
+    }
+
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
-
-        for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+        while true {
+            let (services, servicesVersion) = getLocalServices()
+            let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                if servicesVersion < $0.version {
+                    return $0.version
+                } else {
+                    $0.pendingUpdate = true
+                    return nil
+                }
+            }
+
+            if let expectedServicesVersion {
+                logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                continue
+            }
+
+            logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+            let pendingRequestsBox = Box<Atomic<Int>>(.init(services.count))
+
+            for service in services {
+                let checkID = "service:\(service.serviceID)"
+                let future = consul.agent.check(checkID, status: .passing)
+                future.whenComplete { result in
+                    switch result {
+                    case .success:
+                        self.removePendingRequest(pendingRequestsBox, eventLoop)
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: Have a class instance and call this in its `deinit` -- no need in counter and ensuring we call it on all paths?
created_at: 2025-03-13T15:23:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -87,6 +87,7 @@ final class DiscoveryManager {
     private var logger: Logger { loggerBox.value }
 
     private var lock = NIOLock()
+    private var stateVersion = 0
path: Sources/DistributedSystem/DiscoveryManager.swift
user: login: dimlio
body: Maybe add a comment that it's incremented only on service remove?
So it's move correctly called "removeCount".
created_at: 2025-03-13T15:31:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:41:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -30,10 +30,10 @@ extension Channel {
     }
 }
 
-public final class Box<T> {
+public final class Box<T: ~Copyable>: @unchecked Sendable {
path: Sources/DistributedSystem/DistributedSystem.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:41:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
+            await withCheckedContinuation { continuation in
+                let resumeContinuation = healthUpdateState.withLock {
+                    $0.version = version
+                    if $0.pendingUpdate {
+                        $0.continuation = continuation
+                        return false
+                    } else {
+                        return true
+                    }
+                }
+
+                if resumeContinuation {
+                    continuation.resume(returning: ())
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:41:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
+            await withCheckedContinuation { continuation in
+                let resumeContinuation = healthUpdateState.withLock {
+                    $0.version = version
+                    if $0.pendingUpdate {
+                        $0.continuation = continuation
+                        return false
+                    } else {
+                        return true
+                    }
+                }
+
+                if resumeContinuation {
+                    continuation.resume(returning: ())
+                } else {
+                    logger.debug("waiting for health status update finish")
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:41:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:42:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1326,4 +1326,39 @@ final class DistributedSystemTests: XCTestCase {
 
         distributedSystem.stop()
     }
+
+    func testRemoveService() async throws {
+        let processInfo = ProcessInfo.processInfo
+        let systemName = "\(processInfo.hostName)-ts-\(processInfo.processIdentifier)-\(#line)"
+
+        let (stream, continuation) = AsyncStream<Void>.makeStream()
+
+        let moduleID = DistributedSystem.ModuleIdentifier(1)
+        let serverSystem = DistributedSystemServer(name: systemName)
+        try await serverSystem.start()
+        let serviceID = try await serverSystem.addService(ofType: TestServiceEndpoint.self, toModule: moduleID) { actorSystem in
+            continuation.yield()
+            return try TestServiceEndpoint(DummyServiceImpl(), in: actorSystem)
+        }
+
+        let clientSystem = DistributedSystem(name: systemName)
+        try clientSystem.start()
+
+        clientSystem.connectToServices(
+            TestServiceEndpoint.self,
+            withFilter: { _ in true },
+            clientFactory: { actorSystem, _ in
+                return TestClientEndpoint(Client(), in: actorSystem)
+            },
+            serviceHandler: { serviceEndpoint, _ in }
+        )
+
+        for await _ in stream { break }
+
+        // client and server connected now, restarting server
path: Tests/DistributedSystemTests/DistributedSystemTests.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:42:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -224,5 +288,46 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         }
 
         try await future.get()
+
+        return serviceID
+    }
+
+    @discardableResult
+    public func removeService(_ serviceID: UUID) async throws -> Bool {
+        if let version = super.removeService(serviceID) {
+            await withCheckedContinuation { continuation in
+                let resumeContinuation = healthUpdateState.withLock {
+                    $0.version = version
+                    if $0.pendingUpdate {
+                        $0.continuation = continuation
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:42:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
+            }
+
+            eventLoop.scheduleTask(in: self.healthStatusUpdateInterval) {
+                self.updateHealthStatus(with: eventLoop)
+            }
+        }
+    }
+
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
-
-        for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+        while true {
+            let (services, servicesVersion) = getLocalServices()
+            let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                if servicesVersion < $0.version {
+                    return $0.version
+                } else {
+                    $0.pendingUpdate = true
+                    return nil
+                }
+            }
+
+            if let expectedServicesVersion {
+                logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                continue
+            }
+
+            logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+            let pendingRequestsBox = Box<Atomic<Int>>(.init(services.count))
+
+            for service in services {
+                let checkID = "service:\(service.serviceID)"
+                let future = consul.agent.check(checkID, status: .passing)
+                future.whenComplete { result in
+                    switch result {
+                    case .success:
+                        self.removePendingRequest(pendingRequestsBox, eventLoop)
+                    case let .failure(error):
+                        // dependning on the error we could try to re-register service
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:42:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
+            }
+
+            eventLoop.scheduleTask(in: self.healthStatusUpdateInterval) {
+                self.updateHealthStatus(with: eventLoop)
+            }
+        }
+    }
+
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
-
-        for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+        while true {
+            let (services, servicesVersion) = getLocalServices()
+            let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                if servicesVersion < $0.version {
+                    return $0.version
+                } else {
+                    $0.pendingUpdate = true
+                    return nil
+                }
+            }
+
+            if let expectedServicesVersion {
+                logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                continue
+            }
+
+            logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+            let pendingRequestsBox = Box<Atomic<Int>>(.init(services.count))
+
+            for service in services {
+                let checkID = "service:\(service.serviceID)"
+                let future = consul.agent.check(checkID, status: .passing)
+                future.whenComplete { result in
+                    switch result {
+                    case .success:
+                        self.removePendingRequest(pendingRequestsBox, eventLoop)
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:43:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -87,6 +87,7 @@ final class DiscoveryManager {
     private var logger: Logger { loggerBox.value }
 
     private var lock = NIOLock()
+    private var stateVersion = 0
path: Sources/DistributedSystem/DiscoveryManager.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:43:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +170,95 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
+    private func removePendingRequest(_ pendingRequestsBox: Box<Atomic<Int>>, _ eventLoop: EventLoop) {
+        let pendingRequests = pendingRequestsBox.value.wrappingSubtract(1, ordering: .releasing).newValue
+        if pendingRequests == 0 {
+            logger.trace("schedule next health status update check in \(self.healthStatusUpdateInterval) seconds")
+            let continuation = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return $0.continuation.take()
+            }
+
+            if let continuation {
+                logger.debug("resuming continuation")
+                continuation.resume(returning: ())
+            }
+
+            eventLoop.scheduleTask(in: self.healthStatusUpdateInterval) {
+                self.updateHealthStatus(with: eventLoop)
+            }
+        }
+    }
+
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
-
-        for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+        while true {
+            let (services, servicesVersion) = getLocalServices()
+            let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                if servicesVersion < $0.version {
+                    return $0.version
+                } else {
+                    $0.pendingUpdate = true
+                    return nil
+                }
+            }
+
+            if let expectedServicesVersion {
+                logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                continue
+            }
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-14T10:43:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +174,105 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
+        let services = {
+            while true {
+                let (services, servicesVersion) = getLocalServices()
+
+                let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                    if servicesVersion < $0.version {
+                        return $0.version
+                    } else {
+                        $0.pendingUpdate = true
+                        return nil
+                    }
+                }
+
+                if let expectedServicesVersion {
+                    logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                } else {
+                    logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+                    return services
+                }
+            }
+        }()
+
+        var futures = [EventLoopFuture<Void>]()
 
         for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+            let checkID = Self.makeCheckID(service.serviceID)
+            let promise = eventLoop.makePromise(of: Void.self)
+            let checkFuture = consul.agent.check(checkID, status: .passing)
+            checkFuture.whenComplete { result in
+                switch result {
+                case .success:
+                    promise.succeed()
+                case let .failure(error):
+                    // depending on the error we could try to re-register service
+                    if let error = error as? ConsulError,
+                       case let .httpResponseError(status, _) = error,
+                       case status = HTTPResponseStatus.notFound {
+                        if let serviceName = service.serviceName,
+                           let serviceID = UUID(uuidString: service.serviceID) {
+                            self.logger.error("check '\(checkID)' failed: \(error), trying to register service again")
+                            let serviceMeta = service.serviceMeta ?? [:]
+                            let registerFuture = self.registerService(serviceName, serviceID, metadata: serviceMeta)
+                            registerFuture.whenComplete {
+                                switch $0 {
+                                case .success:
+                                    promise.succeed()
+                                case let .failure(error):
+                                    self.logger.error("failed to re-register service \(serviceName)/\(serviceID): \(error)")
+                                    promise.fail(error)
+                                }
+                            }
+                        } else {
+                            self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                            promise.fail(error)
                         }
                     } else {
-                        self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                        self.logger.error("check '\(checkID)' failed: \(error)")
+                        promise.fail(error)
                     }
-                } else {
-                    self.logger.error("check '\(checkID)' failed: \(error)")
                 }
             }
+
+            futures.append(promise.futureResult)
         }
 
-        eventLoop.scheduleTask(in: healthStatusUpdateInterval) {
-            self.updateHealthStatus(with: eventLoop)
+        let allCompletedFuture = EventLoopFuture.whenAllComplete(futures, on: eventLoop)
+        _ = allCompletedFuture.map { _ in
+            let continuations = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return exchange(&$0.continuations, with: [])
+            }
+
+            if continuations.isEmpty {
+                self.logger.trace("no waiters, schedule next health update update")
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: ```suggestion
                self.logger.trace("no waiters, schedule next health update")
```
created_at: 2025-03-14T13:38:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +174,105 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
+        let services = {
+            while true {
+                let (services, servicesVersion) = getLocalServices()
+
+                let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                    if servicesVersion < $0.version {
+                        return $0.version
+                    } else {
+                        $0.pendingUpdate = true
+                        return nil
+                    }
+                }
+
+                if let expectedServicesVersion {
+                    logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                } else {
+                    logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+                    return services
+                }
+            }
+        }()
+
+        var futures = [EventLoopFuture<Void>]()
 
         for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+            let checkID = Self.makeCheckID(service.serviceID)
+            let promise = eventLoop.makePromise(of: Void.self)
+            let checkFuture = consul.agent.check(checkID, status: .passing)
+            checkFuture.whenComplete { result in
+                switch result {
+                case .success:
+                    promise.succeed()
+                case let .failure(error):
+                    // depending on the error we could try to re-register service
+                    if let error = error as? ConsulError,
+                       case let .httpResponseError(status, _) = error,
+                       case status = HTTPResponseStatus.notFound {
+                        if let serviceName = service.serviceName,
+                           let serviceID = UUID(uuidString: service.serviceID) {
+                            self.logger.error("check '\(checkID)' failed: \(error), trying to register service again")
+                            let serviceMeta = service.serviceMeta ?? [:]
+                            let registerFuture = self.registerService(serviceName, serviceID, metadata: serviceMeta)
+                            registerFuture.whenComplete {
+                                switch $0 {
+                                case .success:
+                                    promise.succeed()
+                                case let .failure(error):
+                                    self.logger.error("failed to re-register service \(serviceName)/\(serviceID): \(error)")
+                                    promise.fail(error)
+                                }
+                            }
+                        } else {
+                            self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                            promise.fail(error)
                         }
                     } else {
-                        self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                        self.logger.error("check '\(checkID)' failed: \(error)")
+                        promise.fail(error)
                     }
-                } else {
-                    self.logger.error("check '\(checkID)' failed: \(error)")
                 }
             }
+
+            futures.append(promise.futureResult)
         }
 
-        eventLoop.scheduleTask(in: healthStatusUpdateInterval) {
-            self.updateHealthStatus(with: eventLoop)
+        let allCompletedFuture = EventLoopFuture.whenAllComplete(futures, on: eventLoop)
+        _ = allCompletedFuture.map { _ in
+            let continuations = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return exchange(&$0.continuations, with: [])
+            }
+
+            if continuations.isEmpty {
+                self.logger.trace("no waiters, schedule next health update update")
+            } else {
+                self.logger.trace("""
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: dimlio
body: I'd have single message for both cases so they are consistent (now you don't print update interval in the first):
```swift
self.logger.trace("resuming \(continuations.count) continuations, schedule next health update in \(...)")
continuations.forEach(\.resume)
```
created_at: 2025-03-14T13:41:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +174,105 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
+        let services = {
+            while true {
+                let (services, servicesVersion) = getLocalServices()
+
+                let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                    if servicesVersion < $0.version {
+                        return $0.version
+                    } else {
+                        $0.pendingUpdate = true
+                        return nil
+                    }
+                }
+
+                if let expectedServicesVersion {
+                    logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                } else {
+                    logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+                    return services
+                }
+            }
+        }()
+
+        var futures = [EventLoopFuture<Void>]()
 
         for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+            let checkID = Self.makeCheckID(service.serviceID)
+            let promise = eventLoop.makePromise(of: Void.self)
+            let checkFuture = consul.agent.check(checkID, status: .passing)
+            checkFuture.whenComplete { result in
+                switch result {
+                case .success:
+                    promise.succeed()
+                case let .failure(error):
+                    // depending on the error we could try to re-register service
+                    if let error = error as? ConsulError,
+                       case let .httpResponseError(status, _) = error,
+                       case status = HTTPResponseStatus.notFound {
+                        if let serviceName = service.serviceName,
+                           let serviceID = UUID(uuidString: service.serviceID) {
+                            self.logger.error("check '\(checkID)' failed: \(error), trying to register service again")
+                            let serviceMeta = service.serviceMeta ?? [:]
+                            let registerFuture = self.registerService(serviceName, serviceID, metadata: serviceMeta)
+                            registerFuture.whenComplete {
+                                switch $0 {
+                                case .success:
+                                    promise.succeed()
+                                case let .failure(error):
+                                    self.logger.error("failed to re-register service \(serviceName)/\(serviceID): \(error)")
+                                    promise.fail(error)
+                                }
+                            }
+                        } else {
+                            self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                            promise.fail(error)
                         }
                     } else {
-                        self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                        self.logger.error("check '\(checkID)' failed: \(error)")
+                        promise.fail(error)
                     }
-                } else {
-                    self.logger.error("check '\(checkID)' failed: \(error)")
                 }
             }
+
+            futures.append(promise.futureResult)
         }
 
-        eventLoop.scheduleTask(in: healthStatusUpdateInterval) {
-            self.updateHealthStatus(with: eventLoop)
+        let allCompletedFuture = EventLoopFuture.whenAllComplete(futures, on: eventLoop)
+        _ = allCompletedFuture.map { _ in
+            let continuations = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return exchange(&$0.continuations, with: [])
+            }
+
+            if continuations.isEmpty {
+                self.logger.trace("no waiters, schedule next health update update")
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-16T16:05:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -156,45 +174,105 @@ public class DistributedSystemServer: DistributedSystem, @unchecked Sendable {
         super.stop()
     }
 
+    @discardableResult
     public func addService(
         ofType type: any ServiceEndpoint.Type,
         toModule moduleID: ModuleIdentifier,
         metadata: [String: String]? = nil,
         _ factory: @escaping ServiceFactory
-    ) async throws {
+    ) async throws -> UUID {
         try await addService(name: type.serviceName, toModule: moduleID, metadata: metadata, factory)
     }
 
     private func updateHealthStatus(with eventLoop: EventLoop) {
-        let services = getLocalServices()
-        logger.trace("update health status for \(services.count) services")
+        let services = {
+            while true {
+                let (services, servicesVersion) = getLocalServices()
+
+                let expectedServicesVersion: Int? = healthUpdateState.withLock {
+                    if servicesVersion < $0.version {
+                        return $0.version
+                    } else {
+                        $0.pendingUpdate = true
+                        return nil
+                    }
+                }
+
+                if let expectedServicesVersion {
+                    logger.debug("services version \(servicesVersion) is less than expected version \(expectedServicesVersion)")
+                } else {
+                    logger.trace("updating health status for \(services.count) services @ \(servicesVersion)")
+                    return services
+                }
+            }
+        }()
+
+        var futures = [EventLoopFuture<Void>]()
 
         for service in services {
-            let checkID = "service:\(service.serviceID)"
-            let future = consul.agent.check(checkID, status: .passing)
-            future.whenFailure { error in
-                if let error = error as? ConsulError,
-                   case let .httpResponseError(status) = error,
-                   case status = HTTPResponseStatus.notFound {
-                    if let serviceName = service.serviceName,
-                       let serviceID = UUID(uuidString: service.serviceID) {
-                        self.logger.error("check '\(checkID)' failed: \(error), register service again")
-                        let serviceMeta = service.serviceMeta ?? [:]
-                        let future = self.registerService(serviceName, serviceID, metadata: serviceMeta)
-                        future.whenFailure { error in
-                            self.logger.error("failed to register service \(serviceName)/\(serviceID): \(error)")
+            let checkID = Self.makeCheckID(service.serviceID)
+            let promise = eventLoop.makePromise(of: Void.self)
+            let checkFuture = consul.agent.check(checkID, status: .passing)
+            checkFuture.whenComplete { result in
+                switch result {
+                case .success:
+                    promise.succeed()
+                case let .failure(error):
+                    // depending on the error we could try to re-register service
+                    if let error = error as? ConsulError,
+                       case let .httpResponseError(status, _) = error,
+                       case status = HTTPResponseStatus.notFound {
+                        if let serviceName = service.serviceName,
+                           let serviceID = UUID(uuidString: service.serviceID) {
+                            self.logger.error("check '\(checkID)' failed: \(error), trying to register service again")
+                            let serviceMeta = service.serviceMeta ?? [:]
+                            let registerFuture = self.registerService(serviceName, serviceID, metadata: serviceMeta)
+                            registerFuture.whenComplete {
+                                switch $0 {
+                                case .success:
+                                    promise.succeed()
+                                case let .failure(error):
+                                    self.logger.error("failed to re-register service \(serviceName)/\(serviceID): \(error)")
+                                    promise.fail(error)
+                                }
+                            }
+                        } else {
+                            self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                            promise.fail(error)
                         }
                     } else {
-                        self.logger.error("check '\(checkID)' failed: \(error), can't register service \(service)")
+                        self.logger.error("check '\(checkID)' failed: \(error)")
+                        promise.fail(error)
                     }
-                } else {
-                    self.logger.error("check '\(checkID)' failed: \(error)")
                 }
             }
+
+            futures.append(promise.futureResult)
         }
 
-        eventLoop.scheduleTask(in: healthStatusUpdateInterval) {
-            self.updateHealthStatus(with: eventLoop)
+        let allCompletedFuture = EventLoopFuture.whenAllComplete(futures, on: eventLoop)
+        _ = allCompletedFuture.map { _ in
+            let continuations = self.healthUpdateState.withLock {
+                $0.pendingUpdate = false
+                return exchange(&$0.continuations, with: [])
+            }
+
+            if continuations.isEmpty {
+                self.logger.trace("no waiters, schedule next health update update")
+            } else {
+                self.logger.trace("""
path: Sources/DistributedSystem/DistributedSystemServer.swift
user: login: ser-0xff
body: Fixed.
created_at: 2025-03-16T16:05:57Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-03-13T13:24:10Z
body: ## [Codecov](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) Report
Attention: Patch coverage is `50.48544%` with `102 lines` in your changes missing coverage. Please review.
> Project coverage is 71.93%. Comparing base [(`d7486c8`)](https://app.codecov.io/gh/ordo-one/package-distributed-system/commit/d7486c81916004776df3e9d446f5d6e21dd24c74?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) to head [(`66ff8a5`)](https://app.codecov.io/gh/ordo-one/package-distributed-system/commit/66ff8a5697346d261d57a94871c33157082d811d?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one).
> Report is 1 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) | Patch % | Lines |
|---|---|---|
| [...es/DistributedSystem/DistributedSystemServer.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDistributedSystemServer.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXN0cmlidXRlZFN5c3RlbVNlcnZlci5zd2lmdA==) | 36.92% | [82 Missing :warning: ](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) |
| [...istributedSystemTests/DistributedSystemTests.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Tests%2FDistributedSystemTests%2FDistributedSystemTests.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-VGVzdHMvRGlzdHJpYnV0ZWRTeXN0ZW1UZXN0cy9EaXN0cmlidXRlZFN5c3RlbVRlc3RzLnN3aWZ0) | 72.00% | [14 Missing :warning: ](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) |
| [Sources/DistributedSystem/DiscoveryManager.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDiscoveryManager.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXNjb3ZlcnlNYW5hZ2VyLnN3aWZ0) | 76.19% | [5 Missing :warning: ](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) |
| [Sources/DistributedSystem/DistributedSystem.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDistributedSystem.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXN0cmlidXRlZFN5c3RlbS5zd2lmdA==) | 80.00% | [1 Missing :warning: ](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) |

<details><summary>Additional details and impacted files</summary>


[![Impacted file tree graph](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104/graphs/tree.svg?width=650&height=150&src=pr&token=vapTcW5IcV&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one)](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one)

```diff
@@            Coverage Diff             @@
##             main     #104      +/-   ##
==========================================
- Coverage   72.20%   71.93%   -0.26%     
==========================================
  Files          30       30              
  Lines        5683     5836     +153     
==========================================
+ Hits         4103     4198      +95     
- Misses       1580     1638      +58     
```

| [Files with missing lines](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) | Coverage Δ | |
|---|---|---|
| [Sources/DistributedSystem/DistributedSystem.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDistributedSystem.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXN0cmlidXRlZFN5c3RlbS5zd2lmdA==) | `77.76% <80.00%> (+0.19%)` | :arrow_up: |
| [Sources/DistributedSystem/DiscoveryManager.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDiscoveryManager.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXNjb3ZlcnlNYW5hZ2VyLnN3aWZ0) | `79.23% <76.19%> (+0.74%)` | :arrow_up: |
| [...istributedSystemTests/DistributedSystemTests.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Tests%2FDistributedSystemTests%2FDistributedSystemTests.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-VGVzdHMvRGlzdHJpYnV0ZWRTeXN0ZW1UZXN0cy9EaXN0cmlidXRlZFN5c3RlbVRlc3RzLnN3aWZ0) | `87.06% <72.00%> (+1.03%)` | :arrow_up: |
| [...es/DistributedSystem/DistributedSystemServer.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDistributedSystemServer.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXN0cmlidXRlZFN5c3RlbVNlcnZlci5zd2lmdA==) | `59.58% <36.92%> (-10.58%)` | :arrow_down: |

... and [1 file with indirect coverage changes](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one)

| [Files with missing lines](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one) | Coverage Δ | |
|---|---|---|
| [Sources/DistributedSystem/DistributedSystem.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDistributedSystem.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXN0cmlidXRlZFN5c3RlbS5zd2lmdA==) | `77.76% <80.00%> (+0.19%)` | :arrow_up: |
| [Sources/DistributedSystem/DiscoveryManager.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDiscoveryManager.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXNjb3ZlcnlNYW5hZ2VyLnN3aWZ0) | `79.23% <76.19%> (+0.74%)` | :arrow_up: |
| [...istributedSystemTests/DistributedSystemTests.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Tests%2FDistributedSystemTests%2FDistributedSystemTests.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-VGVzdHMvRGlzdHJpYnV0ZWRTeXN0ZW1UZXN0cy9EaXN0cmlidXRlZFN5c3RlbVRlc3RzLnN3aWZ0) | `87.06% <72.00%> (+1.03%)` | :arrow_up: |
| [...es/DistributedSystem/DistributedSystemServer.swift](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?src=pr&el=tree&filepath=Sources%2FDistributedSystem%2FDistributedSystemServer.swift&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one#diff-U291cmNlcy9EaXN0cmlidXRlZFN5c3RlbS9EaXN0cmlidXRlZFN5c3RlbVNlcnZlci5zd2lmdA==) | `59.58% <36.92%> (-10.58%)` | :arrow_down: |

... and [1 file with indirect coverage changes](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one)

------

[Continue to review full report in Codecov by Sentry](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?dropdown=coverage&src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one). Last update [d7486c8...66ff8a5](https://app.codecov.io/gh/ordo-one/package-distributed-system/pull/104?dropdown=coverage&src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ordo-one).

</details>
<details><summary>🚀 New features to boost your workflow: </summary>

- ❄ [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-13T13:29:56Z
body: _Pull request had an unknown failure_
[Pull request benchmark comparison [ubuntu-latest] with 'main' run at 2025-03-16T16:13:36+00:00](https://github.com/ordo-one/package-distributed-system/actions/runs/13885118567)

<!-- thollander/actions-comment-pull-request "Pull request benchmark comparison [ubuntu-latest] with" -->
author_association: NONE