Repository: xZetsubou/hass-localtuya
Pull request #584: Better handling of absent subdevices
Author: login: Lurker00
Created: 2025-03-03T13:28:00Z | Merged: 2025-03-06T22:10:21Z

Weak detection of absent sub-devices instead of fuzzy logic with delays:

The lists of offline/online subdevices are parsed and handled immediatelly. A device is considered as absent if it was not mentioned in either list, nor its DPs were updated, for at least two heartbeat periods.

`_connect_subdevices()` no longer checks for `SubdeviceState.ABSENT` because it is now weak state. It does not hurt to try to connect to a subdevice that may be absent anyway.

`pass` operator is intentional: to prevent an attempt to handle this case by a further developer who didn't read the comment in the pytuya.py.

Tested and proved to work.


Diff:
diff --git a/custom_components/localtuya/coordinator.py b/custom_components/localtuya/coordinator.py
index c948e3e54..53ce3d530 100644
--- a/custom_components/localtuya/coordinator.py
+++ b/custom_components/localtuya/coordinator.py
@@ -137,7 +137,7 @@ def is_sleep(self):
         """Return whether the device is sleep or not."""
         if (device_sleep := self._device_config.sleep_time) > 0:
             setattr(self, "low_power", True)
-            last_update = int(time.monotonic()) - self._last_update_time
+            last_update = time.monotonic() - self._last_update_time
             return last_update < device_sleep
 
         return False
@@ -174,8 +174,7 @@ async def _connect_subdevices(self):
         for subdevice in self.sub_devices.values():
             if not self.connected or self.is_closing:
                 break
-            if subdevice.subdevice_state != SubdeviceState.ABSENT:
-                await subdevice.async_connect()
+            await subdevice.async_connect()
 
     async def _make_connection(self):
         """Subscribe localtuya entity events."""
@@ -325,11 +324,11 @@ def _new_entity_handler(entity_id):
 
         # If not connected try to handle the errors.
         if not self.connected and not self.is_closing:
-            if self._task_reconnect is None:
-                self._task_reconnect = asyncio.create_task(self._async_reconnect())
             if update_localkey:
-                # Check if the cloud device info has changed!.
+                # Check if the cloud device info has changed!
                 await self._update_local_key()
+            if self._task_reconnect is None:
+                self._task_reconnect = asyncio.create_task(self._async_reconnect())
 
         self._task_connect = None
 
@@ -497,7 +496,7 @@ async def _shutdown_entities(self, exc=""):
         if self.is_subdevice:
             self.info(f"Sub-device disconnected due to: {exc}")
         elif hasattr(self, "low_power"):
-            m, s = divmod((int(time.monotonic()) - self._last_update_time), 60)
+            m, s = divmod((int(time.monotonic() - self._last_update_time)), 60)
             h, m = divmod(m, 60)
             self.info(f"The device is still out of reach since: {h}h:{m}m:{s}s")
         else:
@@ -510,6 +509,7 @@ async def _update_local_key(self):
         if self._entry.data.get(CONF_NO_CLOUD, True):
             return self.info("Ensure that localkey hasn't changed and it's correct")
 
+        self.info(f"Trying to update local-key...")
         dev_id = self._device_config.id
         cloud_api = self._hass_entry.cloud_data
         await cloud_api.async_get_devices_list(force_update=True)
@@ -601,7 +601,7 @@ def status_updated(self, status: dict):
             # Fake gateways are only used to pass commands no need to update status.
             return
 
-        self._last_update_time = int(time.monotonic())
+        self._last_update_time = time.monotonic()
         self._handle_event(self._status, status)
         self._status.update(status)
         self._dispatch_status()
@@ -645,17 +645,24 @@ def subdevice_state_updated(self, state: SubdeviceState):
         self.subdevice_state = state
 
         # This will trigger if state is absent twice.
-        if old_state == state and state == SubdeviceState.ABSENT:
-            self._subdevice_off_count = 0
-            return self.disconnected("Device is absent")
-        elif state == SubdeviceState.ABSENT:
-            return self.info(f"Sub-device is absent {node_id}")
-        elif old_state == SubdeviceState.ABSENT:
+        if state == SubdeviceState.ABSENT:
+            if old_state == state:
+                delay = time.monotonic() - self._last_update_time
+                if delay >= (HEARTBEAT_INTERVAL * 2):
+                    self._subdevice_off_count = 0
+                    self.disconnected("Device is absent")
+                # Can be >2 subsequent payloads per one request
+                elif delay > HEARTBEAT_INTERVAL:
+                    self.debug(f"Sub-device is absent for {delay:.03f}s")
+            return
+        elif old_state == SubdeviceState.ABSENT and not self.connected:
             self.info(f"Sub-device is back {node_id}")
 
         is_online = state == SubdeviceState.ONLINE
         off_count = self._subdevice_off_count
         self._subdevice_off_count = 0 if is_online else off_count + 1
+        # For sub-devices, the last time it is known as not absent
+        self._last_update_time = time.monotonic()
 
         if is_online:
             return self.info(f"Sub-device is online {node_id}") if off_count else None
diff --git a/custom_components/localtuya/core/pytuya/__init__.py b/custom_components/localtuya/core/pytuya/__init__.py
index 3c59891c5..a70a96151 100644
--- a/custom_components/localtuya/core/pytuya/__init__.py
+++ b/custom_components/localtuya/core/pytuya/__init__.py
@@ -805,7 +805,6 @@ def __init__(
         self.heartbeater: asyncio.Task | None = None
         self._sub_devs_query_task: asyncio.Task | None = None
         self.dps_cache = {}
-        self.sub_devices_states = {"online": [], "offline": []}
         self.local_nonce = b"0123456789abcdef"  # not-so-random random key
         self.remote_nonce = b""
         self.dps_whitelist = UPDATE_DPS_WHITELIST
@@ -846,15 +845,11 @@ def _msg_subdevs_query(self, decoded_message):
         Message: {"online": [cids, ...], "offline": [cids, ...], "nearby": [cids, ...]}
         """
 
-        async def _action():
+        async def _action(on_devs, off_devs):
             try:
-                await asyncio.sleep(2)
-
-                self.debug(f"Sub-Devices States Update: {self.sub_devices_states}")
-                on_devs = self.sub_devices_states.get("online")
-                off_devs = self.sub_devices_states.get("offline")
+                self.debug(f"Sub-Devices States Update: {on_devs=} {off_devs=}")
                 listener = self.listener and self.listener()
-                if listener is None or (on_devs is None and off_devs is None):
+                if listener is None:
                     return
                 for cid, device in listener.sub_devices.items():
                     if cid in on_devs:
@@ -862,6 +857,9 @@ async def _action():
                     elif cid in off_devs:
                         device.subdevice_state_updated(SubdeviceState.OFFLINE)
                     else:
+                        # ABSENT detection is weak, because, with many sub-devices,
+                        # the gateway provides them all in more than 1 reply. This
+                        # should be taken into account in device.subdevice_state_updated()
                         device.subdevice_state_updated(SubdeviceState.ABSENT)
             except asyncio.CancelledError:
                 pass
@@ -869,22 +867,10 @@ async def _action():
                 self._sub_devs_query_task = None
 
         if (data := decoded_message.get("data")) and isinstance(data, dict):
-            devs_states = self.sub_devices_states
-            updated_states = {}
-
-            cached_on_devs = devs_states.get("online", [])
-            cached_off_devs = devs_states.get("offline", [])
-
             on_devs, off_devs = data.get("online", []), data.get("offline", [])
-
-            updated_states["online"] = list(set(cached_on_devs + on_devs))
-            updated_states["offline"] = list(set(cached_off_devs + off_devs))
-
-            if self._sub_devs_query_task is not None:
-                self._sub_devs_query_task.cancel()
-
-            self.sub_devices_states = updated_states
-            self._sub_devs_query_task = self.loop.create_task(_action())
+            self._sub_devs_query_task = self.loop.create_task(
+                _action(on_devs, off_devs)
+            )
 
     def _setup_dispatcher(self) -> MessageDispatcher:
         def _status_update(msg, ack=False):
@@ -1205,13 +1191,8 @@ async def set_dps(self, dps, cid=None):
 
     async def subdevices_query(self):
         """Request a list of sub-devices and their status."""
-        # Avoid empty state parsing!
-        if self._sub_devs_query_task is not None:
-            self._sub_devs_query_task.cancel()
-            self._sub_devs_query_task = None
         # Return payload: {"online": [cid1, ...], "offline": [cid2, ...]}
         # "nearby": [cids, ...] can come in payload.
-        self.sub_devices_states = {"online": [], "offline": []}
         payload = self._generate_payload(
             LAN_EXT_STREAM, rawData={"cids": []}, reqType="subdev_online_stat_query"
         )


Reviews:
user: login: xZetsubou
body: Thanks for the PR, `_action` looks much better.

Just cleaning the codes.
state: COMMENTED
author_association: OWNER
submitted_at: 2025-03-04T23:55:37Z
user: login: Lurker00
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-05T08:57:26Z
user: login: xZetsubou
state: COMMENTED
author_association: OWNER
submitted_at: 2025-03-05T10:07:12Z
user: login: Lurker00
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-05T10:51:33Z
user: login: xZetsubou
state: COMMENTED
author_association: OWNER
submitted_at: 2025-03-05T11:27:27Z

Review comments:
diff_hunk: @@ -645,17 +645,27 @@ def subdevice_state_updated(self, state: SubdeviceState):
         self.subdevice_state = state
 
         # This will trigger if state is absent twice.
-        if old_state == state and state == SubdeviceState.ABSENT:
-            self._subdevice_off_count = 0
-            return self.disconnected("Device is absent")
-        elif state == SubdeviceState.ABSENT:
-            return self.info(f"Sub-device is absent {node_id}")
-        elif old_state == SubdeviceState.ABSENT:
+        if state == SubdeviceState.ABSENT:
+            if old_state == state:
+                delay = time.monotonic() - self._last_update_time
+                if delay >= 2 * HEARTBEAT_INTERVAL:
+                    self._subdevice_off_count = 0
+                    self.disconnected("Device is absent")
+                # Can be >2 subsequent payloads per one request
+                elif delay > HEARTBEAT_INTERVAL:
+                    self.debug(f"Sub-device is absent for {round(delay,3)}s")
+            else:
+                # Can be false alarm, do nothing!
+                pass
+            return
+        elif old_state == SubdeviceState.ABSENT and not self.connected:
path: custom_components/localtuya/coordinator.py
user: login: xZetsubou
body: I understand this this was is intentional however if anyone will change this logic he will needs to check `pytuya` as well. however this looks confused even to me, For readability, let's do it as before with one if statement. 
And used format operator instead of round.

```suggestion
        if old_state == state and state == SubdeviceState.ABSENT:
            delay = time.monotonic() - self._last_update_time
            if delay >= (HEARTBEAT_INTERVAL * 2):
                self._subdevice_off_count = 0
                self.disconnected("Device is absent")
            # Can be >2 subsequent payloads per one request
            elif delay > HEARTBEAT_INTERVAL:
                self.debug(f"Sub-device is absent for {delay:.03f)}s")
            return
        elif old_state == SubdeviceState.ABSENT and not self.connected:
```
created_at: 2025-03-04T23:37:40Z
author_association: OWNER
diff_hunk: @@ -645,17 +645,27 @@ def subdevice_state_updated(self, state: SubdeviceState):
         self.subdevice_state = state
 
         # This will trigger if state is absent twice.
-        if old_state == state and state == SubdeviceState.ABSENT:
-            self._subdevice_off_count = 0
-            return self.disconnected("Device is absent")
-        elif state == SubdeviceState.ABSENT:
-            return self.info(f"Sub-device is absent {node_id}")
-        elif old_state == SubdeviceState.ABSENT:
+        if state == SubdeviceState.ABSENT:
+            if old_state == state:
+                delay = time.monotonic() - self._last_update_time
+                if delay >= 2 * HEARTBEAT_INTERVAL:
+                    self._subdevice_off_count = 0
+                    self.disconnected("Device is absent")
+                # Can be >2 subsequent payloads per one request
+                elif delay > HEARTBEAT_INTERVAL:
+                    self.debug(f"Sub-device is absent for {round(delay,3)}s")
+            else:
+                # Can be false alarm, do nothing!
+                pass
+            return
+        elif old_state == SubdeviceState.ABSENT and not self.connected:
             self.info(f"Sub-device is back {node_id}")
 
         is_online = state == SubdeviceState.ONLINE
         off_count = self._subdevice_off_count
         self._subdevice_off_count = 0 if is_online else off_count + 1
+        # For sub-devices, the last time it is known as not absent
+        self._last_update_time = int(time.monotonic())
path: custom_components/localtuya/coordinator.py
user: login: xZetsubou
body: ```suggestion
        self._last_update_time = time.monotonic()
```
created_at: 2025-03-04T23:37:47Z
author_association: OWNER
diff_hunk: @@ -846,45 +845,34 @@ def _msg_subdevs_query(self, decoded_message):
         Message: {"online": [cids, ...], "offline": [cids, ...], "nearby": [cids, ...]}
         """
 
-        async def _action():
+        async def _action(on_devs, off_devs):
             try:
-                await asyncio.sleep(2)
-
-                self.debug(f"Sub-Devices States Update: {self.sub_devices_states}")
-                on_devs = self.sub_devices_states.get("online")
-                off_devs = self.sub_devices_states.get("offline")
+                self.debug(
+                    f"Sub-Devices States Update: on_devs={on_devs} off_devs={off_devs}"
+                )
path: custom_components/localtuya/core/pytuya/__init__.py
user: login: xZetsubou
body: f format way.
```suggestion
                self.debug(f"Sub-Devices States Update: {on_devs=} {off_devs=}")
```
created_at: 2025-03-04T23:51:07Z
author_association: OWNER
diff_hunk: @@ -645,17 +645,27 @@ def subdevice_state_updated(self, state: SubdeviceState):
         self.subdevice_state = state
 
         # This will trigger if state is absent twice.
-        if old_state == state and state == SubdeviceState.ABSENT:
-            self._subdevice_off_count = 0
-            return self.disconnected("Device is absent")
-        elif state == SubdeviceState.ABSENT:
-            return self.info(f"Sub-device is absent {node_id}")
-        elif old_state == SubdeviceState.ABSENT:
+        if state == SubdeviceState.ABSENT:
+            if old_state == state:
+                delay = time.monotonic() - self._last_update_time
+                if delay >= 2 * HEARTBEAT_INTERVAL:
+                    self._subdevice_off_count = 0
+                    self.disconnected("Device is absent")
+                # Can be >2 subsequent payloads per one request
+                elif delay > HEARTBEAT_INTERVAL:
+                    self.debug(f"Sub-device is absent for {round(delay,3)}s")
+            else:
+                # Can be false alarm, do nothing!
+                pass
+            return
+        elif old_state == SubdeviceState.ABSENT and not self.connected:
path: custom_components/localtuya/coordinator.py
user: login: Lurker00
body: Your suggestion lacks the case `state == SubdeviceState.ABSENT and old_state != state`. In my code, it just returns with the comment `Can be false alarm, do nothing!`.

Also, in your format operator `{delay:.03f)}` the `)` looks odd, must be  `{delay:.03f}`. But I can't find in Python docs information about leading `0` for the precision, only for the width, and I don't want to restart my HA just to check it (there is a test for continuous run at least for a week, and today is 5th day).

So, sorry, I can't just commit your suggestion as it is: it would break it all.
created_at: 2025-03-05T08:57:26Z
author_association: NONE
diff_hunk: @@ -645,17 +645,27 @@ def subdevice_state_updated(self, state: SubdeviceState):
         self.subdevice_state = state
 
         # This will trigger if state is absent twice.
-        if old_state == state and state == SubdeviceState.ABSENT:
-            self._subdevice_off_count = 0
-            return self.disconnected("Device is absent")
-        elif state == SubdeviceState.ABSENT:
-            return self.info(f"Sub-device is absent {node_id}")
-        elif old_state == SubdeviceState.ABSENT:
+        if state == SubdeviceState.ABSENT:
+            if old_state == state:
+                delay = time.monotonic() - self._last_update_time
+                if delay >= 2 * HEARTBEAT_INTERVAL:
+                    self._subdevice_off_count = 0
+                    self.disconnected("Device is absent")
+                # Can be >2 subsequent payloads per one request
+                elif delay > HEARTBEAT_INTERVAL:
+                    self.debug(f"Sub-device is absent for {round(delay,3)}s")
+            else:
+                # Can be false alarm, do nothing!
+                pass
+            return
+        elif old_state == SubdeviceState.ABSENT and not self.connected:
path: custom_components/localtuya/coordinator.py
user: login: xZetsubou
body: Ah, sorry didn't notice that, then do it as you made it but remove unused blocks

```py
        if state == SubdeviceState.ABSENT:
            if old_state == state:
                delay = time.monotonic() - self._last_update_time
                if delay >= 2 * HEARTBEAT_INTERVAL:
                    self._subdevice_off_count = 0
                    self.disconnected("Device is absent")
                # Can be >2 subsequent payloads per one request
                elif delay > HEARTBEAT_INTERVAL: 
                    self.debug(f"Sub-device is absent for {delay:.03f}s")
                return
```

> only for the width

Yes, was your goal to round the number or you want to get the exact time of absent time?  delay will always return float number.
created_at: 2025-03-05T10:07:12Z
author_association: OWNER
diff_hunk: @@ -645,17 +645,27 @@ def subdevice_state_updated(self, state: SubdeviceState):
         self.subdevice_state = state
 
         # This will trigger if state is absent twice.
-        if old_state == state and state == SubdeviceState.ABSENT:
-            self._subdevice_off_count = 0
-            return self.disconnected("Device is absent")
-        elif state == SubdeviceState.ABSENT:
-            return self.info(f"Sub-device is absent {node_id}")
-        elif old_state == SubdeviceState.ABSENT:
+        if state == SubdeviceState.ABSENT:
+            if old_state == state:
+                delay = time.monotonic() - self._last_update_time
+                if delay >= 2 * HEARTBEAT_INTERVAL:
+                    self._subdevice_off_count = 0
+                    self.disconnected("Device is absent")
+                # Can be >2 subsequent payloads per one request
+                elif delay > HEARTBEAT_INTERVAL:
+                    self.debug(f"Sub-device is absent for {round(delay,3)}s")
+            else:
+                # Can be false alarm, do nothing!
+                pass
+            return
+        elif old_state == SubdeviceState.ABSENT and not self.connected:
path: custom_components/localtuya/coordinator.py
user: login: Lurker00
body: > was your goal to round the number

yes, to 3 digits after the dot, i.e. milliseconds, for better readability.

OK, will reduce it.
created_at: 2025-03-05T10:51:33Z
author_association: NONE
diff_hunk: @@ -645,17 +645,24 @@ def subdevice_state_updated(self, state: SubdeviceState):
         self.subdevice_state = state
 
         # This will trigger if state is absent twice.
-        if old_state == state and state == SubdeviceState.ABSENT:
-            self._subdevice_off_count = 0
-            return self.disconnected("Device is absent")
-        elif state == SubdeviceState.ABSENT:
-            return self.info(f"Sub-device is absent {node_id}")
-        elif old_state == SubdeviceState.ABSENT:
+        if state == SubdeviceState.ABSENT:
+            if old_state == state:
+                delay = time.monotonic() - self._last_update_time
+                if delay >= 2 * HEARTBEAT_INTERVAL:
path: custom_components/localtuya/coordinator.py
user: login: xZetsubou
body: readability.

```suggestion
                if delay >= (HEARTBEAT_INTERVAL * 2):
```
created_at: 2025-03-05T11:27:27Z
author_association: OWNER

PR comments:
user: login: Lurker00
created_at: 2025-03-03T13:57:45Z
body: I decided to add the commit c5c8299a7bf613ae62a6b7450b9d7df0bfb3b2ed (and related e5b9689d7ae042741703645f877469e2ee1b3a78) as related. By the current logic, LocalTuya first tries to reconnect (and may call `_update_local_key()` at the end), then tries to update local key from the previous attempt. With an absent device, it leads to two attempts to update local key in a row. I saw the added log message twice in 30 milliseconds. With this order, in case `_update_local_key()` fails, it is called with a delay of the time required for full reconnect process.
author_association: NONE
user: login: Lurker00
created_at: 2025-03-05T14:27:34Z
body: I'll not insist if you'd prefer to roll back the c3387b7cffd2c674795d63de525b0b7f7764e432. It was in my initial set of changes for the previous PR.
author_association: NONE
user: login: xZetsubou
created_at: 2025-03-05T15:26:39Z
body: Yes please revert it, We most not use listener `attr` checks from `pytuya`.

Is there any point of disconnecting low-power device quickly, it wouldn't hurt leave them no?


author_association: OWNER
user: login: Lurker00
created_at: 2025-03-05T20:42:05Z
body: > it wouldn't hurt leave them no?

It does not hurt. But I have log messages around at `info` level, and don't want my T&H sensor to spam every 15 minutes 😄 
author_association: NONE