Repository: zigpy/zha-device-handlers

Issue:
[BUG] Xiaomi lumi.relay.c2acn01  Command Report Attributes 0xFF01
**Describe the bug**
I have updated the lumi.relay.c2acn01 to firmware version 46. After doing this the following error has appeared when receiving Xiaomi manufacturer-specific attribute 0xFF01
```
2021-05-07 13:07:15 DEBUG (MainThread) [zigpy.zcl] [0x5f77:1:0x0000] ZCL deserialize: <ZCLHeader frame_control=<FrameControl frame_type=GLOBAL_COMMAND manufacturer_specific=True is_reply=True disable_default_response=True> manufacturer=4447 tsn=127 command_id=Command.Report_Attributes>
2021-05-07 13:07:15 DEBUG (MainThread) [zigpy.zcl] [0x5f77:1:0x0000] ValueError exception for: <ZCLHeader frame_control=<FrameControl frame_type=GLOBAL_COMMAND manufacturer_specific=True is_reply=True disable_default_response=True> manufacturer=4447 tsn=127 command_id=Command.Report_Attributes> payload: b'01ff424403282305212e0008212e12092100106410006510006e20006f200094200295390a078c41963999eb0c4597390030683b983980bb873c9b2100009c20010a2100000c280000'
2021-05-07 13:07:15 ERROR (MainThread) [zigpy.device] Failed to parse message (b'1c5f117f0a01ff424403282305212e0008212e12092100106410006510006e20006f200094200295390a078c41963999eb0c4597390030683b983980bb873c9b2100009c20010a2100000c280000') on cluster 0, because Data is too short to contain 2 bytes

```

**NOTA:**
As I have been able to see, the frame has a Byte of more
Should be:
```
b'01ff424403282305212e0008212e12092100106410006510006e20006f200094200295390a078c41963999eb0c4597390030683b983980bb873c9b2100009c20010a2100000c2800'
```



Discussion:

Resolving patch:
diff --git a/zhaquirks/xiaomi/__init__.py b/zhaquirks/xiaomi/__init__.py
--- a/zhaquirks/xiaomi/__init__.py
+++ b/zhaquirks/xiaomi/__init__.py
@@ -1,8 +1,9 @@
 """Xiaomi common components for custom device handlers."""
-import binascii
+from __future__ import annotations
+
 import logging
 import math
-from typing import Optional, Union
+from typing import Iterable, Iterator, Optional
 
 from zigpy import types as t
 import zigpy.device
@@ -114,48 +115,98 @@ class BasicCluster(CustomCluster, Basic):
 
     cluster_id = Basic.cluster_id
 
+    def _iter_parse_attr_report(
+        self, data: bytes
+    ) -> Iterator[foundation.Attribute, bytes]:
+        """Yield all interpretations of the first attribute in an Xiaomi report."""
+
+        # Peek at the attribute report
+        attr_id, data = t.uint16_t.deserialize(data)
+        attr_type, data = t.uint8_t.deserialize(data)
+
+        if (
+            attr_id not in (XIAOMI_AQARA_ATTRIBUTE, XIAOMI_MIJA_ATTRIBUTE)
+            or attr_type != 0x42  # "Character String"
+        ):
+            # Assume other attributes are reported correctly
+            data = attr_id.serialize() + attr_type.serialize() + data
+            attribute, data = foundation.Attribute.deserialize(data)
+
+            yield attribute, data
+            return
+
+        # Length of the "string" can be wrong
+        val_len, data = t.uint8_t.deserialize(data)
+
+        # Try every offset. Start with 0 to pass unbroken reports through.
+        for offset in (0, -1, 1):
+            fixed_len = val_len + offset
+
+            if len(data) < fixed_len:
+                continue
+
+            val, final_data = data[:fixed_len], data[fixed_len:]
+            attr_val = t.LVBytes(val)
+            attr_type = 0x41  # The data type should be "Octet String"
+
+            yield foundation.Attribute(
+                attrid=attr_id,
+                value=foundation.TypeValue(python_type=attr_type, value=attr_val),
+            ), final_data
+
+    def _interpret_attr_reports(
+        self, data: bytes
+    ) -> Iterable[tuple[foundation.Attribute]]:
+        """Yield all valid interprations of a Xiaomi attribute report."""
+
+        if not data:
+            yield ()
+            return
+
+        try:
+            parsed = list(self._iter_parse_attr_report(data))
+        except (KeyError, ValueError):
+            return
+
+        for attr, remaining_data in parsed:
+            for remaining_attrs in self._interpret_attr_reports(remaining_data):
+                yield (attr,) + remaining_attrs
+
     def deserialize(self, data):
         """Deserialize cluster data."""
-        try:
-            return super().deserialize(data)
-        except ValueError:
-            hdr, data = foundation.ZCLHeader.deserialize(data)
-            if not (
-                hdr.frame_control.frame_type == foundation.FrameType.GLOBAL_COMMAND
-                and hdr.command_id == 0x0A
-            ):
-                raise
-            msg = "ValueError exception for: %s payload: %s"
-            self.debug(msg, hdr, binascii.hexlify(data))
-            newdata = b""
-            while data:
-                try:
-                    attr, data = foundation.Attribute.deserialize(data)
-                except ValueError:
-                    attr_id, data = t.uint16_t.deserialize(data)
-                    if attr_id not in (XIAOMI_AQARA_ATTRIBUTE, XIAOMI_MIJA_ATTRIBUTE):
-                        raise
-                    attr_type, data = t.uint8_t.deserialize(data)
-                    val_len, data = t.uint8_t.deserialize(data)
-                    val_len = t.uint8_t(val_len - 1)
-                    val, data = data[:val_len], data[val_len:]
-                    newdata += attr_id.serialize()
-                    newdata += attr_type.serialize()
-                    newdata += val_len.serialize() + val
-                    continue
-                newdata += attr.serialize()
-            self.debug("new data: %s", binascii.hexlify(hdr.serialize() + newdata))
-            return super().deserialize(hdr.serialize() + newdata)
+        hdr, data = foundation.ZCLHeader.deserialize(data)
+
+        # Only handle attribute reports differently
+        if (
+            hdr.frame_control.frame_type != foundation.FrameType.GLOBAL_COMMAND
+            or hdr.command_id != foundation.Command.Report_Attributes
+        ):
+            return super().deserialize(hdr.serialize() + data)
+
+        reports = list(self._interpret_attr_reports(data))
+
+        if not reports:
+            _LOGGER.warning("Failed to parse Xiaomi attribute report: %r", data)
+            return super().deserialize(hdr.serialize() + data)
+        elif len(reports) > 1:
+            _LOGGER.warning(
+                "Xiaomi attribute report has multiple valid interpretations: %r",
+                reports,
+            )
+
+        fixed_data = b"".join(attr.serialize() for attr in reports[0])
+
+        return super().deserialize(hdr.serialize() + fixed_data)
 
     def _update_attribute(self, attrid, value):
         if attrid == XIAOMI_AQARA_ATTRIBUTE:
-            attributes = self._parse_aqara_attributes(value.raw)
-            super()._update_attribute(attrid, value.raw)
+            attributes = self._parse_aqara_attributes(value)
+            super()._update_attribute(attrid, value)
             if (
                 MODEL in self._attr_cache
                 and self._attr_cache[MODEL] == "lumi.sensor_switch.aq2"
             ):
-                if value.raw == b"\x04!\xa8C\n!\x00\x00":
+                if value == b"\x04!\xa8C\n!\x00\x00":
                     self.listener_event(ZHA_SEND_EVENT, COMMAND_TRIPLE, [])
         elif attrid == XIAOMI_MIJA_ATTRIBUTE:
             attributes = self._parse_mija_attributes(value)
@@ -254,7 +305,9 @@ def _parse_aqara_attributes(self, value):
             attribute_names.update({11: ILLUMINANCE_MEASUREMENT})
 
         result = {}
-        while value:
+
+        # Some attribute reports end with a stray null byte
+        while value not in (b"", b"\x00"):
             skey = int(value[0])
             svalue, value = foundation.TypeValue.deserialize(value[1:])
             result[skey] = svalue.value
@@ -485,11 +538,11 @@ class OnOffCluster(OnOff, CustomCluster):
 
     def command(
         self,
-        command_id: Union[foundation.Command, int, t.uint8_t],
+        command_id: foundation.Command | int | t.uint8_t,
         *args,
-        manufacturer: Optional[Union[int, t.uint16_t]] = None,
+        manufacturer: Optional[int | t.uint16_t] = None,
         expect_reply: bool = True,
-        tsn: Optional[Union[int, t.uint8_t]] = None
+        tsn: Optional[int | t.uint8_t] = None
     ):
         """Command handler."""
         src_ep = 1


Test patch:
diff --git a/tests/test_xiaomi.py b/tests/test_xiaomi.py
--- a/tests/test_xiaomi.py
+++ b/tests/test_xiaomi.py
@@ -5,6 +5,7 @@
 import pytest
 import zigpy.device
 import zigpy.types as t
+from zigpy.zcl import foundation
 
 import zhaquirks
 from zhaquirks.const import (
@@ -322,3 +323,104 @@ async def test_xiaomi_batt_size(zigpy_device_from_quirk, quirk, batt_size):
     succ, fail = await cluster.read_attributes(("battery_size", "battery_quantity"))
     assert succ["battery_quantity"] == 1
     assert succ["battery_size"] == batt_size
+
+
+@pytest.mark.parametrize(
+    "raw_report",
+    (
+        # https://community.hubitat.com/t/xiaomi-aqara-devices-pairing-keeping-them-connected/623?page=34
+        "02FF4C0600100121BA0B21A813240100000000215D062058",
+        "02FF4C0600100021EC0B21A8012400000000002182002063",
+        "01FF421F0121110D0328130421A8430521F60006240600030000082108140A21E51F",
+        "01FF421A0121C70B03281C0421A84305212B01062403000300000A2120CB",
+        "01FF421F0121C70B0328190421A8430521100106240400040000082109140A2120CB",
+        "01FF421F0121C70B0328180421A8430521100106240600050000082109140A213C50",
+        "01FF421A0121BD0B03281D0421A84305212F01062407000300000A2120CB",
+        # https://community.hubitat.com/t/xiaomi-aqara-zigbee-device-drivers-possibly-may-no-longer-be-maintained/631/print
+        "01FF42090421A8130A212759",
+        (
+            "01FF42296410006510016E20006F20010121E40C03281E05210500082116260A2100009923"
+            "000000009B210000"
+        ),
+        "01FF42220121D10B0328190421A81305212D0006240200000000082104020A21A4B4641000",
+        "01FF42220121D10B03281C0421A81305213A0006240000000000082104020A210367641001",
+        (
+            "01FF42280121B70C0328200421A81305211E00062402000000000A21E18C08210410642000"
+            "962300000000"
+        ),
+        (
+            "01FF42270328240521170007270000000000000000082117010921000A0A2130C064200065"
+            "20336621FA00"
+        ),
+        "02FF4C0600100121B30B21A8012400000000002195002056",
+        "02FF4C0600100121B30B21A8012400000000002195002057",
+        # puddly's logs
+        "01FF421A0121DB0B03280C0421A84305215401062401000000000A2178E0",
+        "01FF421D0121BD0B03280A0421A8330521E801062401000000000A214444641000",
+        "01FF421F0121E50B0328170421A8130521500006240100000000082105140A214761",
+        "01FF42210121950B0328130421A81305214400062401000000000A217CBE6410000B210400",
+        (
+            "01FF42250121630B0421A81305217D2F06240100000000642905006521631D662B4D7F0100"
+            "0A2157DE"
+        ),
+        "02FF4C06001001213C0C21A81324010000000021D1052061",
+        (
+            "050042166C756D692E73656E736F725F6D6F74696F6E2E61713201FF42210121950B032816"
+            "0421A83105214400062401000000000A217CBE6410000B210900"
+        ),
+        # GH Issue #811
+        (
+            "01FF424403282305212E0008212E12092100106410006510006E20006F200094200295390A"
+            "078C41963999EB0C4597390030683B983980BB873C9B2100009C20010A2100000C280000"
+        ),
+        # https://github.com/dresden-elektronik/deconz-rest-plugin/issues/1491#issuecomment-489032272
+        (
+            "01FF422E0121BD0B03281A0421A8430521470106240100010000082108030A216535982128"
+            "00992125009A252900FFFFDC04"
+        ),
+        # https://github.com/dresden-elektronik/deconz-rest-plugin/issues/1411#issuecomment-485724957
+        (
+            "01FF424403280005210F000727000000000000000008212312092100086410006510006E20"
+            "006F20009420089539000000009639B22E1645973988E5C83B9839C013063E9B210000"
+        ),
+        # https://github.com/dresden-elektronik/deconz-rest-plugin/issues/1588
+        (
+            "01FF422E0121770B0328230421A8010521250006240100000000082108030A2161F3982128"
+            "00992100009A25AFFE5B016904"
+        ),
+        # https://github.com/dresden-elektronik/deconz-rest-plugin/issues/1069
+        "02FF4C0600100121D10B21A801240000000000216E002050",
+        "01FF421D0121D10B0328150421A8130521A200062403000000000A210000641000",
+        "01FF421D0121DB0B0328140421A84305219A00062401000000000A21C841641000",
+        "01FF421D0121BD0B0328150421A83305213B00062401000000000A219FF8641000",
+        "01FF421D0121C70B0328130421A81305219200062401000000000A21C96B641000",
+    ),
+)
+def test_attribute_parsing(raw_report):
+    """Test the parsing of various Xiaomi 0xFF01 attribute reports."""
+    raw_report = bytes.fromhex(raw_report)
+
+    hdr = foundation.ZCLHeader.general(
+        manufacturer=4447, tsn=127, command_id=foundation.Command.Report_Attributes
+    )
+    cluster = BasicCluster(mock.MagicMock())
+
+    hdr, reports = cluster.deserialize(hdr.serialize() + raw_report)
+
+    # Keep track of all the data encoded in the attribute report
+    parsed_chunks = []
+
+    for report in reports[0]:
+        # This shouldn't throw an error
+        cluster._update_attribute(report.attrid, report.value.value)
+
+        parsed_chunks.append(report.attrid.serialize())
+        parsed_chunks.append(report.value.value.serialize()[1:])
+
+    # Remove every parsed chunk from the original report bytes
+    for chunk in parsed_chunks:
+        raw_report = raw_report.replace(chunk, b"", 1)
+
+    # The only remaining data should be the data type and the length.
+    # Everything else is passed through unmodified.
+    assert len(raw_report) == 2 * len(reports[0])