Repository: pymeasure/pymeasure

Issue:
`TypeError` on SerialAdapter `read_bytes(-1)`
When I use `self.read_bytes(-1)` from my instrument using a `SerialAdapter`, I get the error:

> C:\Program Files\Python310\lib\ctypes\__init__.py:66: TypeError

With the traceback:

```
..\..\..\pymeasure\instruments\velleman\velleman_k8090.py:247: in read
    response = self.read_bytes(-1)
..\..\..\pymeasure\instruments\instrument.py:153: in read_bytes
    return self.adapter.read_bytes(count, **kwargs)
..\..\..\pymeasure\adapters\adapter.py:123: in read_bytes
    read = self._read_bytes(count, break_on_termchar, **kwargs)
..\..\..\pymeasure\adapters\serial.py:108: in _read_bytes
    return self.connection.read(1e99 if count == -1 else count, **kwargs)
..\..\..\venv\lib\site-packages\serial\serialwin32.py:278: in read
    buf = ctypes.create_string_buffer(n)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

init = 1e+99, size = None

    def create_string_buffer(init, size=None):
        """create_string_buffer(aBytes) -> character array
        create_string_buffer(anInteger) -> character array
        create_string_buffer(aBytes, anInteger) -> character array
        """
        if isinstance(init, bytes):
            if size is None:
                size = len(init)+1
            _sys.audit("ctypes.create_string_buffer", init, size)
            buftype = c_char * size
            buf = buftype()
            buf.value = init
            return buf
        elif isinstance(init, int):
            _sys.audit("ctypes.create_string_buffer", None, init)
            buftype = c_char * init
            buf = buftype()
            return buf
>       raise TypeError(init)
E       TypeError: 1e+99

C:\Program Files\Python310\lib\ctypes\__init__.py:66: TypeError

```

It looks like reading 1e99 bytes no (longer?) works.


Discussion:
Oh.
Creating a that large buffer is beyond many computers...

Thanks for raising that issue.

So we have to reduce the size of the bytes to read.
Maybe we can read a certain number, let's say 1000, and read afterwards all bytes which are in the buffer (via in_waiting).
In total it should be only one timeout time (as we know how many bytes are to read the second time), but we wait at least some time for a device response.
Sounds like a good suggestion.

Won't `in_waiting` always be 0 though, when a timeout occurs?
And if there are 1001 bytes in the buffer, we would immediately read the 1000, followed by the additional 1 byte, but never end up waiting a timeout duration.

I was thinking of reading in chunks instead:

```
            data = bytes()
            buffer_empty = False
            chunk_size = 256
            while not buffer_empty:
                chunk = self.connection.read(chunk_size, **kwargs)
                data += chunk
                buffer_empty = len(chunk) < chunk_size  # If fewer bytes got returned, we had a timeout
            return data
```

I think this results in only a single timeout.

Slightly less engineered would be to read an amount of bytes equal to `in_waiting` + 1000, then we have a single operation and account for unusually full buffers.

Resolving patch:
diff --git a/pymeasure/adapters/serial.py b/pymeasure/adapters/serial.py
--- a/pymeasure/adapters/serial.py
+++ b/pymeasure/adapters/serial.py
@@ -94,7 +94,7 @@ def _read_bytes(self, count, break_on_termchar, **kwargs):
         """Read a certain number of bytes from the instrument.
 
         :param int count: Number of bytes to read. A value of -1 indicates to
-            read from the whole read buffer.
+            read from the whole read buffer (waits for timeout).
         :param bool break_on_termchar: Stop reading at a termination character.
         :param \\**kwargs: Keyword arguments for the connection itself.
         :returns bytes: Bytes response of the instrument (including termination).
@@ -103,9 +103,25 @@ def _read_bytes(self, count, break_on_termchar, **kwargs):
             return self.connection.read_until(self.read_termination.encode(),
                                               count if count > 0 else None,
                                               **kwargs)
+        elif count >= 0:
+            return self.connection.read(count, **kwargs)
         else:
-            # At -1 we read a very large number of bytes, which can be considered the whole buffer.
-            return self.connection.read(1e99 if count == -1 else count, **kwargs)
+            # For -1 we empty the buffer completely
+            return self._read_bytes_until_timeout()
+
+    def _read_bytes_until_timeout(self, chunk_size=256, **kwargs):
+        """Read from the serial until a timeout occurs, regardless of the number of bytes.
+
+        :chunk_size: The number of bytes attempted to in a single transaction.
+            Multiple of these transactions will occur.
+        """
+        # `Serial.readlines()` has an unpredictable timeout, see PR #866
+        data = bytes()
+        while True:
+            chunk = self.connection.read(chunk_size, **kwargs)
+            data += chunk
+            if len(chunk) < chunk_size:  # If fewer bytes got returned, we had a timeout
+                return data
 
     def flush_read_buffer(self):
         """Flush and discard the input buffer."""


Test patch:
diff --git a/tests/adapters/test_serial.py b/tests/adapters/test_serial.py
--- a/tests/adapters/test_serial.py
+++ b/tests/adapters/test_serial.py
@@ -73,6 +73,12 @@ def test_read_bytes_unlimited(adapter):
     assert adapter.read_bytes(-1) == b"basd\x02\nfasdf\n"
 
 
+def test_read_bytes_unlimited_long(adapter):
+    """Test whether all bytes are returned when a lot of data is sent."""
+    adapter.write_bytes(b"abcde" * 50)
+    assert adapter.read_bytes(-1) == b"abcde" * 50
+
+
 @pytest.mark.parametrize("count", (-1, 8))
 def test_read_bytes_break_on_termchar(adapter, count):
     adapter.read_termination = "\n"
diff --git a/tests/adapters/test_serial_with_loopback.py b/tests/adapters/test_serial_with_loopback.py
new file mode 100644
--- /dev/null
+++ b/tests/adapters/test_serial_with_loopback.py
@@ -0,0 +1,68 @@
+import pytest
+
+from serial import Serial
+from time import time
+from pymeasure.adapters import SerialAdapter
+
+
+class TestSerialLoopback:
+    """Test serial adapter with two real but looped COM ports.
+
+    A pair of COM ports that are looped will read the written data from the other.
+    This looping needs to be done outside this script. One suggestion is using the application
+    `com0com <https://com0com.sourceforge.net/>`_.
+    Alternatively two UART adapters could be used, that have their RX/TX physically connected.
+
+    Call PyTest with the argument``--device-address="COM1,COM2"`` to specify the serial addresses,
+    minding the comma for the separation of the two ports.
+    """
+
+    ADAPTER_TIMEOUT = 1.0  # Seconds
+
+    @pytest.fixture()
+    def adapter(self, connected_device_address):
+        device = connected_device_address.split(",")[0]
+        return SerialAdapter(
+            device,
+            baudrate=19200,
+            timeout=self.ADAPTER_TIMEOUT,
+            read_termination=chr(0x0F),
+        )
+
+    @pytest.fixture()
+    def loopback(self, connected_device_address):
+        """See `adapter()`."""
+        device = connected_device_address.split(",")[1]
+        return Serial(device, baudrate=19200)
+
+    def test_read(self, adapter, loopback):
+        """Regular read with fixed number of bytes."""
+        loopback.write(b"abc")
+        result = adapter.read_bytes(3)
+
+        assert len(result) == 3
+
+    @pytest.mark.parametrize("data", [
+        b"a",                               # Short data
+        b"aaaaabbbbbccccceeeee" * 50,       # A lot data
+        b"a" * 256,                         # Exactly one chunk size
+        b"aaaaabbbbb\nccccceeeee" * 50,     # With a newline (should be ignored)
+    ])
+    def test_read_all(self, adapter, loopback, data):
+        """Read with undefined number of bytes - also confirm timeout duration is correct."""
+        loopback.write(data)
+        start = time()
+        result = adapter.read_bytes(-1)
+        elapsed = time() - start
+
+        assert result == data
+        assert self.ADAPTER_TIMEOUT == pytest.approx(elapsed, abs=0.1)
+
+    @pytest.mark.parametrize("chunk", [1, 256, 10000])
+    def test_read_varied_chunk_size(self, adapter, loopback, chunk):
+        """Read with undefined number of bytes with non-default chunk size."""
+        data = b"abcde" * 10
+        loopback.write(data)
+        result = adapter.read_bytes(-1)
+
+        assert result == data