Repository: juergenH87/python-can-j1939 (language: python)

Issue:
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
Continuous Integration
I'm going to try to setup github actions to run the tests on Linux, Mac, and Windows.
Continuous Integration
I'm going to try to setup github actions to run the tests on Linux, Mac, and Windows.
Continuous Integration
I'm going to try to setup github actions to run the tests on Linux, Mac, and Windows.
Continuous Integration
I'm going to try to setup github actions to run the tests on Linux, Mac, and Windows.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.
J1939 memory access
I'm adding memory access support (DM14-16 messaging.)  My use case requires a client, but I may add a server while I'm at it to facilitate testing.


Discussion / hints:

Fix patch:
diff --git a/j1939/__init__.py b/j1939/__init__.py
index a814e1d..d47cc51 100644
--- a/j1939/__init__.py
+++ b/j1939/__init__.py
@@ -4,4 +4,5 @@
 from .name import Name
 from .message_id import MessageId
 from .parameter_group_number import ParameterGroupNumber
-from .diagnostic_messages import *
\ No newline at end of file
+from .diagnostic_messages import *
+from .memory_access import *
\ No newline at end of file
diff --git a/j1939/memory_access.py b/j1939/memory_access.py
new file mode 100644
index 0000000..ccafd8a
--- /dev/null
+++ b/j1939/memory_access.py
@@ -0,0 +1,205 @@
+from enum import Enum
+import queue
+import sys
+import time
+
+import j1939
+
+
+class QueryState(Enum):
+    IDLE = 1
+    WAIT_FOR_SEED = 2
+    WAIT_FOR_DM16 = 3
+    WAIT_FOR_OPER_COMPLETE = 4
+
+
+class Command(Enum):
+    ERASE = 0
+    READ = 1
+    WRITE = 2
+    STATUS_REQUEST = 3
+    OPERATION_COMPLETED = 4
+    OPERATION_FAILED = 5
+    BOOT_LOAD = 6
+    EDCP_GENERATION = 7
+
+
+class ReceiveState(Enum):
+    IDLE = 1
+    WAIT_FOR_KEY = 2
+    WAIT_FOR_CONFIRMATION = 3
+
+
+class Dm15Status(Enum):
+    PROCEED = 0
+    BUSY = 1
+    OPERATION_COMPLETE = 4
+    OPERATION_FAILED = 5
+
+
+class Dm14Query:
+    def __init__(self, ca: j1939.ControllerApplication):
+        """
+        performs memory access queries using DM14-DM18 messaging.  Presently only read queries are supported
+
+        :param obj ca: j1939 controller application
+        """
+
+        self._ca = ca
+        self.state = QueryState.IDLE
+        self._seed_from_key = None
+        self.data_queue = queue.Queue()
+        self.mem_data = None
+
+    def _wait_for_data(self):
+        assert self.state is QueryState.WAIT_FOR_SEED
+        if self.command is Command.WRITE:
+            self._send_dm16()
+            self.state = QueryState.WAIT_FOR_OPER_COMPLETE
+        else:
+            self.state = QueryState.WAIT_FOR_DM16
+            self._ca.unsubscribe(self._parse_dm15)
+            self._ca.subscribe(self._parse_dm16)
+
+    def _send_operation_complete(self):
+        self.object_count = 1
+        self.command = Command.OPERATION_COMPLETED
+        self._send_dm14(0xFFFF)
+
+    def _send_dm14(self, key_or_user_level):
+        self._pgn = j1939.ParameterGroupNumber.PGN.DM14
+        pointer = self.address.to_bytes(length=4, byteorder="little")
+        data = []
+        data.append(self.object_count)
+        data.append(
+            (self.direct << 4) + (self.command.value << 1) + 1
+        )  # (SAE reserved = 1)
+        for octet in pointer:
+            data.append(octet)
+        data.append(key_or_user_level & 0xFF)
+        data.append(key_or_user_level >> 8)
+        self._ca.send_pgn(
+            0, (self._pgn >> 8) & 0xFF, self._dest_address & 0xFF, 6, data
+        )
+
+    def _send_dm16(self):
+        self._pgn = j1939.ParameterGroupNumber.PGN.DM16
+        data = []
+        byte_count = len(self.bytes)
+        data.append(0xFF if byte_count > 7 else byte_count)
+        for i in range(byte_count):
+            data.append(self.bytes[i])
+        self._ca.send_pgn(
+            0, (self._pgn >> 8) & 0xFF, self._dest_address & 0xFF, 6, data
+        )
+
+    def _parse_dm15(self, priority, pgn, sa, timestamp, data):
+        if pgn != j1939.ParameterGroupNumber.PGN.DM15 or sa != self._dest_address:
+            return
+        seed = (data[7] << 8) + data[6]
+        status = (data[1] >> 1) & 7
+        if status is Dm15Status.BUSY.value or status is Dm15Status.OPERATION_FAILED:
+            error = int.from_bytes(data[2:4], byteorder="little", signed=False)
+            self.data_queue.put(None)
+            if error == 0x1000:
+                raise RuntimeError("Key authentication error")
+            else:  # TODO parse error codes more granularly
+                raise RuntimeError(f"Device {sa} busy")
+        length = data[0]
+        if seed == 0xFFFF and length == self.object_count:
+            self._wait_for_data()
+        else:
+            if self.state is QueryState.WAIT_FOR_OPER_COMPLETE:
+                assert status is Command.OPERATION_COMPLETED.value
+                self._send_operation_complete()
+                self.state = QueryState.IDLE
+                self.data_queue.put(self.mem_data)
+            else:
+                assert self.state is QueryState.WAIT_FOR_SEED
+                if self._seed_from_key is not None:
+                    self._send_dm14(self._seed_from_key(seed))
+                else:
+                    self.data_queue.put(None)
+                    raise RuntimeError(
+                        "Key requested from host but no seed-key algorithm has been provided"
+                    )
+
+    def _parse_dm16(self, priority, pgn, sa, timestamp, data):
+        if pgn != j1939.ParameterGroupNumber.PGN.DM16 or sa != self._dest_address:
+            return
+        length = min(data[0], len(data) - 1)
+        # assert object_count == self.object_count
+        self.mem_data = data[1 : length + 1]
+        self._ca.unsubscribe(self._parse_dm16)
+        self._ca.subscribe(self._parse_dm15)
+        self.state = QueryState.WAIT_FOR_OPER_COMPLETE
+
+    def _values_to_bytes(self, values):
+        bytes = []
+        for val in values:
+            bytes.extend(val.to_bytes(self.object_byte_size, byteorder="little"))
+        return bytes
+
+    def _bytes_to_values(self, raw_bytes):
+        values = []
+        for i in range(len(raw_bytes) // self.object_byte_size):
+            values.append(
+                int.from_bytes(
+                    raw_bytes[i : self.object_byte_size - 1],
+                    byteorder="little",
+                    signed=self.signed,
+                )
+            )
+        return values
+
+    def read(
+        self,
+        dest_address,
+        direct,
+        address,
+        object_count,
+        object_byte_size=1,
+        signed=False,
+        return_raw_bytes=False,
+    ):
+        assert object_count > 0
+        self._dest_address = dest_address
+        self.direct = direct
+        self.address = address
+        self.object_count = object_count
+        self.object_byte_size = object_byte_size
+        self.signed = signed
+        self.return_raw_bytes = return_raw_bytes
+        self.command = Command.READ
+        self._ca.subscribe(self._parse_dm15)
+        self._send_dm14(7)
+        self.state = QueryState.WAIT_FOR_SEED
+        # wait for operation completed DM15 message
+        raw_bytes = self.data_queue.get(block=True, timeout=1)
+        if self.return_raw_bytes:
+            return raw_bytes
+        else:
+            return self._bytes_to_values(raw_bytes)
+
+    def write(self, dest_address, direct, address, values, object_byte_size=1):
+        self._dest_address = dest_address
+        self.direct = direct
+        self.address = address
+        self.object_byte_size = object_byte_size
+        self.command = Command.WRITE
+        self.bytes = self._values_to_bytes(values)
+        self.object_count = len(values)
+        self._ca.subscribe(self._parse_dm15)
+        self._send_dm14(7)
+        self.state = QueryState.WAIT_FOR_SEED
+        # wait for operation completed DM15 message
+        try:
+            self.data_queue.get(block=True, timeout=1)
+        except queue.Empty:
+            pass  # expect empty queue for write
+
+    def set_seed_key_algorithm(self, algorithm):
+        self._seed_from_key = algorithm
+
+    def set_seed_key_algorithm(self, algorithm):
+        self._seed_from_key = algorithm


Test patch:
diff --git a/test/feeder.py b/test/feeder.py
index f478478..cdd9269 100644
--- a/test/feeder.py
+++ b/test/feeder.py
@@ -1,9 +1,14 @@
+import logging
 import queue
-
-import time
 import threading
+import time
 
 import j1939
+from j1939.message_id import MessageId
+from j1939.parameter_group_number import ParameterGroupNumber
+
+logger = logging.getLogger(__name__)
+
 
 class AcceptAllCA(j1939.ControllerApplication):
     """CA to accept all messages"""
@@ -14,7 +19,7 @@ def __init__(self, name, device_address_preferred=None):
 
     def message_acceptable(self, dest_address):
         """Indicates if this CA would accept a message
-        (OVERLOADED FUNCTION)        
+        (OVERLOADED FUNCTION)
         This function indicates the acceptance of this CA for the given dest_address.
         """
         return True
@@ -64,18 +69,18 @@ def _send_message(self, can_id, data):
         Checks the message sent and generates the apropriate answer.
         The data is fed from self.can_messages.
         """
-        expected_data = self.can_messages.pop(0)
-        assert (
-            expected_data[0] == Feeder.MsgType.CANTX,
-            "No transmission was expected",
+        logger.info(
+            f'send message ID: {can_id:04x}, data:   {["{:02x}".format(val) for val in data]}'
         )
+        expected_data = self.can_messages.pop(0)
+        assert expected_data[0] == Feeder.MsgType.CANTX
         assert can_id == expected_data[1]
         assert data == expected_data[2]
         self._inject_messages_into_ecu()
 
     def _on_message(self, priority, pgn, sa, timestamp, data):
         """Feed incoming message to this testcase.
-    
+
         :param int priority:
             Priority of the message
         :param int pgn:
@@ -87,6 +92,9 @@ def _on_message(self, priority, pgn, sa, timestamp, data):
         :param bytearray data:
             Data of the PDU
         """
+        logger.info(
+            f'received from sa {sa:02x} pgn {pgn:04x} data: {["{:02x}".format(val) for val in data]}'
+        )
         expected_data = self.pdus.pop(0)
         assert expected_data[0] == Feeder.MsgType.PDU
         assert pgn == expected_data[1]
@@ -95,31 +103,47 @@ def _on_message(self, priority, pgn, sa, timestamp, data):
         else:
             assert data is None
 
-    def accept_all_messages(self):
+    def pdus_from_messages(self):
+        self.pdus = []
+        for message in self.can_messages:
+            if message[0] is Feeder.MsgType.CANRX:
+                pgn = ParameterGroupNumber()
+                pgn.from_message_id(MessageId(can_id=message[1]))
+                self.pdus.append((Feeder.MsgType.PDU, pgn.value & 0xFF00, message[2]))
+
+    def accept_all_messages(
+        self, device_address_preferred=None, bypass_address_claim=False
+    ):
         # install a fake-CA to accept all messages
-        ca = AcceptAllCA(None)
-        self.ecu.add_ca(controller_application = ca)
+        ca = AcceptAllCA(None, device_address_preferred=device_address_preferred)
+
+        self.ecu.add_ca(controller_application=ca)
+        if bypass_address_claim:  # hack
+            ca._device_address = device_address_preferred
+            ca._device_address_state = j1939.ControllerApplication.State.NORMAL
+            self.ecu.subscribe(self._on_message)
+        return ca
 
     def receive(self):
         self.ecu.subscribe(self._on_message)
         self._inject_messages_into_ecu()
         # wait until all messages are processed asynchronously
-        while len(self.pdus)>0:
+        while len(self.pdus) > 0:
             time.sleep(0.500)
-        # wait for final processing    
+        # wait for final processing
         time.sleep(0.100)
         self.ecu.unsubscribe(self._on_message)
 
     def send(self, pdu, source, destination):
         self.ecu.subscribe(self._on_message)
-    
-        # sending from 240 to 155 with prio 6
-        self.ecu.send_pgn(0, pdu[1]>>8, destination, 6, source, pdu[2])
-        
+        self.ecu.send_pgn(0, pdu[1] >> 8, destination, 6, source, pdu[2])
+        self.process_messages()
+
+    def process_messages(self):
         # wait until all messages are processed asynchronously
-        while len(self.can_messages)>0:
+        while len(self.can_messages) > 0:
             time.sleep(0.500)
-        # wait for final processing    
+        # wait for final processing
         time.sleep(0.100)
         self.ecu.unsubscribe(self._on_message)
 
diff --git a/test/test_memory_access.py b/test/test_memory_access.py
new file mode 100644
index 0000000..2cee29a
--- /dev/null
+++ b/test/test_memory_access.py
@@ -0,0 +1,91 @@
+import pytest
+import time
+
+from feeder import Feeder
+import j1939
+
+
+# fmt: off
+read_with_seed_key = [
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x13, 0x03, 0x00, 0x00, 0x92, 0x07, 0x00], 0.0), #DM14 read address 0x92000007
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x00, 0x11, 0xFF, 0xFF, 0xFF, 0xFF, 0x5A, 0xA5], 0.0), #DM15 seed response
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x13, 0x03, 0x00, 0x00, 0x92, 0xA5, 0x5A], 0.0), #DM14 key response
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x01, 0x11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 proceed response
+    (Feeder.MsgType.CANRX, 0x1CD7F9D4, [0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM16 data transfer
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x00, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 operation completed
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x19, 0x03, 0x00, 0x00, 0x92, 0xFF, 0xFF], 0.0), #DM14 operation completed
+]
+
+read_no_seed_key = [
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x13, 0x03, 0x00, 0x00, 0x92, 0x07, 0x00], 0.0), #DM14 read address 0x92000007
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x01, 0x11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 proceed response
+    (Feeder.MsgType.CANRX, 0x1CD7F9D4, [0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM16 data transfer
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x00, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 operation completed
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x19, 0x03, 0x00, 0x00, 0x92, 0xFF, 0xFF], 0.0), #DM14 operation completed
+]
+
+write_with_seed_key = [
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x15, 0x07, 0x00, 0x00, 0x91, 0x07, 0x00], 0.0), #DM14 write address 0x91000007
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x00, 0x11, 0xFF, 0xFF, 0xFF, 0xFF, 0x5A, 0xA5], 0.0), #DM15 seed response
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x15, 0x07, 0x00, 0x00, 0x91, 0xA5, 0x5A], 0.0), #DM14 key response
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x01, 0x11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 proceed response
+    (Feeder.MsgType.CANTX, 0x18D7D4F9, [0x04, 0x44, 0x33, 0x22, 0x11]                  , 0.0), #DM16 data transfer
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x00, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 operation completed
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x19, 0x07, 0x00, 0x00, 0x91, 0xFF, 0xFF], 0.0), #DM14 operation completed
+]
+
+write_no_seed_key = [
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x15, 0x07, 0x00, 0x00, 0x91, 0x07, 0x00], 0.0), #DM14 write address 0x91000007
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x01, 0x11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 proceed response
+    (Feeder.MsgType.CANTX, 0x18D7D4F9, [0x04, 0x44, 0x33, 0x22, 0x11]                  , 0.0), #DM16 data transfer
+    (Feeder.MsgType.CANRX, 0x1CD8F9D4, [0x00, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 0.0), #DM15 operation completed
+    (Feeder.MsgType.CANTX, 0x18D9D4F9, [0x01, 0x19, 0x07, 0x00, 0x00, 0x91, 0xFF, 0xFF], 0.0), #DM14 operation completed
+]
+# fmt: on
+
+
+def key_from_seed(seed):
+    return seed ^ 0xFFFF
+
+
+@pytest.mark.parametrize(
+    argnames=["expected_messages"],
+    argvalues=[[read_with_seed_key], [read_no_seed_key]],
+    ids=["With seed key", "Without seed key"],
+)
+def test_dm14_read(feeder, expected_messages):
+    feeder.can_messages = expected_messages
+    feeder.pdus_from_messages()
+
+    ca = feeder.accept_all_messages(
+        device_address_preferred=0xF9, bypass_address_claim=True
+    )
+
+    dm14 = j1939.Dm14Query(ca)
+    dm14.set_seed_key_algorithm(key_from_seed)
+    dm14.read(0xD4, 1, 0x92000003, 1)
+
+    feeder.process_messages()
+
+
+@pytest.mark.parametrize(
+    argnames=["expected_messages"],
+    argvalues=[[write_with_seed_key], [write_no_seed_key]],
+    ids=["With seed key", "Without seed key"],
+)
+def test_dm14_write(feeder, expected_messages):
+    feeder.can_messages = expected_messages
+    feeder.pdus_from_messages()
+
+    ca = feeder.accept_all_messages(
+        device_address_preferred=0xF9, bypass_address_claim=True
+    )
+
+    dm14 = j1939.Dm14Query(ca)
+    dm14.set_seed_key_algorithm(key_from_seed)
+    values = [0x11223344]
+    dm14.write(0xD4, 1, 0x91000007, values, object_byte_size=4)
+
+    feeder.process_messages()
+
+# TODO: moar test