Repository: da4089/simplefix

Issue:
Add support for alternative end-of-message signalling
The CME (bless 'em) publishes a list of active product definitions using a text file containing (very) many FIX 35=a messages (`secdef.dat`).  Or *almost* FIX messages, except that they don't have a trailing CheckSum (10) field: messages are terminated by an end-of-line.

It'd be useful to parse this file using _simplefix_, but `FixParser` is currently relying on finding a `10=xxx|` field to identify the end of a message.

Add optional support to the parser to terminate messages with an end-of-line instead.

Discussion:

Patch:
diff --git a/simplefix/parser.py b/simplefix/parser.py
index 54bebf7..9190780 100644
--- a/simplefix/parser.py
+++ b/simplefix/parser.py
@@ -28,6 +28,10 @@ from .message import FixMessage, fix_val
 from .data import RAW_DATA_TAGS, RAW_LEN_TAGS
 
 
+# By default, messages are terminated by the Checksum (10) tag.
+DEFAULT_STOP_TAG = 10
+
+
 class FixParser(object):
     """FIX protocol message parser.
 
@@ -60,6 +64,12 @@ class FixParser(object):
 
         # Parsed length of data field.
         self.raw_len = 0
+
+        # Stop tag (default).
+        self.stop_tag = DEFAULT_STOP_TAG
+
+        # Stop character (optional).
+        self.stop_char = None
         return
 
     def add_raw(self, length_tag, value_tag):
@@ -104,6 +114,30 @@ class FixParser(object):
         self.raw_len = 0
         return
 
+    def set_message_terminator(self, tag=None, char=None):
+        """Set the end-of-message detection scheme.
+
+        :param tag: FIX tag number of terminating field.  Default is 10.
+        :param char: Alternative, terminating character.
+
+        By default, messages are terminated by the FIX Checksum (10)
+        field.  This can be changed to use a different tag, or a reserved
+        character using this function.
+
+        Note that only one of 'tag' or 'char' should be set, using a
+        named parameter."""
+
+        if tag is not None and char is not None:
+            raise ValueError("Only supply one of 'tag' or 'char'.")
+        elif tag is not None:
+            self.stop_tag = tag
+            self.stop_char = None
+        else:
+            self.stop_tag = None
+            bs = char.encode() if type(char) == str else char
+            self.stop_char = bs[0]
+        return
+
     def append_buffer(self, buf):
         """Append a byte string to the parser buffer.
 
@@ -142,7 +176,8 @@ class FixParser(object):
         tag = 0
 
         while point < len(self.buf):
-            if in_tag and self.buf[point] == EQUALS_BYTE:
+            c = self.buf[point]
+            if in_tag and c == EQUALS_BYTE:
                 tag_string = self.buf[start:point]
                 point += 1
 
@@ -162,12 +197,25 @@ class FixParser(object):
                     in_tag = False
                     start = point
 
-            elif self.buf[point] == SOH_BYTE:
+            elif c == self.stop_char:
+                if start != point:
+                    value = self.buf[start:point]
+                    self.pairs.append((tag, value))
+                    self.buf = self.buf[point + 1:]
+                else:
+                    self.buf = self.buf[1:]
+                break
+
+            elif c == SOH_BYTE:
                 value = self.buf[start:point]
                 self.pairs.append((tag, value))
                 self.buf = self.buf[point + 1:]
-                point = 0
-                start = point
+
+                if tag == self.stop_tag:
+                    break
+
+                start = 0
+                point = -1
                 in_tag = True
 
                 if tag in self.raw_len_tags:
@@ -175,9 +223,6 @@ class FixParser(object):
 
             point += 1
 
-        if len(self.pairs) == 0:
-            return None
-
         # Check first pair is FIX BeginString.
         while self.pairs and self.pairs[0][0] != 8:
             # Discard pairs until we find the beginning of a message.
@@ -187,19 +232,15 @@ class FixParser(object):
             return None
 
         # Look for checksum.
-        index = 0
-        while index < len(self.pairs) and self.pairs[index][0] != 10:
-            index += 1
-
-        if index == len(self.pairs):
-            return None
+        if self.stop_tag is not None:
+            if self.pairs[-1][0] != self.stop_tag:
+                return None
 
         # Found checksum, so we have a complete message.
         m = FixMessage()
-        pairs = self.pairs[:index + 1]
-        for tag, value in pairs:
+        for tag, value in self.pairs:
             m.append_pair(tag, value)
-        self.pairs = self.pairs[index + 1:]
+        self.pairs = []
 
         return m
 


Test patch:
diff --git a/test/test_parser.py b/test/test_parser.py
index 2df4b11..6da74c8 100644
--- a/test/test_parser.py
+++ b/test/test_parser.py
@@ -258,6 +258,81 @@ class ParserTests(unittest.TestCase):
         msg = parser.get_message()
         checksum = msg.get(10)
         self.assertEqual(checksum, b'169')
+        return
+
+    def test_stop_tag(self):
+        """Test termination using alternative tag number."""
+
+        pkt = FixMessage()
+        pkt.append_pair(8, "FIX.4.2")
+        pkt.append_pair(35, "D")
+        pkt.append_pair(29, "A")
+        pkt.append_pair(20000, "xxx")
+        pkt.append_pair(10, "000")
+        buf = pkt.encode()
+
+        p = FixParser()
+        p.set_message_terminator(tag=20000)
+        p.append_buffer(buf)
+        m = p.get_message()
+
+        self.assertIsNotNone(m)
+        self.assertEqual(b"FIX.4.2", m.get(8))
+        self.assertEqual(b"D", m.get(35))
+        self.assertEqual(b"A", m.get(29))
+        self.assertEqual(b"xxx", m.get(20000))
+        self.assertEqual(False, 10 in m)
+        return
+
+    def test_stop_char_with_field_terminator(self):
+        """Test stop character with field terminator."""
+
+        buf = \
+            b'8=FIX.4.2\x0135=d\x0134=1\x01369=XX\x01\n' + \
+            b'8=FIX.4.2\x0135=d\x0134=2\x01369=XX\x01\n' + \
+            b'8=FIX.4.2\x0135=d\x0134=3\x01369=XX\x01\n'
+
+        p = FixParser()
+        p.set_message_terminator(char='\n')
+        p.append_buffer(buf)
+
+        m = p.get_message()
+        self.assertIsNotNone(m)
+        self.assertEqual(b"1", m.get(34))
+
+        m = p.get_message()
+        self.assertIsNotNone(m)
+        self.assertEqual(b"2", m.get(34))
+
+        m = p.get_message()
+        self.assertIsNotNone(m)
+        self.assertEqual(b"3", m.get(34))
+        return
+
+    def test_stop_char_without_field_terminator(self):
+        """Test stop character without field terminator."""
+
+        buf = \
+            b'8=FIX.4.2\x0135=d\x0134=1\x01369=XX\n' + \
+            b'8=FIX.4.2\x0135=d\x0134=2\x01369=XX\n' + \
+            b'8=FIX.4.2\x0135=d\x0134=3\x01369=XX\n'
+
+        p = FixParser()
+        p.set_message_terminator(char='\n')
+        p.append_buffer(buf)
+
+        m = p.get_message()
+        self.assertIsNotNone(m)
+        self.assertEqual(b"1", m.get(34))
+
+        m = p.get_message()
+        self.assertIsNotNone(m)
+        self.assertEqual(b"2", m.get(34))
+
+        m = p.get_message()
+        self.assertIsNotNone(m)
+        self.assertEqual(b"3", m.get(34))
+        return
 
 
 if __name__ == "__main__":