Repository: jgosmann/dmarc-metrics-exporter (language: python)

Issue:
Fetch gets recognized as untagged IMAP response
Hey,

we currently experience the problem, that our fetched messages get recognized as untagged IMAP responses. The verbose debug log states the following: 
![grafik](https://user-images.githubusercontent.com/12358706/183922894-fb64aa55-64eb-4d8d-8b20-1d18e308b893.png)

We are using a Office365 inbox.

Thank you in advance!

Kind regards
vanBonzen


Discussion / hints:
I won't have time to look into this until at least the middle of next week.

Fix patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 78cc168..c104f27 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -6,7 +6,18 @@ All notable changes to this project will be documented in this file.
 The format is based on `Keep a Changelog <https://keepachangelog.com/en/1.0.0/>`_,
 and this project adheres to `Semantic Versioning <https://semver.org/spec/v2.0.0.html>`_.
 
+[0.6.2] - [unreleased]
+----------------------
+
+Fixed
+^^^^^
+
+* More robust handling of IMAP fetch responses including unknown fields.
+  (`#29 <https://github.com/jgosmann/dmarc-metrics-exporter/issues/29>`_)
+
+
 [0.6.1] - [2022-07-17]
+----------------------
 
 Fixed
 ^^^^^
diff --git a/dmarc_metrics_exporter/imap_parser.py b/dmarc_metrics_exporter/imap_parser.py
index e3347a7..414aae7 100644
--- a/dmarc_metrics_exporter/imap_parser.py
+++ b/dmarc_metrics_exporter/imap_parser.py
@@ -93,10 +93,9 @@ def pair(keyword_expr: Parser, expr: Parser) -> Parser:
     + sp
     + Group(Suppress(nested_lists))
 )
+length = Suppress(Literal(b"<")) + integer + Suppress(Literal(b">"))
 body_section = pair(
-    CaselessLiteral(b"BODY")
-    + section
-    + Opt(Suppress(Literal(b"<")) + integer + Suppress(Literal(b">"))),
+    CaselessLiteral(b"BODY") + section + Opt(length),
     nstring,
 )
 envelope = pair(
@@ -114,6 +113,14 @@ def pair(keyword_expr: Parser, expr: Parser) -> Parser:
 rfc822_text = pair(CaselessLiteral(b"RFC822.TEXT"), nstring)
 rfc822_size = pair(CaselessLiteral(b"RFC822.SIZE"), integer)
 uid = pair(CaselessLiteral(b"UID"), integer)
+unknown_fetch_response_pair = pair(
+    Combine(CharacterSet(b" \t\r\n[<", invert=True)[1, ...])
+    + Opt(
+        Combine(Literal(b"[") + CharacterSet(b"]", invert=True)[0, ...] + Literal(b"]"))
+    )
+    + Opt(length),
+    nil | integer | nstring | nested_lists,
+)
 fetch_response_pair = (
     body_section
     | body_structure
@@ -125,6 +132,7 @@ def pair(keyword_expr: Parser, expr: Parser) -> Parser:
     | rfc822_text
     | rfc822_size
     | uid
+    | unknown_fetch_response_pair
 )
 
 fetch_response_line = (
@@ -159,7 +167,7 @@ def pair(keyword_expr: Parser, expr: Parser) -> Parser:
     | Group(CaselessLiteral(b"UIDNEXT") + sp + integer)
     | Group(CaselessLiteral(b"UIDVALIDITY") + sp + integer)
     | Group(CaselessLiteral(b"UNSEEN") + sp + integer)
-    | Group(atom + Opt(sp + Combine(CharacterSet(b"\r\r]", invert=True)[1, ...])))
+    | Group(atom + Opt(sp + Combine(CharacterSet(b"\r\n]", invert=True)[1, ...])))
 )
 resp_text = Opt(
     Suppress(Literal(b"["))
diff --git a/pyproject.toml b/pyproject.toml
index 892e893..f89ee6c 100755
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,7 +19,7 @@ license = "MIT"
 name = "dmarc-metrics-exporter"
 readme = "README.rst"
 repository = "https://github.com/jgosmann/dmarc-metrics-exporter/"
-version = "0.6.1"
+version = "0.6.2"
 
 [tool.poetry.scripts]
 dmarc-metrics-exporter = "dmarc_metrics_exporter.__main__:run"


Test patch:
diff --git a/dmarc_metrics_exporter/tests/test_imap_parser.py b/dmarc_metrics_exporter/tests/test_imap_parser.py
index b14de31..d9d90db 100644
--- a/dmarc_metrics_exporter/tests/test_imap_parser.py
+++ b/dmarc_metrics_exporter/tests/test_imap_parser.py
@@ -196,6 +196,78 @@ async def test_parses_strings(given_input, expected):
                 ),
             ),
         ),
+        (
+            b"1 FETCH (UNKNOWN NIL)",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN", b"NIL"),),
+            ),
+        ),
+        (
+            b"1 FETCH (UNKNOWN 123)",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN", 123),),
+            ),
+        ),
+        (
+            b'1 FETCH (UNKNOWN "foo")',
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN", b"foo"),),
+            ),
+        ),
+        (
+            b"1 FETCH (UNKNOWN {3}\r\nfoo)",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN", b"foo"),),
+            ),
+        ),
+        (
+            b"1 FETCH (UNKNOWN (foo 123 ({6}\r\nfoobar)))",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN", (b"foo", b"123", (b"foobar",))),),
+            ),
+        ),
+        (
+            b"1 FETCH (UNKNOWN.FOO NIL)",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN.FOO", b"NIL"),),
+            ),
+        ),
+        (
+            b"1 FETCH (UNKNOWN.FOO[FIELD1 FIELD2 (ITEM1 ITEM2)] NIL)",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN.FOO", b"[FIELD1 FIELD2 (ITEM1 ITEM2)]", b"NIL"),),
+            ),
+        ),
+        (
+            b"1 FETCH (UNKNOWN.FOO<42> NIL)",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN.FOO", 42, b"NIL"),),
+            ),
+        ),
+        (
+            b"1 FETCH (UNKNOWN.FOO[FIELD1 FIELD2 (ITEM1 ITEM2)]<42> NIL)",
+            (
+                1,
+                b"FETCH",
+                ((b"UNKNOWN.FOO", b"[FIELD1 FIELD2 (ITEM1 ITEM2)]", 42, b"NIL"),),
+            ),
+        ),
     ],
 )
 async def test_parses_fetch_response_line(given_input, expected):