Repository: rafalp/Misago

Issue:
Message parser doesn't handle CRLF linebreaks
Markup:

```markdown
Lorem ipsum

Dolor met
```

HTML result:

```
<p>Lorem Ipsum<br /><br />Dolor met</p>
```

Expected result:

```
<p>Lorem Ipsum</p>
<p>Dolor met</p>
```

This happens because parser expects LF (`\n`) input and browsers send CRLF (`\r\n`)

We can fix this in two steps:

- Run `.replace("\r\n", "\n").replace("\r", "\n")` to normalize newlines before we send the string deeper into parser.
- Do the `.replace("\n", "\r\n")` in reverse when rendering AST back to HTML/plaintext/etc.
- Replace `\n` with `\r\n` on formatting help page.

Hints from issue comments:
@rafalp: We should also check all `.join(”\n”)` in codebase and change them to CRLF as part of this fix.@raydeal: Hi, I can work on this ticket.

Solution patch:
diff --git a/misago/parser/html.py b/misago/parser/html.py
index e23c882c8..96d257d75 100644
--- a/misago/parser/html.py
+++ b/misago/parser/html.py
@@ -17,7 +17,8 @@ def render_ast_to_html(context: ParserContext, ast: list[dict], metadata: dict)
         node_html = render_ast_node_to_html(context, ast_node, metadata)
         if node_html:
             html += node_html
-    return html
+
+    return html.replace("\n", "\r\n")
 
 
 def render_ast_node_to_html(
diff --git a/misago/parser/parser.py b/misago/parser/parser.py
index 5fc831001..3f493f386 100644
--- a/misago/parser/parser.py
+++ b/misago/parser/parser.py
@@ -46,12 +46,16 @@ class Parser:
         self._reserved_patterns = {}
 
     def __call__(self, markup: str) -> list[dict]:
+        markup = self.normalize_newlines(markup)
         markup = self.reserve_patterns(markup)
         ast = self.parse_blocks(markup, [])
         for post_processor in self.post_processors:
             ast = post_processor(self, ast)
         return ast
 
+    def normalize_newlines(self, markup: str) -> str:
+        return markup.replace("\r\n", "\n").replace("\r", "\n")
+
     def reserve_patterns(self, markup: str) -> str:
         if not "`" in markup:
             return markup
diff --git a/misago/parser/plaintext.py b/misago/parser/plaintext.py
index c51bbe778..fd609a434 100644
--- a/misago/parser/plaintext.py
+++ b/misago/parser/plaintext.py
@@ -33,7 +33,7 @@ def render_ast_to_plaintext(
     ):
         return re.sub("\s\s+", " ", text)
 
-    return text
+    return text.replace("\n", "\r\n")
 
 
 def render_inline_ast_to_plaintext(
diff --git a/misago/parser/views.py b/misago/parser/views.py
index 2faf8adf9..6c84bcb7c 100644
--- a/misago/parser/views.py
+++ b/misago/parser/views.py
@@ -121,13 +121,13 @@ FORMATS = [
     (
         pgettext_lazy("formatting help", "Unordered list"),
         pgettext_lazy(
-            "formatting help", "- Lorem ipsum\n- Dolor met\n- Vulputate lectus"
+            "formatting help", "- Lorem ipsum\r\n- Dolor met\r\n- Vulputate lectus"
         ),
     ),
     (
         pgettext_lazy("formatting help", "Ordered list"),
         pgettext_lazy(
-            "formatting help", "1. Lorem ipsum\n2. Dolor met\n3. Vulputate lectus"
+            "formatting help", "1. Lorem ipsum\r\n2. Dolor met\r\n3. Vulputate lectus"
         ),
     ),
     (
@@ -136,17 +136,17 @@ FORMATS = [
     ),
     (
         pgettext_lazy("formatting help", "Quote text (BBCode)"),
-        pgettext_lazy("formatting help", "[quote]\nQuoted text\n[/quote]"),
+        pgettext_lazy("formatting help", "[quote]\r\nQuoted text\r\n[/quote]"),
     ),
     (
         pgettext_lazy("formatting help", "Quote text with author (BBCode)"),
         pgettext_lazy(
-            "formatting help", '[quote="Quote author"]\nQuoted text\n[/quote]'
+            "formatting help", '[quote="Quote author"]\r\nQuoted text\r\n[/quote]'
         ),
     ),
     (
         pgettext_lazy("formatting help", "Spoiler"),
-        pgettext_lazy("formatting help", "[spoiler]\nSecret text\n[/spoiler]"),
+        pgettext_lazy("formatting help", "[spoiler]\r\nSecret text\r\n[/spoiler]"),
     ),
     (
         pgettext_lazy("formatting help", "Inline code"),
@@ -154,30 +154,30 @@ FORMATS = [
     ),
     (
         pgettext_lazy("formatting help", "Code block"),
-        pgettext_lazy("formatting help", '```\nalert("Hello world!");\n```'),
+        pgettext_lazy("formatting help", '```\r\nalert("Hello world!");\r\n```'),
     ),
     (
         pgettext_lazy("formatting help", "Code block with syntax highlighting"),
-        pgettext_lazy("formatting help", '```python\nalert("Hello world!");\n```'),
+        pgettext_lazy("formatting help", '```python\r\nalert("Hello world!");\r\n```'),
     ),
     (
         pgettext_lazy("formatting help", "Code block (BBCode)"),
-        pgettext_lazy("formatting help", '[code]\nalert("Hello world!");\n[/code]'),
+        pgettext_lazy("formatting help", '[code]\r\nalert("Hello world!");\r\n[/code]'),
     ),
     (
         pgettext_lazy(
             "formatting help", "Code block with syntax highlighting (BBCode)"
         ),
         pgettext_lazy(
-            "formatting help", '[code=python]\nprint("Hello world!");\n[/code]'
+            "formatting help", '[code=python]\r\nprint("Hello world!");\r\n[/code]'
         ),
     ),
     (
         pgettext_lazy("formatting help", "Horizontal rule"),
-        pgettext_lazy("formatting help", "Lorem ipsum\n- - -\nDolor met"),
+        pgettext_lazy("formatting help", "Lorem ipsum\r\n- - -\r\nDolor met"),
     ),
     (
         pgettext_lazy("formatting help", "Horizontal rule (BBCode)"),
-        pgettext_lazy("formatting help", "Lorem ipsum\n[hr]\nDolor met"),
+        pgettext_lazy("formatting help", "Lorem ipsum\r\n[hr]\r\nDolor met"),
     ),
 ]


Test patch:
diff --git a/misago/parser/tests/test_parser.py b/misago/parser/tests/test_parser.py
index 85f1d099d..dba1ab6d2 100644
--- a/misago/parser/tests/test_parser.py
+++ b/misago/parser/tests/test_parser.py
@@ -117,3 +117,67 @@ def test_parser_parses_line_breaks_surrounded_by_spaces():
             ],
         },
     ]
+
+
+def test_parser_parses_crlf_line_break():
+    result = parse("Paragraph with\r\na line break.")
+    assert result == [
+        {
+            "type": "paragraph",
+            "children": [
+                {"type": "text", "text": "Paragraph with"},
+                {"type": "line-break"},
+                {"type": "text", "text": "a line break."},
+            ],
+        },
+    ]
+
+
+def test_parser_parses_crlf_line_breaks_surrounded_by_spaces():
+    result = parse("Paragraph with  \r\n   a line break.")
+    assert result == [
+        {
+            "type": "paragraph",
+            "children": [
+                {"type": "text", "text": "Paragraph with"},
+                {"type": "line-break"},
+                {"type": "text", "text": "a line break."},
+            ],
+        },
+    ]
+
+
+def test_parser_parses_crlf_two_paragraphs():
+    result = parse("First paragraph.\r\n\r\nSecond paragraph.")
+    assert result == [
+        {
+            "type": "paragraph",
+            "children": [
+                {"type": "text", "text": "First paragraph."},
+            ],
+        },
+        {
+            "type": "paragraph",
+            "children": [
+                {"type": "text", "text": "Second paragraph."},
+            ],
+        },
+    ]
+
+
+def test_parser_parses_crlf_skips_extra_linebreaks_between_blocks():
+    result = parse("First paragraph.\r\n\r\n\r\n\r\nSecond paragraph.")
+    assert result == [
+        {
+            "type": "paragraph",
+            "children": [
+                {"type": "text", "text": "First paragraph."},
+            ],
+        },
+        {
+            "type": "paragraph",
+            "children": [
+                {"type": "text", "text": "Second paragraph."},
+            ],
+        },
+    ]