Repository: akaihola/darker

Issue:
`--stdout` doubles newlines
With `"black-formatter.args": ["--stdout", "--isort"]` or `"black-formatter.args": ["-d"]`, it almost works.
However the file in vscode gains double line breaks when using SHIFT+ALT+F to format the file, and with format on save.
This is caused by https://github.com/akaihola/darkgraylib/blob/875dbaf613556f27a4e7949b45fcd798d4ab91f1/src/darkgraylib/utils.py#L17-L22 and commenting out that function to always return `"\n"` makes it work.

What seems to be happening is that you read a file with \r\n line ending and output a file with \r\r\n line endings.

Debug wrapper script:
```py
import subprocess
import sys

print(sys.argv, file=sys.stderr)
cp = subprocess.run(["bin/darker.exe"] + sys.argv[1:], stdout=subprocess.PIPE)

print(cp.stdout[:400], file=sys.stderr)
sys.stdout.buffer.write(cp.stdout)
sys.exit(cp.returncode)
```

Sample file to format
```py
import sys
import collections
```

settings.json
```json
{
    "[python]": {
        "editor.defaultFormatter": "ms-python.black-formatter",
        "editor.formatOnSave": true
    },
    "python.analysis.typeCheckingMode": "off",
    "black-formatter.path": ["py", "-3.11", "./scripts/darkerer.py"],
    "black-formatter.args": ["--stdout", "--isort"],
    "black-formatter.serverTransport": "stdio"
}
```

The black formatter console in vs code shows this:
```
2024-05-20 16:14:39.727 [info] ['./scripts/darkerer.py', '--stdout', '--isort', '--stdin-filename', 'c:\\XXX\\new-file.py', '-']
b'import collections\r\r\nimport sys\r\r\n'

2024-05-20 16:14:39.729 [info] [Trace - 4:14:39 PM] Received response 'textDocument/formatting - (7)' in 1037ms.
```

Running darker on the file normally (`bin/darker --isort new-file.py`) does not create bad line endings.
The problem can be reproduced without VSCode thusly, stdout gets double line endings:
```
cat new-file.py | bin/darker --stdout --isort --stdin-file new-file.py - | py -3.11 -c "import sys; print(sys.stdin.buffer.read())"
b'import collections\r\n\r\nimport sys\r\n\r\n'
```

_Originally posted by @shane-kearns in https://github.com/akaihola/darker/issues/527#issuecomment-2120707306_

Discussion:

Solution patch:
diff --git a/src/darker/__main__.py b/src/darker/__main__.py
index 4669a510e..edc4a1968 100644
--- a/src/darker/__main__.py
+++ b/src/darker/__main__.py
@@ -32,6 +32,7 @@
 )
 from darker.help import LINTING_GUIDE, get_extra_instruction
 from darker.import_sorting import apply_isort, isort
+from darker.terminal import output
 from darker.utils import debug_dump, glob_any
 from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError
 from darkgraylib.command_line import (
@@ -415,7 +416,7 @@ def print_diff(
             n=5,  # Black shows 5 lines of context, do the same
         )
     )
-    print(colorize(diff, "diff", use_color))
+    output(colorize(diff, "diff", use_color), end="\n")
 
 
 def print_source(new: TextDocument, use_color: bool) -> None:
@@ -428,11 +429,11 @@ def print_source(new: TextDocument, use_color: bool) -> None:
                 PythonLexer,
             ) = _import_pygments()  # type: ignore
         except ImportError:
-            print(new.string, end="")
+            output(new.string)
         else:
-            print(highlight(new.string, PythonLexer(), TerminalFormatter()), end="")
+            output(highlight(new.string, PythonLexer(), TerminalFormatter()))
     else:
-        print(new.string, end="")
+        output(new.string)
 
 
 def _import_pygments():  # type: ignore
diff --git a/src/darker/terminal.py b/src/darker/terminal.py
new file mode 100644
index 000000000..be98a6e74
--- /dev/null
+++ b/src/darker/terminal.py
@@ -0,0 +1,10 @@
+"""Terminal output helpers."""
+
+import sys
+
+
+def output(*args: str, end: str = "") -> None:
+    """Print encoded binary output to terminal, with no newline by default."""
+    sys.stdout.buffer.write(*[arg.encode() for arg in args])
+    if end:
+        sys.stdout.buffer.write(end.encode())
diff --git a/src/darker/utils.py b/src/darker/utils.py
index acff93f36..ce8448086 100644
--- a/src/darker/utils.py
+++ b/src/darker/utils.py
@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import Collection, List
 
+from darker.terminal import output
 from darkgraylib.utils import DiffChunk
 
 logger = logging.getLogger(__name__)
@@ -14,14 +15,14 @@ def debug_dump(black_chunks: List[DiffChunk], edited_linenums: List[int]) -> Non
     if logger.getEffectiveLevel() > logging.DEBUG:
         return
     for offset, old_lines, new_lines in black_chunks:
-        print(80 * "-")
+        output(80 * "-", end="\n")
         for delta, old_line in enumerate(old_lines):
             linenum = offset + delta
             edited = "*" if linenum in edited_linenums else " "
-            print(f"{edited}-{linenum:4} {old_line}")
+            output(f"{edited}-{linenum:4} {old_line}", end="\n")
         for _, new_line in enumerate(new_lines):
-            print(f" +     {new_line}")
-    print(80 * "-")
+            output(f" +     {new_line}", end="\n")
+    output(80 * "-", end="\n")
 
 
 def glob_any(path: Path, patterns: Collection[str]) -> bool:


Test patch:
diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py
index dcedaa2d8..446690ee7 100644
--- a/src/darker/tests/test_main.py
+++ b/src/darker/tests/test_main.py
@@ -7,9 +7,12 @@
 import random
 import re
 import string
+import sys
 from argparse import ArgumentError
 from pathlib import Path
+from subprocess import PIPE, CalledProcessError, run  # nosec
 from textwrap import dedent
+from unittest.mock import patch
 
 import pytest
 
@@ -17,11 +20,12 @@
 import darker.import_sorting
 from darker.git import EditedLinenumsDiffer
 from darker.help import LINTING_GUIDE
+from darker.terminal import output
 from darker.tests.examples import A_PY, A_PY_BLACK, A_PY_BLACK_FLYNT, A_PY_BLACK_ISORT
 from darker.tests.test_fstring import FLYNTED_SOURCE, MODIFIED_SOURCE, ORIGINAL_SOURCE
 from darkgraylib.git import RevisionRange
 from darkgraylib.testtools.highlighting_helpers import BLUE, CYAN, RESET, WHITE, YELLOW
-from darkgraylib.utils import TextDocument, joinlines
+from darkgraylib.utils import WINDOWS, TextDocument, joinlines
 
 pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear")
 
@@ -551,6 +555,64 @@ def test_stdout_path_resolution(git_repo, capsys):
     assert capsys.readouterr().out == 'print("foo")\n'
 
 
+@pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["unix", "windows"])
+def test_stdout_newlines(git_repo, capsysbinary, newline):
+    """When using ``--stdout``, newlines are not duplicated.
+
+    See: https://github.com/akaihola/darker/issues/604
+
+    The `git_repo` fixture is used to ensure that the test doesn't run in the Darker
+    repository clone in CI. It helps avoid the Git error message
+    "fatal: Not a valid object name origin/master" in the NixOS CI tests.
+
+    """
+    if WINDOWS and sys.version_info < (3, 10):
+        # See https://bugs.python.org/issue38671
+        Path("new-file.py").touch()
+    code = f"import collections{newline}import sys{newline}".encode()
+    with patch("sys.stdin.buffer.read", return_value=code):
+
+        result = darker.__main__.main(
+            ["--stdout", "--isort", "--stdin-filename=new-file.py", "-"],
+        )
+
+    assert result == 0
+    assert capsysbinary.readouterr().out == code
+
+
+@pytest.mark.parametrize("newline", ["\n", "\r\n"], ids=["unix", "windows"])
+def test_stdout_newlines_subprocess(git_repo, newline):
+    """When using ``--stdout``, newlines are not duplicated.
+
+    See: https://github.com/akaihola/darker/issues/604
+
+    The `git_repo` fixture is used to ensure that the test doesn't run in the Darker
+    repository clone in CI. It helps avoid the Git error message
+    "fatal: Not a valid object name origin/master" in the NixOS CI tests.
+
+    """
+    if WINDOWS and sys.version_info < (3, 10):
+        # See https://bugs.python.org/issue38671
+        Path("new-file.py").touch()
+    code = f"import collections{newline}import sys{newline}".encode()
+    try:
+
+        darker_subprocess = run(  # nosec
+            ["darker", "--stdout", "--isort", "--stdin-filename=new-file.py", "-"],
+            input=code,
+            stdout=PIPE,
+            check=True,
+        )
+
+    except CalledProcessError as e:
+        if e.stdout:
+            output(e.stdout, end="\n")
+        if e.stderr:
+            output(e.stderr, end="\n")
+        raise
+    assert darker_subprocess.stdout == code
+
+
 def test_long_command_length(git_repo):
     """Large amount of changed files does not break Git invocation even on Windows"""
     # For PR #542 - large character count for changed files