Repository: beeware/briefcase

Issue:
Test suite doesn't run on Windows 11
### Describe the bug

Many tests that use the `capsys` fixture fail on Windows 11. It seems like it's because `sys.__stdout__.isatty()` evaluates to `True` when we run the tests with pytest, since the following (horrible) workaround makes everything pass:

```
import sys

@pytest.fixture(autouse=True)
def monkeypatch_stdout(monkeypatch):
    def false(*args, **kwargs): return False
    monkeypatch.setattr(sys.__stdout__, "isatty", false)
```

### Steps to reproduce

1. Use a Windows 11 machine
2. Clone Briefcase and install it in a clean virtual environment
3. Run pytest
4. See error


### Expected behavior

All tests should pass

### Screenshots

_No response_

### Environment

- Operating System: Windows 11
- Python version: 3.12.2
- Software versions:
  - Briefcase: '0.3.19.dev26+g96734be6.d20240520'


### Logs

```
[...]

_______________________________________________________________________ test_variant_with_size ________________________________________________________________________

create_command = <tests.commands.create.conftest.DummyCreateCommand object at 0x000001EBA9515C10>
tmp_path = WindowsPath('C:/Users/yngve/AppData/Local/Temp/pytest-of-yngve/pytest-9/test_variant_with_size0')
capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA9517950>

    def test_variant_with_size(create_command, tmp_path, capsys):
        """If the app specifies a variant with a size, the sized variant is used."""
        create_command.tools.shutil = mock.MagicMock(spec_set=shutil)
    
        # Create the source image
        source_file = tmp_path / "base_path/input/original-3742.png"
        source_file.parent.mkdir(parents=True, exist_ok=True)
        with source_file.open("w", encoding="utf-8") as f:
            f.write("image")
    
        # Try to install the image
        out_path = tmp_path / "output.png"
        create_command.install_image(
            "sample image",
            source={
                "round": "input/original",
            },
            variant="round",
            size="3742",
            target=out_path,
        )

        # The right message was written to output
        expected = (
            "Installing input/original-3742.png as 3742px round sample image... started\n"
            "Installing input/original-3742.png as 3742px round sample image... done\n\n"
        )
>       assert capsys.readouterr().out == expected
E       AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'
E
E         Skipping 57 identical leading characters in diff, use -v to show
E         +  image... done
E         -  image... started
E         - Installing input/original-3742.png as 3742px round sample image... done

tests\commands\create\test_install_image.py:280: AssertionError
________________________________________________________________________ test_unsized_variant _________________________________________________________________________ 

create_command = <tests.commands.create.conftest.DummyCreateCommand object at 0x000001EBA92A7CE0>
tmp_path = WindowsPath('C:/Users/yngve/AppData/Local/Temp/pytest-of-yngve/pytest-9/test_unsized_variant0')
capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA92A7950>

    def test_unsized_variant(create_command, tmp_path, capsys):
        """If the app specifies an unsized variant, it is used."""
        create_command.tools.shutil = mock.MagicMock(spec_set=shutil)

        # Create the source image
        source_file = tmp_path / "base_path/input/original.png"
        source_file.parent.mkdir(parents=True, exist_ok=True)
        with source_file.open("w", encoding="utf-8") as f:
            f.write("image")

        # Try to install the image
        # Unsized variants are an annoying edge case; they get the *variant*
        # as the *size*.
        out_path = tmp_path / "output.png"
        create_command.install_image(
            "sample image",
            source={
                "round": "input/original",
            },
            variant=None,
            size="round",
            target=out_path,
        )

        # The right message was written to output
        expected = (
            "Installing input/original.png as round sample image... started\n"
            "Installing input/original.png as round sample image... done\n\n"
        )
>       assert capsys.readouterr().out == expected
E       AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'
E
E         Skipping 45 identical leading characters in diff, use -v to show
E         +  image... done
E         -  image... started
E         - Installing input/original.png as round sample image... done

tests\commands\create\test_install_image.py:378: AssertionError
_________________________________________________________________________ test_sign_app[True] _________________________________________________________________________

dummy_command = <tests.platforms.macOS.app.test_signing.DummySigningCommand object at 0x000001EBA9BF2C00>
first_app_with_binaries = <com.example.first-app v0.0.1 AppConfig>, verbose = True
tmp_path = WindowsPath('C:/Users/yngve/AppData/Local/Temp/pytest-of-yngve/pytest-9/test_sign_app_True_0')
capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA9C96C30>

    @pytest.mark.parametrize("verbose", [True, False])
    def test_sign_app(dummy_command, first_app_with_binaries, verbose, tmp_path, capsys):
        """An app bundle can be signed."""
        if verbose:
            dummy_command.logger.verbosity = LogLevel.VERBOSE

        # Sign the app
        dummy_command.sign_app(
            first_app_with_binaries, identity="Sekrit identity (DEADBEEF)"
        )

        # A request has been made to sign all the so and dylib files
        # This acts as a test of the discovery process:
        # * It discovers frameworks
        # * It discovers apps
        # * It discovers Mach-O binaries in various forms and guises
        # * It *doesn't* discover directories
        # * It *doesn't* discover non-Mach-O binaries
        # * It traverses in "depth first" order
        app_path = (
            tmp_path
            / "base_path"
            / "build"
            / "first-app"
            / "macos"
            / "app"
            / "First App.app"
        )
        lib_path = app_path / "Contents/Resources/app_packages"
        frameworks_path = app_path / "Contents/Frameworks"
        dummy_command.tools.subprocess.run.assert_has_calls(
            [
                sign_call(tmp_path, lib_path / "subfolder/second_so.so"),
                sign_call(tmp_path, lib_path / "subfolder/second_dylib.dylib"),
                sign_call(tmp_path, lib_path / "special.binary"),
                sign_call(tmp_path, lib_path / "other_binary"),
                sign_call(tmp_path, lib_path / "first_so.so"),
                sign_call(tmp_path, lib_path / "first_dylib.dylib"),
                sign_call(tmp_path, lib_path / "Extras.app/Contents/MacOS/Extras"),
                sign_call(tmp_path, lib_path / "Extras.app"),
                sign_call(
                    tmp_path,
                    frameworks_path / "Extras.framework/Resources/extras.dylib",
                ),
                sign_call(tmp_path, frameworks_path / "Extras.framework"),
                sign_call(tmp_path, app_path),
            ],
            any_order=True,
        )

        # Also check that files are not signed after their parent directory has been
        # signed. Reduce the files mentions in the calls to the dummy command
        # to a list of path objects, then ensure that the call to sign any given file
        # does not occur *after* it's parent directory.
        sign_targets = [
            Path(call.args[0][1]) for call in dummy_command.tools.subprocess.run.mock_calls
        ]

        parents = set()
        for path in sign_targets:
            # Check parent of path is not in parents
            assert path.parent not in parents
            parents.add(path)

        # Output only happens if in debug mode.
        output = capsys.readouterr().out
        if sys.platform == "win32":
            # In practice, we won't ever actually run signing on win32; but to ensure test
            # coverage we need to. However, win32 doesn't handle executable permissions
            # the same as linux/unix, `unknown.binary` is identified as a signing target.
            # We ignore this discrepancy for testing purposes.
>           assert len(output.strip("\n").split("\n")) == (12 if verbose else 1)
E           AssertionError: assert 13 == 12
E            +  where 13 = len(['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...])
E            +    where ['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...] = <built-in method split of str object at 0x000001EBA79D8970>('\n')    
E            +      where <built-in method split of str object at 0x000001EBA79D8970> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...ng build\\first-app\\macos\\app\\First App.app\n     -------------------------------------------------- 100.0% • 00:00'.split  
E            +        where 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...ng build\\first-app\\macos\\app\\First App.app\n     -------------------------------------------------- 100.0% • 00:00' = <built-in method strip of str object at 0x000001EBA79DCFE0>('\n')
E            +          where <built-in method strip of str object at 0x000001EBA79DCFE0> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build... build\\first-app\\macos\\app\\First App.app\n     -------------------------------------------------- 100.0% • 00:00\n'.strip

tests\platforms\macOS\app\test_signing.py:555: AssertionError
__________________________________________________________________ test_sign_app_with_failure[True] ___________________________________________________________________

dummy_command = <tests.platforms.macOS.app.test_signing.DummySigningCommand object at 0x000001EBA92BA0C0>
first_app_with_binaries = <com.example.first-app v0.0.1 AppConfig>, verbose = True, capsys = <_pytest.capture.CaptureFixture object at 0x000001EBA95409E0>

    @pytest.mark.parametrize("verbose", [True, False])
    def test_sign_app_with_failure(dummy_command, first_app_with_binaries, verbose, capsys):
        """If signing a single file in the app fails, the error is surfaced."""
        if verbose:
            dummy_command.logger.verbosity = LogLevel.VERBOSE

        # Sign the app. Signing first_dylib.dylib will fail.
        def _codesign(args, **kwargs):
            if Path(args[1]).name == "first_dylib.dylib":
                raise subprocess.CalledProcessError(
                    returncode=1, cmd=args, stderr=f"{args[1]}: Unknown error"
                )

        dummy_command.tools.subprocess.run.side_effect = _codesign

        # The invocation will raise an error; however, we can't predict exactly which
        # file will raise an error.
        with pytest.raises(
            BriefcaseCommandError, match=r"Unable to code sign .*first_dylib\.dylib"
        ):
            dummy_command.sign_app(
                first_app_with_binaries, identity="Sekrit identity (DEADBEEF)"
            )

        # There has been at least 1 call to sign files. We can't know how many are
        # actually signed, as threads are involved.
        dummy_command.tools.subprocess.run.call_count > 0

        # Output only happens if in debug mode.
        output = capsys.readouterr().out
        if sys.platform == "win32":
            # In practice, we won't ever actually run signing on win32; but to ensure test
            # coverage we need to. However, win32 doesn't handle executable permissions
            # the same as linux/unix, `unknown.binary` is identified as a signing target.
            # We ignore this discrepancy for testing purposes.
>           assert len(output.strip("\n").split("\n")) == (7 if verbose else 1)
E           AssertionError: assert 8 == 7
E            +  where 8 = len(['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...])
E            +    where ['Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary', 'Signing bu...er_binary', 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\first_so.so', ...] = <built-in method split of str object at 0x000001EBA62FF0E0>('\n')    
E            +      where <built-in method split of str object at 0x000001EBA62FF0E0> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...nts\\Resources\\app_packages\\first_dylib.dylib\n   - -----------------------------                      58.3% • 00:01'.split  
E            +        where 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...nts\\Resources\\app_packages\\first_dylib.dylib\n   - -----------------------------                      58.3% • 00:01' = <built-in method strip of str object at 0x000001EBA62FDDF0>('\n')
E            +          where <built-in method strip of str object at 0x000001EBA62FDDF0> = 'Signing build\\first-app\\macos\\app\\First App.app\\Contents\\Resources\\app_packages\\unknown.binary\nSigning build...s\\Resources\\app_packages\\first_dylib.dylib\n   - -----------------------------                      58.3% • 00:01\n'.strip

tests\platforms\macOS\app\test_signing.py:595: AssertionError
======================================================================= short test summary info =======================================================================
FAILED tests/commands/create/test_cleanup_app_content.py::test_no_cleanup[True] - AssertionError: assert 4 == 5
FAILED tests/commands/create/test_cleanup_app_content.py::test_no_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_cleanup[True] - AssertionError: assert 4 == 5
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_cleanup[True] - AssertionError: assert 5 == 6
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_all_files_in_dir_cleanup[True] - AssertionError: assert 7 == 8
FAILED tests/commands/create/test_cleanup_app_content.py::test_all_files_in_dir_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_glob_cleanup[True] - AssertionError: assert 5 == 6
FAILED tests/commands/create/test_cleanup_app_content.py::test_dir_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_glob_cleanup[True] - AssertionError: assert 6 == 7
FAILED tests/commands/create/test_cleanup_app_content.py::test_file_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_deep_glob_cleanup[True] - AssertionError: assert 7 == 8
FAILED tests/commands/create/test_cleanup_app_content.py::test_deep_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_template_glob_cleanup[True] - AssertionError: assert 7 == 8
FAILED tests/commands/create/test_cleanup_app_content.py::test_template_glob_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_cleanup_app_content.py::test_non_existent_cleanup[True] - AssertionError: assert 5 == 6
FAILED tests/commands/create/test_cleanup_app_content.py::test_non_existent_cleanup[False] - AssertionError: assert 3 == 4
FAILED tests/commands/create/test_install_image.py::test_no_requested_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'    
FAILED tests/commands/create/test_install_image.py::test_requested_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'       
FAILED tests/commands/create/test_install_image.py::test_variant_with_no_requested_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'
FAILED tests/commands/create/test_install_image.py::test_variant_with_size - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'    
FAILED tests/commands/create/test_install_image.py::test_unsized_variant - AssertionError: assert 'Installing i...e... done\n\n' == 'Installing i...e... done\n\n'      
FAILED tests/platforms/macOS/app/test_signing.py::test_sign_app[True] - AssertionError: assert 13 == 12
FAILED tests/platforms/macOS/app/test_signing.py::test_sign_app_with_failure[True] - AssertionError: assert 8 == 7
======================================================= 25 failed, 2828 passed, 91 skipped in 131.64s (0:02:11) =======================================================
```

### Additional context

_No response_


Discussion:
Looking at this at PyCon US 2024



Patch:
diff --git a/changes/1823.misc.rst b/changes/1823.misc.rst
new file mode 100644
index 000000000..8708d462b
--- /dev/null
+++ b/changes/1823.misc.rst
@@ -0,0 +1,1 @@
+Fixed failing tests on a Windows 11 development environment.
diff --git a/src/briefcase/console.py b/src/briefcase/console.py
index 4a2081f1f..06f1563a6 100644
--- a/src/briefcase/console.py
+++ b/src/briefcase/console.py
@@ -536,7 +536,7 @@ def is_interactive(self):
         should be specifically disabled in non-interactive sessions.
         """
         # `sys.__stdout__` is used because Rich captures and redirects `sys.stdout`
-        return sys.__stdout__.isatty()
+        return os.isatty(sys.__stdout__.fileno())
 
     @property
     def is_color_enabled(self):


Test patch:
diff --git a/tests/console/conftest.py b/tests/console/conftest.py
index c1ec12941..52bd89683 100644
--- a/tests/console/conftest.py
+++ b/tests/console/conftest.py
@@ -1,4 +1,4 @@
-import sys
+import os
 from unittest import mock
 
 import pytest
@@ -11,7 +11,7 @@ def console(monkeypatch) -> Console:
     console = Console()
     console.input = mock.MagicMock(spec_set=input)
     # default console is always interactive
-    monkeypatch.setattr(sys.__stdout__, "isatty", lambda: True)
+    monkeypatch.setattr(os, "isatty", lambda _: True)
     return console
 
 
@@ -24,5 +24,5 @@ def disabled_console() -> Console:
 
 @pytest.fixture
 def non_interactive_console(console, monkeypatch) -> Console:
-    monkeypatch.setattr(sys.__stdout__, "isatty", lambda: False)
+    monkeypatch.setattr(os, "isatty", lambda _: False)
     yield console