Repository: elastic/apm-agent-python

Issue:
Flaky Test [Initializing / Test / Python-python-3.6-1 / test_send_remote_failover_sync[validating_httpserver0-sending_elasticapm_client0] – tests.client.client_tests]
## Flaky Test

* **Test Name:** `Initializing / Test / Python-python-3.6-1 / test_send_remote_failover_sync[validating_httpserver0-sending_elasticapm_client0] – tests.client.client_tests`
* **Artifact Link:** https://apm-ci.elastic.co/blue/organizations/jenkins/apm-agent-python%2Fapm-agent-python-mbp%2FPR-1632/detail/PR-1632/2/
* **PR:** #1632
* **Commit:** 937b26f18b6eab46a328057362221ba13c7e2a33


### Error details
```
assert not True
 +  where True = <bound method TransportState.did_fail of <elasticapm.transport.base.TransportState object at 0x7f907afa1e48>>()
 +    where <bound method TransportState.did_fail of <elasticapm.transport.base.TransportState object at 0x7f907afa1e48>> = <elasticapm.transport.base.TransportState object at 0x7f907afa1e48>.did_fail
 +      where <elasticapm.transport.base.TransportState object at 0x7f907afa1e48> = <elasticapm.transport.http.Transport object at 0x7f907afa1dd8>.state
 +        where <elasticapm.transport.http.Transport object at 0x7f907afa1dd8> = <elasticapm.base.Client object at 0x7f907af917f0>._transport
```


### Stack Trace
```
should_try = <MagicMock name="should_try" id="140258515296440">
sending_elasticapm_client = <elasticapm.base.Client object at 0x7f907af917f0>
caplog = <_pytest.logging.LogCaptureFixture object at 0x7f907af91d68>

    @pytest.mark.skipif(platform.system() == "Windows", reason="Flaky test on windows")
    @pytest.mark.parametrize(
        "sending_elasticapm_client",
        [{"transport_class": "elasticapm.transport.http.Transport", "async_mode": False}],
        indirect=True,
    )
    @pytest.mark.parametrize("validating_httpserver", [{"app": ContentServer}], indirect=True)
    @mock.patch("elasticapm.transport.base.TransportState.should_try")
    def test_send_remote_failover_sync(should_try, sending_elasticapm_client, caplog):
        sending_elasticapm_client.httpserver.code = 400
        sending_elasticapm_client.httpserver.content = "go away"
        should_try.return_value = True
    
        # test error
        with caplog.at_level("ERROR", "elasticapm.transport"):
            sending_elasticapm_client.capture_message("foo", handled=False)
        sending_elasticapm_client._transport.flush()
        assert sending_elasticapm_client._transport.state.did_fail()
        assert_any_record_contains(caplog.records, "go away")
    
        # test recovery
        sending_elasticapm_client.httpserver.code = 202
        sending_elasticapm_client.capture_message("bar", handled=False)
        sending_elasticapm_client.close()
>       assert not sending_elasticapm_client._transport.state.did_fail()
E       assert not True
E        +  where True = <bound method TransportState.did_fail of <elasticapm.transport.base.TransportState object at 0x7f907afa1e48>>()
E        +    where <bound method TransportState.did_fail of <elasticapm.transport.base.TransportState object at 0x7f907afa1e48>> = <elasticapm.transport.base.TransportState object at 0x7f907afa1e48>.did_fail
E        +      where <elasticapm.transport.base.TransportState object at 0x7f907afa1e48> = <elasticapm.transport.http.Transport object at 0x7f907afa1dd8>.state
E        +        where <elasticapm.transport.http.Transport object at 0x7f907afa1dd8> = <elasticapm.base.Client object at 0x7f907af917f0>._transport

caplog     = <_pytest.logging.LogCaptureFixture object at 0x7f907af91d68>
sending_elasticapm_client = <elasticapm.base.Client object at 0x7f907af917f0>
should_try = <MagicMock name="should_try" id="140258515296440">

tests/client/client_tests.py:257: AssertionError
```




Discussion:

Resolving patch:
diff --git a/elasticapm/transport/base.py b/elasticapm/transport/base.py
--- a/elasticapm/transport/base.py
+++ b/elasticapm/transport/base.py
@@ -180,7 +180,11 @@ def _process_queue(self):
                     # No data on buffer, but due to manual flush we should send
                     # an empty payload with flushed=true query param, but only
                     # to a local APM server (or lambda extension)
-                    self.send("", forced_flush=True)
+                    try:
+                        self.send("", forced_flush=True)
+                        self.handle_transport_success()
+                    except Exception as e:
+                        self.handle_transport_fail(e)
                 self._last_flush = timeit.default_timer()
                 buffer = self._init_buffer()
                 buffer_written = False


Test patch:
diff --git a/tests/client/client_tests.py b/tests/client/client_tests.py
--- a/tests/client/client_tests.py
+++ b/tests/client/client_tests.py
@@ -246,7 +246,7 @@ def test_send_remote_failover_sync(should_try, sending_elasticapm_client, caplog
     # test error
     with caplog.at_level("ERROR", "elasticapm.transport"):
         sending_elasticapm_client.capture_message("foo", handled=False)
-    sending_elasticapm_client._transport.flush()
+    sending_elasticapm_client._transport._flushed.wait(timeout=1)
     assert sending_elasticapm_client._transport.state.did_fail()
     assert_any_record_contains(caplog.records, "go away")
 
diff --git a/tests/transports/test_base.py b/tests/transports/test_base.py
--- a/tests/transports/test_base.py
+++ b/tests/transports/test_base.py
@@ -266,11 +266,12 @@ def test_flushed_arg(sending_elasticapm_client):
     assert sending_elasticapm_client.httpserver.requests[0].args["flushed"] == "true"
 
 
+@pytest.mark.flaky(reruns=3)  # Trying to test automatic flushes is inherently flaky
 @pytest.mark.parametrize("sending_elasticapm_client", [{"api_request_time": "100ms"}], indirect=True)
 def test_flushed_arg_with_wait(sending_elasticapm_client):
     sending_elasticapm_client.begin_transaction("test_type")
     sending_elasticapm_client.end_transaction("test")
-    time.sleep(0.2)
+    time.sleep(0.3)
     sending_elasticapm_client._transport.flush()
 
     assert sending_elasticapm_client.httpserver.requests[1].args["flushed"] == "true"