Repository: getmoto/moto

Issue:
Travis CI jobs failing intermittently
`test_batch.test_submit_job` fails intermittently across all Python versions and regardless of whether `TEST_SERVER_MODE=false` or `TEST_SERVER_MODE=true`.

<details>
<summary>Nose Failure</summary>
<p>

```
======================================================================
FAIL: test_batch.test_submit_job
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.6/lib/python3.7/site-packages/nose/case.py", line 198, in runTest
    self.test(*self.arg)
  File "/home/travis/build/spulec/moto/moto/core/models.py", line 99, in wrapper
    result = func(*args, **kwargs)
  File "/home/travis/build/spulec/moto/moto/core/models.py", line 99, in wrapper
    result = func(*args, **kwargs)
  File "/home/travis/build/spulec/moto/moto/core/models.py", line 99, in wrapper
    result = func(*args, **kwargs)
  [Previous line repeated 2 more times]
  File "/home/travis/build/spulec/moto/tests/test_batch/test_batch.py", line 750, in test_submit_job
    [event["message"] for event in resp["events"]].should.equal(["hello"])
  File "/home/travis/virtualenv/python3.7.6/lib/python3.7/site-packages/sure/__init__.py", line 387, in wrapper
    raise AssertionError(e)
AssertionError: given
X = []
    and
Y = ['hello']
Y has 1 items whereas X has only 0
>>  raise AssertionError(e)
```

</p>
</details>  

The test is still failing occasionally after the recent switch from nose to pytest.

<details>
<summary>Pytest Failure</summary>
<p>

```
=================================== FAILURES ===================================
_______________________________ test_submit_job ________________________________

self = <sure.AssertionBuilder object at 0x7fd94262df60>, args = (['hello'],)
kw = {}

    @wraps(func)
    def wrapper(self, *args, **kw):
        try:
>           value = func(self, *args, **kw)

../../../virtualenv/python3.6.10/lib/python3.6/site-packages/sure/__init__.py:385: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <sure.AssertionBuilder object at 0x7fd94262df60>, what = ['hello']
epsilon = None

    @assertionmethod
    def equal(self, what, epsilon=None):
        """compares given object ``X``  with an expected ``Y`` object.
    
        It primarily assures that the compared objects are absolute equal ``==``.
    
        :param what: the expected value
        :param epsilon: a delta to leverage upper-bound floating point permissiveness
        """
    
        try:
            comparison = DeepComparison(self.obj, what, epsilon).compare()
            error = False
        except AssertionError as e:
            error = e
            comparison = None
    
        if isinstance(comparison, DeepExplanation):
            error = comparison.get_assertion(self.obj, what)
    
        if self.negative:
            if error:
                return True
    
            msg = '%s should differ from %s, but is the same thing'
            raise AssertionError(msg % (safe_repr(self.obj), safe_repr(what)))
    
        else:
            if not error:
                return True
>           raise error
E           AssertionError: given
E           X = []
E               and
E           Y = ['hello']
E           Y has 1 items whereas X has only 0

../../../virtualenv/python3.6.10/lib/python3.6/site-packages/sure/__init__.py:673: AssertionError
```

</p>
</details>

This "flipper test" was mentioned and acknowledged as a problem in #3161, but I wasn't able to find an existing issue tracking it.  I will investigate.


Discussion:
Hey @bpandola, just as an FYI: this was my attempt at fixing it: https://github.com/spulec/moto/pull/3146

That obviously wasn't the solution, hopefully you have better luck/investigator skills ;) 

Resolving patch:
diff --git a/moto/batch/models.py b/moto/batch/models.py
--- a/moto/batch/models.py
+++ b/moto/batch/models.py
@@ -392,7 +392,6 @@ def run(self):
         """
         try:
             self.job_state = "PENDING"
-            time.sleep(1)
 
             image = self.job_definition.container_properties.get(
                 "image", "alpine:latest"
@@ -425,8 +424,8 @@ def run(self):
 
             self.job_state = "RUNNABLE"
             # TODO setup ecs container instance
-            time.sleep(1)
 
+            self.job_started_at = datetime.datetime.now()
             self.job_state = "STARTING"
             log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON)
             container = self.docker_client.containers.run(
@@ -440,58 +439,24 @@ def run(self):
                 privileged=privileged,
             )
             self.job_state = "RUNNING"
-            self.job_started_at = datetime.datetime.now()
             try:
-                # Log collection
-                logs_stdout = []
-                logs_stderr = []
                 container.reload()
-
-                # Dodgy hack, we can only check docker logs once a second, but we want to loop more
-                # so we can stop if asked to in a quick manner, should all go away if we go async
-                # There also be some dodgyness when sending an integer to docker logs and some
-                # events seem to be duplicated.
-                now = datetime.datetime.now()
-                i = 1
                 while container.status == "running" and not self.stop:
-                    time.sleep(0.2)
-                    if i % 5 == 0:
-                        logs_stderr.extend(
-                            container.logs(
-                                stdout=False,
-                                stderr=True,
-                                timestamps=True,
-                                since=datetime2int(now),
-                            )
-                            .decode()
-                            .split("\n")
-                        )
-                        logs_stdout.extend(
-                            container.logs(
-                                stdout=True,
-                                stderr=False,
-                                timestamps=True,
-                                since=datetime2int(now),
-                            )
-                            .decode()
-                            .split("\n")
-                        )
-                        now = datetime.datetime.now()
-                        container.reload()
-                    i += 1
+                    container.reload()
 
                 # Container should be stopped by this point... unless asked to stop
                 if container.status == "running":
                     container.kill()
 
-                self.job_stopped_at = datetime.datetime.now()
-                # Get final logs
+                # Log collection
+                logs_stdout = []
+                logs_stderr = []
                 logs_stderr.extend(
                     container.logs(
                         stdout=False,
                         stderr=True,
                         timestamps=True,
-                        since=datetime2int(now),
+                        since=datetime2int(self.job_started_at),
                     )
                     .decode()
                     .split("\n")
@@ -501,14 +466,12 @@ def run(self):
                         stdout=True,
                         stderr=False,
                         timestamps=True,
-                        since=datetime2int(now),
+                        since=datetime2int(self.job_started_at),
                     )
                     .decode()
                     .split("\n")
                 )
 
-                self.job_state = "SUCCEEDED" if not self.stop else "FAILED"
-
                 # Process logs
                 logs_stdout = [x for x in logs_stdout if len(x) > 0]
                 logs_stderr = [x for x in logs_stderr if len(x) > 0]
@@ -532,6 +495,8 @@ def run(self):
                 self._log_backend.create_log_stream(log_group, stream_name)
                 self._log_backend.put_log_events(log_group, stream_name, logs, None)
 
+                self.job_state = "SUCCEEDED" if not self.stop else "FAILED"
+
             except Exception as err:
                 logger.error(
                     "Failed to run AWS Batch container {0}. Error {1}".format(


Test patch:
diff --git a/tests/test_batch/test_batch.py b/tests/test_batch/test_batch.py
--- a/tests/test_batch/test_batch.py
+++ b/tests/test_batch/test_batch.py
@@ -725,18 +725,7 @@ def test_submit_job():
     )
     job_id = resp["jobId"]
 
-    future = datetime.datetime.now() + datetime.timedelta(seconds=30)
-
-    while datetime.datetime.now() < future:
-        time.sleep(1)
-        resp = batch_client.describe_jobs(jobs=[job_id])
-
-        if resp["jobs"][0]["status"] == "FAILED":
-            raise RuntimeError("Batch job failed")
-        if resp["jobs"][0]["status"] == "SUCCEEDED":
-            break
-    else:
-        raise RuntimeError("Batch job timed out")
+    _wait_for_job_status(batch_client, job_id, "SUCCEEDED")
 
     resp = logs_client.describe_log_streams(
         logGroupName="/aws/batch/job", logStreamNamePrefix="sayhellotomylittlefriend"
@@ -798,26 +787,13 @@ def test_list_jobs():
     )
     job_id2 = resp["jobId"]
 
-    future = datetime.datetime.now() + datetime.timedelta(seconds=30)
-
     resp_finished_jobs = batch_client.list_jobs(
         jobQueue=queue_arn, jobStatus="SUCCEEDED"
     )
 
     # Wait only as long as it takes to run the jobs
-    while datetime.datetime.now() < future:
-        resp = batch_client.describe_jobs(jobs=[job_id1, job_id2])
-
-        any_failed_jobs = any([job["status"] == "FAILED" for job in resp["jobs"]])
-        succeeded_jobs = all([job["status"] == "SUCCEEDED" for job in resp["jobs"]])
-
-        if any_failed_jobs:
-            raise RuntimeError("A Batch job failed")
-        if succeeded_jobs:
-            break
-        time.sleep(0.5)
-    else:
-        raise RuntimeError("Batch jobs timed out")
+    for job_id in [job_id1, job_id2]:
+        _wait_for_job_status(batch_client, job_id, "SUCCEEDED")
 
     resp_finished_jobs2 = batch_client.list_jobs(
         jobQueue=queue_arn, jobStatus="SUCCEEDED"
@@ -854,13 +830,13 @@ def test_terminate_job():
     queue_arn = resp["jobQueueArn"]
 
     resp = batch_client.register_job_definition(
-        jobDefinitionName="sleep10",
+        jobDefinitionName="echo-sleep-echo",
         type="container",
         containerProperties={
             "image": "busybox:latest",
             "vcpus": 1,
             "memory": 128,
-            "command": ["sleep", "10"],
+            "command": ["sh", "-c", "echo start && sleep 30 && echo stop"],
         },
     )
     job_def_arn = resp["jobDefinitionArn"]
@@ -870,13 +846,43 @@ def test_terminate_job():
     )
     job_id = resp["jobId"]
 
-    time.sleep(2)
+    _wait_for_job_status(batch_client, job_id, "RUNNING")
 
     batch_client.terminate_job(jobId=job_id, reason="test_terminate")
 
-    time.sleep(2)
+    _wait_for_job_status(batch_client, job_id, "FAILED")
 
     resp = batch_client.describe_jobs(jobs=[job_id])
     resp["jobs"][0]["jobName"].should.equal("test1")
     resp["jobs"][0]["status"].should.equal("FAILED")
     resp["jobs"][0]["statusReason"].should.equal("test_terminate")
+
+    resp = logs_client.describe_log_streams(
+        logGroupName="/aws/batch/job", logStreamNamePrefix="echo-sleep-echo"
+    )
+    len(resp["logStreams"]).should.equal(1)
+    ls_name = resp["logStreams"][0]["logStreamName"]
+
+    resp = logs_client.get_log_events(
+        logGroupName="/aws/batch/job", logStreamName=ls_name
+    )
+    # Events should only contain 'start' because we interrupted
+    # the job before 'stop' was written to the logs.
+    resp["events"].should.have.length_of(1)
+    resp["events"][0]["message"].should.equal("start")
+
+
+def _wait_for_job_status(client, job_id, status, seconds_to_wait=30):
+    wait_time = datetime.datetime.now() + datetime.timedelta(seconds=seconds_to_wait)
+    last_job_status = None
+    while datetime.datetime.now() < wait_time:
+        resp = client.describe_jobs(jobs=[job_id])
+        last_job_status = resp["jobs"][0]["status"]
+        if last_job_status == status:
+            break
+    else:
+        raise RuntimeError(
+            "Time out waiting for job status {status}!\n Last status: {last_status}".format(
+                status=status, last_status=last_job_status
+            )
+        )