Repository: locustio/locust

Issue:
Locust exits with 1 on timeouts / errors
Depending on how others are deploying this tool from the command line it may be a non-issue, but it's causing problems for us. Returning non-zero with the same code that we use when there are misconfigurations for, say, one 500 response code out of thousands is not subtle enough.

By the time we exit in this case, the entire test was already completed and requests were sent. Whatever downstream metrics systems will have valid data, etc. If we retry based on a non-zero exit code, we run the test twice. Users of the tool should probably be doing something more intelligent downstream to determine if the test was a failure or not than if there was at least 1 error. So why exit with an error code?

Two initial ideas that could improve the behavior (either or both would work):
1) Use a return code that's unique to this kind of "failure but not really", so it can be handled and ignored / logged with the correct criticality
2) Add a new command line parameter that is a threshold for failure (percent using stats.fail_ratio and/or absolute compared against num_requests)

Is this bothering anyone else?
([ref](https://github.com/locustio/locust/blob/v0.8a1/locust/main.py#L443))

Comments:

Patch:
diff --git a/locust/main.py b/locust/main.py
index fe3186d8..a42308b4 100644
--- a/locust/main.py
+++ b/locust/main.py
@@ -282,6 +282,16 @@ def parse_options():
         help="show program's version number and exit"
     )
 
+    # set the exit code to post on errors
+    parser.add_option(
+        '--exit-code-on-error',
+        action='store',
+        type="int",
+        dest='exit_code_on_error',
+        default=1,
+        help="sets the exit code to post on error"
+    )
+
     # Finalize
     # Return three-tuple of parser + the output from parse_args (opt obj, args)
     opts, args = parser.parse_args()
@@ -554,7 +564,7 @@ def main():
         main_greenlet.join()
         code = 0
         if len(runners.locust_runner.errors):
-            code = 1
+            code = options.exit_code_on_error
         shutdown(code=code)
     except KeyboardInterrupt as e:
         shutdown(0)
diff --git a/locust/stats.py b/locust/stats.py
index 4ba20644..7ec6a9b7 100644
--- a/locust/stats.py
+++ b/locust/stats.py
@@ -121,6 +121,7 @@ class RequestStats(object):
         """
         self.start_time = time.time()
         self.total.reset()
+        self.errors = {}
         for r in six.itervalues(self.entries):
             r.reset()
     
@@ -276,7 +277,7 @@ class StatsEntry(object):
     @property
     def fail_ratio(self):
         try:
-            return float(self.num_failures) / (self.num_requests + self.num_failures)
+            return float(self.num_failures) / self.num_requests
         except ZeroDivisionError:
             if self.num_failures > 0:
                 return 1.0
@@ -734,3 +735,25 @@ def distribution_csv():
             rows.append('"%s",0,"N/A","N/A","N/A","N/A","N/A","N/A","N/A","N/A","N/A"' % s.name)
 
     return "\n".join(rows)
+
+def failures_csv():
+    """"Return the contents of the 'failures' tab as a CSV."""
+    from . import runners
+
+    rows = [
+        ",".join((
+            '"Method"',
+            '"Name"',
+            '"Error"',
+            '"Occurences"',
+        ))
+    ]
+
+    for s in sort_stats(runners.locust_runner.stats.errors):
+        rows.append('"%s","%s","%s",%i' % (
+            s.method,
+            s.name,
+            s.error,
+            s.occurences,
+        ))
+    return "\n".join(rows)
\ No newline at end of file
diff --git a/locust/templates/index.html b/locust/templates/index.html
index ce4e6b72..9623eec7 100644
--- a/locust/templates/index.html
+++ b/locust/templates/index.html
@@ -148,6 +148,7 @@
                     <div style="margin-top:20px;">
                         <a href="./stats/requests/csv">Download request statistics CSV</a><br>
                         <a href="./stats/distribution/csv">Download response time distribution CSV</a><br>
+                        <a href="./stats/failures/csv">Download failures CSV</a><br>
                         <a href="./exceptions/csv">Download exceptions CSV</a>
                     </div>
                 </div>
diff --git a/locust/web.py b/locust/web.py
index aa31599e..97c87f1a 100644
--- a/locust/web.py
+++ b/locust/web.py
@@ -17,7 +17,7 @@ from six.moves import StringIO, xrange
 
 from . import runners
 from .runners import MasterLocustRunner
-from .stats import distribution_csv, median_from_dict, requests_csv, sort_stats
+from .stats import distribution_csv, failures_csv, median_from_dict, requests_csv, sort_stats
 from .util.cache import memoize
 
 logger = logging.getLogger(__name__)
@@ -69,6 +69,7 @@ def stop():
 @app.route("/stats/reset")
 def reset_stats():
     runners.locust_runner.stats.reset_all()
+    runners.locust_runner.exceptions = {}
     return "ok"
     
 @app.route("/stats/requests/csv")
@@ -89,6 +90,15 @@ def distribution_stats_csv():
     response.headers["Content-disposition"] = disposition
     return response
 
+@app.route("/stats/failures/csv")
+def failures_stats_csv():
+    response = make_response(failures_csv())
+    file_name = "failures_{0}.csv".format(time())
+    disposition = "attachment;filename={0}".format(file_name)
+    response.headers["Content-type"] = "text/csv"
+    response.headers["Content-disposition"] = disposition
+    return response
+
 @app.route('/stats/requests')
 @memoize(timeout=DEFAULT_CACHE_TIME, dynamic_timeout=True)
 def request_stats():


Test patch:
diff --git a/locust/test/test_stats.py b/locust/test/test_stats.py
index 4b45aeed..fcfee1cc 100644
--- a/locust/test/test_stats.py
+++ b/locust/test/test_stats.py
@@ -252,6 +252,18 @@ class TestStatsEntryResponseTimesCache(unittest.TestCase):
         ))
 
 
+class TestStatsEntry(unittest.TestCase):
+    def setUp(self, *args, **kwargs):
+        super(TestStatsEntry, self).setUp(*args, **kwargs)
+        self.stats = RequestStats()
+
+    def test_fail_ratio_with_failures(self):
+        s = StatsEntry(self.stats, "/", "GET")
+        s.num_requests = 10
+        s.num_failures = 5
+        self.assertAlmostEqual(s.fail_ratio, 0.5)
+
+
 class TestRequestStatsWithWebserver(WebserverTestCase):
     def test_request_stats_content_length(self):
         class MyLocust(HttpLocust):
diff --git a/locust/test/test_web.py b/locust/test/test_web.py
index ed3f07eb..14a02d7c 100644
--- a/locust/test/test_web.py
+++ b/locust/test/test_web.py
@@ -95,12 +95,40 @@ class TestWebUI(LocustTestCase):
         # verify that the 95%, 98%, 99% and 100% percentiles are 1200
         for value in total_cols[-4:]:
             self.assertEqual('1200', value)
+
+    def test_failure_stats_csv(self):
+        stats.global_stats.log_error("GET", "/", Exception("Error1337"))
+        response = requests.get("http://127.0.0.1:%i/stats/failures/csv" % self.web_port)
+        self.assertEqual(200, response.status_code)
     
     def test_request_stats_with_errors(self):
         stats.global_stats.log_error("GET", "/", Exception("Error1337"))
         response = requests.get("http://127.0.0.1:%i/stats/requests" % self.web_port)
         self.assertEqual(200, response.status_code)
         self.assertIn("Error1337", response.text)
+
+    def test_reset_stats(self):
+        try:
+            raise Exception(u"A cool test exception")
+        except Exception as e:
+            tb = sys.exc_info()[2]
+            runners.locust_runner.log_exception("local", str(e), "".join(traceback.format_tb(tb)))
+            runners.locust_runner.log_exception("local", str(e), "".join(traceback.format_tb(tb)))
+
+        stats.global_stats.log_request("GET", "/test", 120, 5612)
+        stats.global_stats.log_error("GET", "/", Exception("Error1337"))
+
+        response = requests.get("http://127.0.0.1:%i/stats/reset" % self.web_port)
+
+        self.assertEqual(200, response.status_code)
+
+        self.assertEqual({}, stats.global_stats.errors)
+        self.assertEqual({}, runners.locust_runner.exceptions)
+        
+        self.assertEqual(0, stats.global_stats.get("/", "GET").num_requests)
+        self.assertEqual(0, stats.global_stats.get("/", "GET").num_failures)
+        self.assertEqual(0, stats.global_stats.get("/test", "GET").num_requests)
+        self.assertEqual(0, stats.global_stats.get("/test", "GET").num_failures)
     
     def test_exceptions(self):
         try: