Repository: hedyorg/hedy

Issue:
Modify E2E script to make it a stress tester
- Pass target environment & concurrent users.
- Use a certain domain that won't trigger transactional emails.
- Collect & print statistics.


Discussion:

Resolving patch:
diff --git a/app.py b/app.py
--- a/app.py
+++ b/app.py
@@ -175,6 +175,12 @@ def redirect_ab (request, session):
     redirect_flag = (hash_user_or_session (user_identifier) % 100) < redirect_proportion
     return redirect_flag
 
+if os.getenv('IS_PRODUCTION'):
+    @app.before_request
+    def reject_e2e_requests():
+        if utils.is_testing_request (request):
+            return 'No E2E tests are allowed in production', 400
+
 # If present, PROXY_TO_TEST_ENV should be the name of the target environment
 if os.getenv ('PROXY_TO_TEST_ENV') and not os.getenv ('IS_TEST_ENV'):
     @app.before_request
diff --git a/auth.py b/auth.py
--- a/auth.py
+++ b/auth.py
@@ -4,7 +4,7 @@
 import urllib
 from flask import request, make_response, jsonify, redirect
 from flask_helpers import render_template
-from utils import type_check, object_check, timems, times, db_get, db_set, db_del, db_del_many, db_scan, db_describe, db_get_many, extract_bcrypt_rounds
+from utils import type_check, object_check, timems, times, db_get, db_set, db_del, db_del_many, db_scan, db_describe, db_get_many, extract_bcrypt_rounds, is_testing_request
 import datetime
 from functools import wraps
 from config import config
@@ -165,7 +165,7 @@ def signup ():
         username = body ['username'].strip ().lower ()
         email = body ['email'].strip ().lower ()
 
-        if env and 'subscribe' in body and body ['subscribe'] == True:
+        if not is_testing_request (request) and 'subscribe' in body and body ['subscribe'] == True:
             # If we have a Mailchimp API key, we use it to add the subscriber through the API
             if os.getenv ('MAILCHIMP_API_KEY') and os.getenv ('MAILCHIMP_AUDIENCE_ID'):
                 # The first domain in the path is the server name, which is contained in the Mailchimp API key
@@ -209,8 +209,8 @@ def signup ():
         cookie = make_salt ()
         db_set ('tokens', {'id': cookie, 'username': user ['username'], 'ttl': times () + session_length})
 
-        # If on local environment, we return email verification token directly instead of emailing it, for test purposes.
-        if not env:
+        # If this is an e2e test, we return the email verification token directly instead of emailing it.
+        if is_testing_request (request):
             resp = make_response ({'username': username, 'token': hashed_token})
         # Otherwise, we send an email with a verification link and we return an empty body
         else:
@@ -283,7 +283,7 @@ def change_password (user):
         hashed = hash (body ['new_password'], make_salt ())
 
         db_set ('users', {'username': user ['username'], 'password': hashed})
-        if env:
+        if not is_testing_request (request):
             send_email_template ('change_password', user ['email'], requested_lang (), None)
 
         return '', 200
@@ -320,8 +320,8 @@ def update_profile (user):
                 token = make_salt ()
                 hashed_token = hash (token, make_salt ())
                 db_set ('users', {'username': user ['username'], 'email': email, 'verification_pending': hashed_token})
-                if not env:
-                   # If on local environment, we return email verification token directly instead of emailing it, for test purposes.
+                # If this is an e2e test, we return the email verification token directly instead of emailing it.
+                if is_testing_request (request):
                    resp = {'username': user ['username'], 'token': hashed_token}
                 else:
                     send_email_template ('welcome_verify', email, requested_lang (), os.getenv ('BASE_URL') + '/auth/verify?username=' + urllib.parse.quote_plus (user['username']) + '&token=' + urllib.parse.quote_plus (hashed_token))
@@ -374,8 +374,8 @@ def recover ():
 
         db_set ('tokens', {'id': user ['username'], 'token': hashed, 'ttl': times () + session_length})
 
-        if not env:
-            # If on local environment, we return email verification token directly instead of emailing it, for test purposes.
+        if is_testing_request (request):
+            # If this is an e2e test, we return the email verification token directly instead of emailing it.
             return jsonify ({'username': user ['username'], 'token': token}), 200
         else:
             send_email_template ('recover_password', user ['email'], requested_lang (), os.getenv ('BASE_URL') + '/reset?username=' + urllib.parse.quote_plus (user ['username']) + '&token=' + urllib.parse.quote_plus (token))
@@ -409,7 +409,7 @@ def reset ():
         db_set ('users', {'username': body ['username'], 'password': hashed})
         user = db_get ('users', {'username': body ['username']})
 
-        if env:
+        if not is_testing_request (request):
             send_email_template ('reset_password', user ['email'], requested_lang (), None)
 
         return '', 200
diff --git a/utils.py b/utils.py
--- a/utils.py
+++ b/utils.py
@@ -274,6 +274,9 @@ def slash_join(*args):
         ret.append(arg.lstrip('/') if ret else arg)
     return ''.join(ret)
 
+def is_testing_request(request):
+    return bool ('X-Testing' in request.headers and request.headers ['X-Testing'])
+
 def extract_bcrypt_rounds (hash):
     return int (re.match ('\$2b\$\d+', hash) [0].replace ('$2b$', ''))
 


Test patch:
diff --git a/e2e_tests.py b/e2e_tests.py
--- a/e2e_tests.py
+++ b/e2e_tests.py
@@ -4,19 +4,34 @@
 from utils import type_check, timems
 import urllib.parse
 from config import config
+import sys
+import threading
 
-host = 'http://localhost:' + str (config ['port']) + '/'
+# USAGE: python e2e_tests.py [CONCURRENT_TESTS] [alpha|test]
+# Concurrent tests are a way to stress test an environment
+# When no environment is specified, the tests are run against localhost
+
+import argparse
+args = argparse.ArgumentParser ()
+args.add_argument ('--concurrent', help='how many tests to run at the same time (stress testing), default is 1', type=int)
+args.add_argument ('--host', help='Host against which to run the tests (optional), default is localhost')
+args = args.parse_args ()
 
-t0 = timems ()
+host = 'http://localhost:' + str (config ['port']) + '/'
+hosts = {'alpha': 'https://hedy-alpha.herokuapp.com/', 'test': 'https://hedy-test.herokuapp.com/'}
 
-state = {'headers': {}}
+if args.host:
+    if not args.host in hosts:
+        raise Exception ('No such host')
+    host = hosts [sys.argv [2]]
 
-# test structure: tag method path headers body code
-def request(test, counter):
+# test structure: [tag, method, path, headers, body, code, after_test_function]
+def request (state, test, counter, username):
 
     start = timems ()
 
-    print ('Start #' + str (counter) + ': ' + test [0])
+    if isinstance (threading.current_thread (), threading._MainThread):
+        print ('Start #' + str (counter) + ': ' + test [0])
 
     # If no explicit cookie passed, use the one from the state
     if not 'cookie' in test [3] and 'cookie' in state ['headers']:
@@ -35,6 +50,10 @@ def request(test, counter):
     if type_check (test[4], 'dict'):
         test[3] ['content-type'] = 'application/json'
         test[4] = json.dumps (test [4])
+
+    # We pass the X-Testing header to let the server know that this is a request coming from an E2E test, thus no transactional emails should be sent.
+    test [3] ['X-Testing'] = '1'
+
     r = getattr (requests, test [1]) (host + test [2], headers=test[3], data=test[4])
 
     if 'Content-Type' in r.headers and r.headers ['Content-Type'] == 'application/json':
@@ -59,33 +78,44 @@ def request(test, counter):
         raise Exception ('A test failed!')
 
     if len (test) == 7:
-        test [6] (state, output)
+        test [6] (state, output, username)
 
-    print ('Done  #' + str (counter) + ': ' + test [0] + ' - ' + str (r.status_code) + ' (' + str (timems () - start) + 'ms)')
+    if isinstance (threading.current_thread (), threading._MainThread):
+        print ('Done  #' + str (counter) + ': ' + test [0] + ' - ' + str (r.status_code) + ' (' + str (timems () - start) + 'ms)')
 
     return output
 
-def run_suite(tests):
-    if not type_check(tests, 'list'):
+def run_suite (suite):
+    # We use a random username so that if a test fails, we don't have to do a cleaning of the DB so that the test suite can run again
+    # This also allows us to run concurrent tests without having username conflicts.
+    username = 'user' + str (random.randint (10000, 100000))
+    tests = suite (username)
+    state = {'headers': {}}
+    t0 = timems ()
+
+    if not type_check (tests, 'list'):
         return print ('Invalid test suite, must be a list.')
     counter = 1
 
-    def run_test(test, counter):
-        result = request(test, counter)
+    def run_test (test, counter):
+        result = request (state, test, counter, username)
 
     for test in tests:
         # If test is nested, run a nested loop
-        if not (type_check(test[0], 'str')):
+        if not (type_check (test[0], 'str')):
             for subtest in test:
-                run_test(subtest, counter)
+                run_test (subtest, counter)
                 counter += 1
         else:
-           run_test(test, counter)
+           run_test (test, counter)
            counter += 1
 
-    print ('Test suite successful! (' + str (timems () - t0) + 'ms)')
+    if isinstance (threading.current_thread (), threading._MainThread):
+        print ('Test suite successful! (' + str (timems () - t0) + 'ms)')
+    else:
+        return timems () - t0
 
-def invalidMap(tag, method, path, bodies):
+def invalidMap (tag, method, path, bodies):
     output = []
     counter = 1
     for body in bodies:
@@ -93,95 +123,120 @@ def invalidMap(tag, method, path, bodies):
         counter += 1
     return output
 
-# We use a random username so that if a test fails, we don't have to do a cleaning of the DB so that the test suite can run again
-username = 'user' + str (random.randint (10000, 100000))
-
-def successfulSignup(state, response):
+def successfulSignup (state, response, username):
     if not 'token' in response ['body']:
         raise Exception ('No token present')
     state ['token'] = response ['body'] ['token']
 
 # We define apres functions here because multiline lambdas are not supported by python
-def successfulLogin(state, response):
+def successfulLogin (state, response, username):
     state ['headers'] ['cookie'] = response ['headers'] ['Set-Cookie']
 
-def getProfile1(state, response):
+def getProfile1 (state, response, username):
     profile = response ['body']
     if profile ['username'] != username:
         raise Exception ('Invalid username (getProfile1)')
-    if profile ['email'] != username + '@domain.com':
+    if profile ['email'] != username + '@e2e-testing.com':
         raise Exception ('Invalid username (getProfile1)')
     if not profile ['session_expires_at']:
         raise Exception ('No session_expires_at (getProfile1)')
     expire = profile ['session_expires_at'] - config ['session'] ['session_length'] * 60 * 1000 - timems ()
     if expire > 0:
         raise Exception ('Invalid session_expires_at (getProfile1), too large')
-    # We give the server up to 10ms to respond to the query
-    if expire < -10:
+    # We give the server up to 2s to respond to the query
+    if expire < -2000:
         raise Exception ('Invalid session_expires_at (getProfile1), too small')
 
-def getProfile2(state, response):
+def getProfile2 (state, response, username):
     profile = response ['body']
     if profile ['country'] != 'NL':
         raise Exception ('Invalid country (getProfile2)')
-    if profile ['email'] != username + '@domain2.com':
+    if profile ['email'] != username + '@e2e-testing2.com':
         raise Exception ('Invalid country (getProfile2)')
     if not 'verification_pending' in profile or profile ['verification_pending'] != True:
         raise Exception ('Invalid verification_pending (getProfile2)')
 
-def getProfile3(state, response):
+def getProfile3 (state, response, username):
     profile = response ['body']
     if 'verification_pending' in profile:
         raise Exception ('Invalid verification_pending (getProfile3)')
 
-def emailChange(state, response):
+def emailChange (state, response, username):
     if not type_check (response ['body'] ['token'], 'str'):
         raise Exception ('Invalid country (emailChange)')
     if response ['body'] ['username'] != username:
         raise Exception ('Invalid username (emailChange)')
     state ['token2'] = response ['body'] ['token']
 
-def recoverPassword(state, response):
+def recoverPassword (state, response, username):
     if not 'token' in response ['body']:
         raise Exception ('No token present')
     state ['token'] = response ['body'] ['token']
 
-suite = [
-    ['get root', 'get', '/', {}, '', 200],
-    invalidMap ('signup', 'post', '/auth/signup', ['', [], {}, {'username': 1}, {'username': 'user@me', 'password': 'foobar', 'email': 'a@a.com'}, {'username:': 'user: me', 'password': 'foobar', 'email': 'a@a.co'}, {'username': 't'}, {'username': '    t    '}, {'username': username}, {'username': username, 'password': 1}, {'username': username, 'password': 'foo'}, {'username': username, 'password': 'foobar'}, {'username': username, 'password': 'foobar', 'email': 'me@something'}]),
-    ['valid signup', 'post', '/auth/signup', {}, {'username': username, 'password': 'foobar', 'email': username + '@domain.com'}, 200, successfulSignup],
-    invalidMap ('login', 'post', '/auth/login', ['', [], {}, {'username': 1}, {'username': 'user@me'}, {'username:': 'user: me'}]),
-    ['valid login, invalid credentials', 'post', '/auth/login', {}, {'username': username, 'password': 'password'}, 403],
-    ['verify email (missing fields)', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': 'foobar', 'token': state ['token']}), {}, '', 403],
-    ['verify email (invalid username)', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': 'foobar', 'token': state ['token']}), {}, '', 403],
-    ['verify email (invalid token)', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': username, 'token': 'foobar'}), {}, '', 403],
-    ['verify email', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': username, 'token': state ['token']}), {}, '', 302],
-    ['valid login', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar'}, 200, successfulLogin],
-    invalidMap ('change password', 'post', '/auth/change_password', ['', [], {}, {'foo': 'bar'}, {'old_password': 1}, {'old_password': 'foobar'}, {'old_password': 'foobar', 'new_password': 1}, {'old_password': 'foobar', 'new_password': 'short'}]),
-    ['change password', 'post', '/auth/change_password', {}, {'old_password': 'foobar', 'new_password': 'foobar2'}, 200],
-    ['invalid login after password change', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar'}, 403],
-    ['valid login after password change', 'post', '/auth/login', {}, {'username': username + '@domain.com', 'password': 'foobar2'}, 200, successfulLogin],
-    ['logout', 'post', '/auth/logout', {}, {}, 200],
-    ['check that session is no longer valid', 'get', '/profile', {}, '', 403],
-    ['login again', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar2'}, 200, successfulLogin],
-    invalidMap ('change password', 'post', '/auth/change_password', ['', [], {}, {'foo': 'bar'}, {'old_password': 1}, {'old_password': 'foobar'}, {'old_password': 'foobar', 'new_password': 1}]),
-    ['get profile before profile update', 'get', '/profile', {}, {}, 200, getProfile1],
-    invalidMap ('update profile', 'post', '/profile', ['', [], {'email': 'foobar'}, {'birth_year': 'a'}, {'birth_year': 20}, {'country': 'Netherlands'}, {'gender': 0}, {'gender': 'a'}]),
-    ['change profile with same email', 'post', '/profile', {}, {'email': username + '@domain.com', 'country': 'US'}, 200],
-    ['change profile with different email', 'post', '/profile', {}, {'email': username + '@domain2.com', 'country': 'NL'}, 200, emailChange],
-    ['get profile after profile update', 'get', '/profile', {}, {}, 200, getProfile2],
-    ['verify email after email change', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': username, 'token': state ['token2']}), {}, '', 302],
-    ['get profile after email verification', 'get', '/profile', {}, {}, 200, getProfile3],
-    invalidMap ('recover password', 'post', '/auth/recover', ['', [], {}, {'username': 1}]),
-    ['recover password, invalid user', 'post', '/auth/recover', {}, {'username': 'nosuch'}, 403],
-    ['recover password', 'post', '/auth/recover', {}, {'username': username}, 200, recoverPassword],
-    invalidMap ('reset password', 'post', '/auth/reset', ['', [], {}, {'username': 1}, {'username': 'foobar', 'token': 1}, {'username': 'foobar', 'token': 'some'}, {'username': 'foobar', 'token': 'some', 'password': 1}, {'username': 'foobar', 'token': 'some', 'password': 'short'}]),
-    ['reset password, invalid username', 'post', '/auth/reset', {}, lambda state: {'username': 'foobar', 'token': state ['token'], 'password': 'foobar'}, 403],
-    ['reset password, invalid token', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': 'foobar', 'password': 'foobar'}, 403],
-    ['reset password, invalid password', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': state ['token'], 'password': 'short'}, 400],
-    ['reset password', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': state ['token'], 'password': 'foobar3'}, 200],
-    ['login again after reset', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar3'}, 200],
-    ['destroy account', 'post', '/auth/destroy', {}, {}, 200]
-]
-
-run_suite (suite)
+def suite (username):
+    return [
+        ['get root', 'get', '/', {}, '', 200],
+            invalidMap ('signup', 'post', '/auth/signup', ['', [], {}, {'username': 1}, {'username': 'user@me', 'password': 'foobar', 'email': 'a@a.com'}, {'username:': 'user: me', 'password': 'foobar', 'email': 'a@a.co'}, {'username': 't'}, {'username': '    t    '}, {'username': username}, {'username': username, 'password': 1}, {'username': username, 'password': 'foo'}, {'username': username, 'password': 'foobar'}, {'username': username, 'password': 'foobar', 'email': 'me@something'}]),
+        ['valid signup', 'post', '/auth/signup', {}, {'username': username, 'password': 'foobar', 'email': username + '@e2e-testing.com'}, 200, successfulSignup],
+        invalidMap ('login', 'post', '/auth/login', ['', [], {}, {'username': 1}, {'username': 'user@me'}, {'username:': 'user: me'}]),
+        ['valid login, invalid credentials', 'post', '/auth/login', {}, {'username': username, 'password': 'password'}, 403],
+        ['verify email (missing fields)', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': 'foobar', 'token': state ['token']}), {}, '', 403],
+        ['verify email (invalid username)', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': 'foobar', 'token': state ['token']}), {}, '', 403],
+        ['verify email (invalid token)', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': username, 'token': 'foobar'}), {}, '', 403],
+        ['verify email', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': username, 'token': state ['token']}), {}, '', 302],
+        ['valid login', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar'}, 200, successfulLogin],
+        invalidMap ('change password', 'post', '/auth/change_password', ['', [], {}, {'foo': 'bar'}, {'old_password': 1}, {'old_password': 'foobar'}, {'old_password': 'foobar', 'new_password': 1}, {'old_password': 'foobar', 'new_password': 'short'}]),
+        ['change password', 'post', '/auth/change_password', {}, {'old_password': 'foobar', 'new_password': 'foobar2'}, 200],
+        ['invalid login after password change', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar'}, 403],
+        ['valid login after password change', 'post', '/auth/login', {}, {'username': username + '@e2e-testing.com', 'password': 'foobar2'}, 200, successfulLogin],
+        ['logout', 'post', '/auth/logout', {}, {}, 200],
+        ['check that session is no longer valid', 'get', '/profile', {}, '', 403],
+        ['login again', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar2'}, 200, successfulLogin],
+        invalidMap ('change password', 'post', '/auth/change_password', ['', [], {}, {'foo': 'bar'}, {'old_password': 1}, {'old_password': 'foobar'}, {'old_password': 'foobar', 'new_password': 1}]),
+        ['get profile before profile update', 'get', '/profile', {}, {}, 200, getProfile1],
+        invalidMap ('update profile', 'post', '/profile', ['', [], {'email': 'foobar'}, {'birth_year': 'a'}, {'birth_year': 20}, {'country': 'Netherlands'}, {'gender': 0}, {'gender': 'a'}]),
+        ['change profile with same email', 'post', '/profile', {}, {'email': username + '@e2e-testing.com', 'country': 'US'}, 200],
+        ['change profile with different email', 'post', '/profile', {}, {'email': username + '@e2e-testing2.com', 'country': 'NL'}, 200, emailChange],
+        ['get profile after profile update', 'get', '/profile', {}, {}, 200, getProfile2],
+        ['verify email after email change', 'get', lambda state: '/auth/verify?' + urllib.parse.urlencode ({'username': username, 'token': state ['token2']}), {}, '', 302],
+        ['get profile after email verification', 'get', '/profile', {}, {}, 200, getProfile3],
+        invalidMap ('recover password', 'post', '/auth/recover', ['', [], {}, {'username': 1}]),
+        ['recover password, invalid user', 'post', '/auth/recover', {}, {'username': 'nosuch'}, 403],
+        ['recover password', 'post', '/auth/recover', {}, {'username': username}, 200, recoverPassword],
+        invalidMap ('reset password', 'post', '/auth/reset', ['', [], {}, {'username': 1}, {'username': 'foobar', 'token': 1}, {'username': 'foobar', 'token': 'some'}, {'username': 'foobar', 'token': 'some', 'password': 1}, {'username': 'foobar', 'token': 'some', 'password': 'short'}]),
+        ['reset password, invalid username', 'post', '/auth/reset', {}, lambda state: {'username': 'foobar', 'token': state ['token'], 'password': 'foobar'}, 403],
+        ['reset password, invalid token', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': 'foobar', 'password': 'foobar'}, 403],
+        ['reset password, invalid password', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': state ['token'], 'password': 'short'}, 400],
+        ['reset password', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': state ['token'], 'password': 'foobar3'}, 200],
+        ['login again after reset', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar3'}, 200],
+        ['destroy account', 'post', '/auth/destroy', {}, {}, 200]
+    ]
+
+if not args.concurrent:
+    run_suite (suite)
+else:
+    counter = 0
+    threads = []
+    results = []
+    errors  = []
+
+    def thread_function (counter):
+        try:
+            ms = run_suite (suite)
+            print ('Finished concurrent test #' + str (counter))
+            results.append (ms)
+        except Exception as e:
+            print ('Concurrent test #' + str (counter), "finished with error")
+            errors.append (e)
+
+    print ('Starting', args.concurrent, 'concurrent tests')
+    while counter < args.concurrent:
+        counter += 1
+        thread = threading.Thread (target=thread_function, args=[counter])
+        thread.start ()
+        threads.append (thread)
+
+    for thread in threads:
+        # join waits until all threads are done
+        thread.join ()
+    print ('Done with', args.concurrent, 'concurrent tests,', len (results), 'OK,', len (errors), 'errors,', round (sum (results) / (len (results) * 1000), 2), 'seconds average')