Repository: hedyorg/hedy

Issue:
Split `db_set` into `db_create` and `db_update`
Benefits:
- Avoid race conditions like the one fixed in #425 
- Performance: avoid doing a `db_get` every time we invoke `db_set`.


Discussion:

Resolving patch:
diff --git a/app.py b/app.py
--- a/app.py
+++ b/app.py
@@ -20,7 +20,7 @@
 from werkzeug.urls import url_encode
 from config import config
 from auth import auth_templates, current_user, requires_login, is_admin, is_teacher
-from utils import db_get, db_get_many, db_set, timems, type_check, object_check, db_del, load_yaml, load_yaml_rt, dump_yaml_rt, version
+from utils import db_get, db_get_many, db_create, db_update, timems, type_check, object_check, db_del, load_yaml, load_yaml_rt, dump_yaml_rt, version
 import utils
 
 # app.py
@@ -686,6 +686,11 @@ def render_main_menu(current_page):
 
 # *** PROGRAMS ***
 
+@app.route('/programs_list', methods=['GET'])
+@requires_login
+def list_programs (user):
+    return {'programs': db_get_many ('programs', {'username': user ['username']}, True)}
+
 # Not very restful to use a GET to delete something, but indeed convenient; we can do it with a single link and avoiding AJAX.
 @app.route('/programs/delete/<program_id>', methods=['GET'])
 @requires_login
@@ -697,7 +702,7 @@ def delete_program (user, program_id):
     program_count = 0
     if 'program_count' in user:
         program_count = user ['program_count']
-    db_set ('users', {'username': user ['username'], 'program_count': program_count - 1})
+    db_update ('users', {'username': user ['username'], 'program_count': program_count - 1})
     return redirect ('/programs')
 
 @app.route('/programs', methods=['POST'])
@@ -759,12 +764,12 @@ def save_program (user):
     if 'adventure_name' in body:
         stored_program ['adventure_name'] = body ['adventure_name']
 
-    db_set('programs', stored_program)
+    db_create('programs', stored_program)
 
     program_count = 0
     if 'program_count' in user:
         program_count = user ['program_count']
-    db_set('users', {'username': user ['username'], 'program_count': program_count + 1})
+    db_update('users', {'username': user ['username'], 'program_count': program_count + 1})
 
     return jsonify({'name': name})
 
diff --git a/auth.py b/auth.py
--- a/auth.py
+++ b/auth.py
@@ -4,7 +4,7 @@
 import urllib
 from flask import request, make_response, jsonify, redirect
 from flask_helpers import render_template
-from utils import type_check, object_check, timems, times, db_get, db_set, db_del, db_del_many, db_scan, db_describe, db_get_many, extract_bcrypt_rounds, is_testing_request, valid_email
+from utils import type_check, object_check, timems, times, db_get, db_create, db_update, db_del, db_del_many, db_scan, db_describe, db_get_many, extract_bcrypt_rounds, is_testing_request, valid_email
 import datetime
 from functools import wraps
 from config import config
@@ -107,11 +107,11 @@ def login ():
             new_hash = hash (body ['password'], make_salt ())
 
         cookie = make_salt ()
-        db_set ('tokens', {'id': cookie, 'username': user ['username'], 'ttl': times () + session_length})
+        db_create ('tokens', {'id': cookie, 'username': user ['username'], 'ttl': times () + session_length})
         if new_hash:
-            db_set ('users', {'username': user ['username'], 'password': new_hash, 'last_login': timems ()})
+            db_update ('users', {'username': user ['username'], 'password': new_hash, 'last_login': timems ()})
         else:
-            db_set ('users', {'username': user ['username'], 'last_login': timems ()})
+            db_update ('users', {'username': user ['username'], 'last_login': timems ()})
         resp = make_response ({})
         # We set the cookie to expire in a year, just so that the browser won't invalidate it if the same cookie gets renewed by constant use.
         # The server will decide whether the cookie expires.
@@ -203,11 +203,11 @@ def signup ():
         if 'gender' in body:
             user ['gender'] = body ['gender']
 
-        db_set ('users', user)
+        db_create ('users', user)
 
         # We automatically login the user
         cookie = make_salt ()
-        db_set ('tokens', {'id': cookie, 'username': user ['username'], 'ttl': times () + session_length})
+        db_create ('tokens', {'id': cookie, 'username': user ['username'], 'ttl': times () + session_length})
 
         # If this is an e2e test, we return the email verification token directly instead of emailing it.
         if is_testing_request (request):
@@ -243,7 +243,7 @@ def verify_email ():
         if token != user ['verification_pending']:
             return 'invalid username/token', 403
 
-        db_set ('users', {'username': username, 'verification_pending': None})
+        db_update ('users', {'username': username, 'verification_pending': None})
         return redirect ('/')
 
     @app.route ('/auth/logout', methods=['POST'])
@@ -282,7 +282,7 @@ def change_password (user):
 
         hashed = hash (body ['new_password'], make_salt ())
 
-        db_set ('users', {'username': user ['username'], 'password': hashed})
+        db_update ('users', {'username': user ['username'], 'password': hashed})
         if not is_testing_request (request):
             send_email_template ('change_password', user ['email'], requested_lang (), None)
 
@@ -319,7 +319,7 @@ def update_profile (user):
                     return 'email exists', 403
                 token = make_salt ()
                 hashed_token = hash (token, make_salt ())
-                db_set ('users', {'username': user ['username'], 'email': email, 'verification_pending': hashed_token})
+                db_update ('users', {'username': user ['username'], 'email': email, 'verification_pending': hashed_token})
                 # If this is an e2e test, we return the email verification token directly instead of emailing it.
                 if is_testing_request (request):
                    resp = {'username': user ['username'], 'token': hashed_token}
@@ -327,11 +327,11 @@ def update_profile (user):
                     send_email_template ('welcome_verify', email, requested_lang (), os.getenv ('BASE_URL') + '/auth/verify?username=' + urllib.parse.quote_plus (user['username']) + '&token=' + urllib.parse.quote_plus (hashed_token))
 
         if 'country' in body:
-            db_set ('users', {'username': user ['username'], 'country': body ['country']})
+            db_update ('users', {'username': user ['username'], 'country': body ['country']})
         if 'birth_year' in body:
-            db_set ('users', {'username': user ['username'], 'birth_year': body ['birth_year']})
+            db_update ('users', {'username': user ['username'], 'birth_year': body ['birth_year']})
         if 'gender' in body:
-            db_set ('users', {'username': user ['username'], 'gender': body ['gender']})
+            db_update ('users', {'username': user ['username'], 'gender': body ['gender']})
 
         return jsonify (resp)
 
@@ -372,7 +372,7 @@ def recover ():
         token = make_salt ()
         hashed = hash (token, make_salt ())
 
-        db_set ('tokens', {'id': user ['username'], 'token': hashed, 'ttl': times () + session_length})
+        db_create ('tokens', {'id': user ['username'], 'token': hashed, 'ttl': times () + session_length})
 
         if is_testing_request (request):
             # If this is an e2e test, we return the email verification token directly instead of emailing it.
@@ -406,7 +406,7 @@ def reset ():
 
         hashed = hash (body ['password'], make_salt ())
         token = db_del ('tokens', {'id': body ['username']})
-        db_set ('users', {'username': body ['username'], 'password': hashed})
+        db_update ('users', {'username': body ['username'], 'password': hashed})
         user = db_get ('users', {'username': body ['username']})
 
         if not is_testing_request (request):
@@ -436,7 +436,7 @@ def mark_as_teacher ():
         if not user:
             return 'invalid username', 400
 
-        db_set ('users', {'username': user ['username'], 'is_teacher': 1 if body ['is_teacher'] else 0})
+        db_update ('users', {'username': user ['username'], 'is_teacher': 1 if body ['is_teacher'] else 0})
 
         return '', 200
 
@@ -466,7 +466,7 @@ def change_user_email():
         hashed_token = hash (token, make_salt ())
 
         # We assume that this email is not in use by any other users. In other words, we trust the admin to enter a valid, not yet used email address.
-        db_set ('users', {'username': user ['username'], 'email': body ['email'], 'verification_pending': hashed_token})
+        db_update ('users', {'username': user ['username'], 'email': body ['email'], 'verification_pending': hashed_token})
 
         # If this is an e2e test, we return the email verification token directly instead of emailing it.
         if is_testing_request (request):
diff --git a/utils.py b/utils.py
--- a/utils.py
+++ b/utils.py
@@ -122,27 +122,32 @@ def dump_yaml_rt(data):
 db_prefix = os.getenv ('AWS_DYNAMODB_TABLE_PREFIX')
 
 # Encode a dict so that it has the format expected by DynamoDB
-def db_encode (data, *args):
-    # args may contain a boolean flag which we use to detect whether we should format the payload for update_item
-    # when len (args) is truthy, we know we're dealing with update_item
+def db_encode (data):
+    # update is a boolean flag which we use to detect whether we should format the payload for update_item
     processed_data = {}
     for key in data:
         if type_check (data [key], 'str'):
-            if len (args):
-                processed_data [key] = {'Value': {'S': data [key]}}
-            else:
-                processed_data [key] = {'S': data [key]}
+            processed_data [key] = {'S': data [key]}
         elif type_check (data [key], 'int'):
             # Note we convert the value into a string
-            if len (args):
-                processed_data [key] = {'Value': {'N': str (data [key])}}
-            else:
-                processed_data [key] = {'N': str (data [key])}
+            processed_data [key] = {'N': str (data [key])}
         elif data [key] == None:
-            if len (args):
-                processed_data [key] = {'Action': 'DELETE'}
-            else:
-                processed_data [key] = {'NULL': True}
+            processed_data [key] = {'NULL': True}
+        else:
+            raise ValueError ('Unsupported type passed to db_put')
+    return processed_data
+
+# Encode a dict so that it has the format expected by DynamoDB
+def db_encode_updates (data):
+    processed_data = {}
+    for key in data:
+        if type_check (data [key], 'str'):
+            processed_data [key] = {'Value': {'S': data [key]}}
+        elif type_check (data [key], 'int'):
+            # Note we convert the value into a string
+            processed_data [key] = {'Value': {'N': str (data [key])}}
+        elif data [key] == None:
+            processed_data [key] = {'Action': 'DELETE'}
         else:
             raise ValueError ('Unsupported type passed to db_put')
     return processed_data
@@ -161,30 +166,30 @@ def db_decode (data):
             raise ValueError ('Unsupported type passed to db_put')
     return processed_data
 
+db_main_indexes = {
+   'users': 'username',
+   'tokens': 'id',
+   'programs': 'id'
+}
+
 # This function takes a dict `data` and returns a new dict with only the key/value for the index key for the table.
-# If *remove is truthy, then the index key is removed instead, leaving the rest of the keys intact.
-def db_key (table, data, *remove):
+# If remove is truthy, then the index key is removed instead, leaving the rest of the keys intact.
+def db_key (table, data, remove=False):
     processed_data = {}
-    if len (remove):
+    if remove:
         for key in data:
-            if table == 'users' and key == 'username':
-                continue
-            if (table == 'tokens' or table == 'programs') and key == 'id':
-                continue
-            processed_data [key] = data [key]
+            if key != db_main_indexes [table]:
+                processed_data [key] = data [key]
     else:
-        if table == 'users':
-            processed_data ['username'] = data ['username']
-        if table == 'tokens' or table == 'programs':
-            processed_data ['id'] = data ['id']
+        processed_data [db_main_indexes [table]] = data [db_main_indexes [table]]
     return processed_data
 
 # Gets an item by index from the database. If not_primary is truthy, the search is done by a field that should be set as a secondary index.
 @querylog.timed
-def db_get (table, data, *not_primary):
+def db_get (table, data, not_primary=False):
     querylog.log_counter('db_get:' + table)
     # If we're querying by something else than the primary key of the table, we assume that data contains only one field, that on which we want to search. We also require that field to have an index.
-    if len (not_primary):
+    if not_primary:
         field = list (data.keys ()) [0]
         result = db.query (TableName = db_prefix + '-' + table, IndexName = field + '-index', KeyConditionExpression = field + ' = :value', ExpressionAttributeValues = {':value': {'S': data [field]}})
         if len (result ['Items']):
@@ -199,9 +204,9 @@ def db_get (table, data, *not_primary):
 
 # Gets an item by index from the database. If not_primary is truthy, the search is done by a field that should be set as a secondary index.
 @querylog.timed
-def db_get_many (table, data, *not_primary):
+def db_get_many (table, data, not_primary=False):
     querylog.log_counter('db_get_many:' + table)
-    if len (not_primary):
+    if not_primary:
         field = list (data.keys ()) [0]
         # We use ScanIndexForward = False to get the latest items from the Programs table
         result = db.query (TableName = db_prefix + '-' + table, IndexName = field + '-index', KeyConditionExpression = field + ' = :value', ExpressionAttributeValues = {':value': {'S': data [field]}}, ScanIndexForward = False)
@@ -213,15 +218,17 @@ def db_get_many (table, data, *not_primary):
         data.append (db_decode (item))
     return data
 
-# Creates or updates an item by primary key.
+# Creates an item.
 @querylog.timed
-def db_set (table, data):
-    querylog.log_counter('db_set:' + table)
-    if db_get (table, data):
-        result = db.update_item (TableName = db_prefix + '-' + table, Key = db_encode (db_key (table, data)), AttributeUpdates = db_encode (db_key (table, data, True), True))
-    else:
-        result = db.put_item (TableName = db_prefix + '-' + table, Item = db_encode (data))
-    return result
+def db_create (table, data):
+    querylog.log_counter('db_create:' + table)
+    return db.put_item (TableName = db_prefix + '-' + table, Item = db_encode (data))
+
+# Updates an item by primary key.
+@querylog.timed
+def db_update (table, data):
+    querylog.log_counter('db_update:' + table)
+    return db.update_item (TableName = db_prefix + '-' + table, Key = db_encode (db_key (table, data)), AttributeUpdates = db_encode_updates (db_key (table, data, True)))
 
 # Deletes an item by primary key.
 @querylog.timed
@@ -231,11 +238,11 @@ def db_del (table, data):
 
 # Deletes multiple items.
 @querylog.timed
-def db_del_many (table, data, *not_primary):
+def db_del_many (table, data, not_primary=False):
     querylog.log_counter('db_del_many:' + table)
     # We define a recursive function in case the number of results is very large and cannot be returned with a single call to db_get_many.
     def batch ():
-        to_delete = db_get_many (table, data, *not_primary)
+        to_delete = db_get_many (table, data, not_primary)
         if len (to_delete) == 0:
             return
         for item in to_delete:


Test patch:
diff --git a/e2e_tests.py b/e2e_tests.py
--- a/e2e_tests.py
+++ b/e2e_tests.py
@@ -209,6 +209,30 @@ def checkMainSessionVarsAgain (state, response, username):
     if response ['body'] ['session'] ['test_session'] != state ['test_session']:
         raise Exception ('test_session not received by main')
 
+def retrieveProgramsBefore (state, response, username):
+    if not type_check (response ['body'], 'dict'):
+        raise Exception ('Invalid response body')
+    if not 'programs' in response ['body'] or not type_check (response ['body'] ['programs'], 'list'):
+        raise Exception ('Invalid programs list')
+    if len (response ['body'] ['programs']) != 0:
+        raise Exception ('Programs list should be empty')
+
+def retrieveProgramsAfter (state, response, username):
+    if not type_check (response ['body'], 'dict'):
+        raise Exception ('Invalid response body')
+    if not 'programs' in response ['body'] or not type_check (response ['body'] ['programs'], 'list'):
+        raise Exception ('Invalid programs list')
+    if len (response ['body'] ['programs']) != 1:
+        raise Exception ('Programs list should contain one program')
+    program = response ['body'] ['programs'] [0]
+    state ['program'] = program
+    if not type_check (program, 'dict'):
+        raise Exception ('Invalid program type')
+    if not 'code' in program or program ['code'] != 'print Hello world':
+        raise Exception ('Invalid program.code')
+    if not 'level' in program or program ['level'] != 1:
+        raise Exception ('Invalid program.level')
+
 def suite (username):
     return [
         ['get session vars from main', 'get', '/session_main', {}, {}, 200, checkMainSessionVars],
@@ -248,6 +272,11 @@ def suite (username):
         ['reset password, invalid password', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': state ['token'], 'password': 'short'}, 400],
         ['reset password', 'post', '/auth/reset', {}, lambda state: {'username': username, 'token': state ['token'], 'password': 'foobar3'}, 200],
         ['login again after reset', 'post', '/auth/login', {}, {'username': username, 'password': 'foobar3'}, 200],
+        ['retrieve programs before saving any', 'get', '/programs_list', {}, {}, 200, retrieveProgramsBefore],
+        ['create program', 'post', '/programs', {}, {'code': 'print Hello world', 'name': 'Program 1', 'level': 1}, 200],
+        ['retrieve programs after saving one', 'get', '/programs_list', {}, {}, 200, retrieveProgramsAfter],
+        ['delete program', 'get', lambda state: '/programs/delete/' + state ['program'] ['id'], {}, {}, 302],
+        ['retrieve programs after deleting saved program', 'get', '/programs_list', {}, {}, 200, retrieveProgramsBefore],
         ['destroy account', 'post', '/auth/destroy', {}, {}, 200]
     ]