Repository: StackStorm/st2

Issue:
Concurrent auth failure due to duplicate DB write conflict
With RBAC enabled, when there are multiple concurrent auth operations, the auth fails because RBACRemoteGroupToRoleSyncer sync operation error on duplicate DB write conflict.

```
ERROR base [-] Conflict while trying to save in DB.Traceback (most recent call last):File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2common/persistence/base.py", line 173, in add_or_updatemodel_object = cls._get_impl().add_or_update(model_object)
File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2common/models/db/__init__.py", line 314, in add_or_updateinstance.save()
File "/opt/stackstorm/st2/lib/python2.7/site-packages/mongoengine/document.py", line 408,
NotUniqueError(message % six.text_type(err))NotUniqueError: Tried to save duplicate unique keys (E11000 duplicate key error collection: st2.user_role_assignment_d_b index: role_1_user_1 dup key: { : "base_permissions", : "foobar" })
ERROR handlers [-] Failed to synchronize remote groups for user "foobar"
Traceback (most recent call last):File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2auth/handlers.py", line 202, in handle_auth syncer.sync(user_db=user_db, groups=user_groups) File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2common/rbac/syncer.py", line 423, in sync is_remote=True)
File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2common/services/rbac.py", line 207, in assign_role_to_userrole_assignment_db = UserRoleAssignment.add_or_update(role_assignment_db)
File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2common/persistence/base.py", line 185, in add_or_updatemodel_object=model_object)
StackStormDBObjectConflictError: Tried to save duplicate unique keys (E11000 duplicate key error collection: st2.user_role_assignment_d_b index: role_1_user_1 dup key
```


Discussion:
This DB conflict occurs at https://github.com/StackStorm/st2/blob/v2.3/st2common/st2common/rbac/syncer.py#L421.

Resolving patch:
diff --git a/st2common/st2common/rbac/syncer.py b/st2common/st2common/rbac/syncer.py
--- a/st2common/st2common/rbac/syncer.py
+++ b/st2common/st2common/rbac/syncer.py
@@ -456,7 +456,8 @@ def sync(self, user_db, groups):
                 assignment_db = rbac_services.assign_role_to_user(role_db=role_db, user_db=user_db,
                                                                   description=description,
                                                                   is_remote=True,
-                                                                  source=mapping_db.source)
+                                                                  source=mapping_db.source,
+                                                                  ignore_already_exists_error=True)
                 assert assignment_db.is_remote is True
                 created_assignments_dbs.append(assignment_db)
 
diff --git a/st2common/st2common/services/rbac.py b/st2common/st2common/services/rbac.py
--- a/st2common/st2common/services/rbac.py
+++ b/st2common/st2common/services/rbac.py
@@ -14,7 +14,9 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
 from mongoengine.queryset.visitor import Q
+from mongoengine import NotUniqueError
 
 from st2common.rbac.types import PermissionType
 from st2common.rbac.types import ResourceType
@@ -27,6 +29,7 @@
 from st2common.models.db.rbac import UserRoleAssignmentDB
 from st2common.models.db.rbac import PermissionGrantDB
 from st2common.models.db.rbac import GroupToRoleMappingDB
+from st2common.exceptions.db import StackStormDBObjectConflictError
 
 
 __all__ = [
@@ -186,7 +189,8 @@ def delete_role(name):
     return result
 
 
-def assign_role_to_user(role_db, user_db, description=None, is_remote=False, source=None):
+def assign_role_to_user(role_db, user_db, description=None, is_remote=False, source=None,
+                        ignore_already_exists_error=False):
     """
     Assign role to a user.
 
@@ -205,11 +209,22 @@ def assign_role_to_user(role_db, user_db, description=None, is_remote=False, sou
     :param source: Source from where this assignment comes from. For example, path of a file if
                    it's a local assignment or mapping or "API".
     :type source: ``str``
+
+    :param: ignore_already_exists_error: True to ignore error if an assignment already exists.
+    :type ignore_already_exists_error: ``bool``
     """
     role_assignment_db = UserRoleAssignmentDB(user=user_db.name, role=role_db.name, source=source,
                                               description=description, is_remote=is_remote)
 
-    role_assignment_db = UserRoleAssignment.add_or_update(role_assignment_db)
+    try:
+        role_assignment_db = UserRoleAssignment.add_or_update(role_assignment_db)
+    except (NotUniqueError, StackStormDBObjectConflictError) as e:
+        if not ignore_already_exists_error:
+            raise e
+
+        role_assignment_db = UserRoleAssignment.query(user=user_db.name, role=role_db.name,
+                                                      source=source,
+                                                      description=description).first()
 
     return role_assignment_db
 


Test patch:
diff --git a/st2auth/tests/unit/test_handlers.py b/st2auth/tests/unit/test_handlers.py
--- a/st2auth/tests/unit/test_handlers.py
+++ b/st2auth/tests/unit/test_handlers.py
@@ -16,6 +16,7 @@
 import base64
 
 import mock
+import eventlet
 from oslo_config import cfg
 
 from st2tests.base import CleanDbTestCase
@@ -79,6 +80,10 @@ def setUp(self):
         user_1_db = User.add_or_update(user_1_db)
         self.users['user_1'] = user_1_db
 
+        user_2_db = UserDB(name='buser')
+        user_2_db = User.add_or_update(user_2_db)
+        self.users['user_2'] = user_2_db
+
         # Insert mock local role assignments
         role_db = create_role(name='mock_local_role_1')
         user_db = self.users['user_1']
@@ -341,6 +346,27 @@ def test_group_to_role_sync_is_performed_on_successful_auth_with_groups_and_mapp
         self.assertEqual(role_dbs[2], self.roles['mock_role_3'])
         self.assertEqual(role_dbs[3], self.roles['mock_role_4'])
 
+    def test_group_to_role_sync_concurrent_auth(self):
+        # Verify that there is no race and group sync during concurrent auth works fine
+        # Enable group sync
+        cfg.CONF.set_override(group='rbac', name='sync_remote_groups', override=True)
+        cfg.CONF.set_override(group='rbac', name='sync_remote_groups', override=True)
+
+        h = handlers.StandaloneAuthHandler()
+        request = {}
+
+        def handle_auth():
+            token = h.handle_auth(request, headers={}, remote_addr=None, remote_user=None,
+                                  authorization=('basic', DUMMY_CREDS))
+            self.assertEqual(token.user, 'auser')
+
+        thread_pool = eventlet.GreenPool(20)
+
+        for i in range(0, 20):
+            thread_pool.spawn(handle_auth)
+
+        thread_pool.waitall()
+
     @mock.patch.object(RBACRemoteGroupToRoleSyncer, 'sync',
                       mock.Mock(side_effect=Exception('throw')))
     def test_group_to_role_sync_error_non_fatal_on_succesful_auth(self):
diff --git a/st2common/tests/unit/services/test_rbac.py b/st2common/tests/unit/services/test_rbac.py
--- a/st2common/tests/unit/services/test_rbac.py
+++ b/st2common/tests/unit/services/test_rbac.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
 from pymongo import MongoClient
 
 from st2tests.base import CleanDbTestCase
@@ -27,6 +28,7 @@
 from st2common.models.db.auth import UserDB
 from st2common.models.db.rbac import UserRoleAssignmentDB
 from st2common.models.db.rule import RuleDB
+from st2common.exceptions.db import StackStormDBObjectConflictError
 
 
 class RBACServicesTestCase(CleanDbTestCase):
@@ -247,6 +249,30 @@ def test_grant_duplicate_role(self):
         role_dbs = user_db.get_roles()
         self.assertItemsEqual(role_dbs, [])
 
+    def test_assign_role_to_user_ignore_already_exists_error(self):
+        user_db = UserDB(name='test-user-10')
+        user_db = User.add_or_update(user_db)
+
+        role_assignment_db_1 = rbac_services.assign_role_to_user(
+            role_db=self.roles['custom_role_1'], user_db=user_db,
+            source='assignments/%s_10.yaml' % user_db.name)
+
+        # 1. Without ignore errors
+        self.assertRaises(StackStormDBObjectConflictError, rbac_services.assign_role_to_user,
+            role_db=self.roles['custom_role_1'], user_db=user_db,
+            source='assignments/%s_10.yaml' % user_db.name)
+
+        # 2. With ignore errors
+        role_assignment_db_2 = rbac_services.assign_role_to_user(
+            role_db=self.roles['custom_role_1'], user_db=user_db,
+            source='assignments/%s_10.yaml' % user_db.name,
+            ignore_already_exists_error=True)
+
+        self.assertEqual(role_assignment_db_1, role_assignment_db_2)
+        self.assertEqual(role_assignment_db_1.id, role_assignment_db_2.id)
+        self.assertEqual(role_assignment_db_1.user, role_assignment_db_2.user)
+        self.assertEqual(role_assignment_db_1.role, role_assignment_db_2.role)
+
     def test_get_all_permission_grants_for_user(self):
         user_db = self.users['1_custom_role']
         role_db = self.roles['custom_role_1']