Repository: oppia/oppia

Issue:
RuntimeError: Key has already been set in this batch
<!--
  - Before filing a new issue, please do a quick search to check that it hasn't
  - already been filed on the [issue tracker](https://github.com/oppia/oppia/issues)._
  -->

This error occurred recently in production

```
RuntimeError: Key has already been set in this batch: b'XXX'

at .wrapper ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/_cache.py:185 )
at ._advance_tasklet ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/tasklets.py:334 )
at .global_lock_for_read ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/_cache.py:607 )
at ._advance_tasklet ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/tasklets.py:334 )
at .lookup ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/_datastore_api.py:150 )
at ._advance_tasklet ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/tasklets.py:334 )
at .get ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/key.py:894 )
at ._advance_tasklet ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/tasklets.py:334 )
at .check_success ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/tasklets.py:161 )
at .result ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/tasklets.py:214 )
at .<listcomp> ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/model.py:6380 )
at .get_multi ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/model.py:6380 )
at .positional_wrapper ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/utils.py:153 )
at .wrapper ( /layers/google.python.pip/pip/lib/python3.7/site-packages/google/cloud/ndb/_options.py:102 )
at .get_multi ( /workspace/core/platform/datastore/cloud_datastore_services.py:89 )
at .get_multi ( /workspace/core/storage/base_model/gae_models.py:326 )
at .get_users_settings ( /workspace/core/domain/user_services.py:160 )
at .get_usernames ( /workspace/core/domain/user_services.py:1040 )
at .get ( /workspace/core/controllers/editor.py:700 )
at .test_can_play ( /workspace/core/controllers/acl_decorators.py:216 )
at .dispatch ( /workspace/third_party/python_libs/webapp2.py:604 )
```



Discussion:
Based on https://github.com/googleapis/python-ndb/issues/743, it looks like a race condition when doing a get_multi(). Next step here would be to use the same workaround the author used i.e. adding a simple catch and retry mechanism for get_multi calls.

Resolving patch:
diff --git a/core/platform/datastore/cloud_datastore_services.py b/core/platform/datastore/cloud_datastore_services.py
--- a/core/platform/datastore/cloud_datastore_services.py
+++ b/core/platform/datastore/cloud_datastore_services.py
@@ -19,6 +19,7 @@
 from __future__ import annotations
 
 import contextlib
+import logging
 
 from core.platform import models
 
@@ -52,6 +53,7 @@
 TextProperty = ndb.TextProperty
 
 TYPE_MODEL_SUBCLASS = TypeVar('TYPE_MODEL_SUBCLASS', bound=Model) # pylint: disable=invalid-name
+MAX_GET_RETRIES = 3
 
 CLIENT = ndb.Client()
 
@@ -85,8 +87,17 @@ def get_multi(keys: List[Key]) -> List[Optional[TYPE_MODEL_SUBCLASS]]:
     Returns:
         list(datastore_services.Model | None). List whose items are either a
         Model instance or None if the corresponding key wasn't found.
+
+    Raises:
+        Exception. If ndb.get_multi fails for MAX_GET_RETRIES.
     """
-    return ndb.get_multi(keys)
+    for unused_i in range(0, MAX_GET_RETRIES):
+        try:
+            return ndb.get_multi(keys)
+        except Exception as e:
+            logging.exception('Exception raised: %s', e)
+            continue
+    raise Exception('get_multi failed after %s retries' % MAX_GET_RETRIES)
 
 
 def update_timestamps_multi(


Test patch:
diff --git a/core/platform/datastore/cloud_datastore_services_test.py b/core/platform/datastore/cloud_datastore_services_test.py
--- a/core/platform/datastore/cloud_datastore_services_test.py
+++ b/core/platform/datastore/cloud_datastore_services_test.py
@@ -17,12 +17,15 @@
 from __future__ import annotations
 
 import datetime
+import logging
 
 from core import feconf
 from core.platform import models
 from core.platform.datastore import cloud_datastore_services
 from core.tests import test_utils
 
+from google.cloud import ndb
+
 from typing import Sequence, Tuple
 
 MYPY = False
@@ -141,6 +144,38 @@ def test_fetch_multiple_entities_throws_error_on_duplicate_parameters(
                 ]
             )
 
+    def test_get_multi_throws_error_on_failure(
+        self
+    ) -> None:
+        observed_log_messages = []
+
+        def _mock_logging_function(msg: str, *args: str) -> None:
+
+            """Mocks logging.exception()."""
+            observed_log_messages.append(msg % args)
+        dummy_keys = [
+            ndb.Key('model1', 'id1'),
+            ndb.Key('model2', 'id2'),
+            ndb.Key('model3', 'id3')
+        ]
+        error_msg = (
+            'get_multi failed after %s retries' %
+            cloud_datastore_services.MAX_GET_RETRIES
+        )
+        with self.swap_to_always_raise(
+            ndb,
+            'get_multi',
+            Exception('Mock key error')
+        ), self.swap(logging, 'exception', _mock_logging_function):
+            with self.assertRaisesRegex(Exception, error_msg):
+                cloud_datastore_services.get_multi(dummy_keys)
+        self.assertEqual(
+            observed_log_messages,
+            ['Exception raised: Mock key error',
+            'Exception raised: Mock key error',
+            'Exception raised: Mock key error']
+        )
+
     def test_ndb_query_with_filters(self) -> None:
         user_query_model1 = user_models.UserQueryModel(
             id='query_id1',