Repository: oppia/oppia

Issue:
Exploration migration job throws error because of multiple contribution models entity for oppiamigrationbot 
**Observed behavior:**
The user contribution models entity for oppiamigrationbot has tens of thousands of entries because of that the exploration migration job sometimes use to throw an error (`BadRequestError: Too many indexed properties`) but the errors seem to resolve themselves after a while.

**Fix:**
To fix this issue while doing exploration migration let's not update the user_model if the contributor is the oppia bot. 
<details>
<summary><strong>Error log:</strong></summary>

```
Traceback (most recent call last): (/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py:517)
  File "/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py", line 503, in handle
    tstate.input_reader, shard_state, tstate, ctx)
  File "/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py", line 593, in _process_inputs
    entity, input_reader, ctx, tstate):
  File "/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py", line 635, in _process_datum
    for output in result:
  File "/<path>/core/domain/exp_jobs_one_off.py", line 262, in map
    feconf.CURRENT_STATE_SCHEMA_VERSION))
  File "/<path>/core/domain/exp_services.py", line 1201, in update_exploration
    user_services.add_edited_exploration_id(committer_id, exploration.id)
  File "/<path>/core/domain/user_services.py", line 1401, in add_edited_exploration_id
    _save_user_contributions(user_contributions)
  File "/<path>/core/domain/user_services.py", line 1415, in _save_user_contributions
    edited_exploration_ids=user_contributions.edited_exploration_ids,
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/model.py", line 3458, in _put
    return self._put_async(**ctx_options).get_result()
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 383, in get_result
    self.check_success()
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
    value = gen.throw(exc.__class__, exc, tb)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 824, in put
    key = yield self._put_batcher.add(entity, options)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
    value = gen.throw(exc.__class__, exc, tb)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 358, in _put_tasklet
    keys = yield self._conn.async_put(options, datastore_entities)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 513, in _on_rpc_completion
    result = rpc.get_result()
  File "/<path>/python27/python27_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 615, in get_result
    return self.__get_result_hook(self)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1887, in __put_hook
    self.check_rpc_success(rpc)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1379, in check_rpc_success
    raise _ToDatastoreError(err)
BadRequestError: Too many indexed properties
```
</details>

****Additional thoughts:****
For cleanliness, we should clear the edited_exp_ids entries in the bot too, and add a check to the prod validation job that that entity is treated specially


Exploration migration job throws error because of multiple contribution models entity for oppiamigrationbot 
**Observed behavior:**
The user contribution models entity for oppiamigrationbot has tens of thousands of entries because of that the exploration migration job sometimes use to throw an error (`BadRequestError: Too many indexed properties`) but the errors seem to resolve themselves after a while.

**Fix:**
To fix this issue while doing exploration migration let's not update the user_model if the contributor is the oppia bot. 
<details>
<summary><strong>Error log:</strong></summary>

```
Traceback (most recent call last): (/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py:517)
  File "/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py", line 503, in handle
    tstate.input_reader, shard_state, tstate, ctx)
  File "/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py", line 593, in _process_inputs
    entity, input_reader, ctx, tstate):
  File "/<path>/third_party/gae-mapreduce-1.9.22.0/mapreduce/handlers.py", line 635, in _process_datum
    for output in result:
  File "/<path>/core/domain/exp_jobs_one_off.py", line 262, in map
    feconf.CURRENT_STATE_SCHEMA_VERSION))
  File "/<path>/core/domain/exp_services.py", line 1201, in update_exploration
    user_services.add_edited_exploration_id(committer_id, exploration.id)
  File "/<path>/core/domain/user_services.py", line 1401, in add_edited_exploration_id
    _save_user_contributions(user_contributions)
  File "/<path>/core/domain/user_services.py", line 1415, in _save_user_contributions
    edited_exploration_ids=user_contributions.edited_exploration_ids,
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/model.py", line 3458, in _put
    return self._put_async(**ctx_options).get_result()
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 383, in get_result
    self.check_success()
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
    value = gen.throw(exc.__class__, exc, tb)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 824, in put
    key = yield self._put_batcher.add(entity, options)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
    value = gen.throw(exc.__class__, exc, tb)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 358, in _put_tasklet
    keys = yield self._conn.async_put(options, datastore_entities)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 513, in _on_rpc_completion
    result = rpc.get_result()
  File "/<path>/python27/python27_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 615, in get_result
    return self.__get_result_hook(self)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1887, in __put_hook
    self.check_rpc_success(rpc)
  File "/<path>/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1379, in check_rpc_success
    raise _ToDatastoreError(err)
BadRequestError: Too many indexed properties
```
</details>

****Additional thoughts:****
For cleanliness, we should clear the edited_exp_ids entries in the bot too, and add a check to the prod validation job that that entity is treated specially




Discussion:
[Blocking bug for the August release.]
[Blocking bug for the August release.]

Resolving patch:
diff --git a/core/controllers/admin.py b/core/controllers/admin.py
--- a/core/controllers/admin.py
+++ b/core/controllers/admin.py
@@ -159,6 +159,9 @@ def post(self):
             elif self.payload.get('action') == 'clear_search_index':
                 search_services.clear_collection_search_index()
                 search_services.clear_exploration_search_index()
+            elif self.payload.get('action') == (
+                    'flush_migration_bot_contribution_data'):
+                user_services.flush_migration_bot_contributions_model()
             elif self.payload.get('action') == 'save_config_properties':
                 new_config_property_values = self.payload.get(
                     'new_config_property_values')
diff --git a/core/domain/exp_services.py b/core/domain/exp_services.py
--- a/core/domain/exp_services.py
+++ b/core/domain/exp_services.py
@@ -836,13 +836,13 @@ def update_exploration(
     discard_draft(exploration_id, committer_id)
     # Update summary of changed exploration.
     update_exploration_summary(exploration.id, committer_id)
-    user_services.add_edited_exploration_id(committer_id, exploration.id)
-    user_services.record_user_edited_an_exploration(committer_id)
 
-    if (not rights_manager.is_exploration_private(exploration.id) and
-            committer_id != feconf.MIGRATION_BOT_USER_ID):
-        user_services.update_first_contribution_msec_if_not_set(
-            committer_id, utils.get_current_time_in_millisecs())
+    if committer_id != feconf.MIGRATION_BOT_USER_ID:
+        user_services.add_edited_exploration_id(committer_id, exploration.id)
+        user_services.record_user_edited_an_exploration(committer_id)
+        if not rights_manager.is_exploration_private(exploration.id):
+            user_services.update_first_contribution_msec_if_not_set(
+                committer_id, utils.get_current_time_in_millisecs())
 
 
 def create_exploration_summary(exploration_id, contributor_id_to_add):
diff --git a/core/domain/user_services.py b/core/domain/user_services.py
--- a/core/domain/user_services.py
+++ b/core/domain/user_services.py
@@ -1102,6 +1102,16 @@ def get_email_preferences(user_id):
             email_preferences_model.subscription_notifications)
 
 
+def flush_migration_bot_contributions_model():
+    """Cleans migration bot contributions model."""
+    user_contributions = get_user_contributions(
+        feconf.MIGRATION_BOT_USER_ID, strict=False)
+
+    if user_contributions is not None:
+        user_contributions.edited_exploration_ids = []
+        user_contributions.created_exploration_ids = []
+        _save_user_contributions(user_contributions)
+
 def get_users_email_preferences(user_ids):
     """Get email preferences for the list of users.
 


Test patch:
diff --git a/core/controllers/admin_test.py b/core/controllers/admin_test.py
--- a/core/controllers/admin_test.py
+++ b/core/controllers/admin_test.py
@@ -31,6 +31,7 @@
 from core.domain import stats_domain
 from core.domain import stats_services
 from core.domain import topic_services
+from core.domain import user_services
 from core.platform import models
 from core.platform.taskqueue import gae_taskqueue_services as taskqueue_services
 from core.tests import test_utils
@@ -184,6 +185,36 @@ def _mock_logging_function(msg, *args):
 
         self.logout()
 
+    def test_flush_migration_bot_contributions_action(self):
+        created_exploration_ids = ['exp_1', 'exp_2']
+        edited_exploration_ids = ['exp_3', 'exp_4']
+        user_services.create_user_contributions(
+            feconf.MIGRATION_BOT_USER_ID, created_exploration_ids,
+            edited_exploration_ids)
+
+        migration_bot_contributions_model = (
+            user_services.get_user_contributions(feconf.MIGRATION_BOT_USER_ID))
+        self.assertEqual(
+            migration_bot_contributions_model.created_exploration_ids,
+            created_exploration_ids)
+        self.assertEqual(
+            migration_bot_contributions_model.edited_exploration_ids,
+            edited_exploration_ids)
+        self.login(self.ADMIN_EMAIL, is_super_admin=True)
+        csrf_token = self.get_new_csrf_token()
+
+        self.post_json(
+            '/adminhandler', {
+                'action': 'flush_migration_bot_contribution_data'
+            }, csrf_token=csrf_token)
+
+        migration_bot_contributions_model = (
+            user_services.get_user_contributions(feconf.MIGRATION_BOT_USER_ID))
+        self.assertEqual(
+            migration_bot_contributions_model.created_exploration_ids, [])
+        self.assertEqual(
+            migration_bot_contributions_model.edited_exploration_ids, [])
+
     def test_admin_topics_csv_download_handler(self):
         self.login(self.ADMIN_EMAIL, is_super_admin=True)
         response = self.get_custom_response(
diff --git a/core/domain/exp_services_test.py b/core/domain/exp_services_test.py
--- a/core/domain/exp_services_test.py
+++ b/core/domain/exp_services_test.py
@@ -668,6 +668,53 @@ def test_update_exploration_by_migration_bot(self):
                     'new_value': 'New title'
                 })], 'Did migration.')
 
+    def test_update_exploration_by_migration_bot_not_updates_contribution_model(
+            self):
+        user_services.create_user_contributions(
+            feconf.MIGRATION_BOT_USER_ID, [], [])
+        self.save_new_valid_exploration(
+            self.EXP_ID, self.owner_id, end_state_name='end')
+        rights_manager.publish_exploration(self.owner, self.EXP_ID)
+
+        migration_bot_contributions_model = (
+            user_services.get_user_contributions(feconf.MIGRATION_BOT_USER_ID))
+        self.assertTrue(
+            self.EXP_ID not in (
+                migration_bot_contributions_model.edited_exploration_ids))
+
+        exp_services.update_exploration(
+            feconf.MIGRATION_BOT_USER_ID, self.EXP_ID, [
+                exp_domain.ExplorationChange({
+                    'cmd': 'edit_exploration_property',
+                    'property_name': 'title',
+                    'new_value': 'New title'
+                })], 'Did migration.')
+
+        migration_bot_contributions_model = (
+            user_services.get_user_contributions(feconf.MIGRATION_BOT_USER_ID))
+        self.assertTrue(
+            self.EXP_ID not in (
+                migration_bot_contributions_model.edited_exploration_ids))
+
+    def test_update_exploration_by_migration_bot_not_updates_settings_model(
+            self):
+        self.save_new_valid_exploration(
+            self.EXP_ID, self.owner_id, end_state_name='end')
+        rights_manager.publish_exploration(self.owner, self.EXP_ID)
+
+        exp_services.update_exploration(
+            feconf.MIGRATION_BOT_USER_ID, self.EXP_ID, [
+                exp_domain.ExplorationChange({
+                    'cmd': 'edit_exploration_property',
+                    'property_name': 'title',
+                    'new_value': 'New title'
+                })], 'Did migration.')
+
+        migration_bot_settings_model = (
+            user_services.get_user_settings_from_username(
+                feconf.MIGRATION_BOT_USERNAME))
+        self.assertEqual(migration_bot_settings_model, None)
+
     def test_get_multiple_explorations_from_model_by_id(self):
         rights_manager.create_new_exploration_rights(
             'exp_id_1', self.owner_id)
diff --git a/core/domain/user_services_test.py b/core/domain/user_services_test.py
--- a/core/domain/user_services_test.py
+++ b/core/domain/user_services_test.py
@@ -1331,3 +1331,27 @@ def test_cannot_update_dashboard_stats_log_with_invalid_schema_version(
             'Sorry, we can only process v1-v%d dashboard stats schemas at '
             'present.' % feconf.CURRENT_DASHBOARD_STATS_SCHEMA_VERSION):
             user_services.update_dashboard_stats_log(self.owner_id)
+
+    def test_flush_migration_bot_contributions_model(self):
+        created_exploration_ids = ['exp_1', 'exp_2']
+        edited_exploration_ids = ['exp_3', 'exp_4']
+        user_services.create_user_contributions(
+            feconf.MIGRATION_BOT_USER_ID, created_exploration_ids,
+            edited_exploration_ids)
+
+        migration_bot_contributions_model = (
+            user_services.get_user_contributions(feconf.MIGRATION_BOT_USER_ID))
+        self.assertEqual(
+            migration_bot_contributions_model.created_exploration_ids,
+            created_exploration_ids)
+        self.assertEqual(
+            migration_bot_contributions_model.edited_exploration_ids,
+            edited_exploration_ids)
+
+        user_services.flush_migration_bot_contributions_model()
+        migration_bot_contributions_model = (
+            user_services.get_user_contributions(feconf.MIGRATION_BOT_USER_ID))
+        self.assertEqual(
+            migration_bot_contributions_model.created_exploration_ids, [])
+        self.assertEqual(
+            migration_bot_contributions_model.edited_exploration_ids, [])