Repository: oppia/oppia

Issue:
Errors when running GenerateV1StatisticsJob and ExplorationStateIdMappingJob in production
Two errors show up when running the GenerateV1StatisticsJob in production:

```
Traceback (most recent call last): (/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py:508)
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 494, in handle
    tstate.input_reader, shard_state, tstate, ctx)
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 583, in _process_inputs
    entity, input_reader, ctx, tstate):
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 625, in _process_datum
    for output in result:
  File "/core/domain/stats_jobs_one_off.py", line 345, in reduce
    change_dict['state_name'])
KeyError: u'END'
```

```
Traceback (most recent call last): (/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py:508)
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 494, in handle
    tstate.input_reader, shard_state, tstate, ctx)
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 583, in _process_inputs
    entity, input_reader, ctx, tstate):
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 625, in _process_datum
    for output in result:
  File "/core/domain/stats_jobs_one_off.py", line 432, in reduce
    dest_states.append(init_state.interaction.default_outcome.dest)
AttributeError: 'NoneType' object has no attribute 'dest'
```

The ExplorationStateIdMappingJob also continues to exhibit this error in production:

```
Traceback (most recent call last): (/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py:508)
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 494, in handle
    tstate.input_reader, shard_state, tstate, ctx)
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 583, in _process_inputs
    entity, input_reader, ctx, tstate):
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/handlers.py", line 634, in _process_datum
    output_writer.write(output)
  File "/third_party/gae-mapreduce-1.9.17.0/mapreduce/output_writers.py", line 1145, in write
    if len(data) != 2:
TypeError: object of type 'exceptions.Exception' has no len()
```


Discussion:

Resolving patch:
diff --git a/core/domain/stats_jobs_one_off.py b/core/domain/stats_jobs_one_off.py
--- a/core/domain/stats_jobs_one_off.py
+++ b/core/domain/stats_jobs_one_off.py
@@ -341,9 +341,25 @@ def reduce(exp_id, stringified_values):
                             'state_name']] = (
                                 stats_domain.StateStats.create_default())
                     elif change_dict['cmd'] == exp_domain.CMD_DELETE_STATE:
+                        if change_dict['state_name'] not in state_stats_mapping:
+                            yield (
+                                'ERROR during state deletion: State name %s not'
+                                ' in state stats mapping of exploration with ID'
+                                ' %s and version %s' % (
+                                    change_dict['state_name'], exp_id, version))
+                            return
                         state_stats_mapping.pop(
                             change_dict['state_name'])
                     elif change_dict['cmd'] == exp_domain.CMD_RENAME_STATE:
+                        if change_dict['old_state_name'] not in (
+                                state_stats_mapping):
+                            yield (
+                                'ERROR during state rename: State name %s not '
+                                'in state stats mapping of exploration with ID '
+                                '%s and version %s' % (
+                                    change_dict['old_state_name'], exp_id,
+                                    version))
+                            return
                         state_stats_mapping[change_dict[
                             'new_state_name']] = (
                                 state_stats_mapping.pop(
@@ -429,7 +445,9 @@ def reduce(exp_id, stringified_values):
                 dest_states = [
                     answer_group.outcome.dest
                     for answer_group in init_state.interaction.answer_groups]
-                dest_states.append(init_state.interaction.default_outcome.dest)
+                if init_state.interaction.default_outcome is not None:
+                    dest_states.append(
+                        init_state.interaction.default_outcome.dest)
                 for dest_state in dest_states:
                     if dest_state != versioned_exploration.init_state_name:
                         # Some older explorations had the pseudo-END state as a
@@ -461,9 +479,58 @@ def reduce(exp_id, stringified_values):
                 exploration_stats = stats_domain.ExplorationStats(
                     exp_id, version, num_starts, 0, num_actual_starts, 0,
                     num_completions, 0, state_stats_mapping)
+
+            # Check if the number of actual starts is greater than the number of
+            # starts and update num_starts to be the maximum of the
+            # first hit count of all states of the exploration.
+            if exploration_stats.num_actual_starts_v1 > (
+                    exploration_stats.num_starts_v1):
+                exploration_stats.num_starts_v1 = max([
+                    exploration_stats.state_stats_mapping[
+                        state_name].first_hit_count_v1
+                    for state_name in exploration_stats.state_stats_mapping])
+
             exploration_stats_by_version_updated.append(
                 exploration_stats.to_dict())
 
+        refresh = False
+        # Check if number of actual starts is still greater than the number of
+        # starts for any version of the exploration.
+        for exploration_stats_dict in exploration_stats_by_version_updated:
+            if exploration_stats_dict['num_actual_starts_v1'] > (
+                    exploration_stats_dict['num_starts_v1']):
+                yield (
+                    'LOG: Refreshing statistics due to actual starts being '
+                    'greater than starts for exploration with ID %s ' % (
+                        exp_id))
+                refresh = True
+                break
+
+        # Check if number of actual starts is lesser than the number of
+        # completions for any version of the exploration.
+        for exploration_stats_dict in exploration_stats_by_version_updated:
+            if exploration_stats_dict['num_actual_starts_v1'] < (
+                    exploration_stats_dict['num_completions_v1']):
+                yield (
+                    'LOG: Refreshing statistics due to actual starts being '
+                    'lesser than completions for exploration with ID %s ' % (
+                        exp_id))
+                refresh = True
+                break
+
+        # Refresh statistics for all versions of the exploration.
+        if refresh:
+            for index, exploration_stats_dict in enumerate(
+                    exploration_stats_by_version_updated):
+                default_state_stats_mapping = {}
+                for state_name in exploration_stats_dict['state_stats_mapping']:
+                    default_state_stats_mapping[state_name] = (
+                        stats_domain.StateStats.create_default())
+                exploration_stats_by_version_updated[index] = (
+                    stats_domain.ExplorationStats.create_default(
+                        exp_id, index + 1,
+                        default_state_stats_mapping).to_dict())
+
         # Save all the ExplorationStats models to storage.
         stats_models.ExplorationStatsModel.save_multi(
             exploration_stats_by_version_updated)


Test patch:
diff --git a/core/domain/stats_jobs_one_off_test.py b/core/domain/stats_jobs_one_off_test.py
--- a/core/domain/stats_jobs_one_off_test.py
+++ b/core/domain/stats_jobs_one_off_test.py
@@ -470,3 +470,50 @@ def test_none_version_uses_snapshot_timestamp(self):
         self.assertEqual(
             exploration_stats.state_stats_mapping[
                 'New 2'].total_hit_count_v1, 1)
+
+    def test_higher_completions_over_actual_starts_refreshes_stats(self):
+        stats_models.CompleteExplorationEventLogEntryModel.create(
+            self.exp_id, self.exploration.version, 'End', 'session_id3',
+            10, {}, feconf.PLAY_TYPE_NORMAL)
+
+        job_id = stats_jobs_one_off.GenerateV1StatisticsJob.create_new()
+        stats_jobs_one_off.GenerateV1StatisticsJob.enqueue(job_id)
+
+        self.assertEqual(self.count_jobs_in_taskqueue(
+            taskqueue_services.QUEUE_NAME_ONE_OFF_JOBS), 1)
+        self.process_and_flush_pending_tasks()
+
+        exploration_stats = stats_services.get_exploration_stats_by_id(
+            self.exp_id, self.exploration.version)
+        self.assertEqual(exploration_stats.num_starts_v1, 0)
+        self.assertEqual(exploration_stats.num_completions_v1, 0)
+        self.assertEqual(exploration_stats.num_actual_starts_v1, 0)
+
+        self.assertEqual(
+            exploration_stats.state_stats_mapping['Home'].first_hit_count_v1, 0)
+        self.assertEqual(
+            exploration_stats.state_stats_mapping['End'].first_hit_count_v1, 0)
+
+        self.assertEqual(
+            exploration_stats.state_stats_mapping['Home'].total_hit_count_v1, 0)
+        self.assertEqual(
+            exploration_stats.state_stats_mapping['End'].total_hit_count_v1, 0)
+
+        self.assertEqual(
+            exploration_stats.state_stats_mapping['Home'].num_completions_v1, 0)
+        self.assertEqual(
+            exploration_stats.state_stats_mapping['End'].num_completions_v1, 0)
+
+        self.assertEqual(
+            exploration_stats.state_stats_mapping[
+                'Home'].total_answers_count_v1, 0)
+        self.assertEqual(
+            exploration_stats.state_stats_mapping['End'].total_answers_count_v1,
+            0)
+
+        self.assertEqual(
+            exploration_stats.state_stats_mapping[
+                'Home'].useful_feedback_count_v1, 0)
+        self.assertEqual(
+            exploration_stats.state_stats_mapping[
+                'End'].useful_feedback_count_v1, 0)