Repository: oppia/oppia

Issue:
Changing a state's interaction should stop old answers from appearing in the stats tab
This was found as part of the regression testpass for the answer migration.

When answers are submitted to a specific state S with interaction type A, the answers still show up for S's statistics tab even if the interaction type of that state changes to type B (though no answers will show up in the statistics tab after switching the interaction until after another run of the answer summarizers job completes). Old answers submitted to interaction A will stop appearing in the stats tab once a new answer has been submitted for type B. Although this is WAI, there is slight potential user confusion in the gap of time where old interaction answers show up for the state, especially since there have not been answers submitted to the state's new interaction.


Discussion:

Resolving patch:
diff --git a/core/domain/stats_jobs_continuous.py b/core/domain/stats_jobs_continuous.py
--- a/core/domain/stats_jobs_continuous.py
+++ b/core/domain/stats_jobs_continuous.py
@@ -607,6 +607,25 @@ def reduce(key, stringified_values):
         # will refer to the most relevant answers.
         latest_version = versions[0]
         latest_interaction_id = versioned_interaction_ids[latest_version][0]
+
+        if exploration_version == VERSION_ALL:
+            # If aggregating across all versions, verify that the latest answer
+            # version is equal to the latest version of the exploration,
+            # otherwise ignore all answers since none of them can be applied to
+            # the latest version.
+            exp = exp_services.get_exploration_by_id(exploration_id)
+            if state_name in exp.states:
+                loaded_interaction_id = exp.states[state_name].interaction.id
+                # Only check if the version mismatches if the new version has a
+                # different interaction ID.
+                if latest_interaction_id != loaded_interaction_id and (
+                        latest_version != exp.version):
+                    yield (
+                        'Ignoring answers submitted to version %s and below '
+                        'since the latest exploration version is %s' % (
+                            latest_version, exp.version))
+                    versions = []
+
         # In the VERSION_ALL case, we only take into account the most recent
         # consecutive block of versions with the same interaction ID as the
         # current version, and ignore all versions prior to this block. This


Test patch:
diff --git a/core/domain/stats_jobs_continuous_test.py b/core/domain/stats_jobs_continuous_test.py
--- a/core/domain/stats_jobs_continuous_test.py
+++ b/core/domain/stats_jobs_continuous_test.py
@@ -665,7 +665,7 @@ def test_answers_across_multiple_exploration_versions(self):
                 calculation_output_all,
                 expected_calculation_output_all_answers)
 
-    def test_answers_across_multiple_exp_versions_different_interactions(self):
+    def test_ignores_old_answers_if_new_interaction_has_no_new_answers(self):
         """Similar to test_answers_across_multiple_exploration_versions except
         the exploration has changed interactions in the new versions. The
         aggregation job should not include answers corresponding to exploration
@@ -705,6 +705,196 @@ def test_answers_across_multiple_exp_versions_different_interactions(self):
                 'new_value': 'NumericInput',
             }], 'Change to NumericInput')
 
+            exp = exp_services.get_exploration_by_id(exp_id)
+            self.assertEqual(exp.version, 2)
+
+            # Run the answers aggregation job.
+            ModifiedInteractionAnswerSummariesAggregator.start_computation()
+            self.assertEqual(
+                self.count_jobs_in_taskqueue(
+                    taskqueue_services.QUEUE_NAME_CONTINUOUS_JOBS), 1)
+            self.process_and_flush_pending_tasks()
+            self.assertEqual(
+                self.count_jobs_in_taskqueue(
+                    taskqueue_services.QUEUE_NAME_CONTINUOUS_JOBS), 0)
+
+            calc_id = 'Top10AnswerFrequencies'
+
+            # Check the output of the job.
+            calc_output_latest_version_domain_object = (
+                stats_jobs_continuous.InteractionAnswerSummariesAggregator.get_calc_output( # pylint: disable=line-too-long
+                    exp_id, init_state_name, calc_id,
+                    exploration_version='2'))
+            calc_output_first_version_domain_object = (
+                stats_jobs_continuous.InteractionAnswerSummariesAggregator.get_calc_output( # pylint: disable=line-too-long
+                    exp_id, init_state_name, calc_id,
+                    exploration_version='1'))
+            calc_output_all_domain_object = (
+                stats_jobs_continuous.InteractionAnswerSummariesAggregator.get_calc_output( # pylint: disable=line-too-long
+                    exp_id, init_state_name, calc_id))
+
+            # Since no answers were submitted to the latest version of the
+            # exploration, there should be no calculated output for it.
+            self.assertIsNone(calc_output_latest_version_domain_object)
+
+            # Top answers will still be computed for the first version.
+            self.assertEqual(
+                'Top10AnswerFrequencies',
+                calc_output_first_version_domain_object.calculation_id)
+            calculation_output_first = (
+                calc_output_first_version_domain_object.calculation_output)
+            expected_calculation_output_first_answer = [{
+                'answer': 'verb',
+                'frequency': 2
+            }, {
+                'answer': '2',
+                'frequency': 1
+            }]
+            self.assertEqual(
+                calculation_output_first,
+                expected_calculation_output_first_answer)
+
+            self.assertEqual(
+                'Top10AnswerFrequencies',
+                calc_output_all_domain_object.calculation_id)
+
+            # No answers should be aggregated since all past answers do not
+            # match the newly submitted interaction ID.
+            calculation_output_all = (
+                calc_output_all_domain_object.calculation_output)
+            self.assertEqual(calculation_output_all, [])
+
+    def test_uses_old_answers_if_updated_exploration_has_same_interaction(self):
+        """Similar to
+        test_ignores_old_answers_if_new_interaction_has_no_new_answers except
+        this is demonstrating that if an exploration is updated and no new
+        answers are submitted to the new version, but the interaction ID is the
+        same then old answers should still be aggregated.
+        """
+        with self.swap(
+            jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS',
+            self.ALL_CC_MANAGERS_FOR_TESTS):
+
+            # Setup example exploration.
+            exp_id = 'eid'
+            exp = self.save_new_valid_exploration(exp_id, 'fake@user.com')
+            init_state_name = exp.init_state_name
+
+            time_spent = 5.0
+            params = {}
+
+            # Add a few different answers.
+            event_services.AnswerSubmissionEventHandler.record(
+                exp_id, 1, init_state_name, 'TextInput', 0, 0,
+                exp_domain.EXPLICIT_CLASSIFICATION, 'session1', time_spent,
+                params, 'verb')
+            event_services.AnswerSubmissionEventHandler.record(
+                exp_id, 1, init_state_name, 'TextInput', 0, 0,
+                exp_domain.EXPLICIT_CLASSIFICATION, 'session1', time_spent,
+                params, '2')
+            event_services.AnswerSubmissionEventHandler.record(
+                exp_id, 1, init_state_name, 'TextInput', 0, 0,
+                exp_domain.EXPLICIT_CLASSIFICATION, 'session1', time_spent,
+                params, 'verb')
+
+            # Change something other than the interaction ID.
+            exp_services.update_exploration('fake@user.com', exp_id, [{
+                'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY,
+                'state_name': init_state_name,
+                'property_name': exp_domain.STATE_PROPERTY_CONTENT,
+                'new_value': {
+                    'html': 'New content',
+                    'audio_translations': {}
+                },
+            }], 'Change state content')
+
+            exp = exp_services.get_exploration_by_id(exp_id)
+            self.assertEqual(exp.version, 2)
+
+            # Run the answers aggregation job.
+            ModifiedInteractionAnswerSummariesAggregator.start_computation()
+            self.assertEqual(
+                self.count_jobs_in_taskqueue(
+                    taskqueue_services.QUEUE_NAME_CONTINUOUS_JOBS), 1)
+            self.process_and_flush_pending_tasks()
+            self.assertEqual(
+                self.count_jobs_in_taskqueue(
+                    taskqueue_services.QUEUE_NAME_CONTINUOUS_JOBS), 0)
+
+            calc_id = 'Top10AnswerFrequencies'
+
+            # Extract the output from the job.
+            calc_output_latest_version_domain_object = (
+                stats_jobs_continuous.InteractionAnswerSummariesAggregator.get_calc_output( # pylint: disable=line-too-long
+                    exp_id, init_state_name, calc_id,
+                    exploration_version='2'))
+            calc_output_all_domain_object = (
+                stats_jobs_continuous.InteractionAnswerSummariesAggregator.get_calc_output( # pylint: disable=line-too-long
+                    exp_id, init_state_name, calc_id))
+
+            # Since no answers were submitted to the latest version of the
+            # exploration, there should be no calculated output for it.
+            self.assertIsNone(calc_output_latest_version_domain_object)
+
+            self.assertEqual(
+                'Top10AnswerFrequencies',
+                calc_output_all_domain_object.calculation_id)
+            calculation_output_all = (
+                calc_output_all_domain_object.calculation_output)
+            expected_calculation_output_all_answers = [{
+                'answer': 'verb',
+                'frequency': 2
+            }, {
+                'answer': '2',
+                'frequency': 1
+            }]
+            self.assertEqual(
+                calculation_output_all,
+                expected_calculation_output_all_answers)
+
+    def test_answers_across_multiple_exp_versions_different_interactions(self):
+        """Same as
+        test_ignores_old_answers_if_new_interaction_has_no_new_answers except
+        this also adds additional answers after changing the interaction a few
+        times to ensure the aggregation job does not include answers across
+        interaction changes, even if the interaction reverts back to a past
+        interaction type with answers submitted to both versions of the
+        exploration.
+        """
+        with self.swap(
+            jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS',
+            self.ALL_CC_MANAGERS_FOR_TESTS):
+
+            # Setup example exploration.
+            exp_id = 'eid'
+            exp = self.save_new_valid_exploration(exp_id, 'fake@user.com')
+            init_state_name = exp.init_state_name
+
+            time_spent = 5.0
+            params = {}
+
+            # Add a few different answers.
+            event_services.AnswerSubmissionEventHandler.record(
+                exp_id, 1, init_state_name, 'TextInput', 0, 0,
+                exp_domain.EXPLICIT_CLASSIFICATION, 'session1', time_spent,
+                params, 'verb')
+            event_services.AnswerSubmissionEventHandler.record(
+                exp_id, 1, init_state_name, 'TextInput', 0, 0,
+                exp_domain.EXPLICIT_CLASSIFICATION, 'session1', time_spent,
+                params, '2')
+            event_services.AnswerSubmissionEventHandler.record(
+                exp_id, 1, init_state_name, 'TextInput', 0, 0,
+                exp_domain.EXPLICIT_CLASSIFICATION, 'session1', time_spent,
+                params, 'verb')
+
+            # Change the interaction ID.
+            exp_services.update_exploration('fake@user.com', exp_id, [{
+                'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY,
+                'state_name': init_state_name,
+                'property_name': exp_domain.STATE_PROPERTY_INTERACTION_ID,
+                'new_value': 'NumericInput',
+            }], 'Change to NumericInput')
+
             # Submit an answer to the numeric interaction.
             event_services.AnswerSubmissionEventHandler.record(
                 exp_id, 2, init_state_name, 'NumericInput', 0, 0,