Repository: oppia/oppia

Issue:
Running the e2e test script results in residual changes in the constants.ts file.
**Describe the bug**

When the e2e test script is run without skip-install/skip-build (and whether it exits successfully or not), this results in a local change being made to assets/constants.ts: DEV_MODE is changed from true to false.

This is not great: tests should not be changing code, and the code changes can get included by mistake in pushed commits. The e2e tests should reset the constants.ts file prior to exiting, regardless of whether the exit is a natural one or one caused by an exception.

**To Reproduce**
Steps to reproduce the behavior:
 1. Run `python -m scripts.run_e2e_tests --suite="preferences" --prod_env`. (You can replace "preferences" with any suite name.)
 2. Run `git status` when the tests finish.

**Observed behavior**
There is a single file changed: assets/constants.ts.

**Expected behavior**
No change should happen in any local files.
Running the e2e test script results in residual changes in the constants.ts file.
**Describe the bug**

When the e2e test script is run without skip-install/skip-build (and whether it exits successfully or not), this results in a local change being made to assets/constants.ts: DEV_MODE is changed from true to false.

This is not great: tests should not be changing code, and the code changes can get included by mistake in pushed commits. The e2e tests should reset the constants.ts file prior to exiting, regardless of whether the exit is a natural one or one caused by an exception.

**To Reproduce**
Steps to reproduce the behavior:
 1. Run `python -m scripts.run_e2e_tests --suite="preferences" --prod_env`. (You can replace "preferences" with any suite name.)
 2. Run `git status` when the tests finish.

**Observed behavior**
There is a single file changed: assets/constants.ts.

**Expected behavior**
No change should happen in any local files.


Discussion:
/cc @kaylahardie @MegrezZhu 
/cc @kaylahardie @MegrezZhu 

Resolving patch:
diff --git a/scripts/build.py b/scripts/build.py
--- a/scripts/build.py
+++ b/scripts/build.py
@@ -198,7 +198,7 @@ def generate_app_yaml(deploy_mode=False, maintenance_mode=False):
         prod_yaml_file.write(content)
 
 
-def modify_constants(prod_env, maintenance_mode):
+def modify_constants(prod_env=False, maintenance_mode=False):
     """Modify constants.ts and feconf.py.
 
     Args:
@@ -219,6 +219,11 @@ def modify_constants(prod_env, maintenance_mode):
         enable_maintenance_mode_variable)
 
 
+def set_constants_to_default():
+    """Set variables in constants.ts and feconf.py to default values."""
+    modify_constants(prod_env=False, maintenance_mode=False)
+
+
 def _minify(source_path, target_path):
     """Runs the given file through a minifier and outputs it to target_path.
 
@@ -1317,7 +1322,8 @@ def main(args=None):
                 'minify_third_party_libs_only should not be '
                 'set in non-prod env.')
 
-    modify_constants(options.prod_env, options.maintenance_mode)
+    modify_constants(
+        prod_env=options.prod_env, maintenance_mode=options.maintenance_mode)
     if options.prod_env:
         minify_third_party_libs(THIRD_PARTY_GENERATED_DEV_DIR)
         hashes = generate_hashes()
diff --git a/scripts/start.py b/scripts/start.py
--- a/scripts/start.py
+++ b/scripts/start.py
@@ -75,12 +75,15 @@
 
 
 def cleanup():
-    """Function for waiting for the servers to go down."""
+    """Wait for the servers to go down and set constants back to default
+    values.
+    """
     common.print_each_string_after_two_new_lines([
         'INFORMATION',
         'Cleaning up the servers.'])
     while common.is_port_open(PORT_NUMBER_FOR_GAE_SERVER):
         time.sleep(1)
+    build.set_constants_to_default()
 
 
 def main(args=None):


Test patch:
diff --git a/scripts/build_test.py b/scripts/build_test.py
--- a/scripts/build_test.py
+++ b/scripts/build_test.py
@@ -793,7 +793,7 @@ def test_modify_constants(self):
             tmp.write(u'ENABLE_MAINTENANCE_MODE = False')
 
         with constants_path_swap, feconf_path_swap:
-            build.modify_constants(True, False)
+            build.modify_constants(prod_env=True, maintenance_mode=False)
             with python_utils.open_file(
                 mock_constants_path, 'r') as constants_file:
                 self.assertEqual(
@@ -805,7 +805,7 @@ def test_modify_constants(self):
                 self.assertEqual(
                     feconf_file.read(), 'ENABLE_MAINTENANCE_MODE = False')
 
-            build.modify_constants(False, True)
+            build.modify_constants(prod_env=False, maintenance_mode=True)
             with python_utils.open_file(
                 mock_constants_path, 'r') as constants_file:
                 self.assertEqual(
@@ -820,6 +820,41 @@ def test_modify_constants(self):
         constants_temp_file.close()
         feconf_temp_file.close()
 
+    def test_set_constants_to_default(self):
+        mock_constants_path = 'mock_app_dev.yaml'
+        mock_feconf_path = 'mock_app.yaml'
+        constants_path_swap = self.swap(
+            common, 'CONSTANTS_FILE_PATH', mock_constants_path)
+        feconf_path_swap = self.swap(common, 'FECONF_PATH', mock_feconf_path)
+
+        constants_temp_file = tempfile.NamedTemporaryFile()
+        constants_temp_file.name = mock_constants_path
+        with python_utils.open_file(mock_constants_path, 'w') as tmp:
+            tmp.write(u'export = {\n')
+            tmp.write(u'  "DEV_MODE": false\n')
+            tmp.write(u'};')
+
+        feconf_temp_file = tempfile.NamedTemporaryFile()
+        feconf_temp_file.name = mock_feconf_path
+        with python_utils.open_file(mock_feconf_path, 'w') as tmp:
+            tmp.write(u'ENABLE_MAINTENANCE_MODE = True')
+
+        with constants_path_swap, feconf_path_swap:
+            build.set_constants_to_default()
+            with python_utils.open_file(
+                mock_constants_path, 'r') as constants_file:
+                self.assertEqual(
+                    constants_file.read(),
+                    'export = {\n'
+                    '  "DEV_MODE": true\n'
+                    '};')
+            with python_utils.open_file(mock_feconf_path, 'r') as feconf_file:
+                self.assertEqual(
+                    feconf_file.read(), 'ENABLE_MAINTENANCE_MODE = False')
+
+        constants_temp_file.close()
+        feconf_temp_file.close()
+
     def test_safe_delete_file(self):
         temp_file = tempfile.NamedTemporaryFile()
         temp_file.name = 'some_file.txt'
@@ -892,7 +927,7 @@ def mock_build_using_webpack(config_path):
         def mock_ensure_files_exist(unused_filepaths):
             check_function_calls['ensure_files_exist_gets_called'] = True
 
-        def mock_modify_constants(unused_prod_env, unused_maintenance_mode):
+        def mock_modify_constants(prod_env, maintenance_mode):  # pylint: disable=unused-argument
             check_function_calls['modify_constants_gets_called'] = True
 
         def mock_compare_file_count(unused_first_dir, unused_second_dir):
@@ -936,7 +971,7 @@ def mock_build_using_webpack(config_path):
         def mock_ensure_files_exist(unused_filepaths):
             check_function_calls['ensure_files_exist_gets_called'] = True
 
-        def mock_modify_constants(unused_prod_env, unused_maintenance_mode):
+        def mock_modify_constants(prod_env, maintenance_mode):  # pylint: disable=unused-argument
             check_function_calls['modify_constants_gets_called'] = True
 
         def mock_compare_file_count(unused_first_dir, unused_second_dir):
@@ -970,7 +1005,7 @@ def test_build_with_watcher(self):
         def mock_ensure_files_exist(unused_filepaths):
             check_function_calls['ensure_files_exist_gets_called'] = True
 
-        def mock_modify_constants(unused_prod_env, unused_maintenance_mode):
+        def mock_modify_constants(prod_env, maintenance_mode):  # pylint: disable=unused-argument
             check_function_calls['modify_constants_gets_called'] = True
 
         ensure_files_exist_swap = self.swap(
@@ -1023,7 +1058,7 @@ def test_only_minify_third_party_libs_in_dev_mode(self):
         def mock_ensure_files_exist(unused_filepaths):
             check_function_calls['ensure_files_exist_gets_called'] = True
 
-        def mock_modify_constants(unused_prod_env, unused_maintenance_mode):
+        def mock_modify_constants(unused_prod_env, maintenance_mode):  # pylint: disable=unused-argument
             check_function_calls['ensure_modify_constants_gets_called'] = True
 
         ensure_files_exist_swap = self.swap(
diff --git a/scripts/run_e2e_tests.py b/scripts/run_e2e_tests.py
--- a/scripts/run_e2e_tests.py
+++ b/scripts/run_e2e_tests.py
@@ -163,7 +163,9 @@ def ensure_screenshots_dir_is_removed():
 
 
 def cleanup():
-    """Kill the running subprocesses and server fired in this program."""
+    """Kill the running subprocesses and server fired in this program, set
+    constants back to default values.
+    """
     google_app_engine_path = '%s/' % common.GOOGLE_APP_ENGINE_HOME
     webdriver_download_path = '%s/downloads' % WEBDRIVER_HOME_PATH
     if common.is_windows_os():
@@ -178,6 +180,7 @@ def cleanup():
 
     for p in processes_to_kill:
         common.kill_processes_based_on_regex(p)
+    build.set_constants_to_default()
 
 
 def is_oppia_server_already_running():
@@ -473,7 +476,9 @@ def main(args=None):
     update_community_dashboard_status_in_feconf_file(
         FECONF_FILE_PATH, parsed_args.community_dashboard_enabled)
 
-    if not parsed_args.skip_build:
+    if parsed_args.skip_build:
+        build.modify_constants(prod_env=parsed_args.prod_env)
+    else:
         build_js_files(
             dev_mode, deparallelize_terser=parsed_args.deparallelize_terser)
     version = (
diff --git a/scripts/run_e2e_tests_test.py b/scripts/run_e2e_tests_test.py
--- a/scripts/run_e2e_tests_test.py
+++ b/scripts/run_e2e_tests_test.py
@@ -135,11 +135,17 @@ def mock_rmdir(unused_path):
             run_e2e_tests.ensure_screenshots_dir_is_removed()
 
     def test_cleanup_when_no_subprocess(self):
+
+        def mock_kill_process_based_on_regex(unused_regex):
+            return
+
         def mock_is_windows_os():
             return False
 
-        subprocess_swap = self.swap(run_e2e_tests, 'SUBPROCESSES', [])
+        def mock_set_constants_to_default():
+            return
 
+        subprocess_swap = self.swap(run_e2e_tests, 'SUBPROCESSES', [])
 
         google_app_engine_path = '%s/' % (
             common.GOOGLE_APP_ENGINE_HOME)
@@ -149,8 +155,6 @@ def mock_is_windows_os():
             ('.*%s.*' % re.escape(google_app_engine_path),),
             ('.*%s.*' % re.escape(webdriver_download_path),)
         ]
-        def mock_kill_process_based_on_regex(unused_regex):
-            return
 
         swap_kill_process = self.swap_with_checks(
             common, 'kill_processes_based_on_regex',
@@ -158,8 +162,11 @@ def mock_kill_process_based_on_regex(unused_regex):
             expected_args=process_pattern)
         swap_is_windows = self.swap_with_checks(
             common, 'is_windows_os', mock_is_windows_os)
+        swap_set_constants_to_default = self.swap_with_checks(
+            build, 'set_constants_to_default', mock_set_constants_to_default)
         with swap_kill_process, subprocess_swap, swap_is_windows:
-            run_e2e_tests.cleanup()
+            with swap_set_constants_to_default:
+                run_e2e_tests.cleanup()
 
     def test_cleanup_when_subprocesses_exist(self):
 
@@ -168,21 +175,30 @@ def mock_kill_process_based_on_regex(unused_regex):
             return True
         mock_kill_process_based_on_regex.called_times = 0
 
+        def mock_set_constants_to_default():
+            return
+
         mock_processes = [MockProcessClass(), MockProcessClass()]
         subprocess_swap = self.swap(
             run_e2e_tests, 'SUBPROCESSES', mock_processes)
         swap_kill_process = self.swap_with_checks(
             common, 'kill_processes_based_on_regex',
             mock_kill_process_based_on_regex)
-        with subprocess_swap, swap_kill_process:
+        swap_set_constants_to_default = self.swap_with_checks(
+            build, 'set_constants_to_default', mock_set_constants_to_default)
+        with subprocess_swap, swap_kill_process, swap_set_constants_to_default:
             run_e2e_tests.cleanup()
         self.assertEqual(
             mock_kill_process_based_on_regex.called_times, len(mock_processes))
 
     def test_cleanup_on_windows(self):
+
         def mock_is_windows_os():
             return True
 
+        def mock_set_constants_to_default():
+            return
+
         subprocess_swap = self.swap(run_e2e_tests, 'SUBPROCESSES', [])
 
         google_app_engine_path = '%s/' % (
@@ -205,8 +221,11 @@ def mock_kill_process_based_on_regex(unused_regex):
             expected_args=expected_pattern)
         swap_is_windows = self.swap_with_checks(
             common, 'is_windows_os', mock_is_windows_os)
+        swap_set_constants_to_default = self.swap_with_checks(
+            build, 'set_constants_to_default', mock_set_constants_to_default)
         with swap_kill_process, subprocess_swap, swap_is_windows:
-            run_e2e_tests.cleanup()
+            with swap_set_constants_to_default:
+                run_e2e_tests.cleanup()
 
     def test_is_oppia_server_already_running_when_ports_closed(self):
         def mock_is_port_open(unused_port):
@@ -799,6 +818,97 @@ def mock_exit(unused_code):
                         with get_parameters_swap, popen_swap, exit_swap:
                             run_e2e_tests.main(args=[])
 
+    def test_start_tests_skip_build(self):
+
+        def mock_is_oppia_server_already_running(*unused_args):
+            return False
+
+        def mock_setup_and_install_dependencies(unused_arg):
+            return
+
+        def mock_register(unused_func):
+            return
+
+        def mock_cleanup():
+            return
+
+        def mock_modify_constants(prod_env, maintenance_mode=False):  # pylint: disable=unused-argument
+            return
+
+        def mock_start_webdriver_manager(unused_arg):
+            return
+
+        def mock_start_google_app_engine_server(unused_arg, unused_log_level):
+            return
+
+        def mock_wait_for_port_to_be_open(unused_port):
+            return
+
+        def mock_ensure_screenshots_dir_is_removed():
+            return
+
+        def mock_get_e2e_test_parameters(
+                unused_sharding_instances, unused_suite, unused_dev_mode):
+            return ['commands']
+
+        def mock_popen(unused_commands):
+            def mock_communicate():
+                return
+            result = MockProcessClass()
+            result.communicate = mock_communicate # pylint: disable=attribute-defined-outside-init
+            result.returncode = 0 # pylint: disable=attribute-defined-outside-init
+            return result
+
+        def mock_exit(unused_code):
+            return
+
+        check_swap = self.swap_with_checks(
+            run_e2e_tests, 'is_oppia_server_already_running',
+            mock_is_oppia_server_already_running)
+        setup_and_install_swap = self.swap_with_checks(
+            run_e2e_tests, 'setup_and_install_dependencies',
+            mock_setup_and_install_dependencies, expected_args=[(True,)])
+        register_swap = self.swap_with_checks(
+            atexit, 'register', mock_register, expected_args=[(mock_cleanup,)])
+        cleanup_swap = self.swap(run_e2e_tests, 'cleanup', mock_cleanup)
+        modify_constants_swap = self.swap_with_checks(
+            build, 'modify_constants', mock_modify_constants,
+            expected_kwargs=[{'prod_env': False}])
+        start_webdriver_swap = self.swap_with_checks(
+            run_e2e_tests, 'start_webdriver_manager',
+            mock_start_webdriver_manager,
+            expected_args=[(run_e2e_tests.CHROME_DRIVER_VERSION,)])
+        start_google_app_engine_server_swap = self.swap_with_checks(
+            run_e2e_tests, 'start_google_app_engine_server',
+            mock_start_google_app_engine_server,
+            expected_args=[(True, 'critical')])
+        wait_swap = self.swap_with_checks(
+            run_e2e_tests, 'wait_for_port_to_be_open',
+            mock_wait_for_port_to_be_open,
+            expected_args=[
+                (run_e2e_tests.WEB_DRIVER_PORT,),
+                (run_e2e_tests.GOOGLE_APP_ENGINE_PORT,)])
+        ensure_screenshots_dir_is_removed_swap = self.swap_with_checks(
+            run_e2e_tests, 'ensure_screenshots_dir_is_removed',
+            mock_ensure_screenshots_dir_is_removed)
+        get_parameters_swap = self.swap_with_checks(
+            run_e2e_tests, 'get_e2e_test_parameters',
+            mock_get_e2e_test_parameters, expected_args=[(3, 'full', True)])
+        popen_swap = self.swap_with_checks(
+            subprocess, 'Popen', mock_popen, expected_args=[([
+                common.NODE_BIN_PATH, '--unhandled-rejections=strict',
+                run_e2e_tests.PROTRACTOR_BIN_PATH,
+                'commands'],)])
+        exit_swap = self.swap_with_checks(
+            sys, 'exit', mock_exit, expected_args=[(0,)])
+        with check_swap, setup_and_install_swap, register_swap, cleanup_swap:
+            with modify_constants_swap, start_webdriver_swap:
+                with start_google_app_engine_server_swap:
+                    with wait_swap, ensure_screenshots_dir_is_removed_swap:
+                        with get_parameters_swap, popen_swap, exit_swap:
+                            run_e2e_tests.main(
+                                args=['--skip-install', '--skip-build'])
+
     def test_start_tests_in_debug_mode(self):
 
         def mock_is_oppia_server_already_running(*unused_args):