Repository: oppia/oppia

Issue:
Webpack is timeouting in e2e tests
Webpack for some reason takes more than 10 minutes to compile in some of our e2e tests

https://app.circleci.com/pipelines/github/oppia/oppia/8919/workflows/840dbb95-3b95-4e85-8653-5a3b48757dfc/jobs/43052
Webpack is timeouting in e2e tests
Webpack for some reason takes more than 10 minutes to compile in some of our e2e tests

https://app.circleci.com/pipelines/github/oppia/oppia/8919/workflows/840dbb95-3b95-4e85-8653-5a3b48757dfc/jobs/43052


Discussion:
Something similar happened in #8976 ; see https://github.com/oppia/oppia/pull/8976#issuecomment-610718690
Something similar happened in #8976 ; see https://github.com/oppia/oppia/pull/8976#issuecomment-610718690

Resolving patch:
diff --git a/scripts/build.py b/scripts/build.py
--- a/scripts/build.py
+++ b/scripts/build.py
@@ -82,6 +82,7 @@
 UGLIFY_FILE = os.path.join('node_modules', 'uglify-js', 'bin', 'uglifyjs')
 WEBPACK_FILE = os.path.join('node_modules', 'webpack', 'bin', 'webpack.js')
 WEBPACK_PROD_CONFIG = 'webpack.prod.config.ts'
+WEBPACK_TERSER_CONFIG = 'webpack.terser.config.ts'
 
 # Files with these extensions shouldn't be moved to build directory.
 FILE_EXTENSIONS_TO_IGNORE = ('.py', '.pyc', '.stylelintrc', '.ts')
@@ -145,6 +146,9 @@
 _PARSER.add_argument(
     '--minify_third_party_libs_only', action='store_true', default=False,
     dest='minify_third_party_libs_only')
+_PARSER.add_argument(
+    '--deparallelize_terser', action='store_true', default=False,
+    dest='deparallelize_terser')
 
 
 def generate_app_yaml(deploy_mode=False):
@@ -571,19 +575,20 @@ def build_third_party_libs(third_party_directory_path):
             dependency_filepaths['fonts'], WEBFONTS_DIR))
 
 
-def build_using_webpack():
+def build_using_webpack(config_path):
     """Execute webpack build process. This takes all TypeScript files we have in
     /templates and generates JS bundles according the require() imports
     and also compiles HTML pages into the /backend_prod_files/webpack_bundles
     folder. The files are later copied into /build/webpack_bundles.
 
-    The settings for this are specified in webpack.prod.config.ts.
+    Args:
+        config_path: str. Webpack config to be used for building.
     """
 
     python_utils.PRINT('Building webpack')
 
     cmd = '%s %s --config %s' % (
-        common.NODE_BIN_PATH, WEBPACK_FILE, WEBPACK_PROD_CONFIG)
+        common.NODE_BIN_PATH, WEBPACK_FILE, config_path)
     subprocess.check_call(cmd, shell=True)
 
 
@@ -1265,7 +1270,10 @@ def main(args=None):
         minify_third_party_libs(THIRD_PARTY_GENERATED_DEV_DIR)
         if not options.minify_third_party_libs_only:
             hashes = generate_hashes()
-            build_using_webpack()
+            if options.deparallelize_terser:
+                build_using_webpack(WEBPACK_TERSER_CONFIG)
+            else:
+                build_using_webpack(WEBPACK_PROD_CONFIG)
             generate_app_yaml(deploy_mode=options.deploy_mode)
             generate_build_directory(hashes)
 


Test patch:
diff --git a/scripts/build_test.py b/scripts/build_test.py
--- a/scripts/build_test.py
+++ b/scripts/build_test.py
@@ -796,7 +796,10 @@ def test_build_with_prod_env(self):
             'compare_file_count_gets_called': True
         }
 
-        def mock_build_using_webpack():
+        expected_config_path = build.WEBPACK_PROD_CONFIG
+
+        def mock_build_using_webpack(config_path):
+            self.assertEqual(config_path, expected_config_path)
             check_function_calls['build_using_webpack_gets_called'] = True
 
         def mock_ensure_files_exist(unused_filepaths):
@@ -818,6 +821,43 @@ def mock_compare_file_count(unused_first_dir, unused_second_dir):
 
         self.assertEqual(check_function_calls, expected_check_function_calls)
 
+    def test_build_with_deparallelize_terser(self):
+        check_function_calls = {
+            'build_using_webpack_gets_called': False,
+            'ensure_files_exist_gets_called': False,
+            'compare_file_count_gets_called': False
+        }
+        expected_check_function_calls = {
+            'build_using_webpack_gets_called': True,
+            'ensure_files_exist_gets_called': True,
+            'compare_file_count_gets_called': True
+        }
+
+        expected_config_path = build.WEBPACK_TERSER_CONFIG
+
+        def mock_build_using_webpack(config_path):
+            self.assertEqual(config_path, expected_config_path)
+            check_function_calls['build_using_webpack_gets_called'] = True
+
+        def mock_ensure_files_exist(unused_filepaths):
+            check_function_calls['ensure_files_exist_gets_called'] = True
+
+        def mock_compare_file_count(unused_first_dir, unused_second_dir):
+            check_function_calls['compare_file_count_gets_called'] = True
+
+        ensure_files_exist_swap = self.swap(
+            build, '_ensure_files_exist', mock_ensure_files_exist)
+        build_using_webpack_swap = self.swap(
+            build, 'build_using_webpack', mock_build_using_webpack)
+        compare_file_count_swap = self.swap(
+            build, '_compare_file_count', mock_compare_file_count)
+
+        with ensure_files_exist_swap, build_using_webpack_swap, (
+            compare_file_count_swap):
+            build.main(args=['--prod_env', '--deparallelize_terser'])
+
+        self.assertEqual(check_function_calls, expected_check_function_calls)
+
     def test_build_with_watcher(self):
         check_function_calls = {
             'ensure_files_exist_gets_called': False,
@@ -866,6 +906,6 @@ def mock_check_call(cmd, **unused_kwargs):
             self.assertEqual(cmd, expected_command)
 
         with self.swap(subprocess, 'check_call', mock_check_call):
-            build.build_using_webpack()
+            build.build_using_webpack(build.WEBPACK_PROD_CONFIG)
 
 # pylint: enable=protected-access
diff --git a/scripts/run_e2e_tests.py b/scripts/run_e2e_tests.py
--- a/scripts/run_e2e_tests.py
+++ b/scripts/run_e2e_tests.py
@@ -126,6 +126,11 @@
          'https://www.protractortest.org/#/debugging#disabled-control-flow',
     action='store_true')
 
+_PARSER.add_argument(
+    '--deparallelize_terser',
+    help='Disable parallelism on terser plugin in webpack. Use with prod_env.',
+    action='store_true')
+
 # This list contains the sub process triggered by this script. This includes
 # the oppia web server.
 SUBPROCESSES = []
@@ -257,17 +262,22 @@ def setup_and_install_dependencies(skip_install):
         install_chrome_on_travis.main(args=[])
 
 
-def build_js_files(dev_mode_setting):
+def build_js_files(dev_mode_setting, deparallelize_terser=False):
     """Build the javascript files.
 
     Args:
         dev_mode_setting: bool. Represents whether to run the related commands
         in dev mode.
+        deparallelize_terser: bool. Represents whether to use webpack
+        compilation config that disables parallelism on terser plugin.
     """
     update_dev_mode_in_constants_js(CONSTANT_FILE_PATH, dev_mode_setting)
     if not dev_mode_setting:
         python_utils.PRINT('  Generating files for production mode...')
-        build.main(args=['--prod_env'])
+        if deparallelize_terser:
+            build.main(args=['--prod_env', '--deparallelize_terser'])
+        else:
+            build.main(args=['--prod_env'])
     else:
         # The 'hashes.json' file is used by the `url-interpolation` service.
         if not os.path.isfile(HASHES_FILE_PATH):
@@ -431,7 +441,8 @@ def main(args=None):
 
     dev_mode = not parsed_args.prod_env
     if not parsed_args.skip_build:
-        build_js_files(dev_mode)
+        build_js_files(
+            dev_mode, deparallelize_terser=parsed_args.deparallelize_terser)
     start_webdriver_manager()
 
     start_google_app_engine_server(dev_mode)
diff --git a/scripts/run_e2e_tests_test.py b/scripts/run_e2e_tests_test.py
--- a/scripts/run_e2e_tests_test.py
+++ b/scripts/run_e2e_tests_test.py
@@ -581,6 +581,32 @@ def mock_update_dev_mode_in_constants_js(
                     with build_main_swap:
                         run_e2e_tests.build_js_files(False)
 
+    def test_build_js_files_in_prod_mode_with_deparallelize_terser(self):
+
+        def mock_update_dev_mode_in_constants_js(
+                unused_filename, unused_dev_mode):
+            pass
+
+        update_dev_mode_in_constants_js_swap = self.swap_with_checks(
+            run_e2e_tests, 'update_dev_mode_in_constants_js',
+            mock_update_dev_mode_in_constants_js,
+            expected_args=[(self.mock_constant_file_path, False)])
+
+        run_cmd_swap = self.swap_with_checks(
+            common, 'run_cmd', self.mock_run_cmd, called=False)
+
+        build_main_swap = self.swap_with_checks(
+            build, 'main', self.mock_build_main,
+            expected_kwargs=[{'args': [
+                '--prod_env', '--deparallelize_terser']}])
+
+        with self.constant_file_path_swap:
+            with self.node_bin_path_swap, self.webpack_bin_path_swap:
+                with update_dev_mode_in_constants_js_swap, run_cmd_swap:
+                    with build_main_swap:
+                        run_e2e_tests.build_js_files(
+                            False, deparallelize_terser=True)
+
     def test_tweak_webdriver_manager_on_x64_machine(self):
 
         def mock_is_windows():
@@ -811,7 +837,7 @@ def mock_register(unused_func):
         def mock_cleanup():
             return
 
-        def mock_build_js_files(unused_arg):
+        def mock_build_js_files(unused_arg, deparallelize_terser=False): # pylint: disable=unused-argument
             return
 
         def mock_start_webdriver_manager():
@@ -902,7 +928,7 @@ def mock_register(unused_func):
         def mock_cleanup():
             return
 
-        def mock_build_js_files(unused_arg):
+        def mock_build_js_files(unused_arg, deparallelize_terser=False): # pylint: disable=unused-argument
             return
 
         def mock_start_webdriver_manager():