Repository: oppia/oppia

Issue:
Assert Chrome Version in CI Checks
**Is your feature request related to a problem? Please describe.**
This is a mitigation from the [Lighthouse failures postmortem](https://docs.google.com/document/d/1xHMMY4Xy2bcq6Urd7e1k3sk0RMl_Xu5TRycsJof9B58/edit#).

**Describe the solution you'd like**
Create a script to assert that the available version of Chrome matches what we expect. Then call this script in every CI check that uses Chrome.

**Describe alternatives you've considered**
1. **Just make sure that we install Chrome in every CI check.** This leaves open the possibility that the installation failed or that an existing Chrome version overrides the one we installed.
2. **Perform this check in the various CI check script files (e.g. `run_e2e_tests.py`).** We don't want to enforce this when running on developers' local machines, so we'd have to figure out whether we were running on CI. This could be through a flag or an automatic check. A flag is no better than calling another script and introduces more redundant code. An automatic check introduces more ways for this assertion to fail. It's better to make the tests declarative.
Assert Chrome Version in CI Checks
**Is your feature request related to a problem? Please describe.**
This is a mitigation from the [Lighthouse failures postmortem](https://docs.google.com/document/d/1xHMMY4Xy2bcq6Urd7e1k3sk0RMl_Xu5TRycsJof9B58/edit#).

**Describe the solution you'd like**
Create a script to assert that the available version of Chrome matches what we expect. Then call this script in every CI check that uses Chrome.

**Describe alternatives you've considered**
1. **Just make sure that we install Chrome in every CI check.** This leaves open the possibility that the installation failed or that an existing Chrome version overrides the one we installed.
2. **Perform this check in the various CI check script files (e.g. `run_e2e_tests.py`).** We don't want to enforce this when running on developers' local machines, so we'd have to figure out whether we were running on CI. This could be through a flag or an automatic check. A flag is no better than calling another script and introduces more redundant code. An automatic check introduces more ways for this assertion to fail. It's better to make the tests declarative.


Discussion:

Resolving patch:
diff --git a/scripts/common.py b/scripts/common.py
--- a/scripts/common.py
+++ b/scripts/common.py
@@ -214,7 +214,8 @@ def run_cmd(cmd_tokens):
     Returns:
         str. The output of the command.
     """
-    return subprocess.check_output(cmd_tokens).strip()
+    return subprocess.check_output(
+        cmd_tokens, stderr=subprocess.STDOUT).strip()
 
 
 def ensure_directory_exists(d):
diff --git a/scripts/install_chrome_for_ci.py b/scripts/install_chrome_for_ci.py
new file mode 100644
--- /dev/null
+++ b/scripts/install_chrome_for_ci.py
@@ -0,0 +1,89 @@
+# Copyright 2021 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Install Chrome for CI server.
+
+This is meant to be run as part of CI setup. It installs the correct
+version of Chrome for our CI tests and checks that the correct version
+was installed.
+"""
+
+from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
+
+import re
+
+import python_utils
+from scripts import common
+
+
+CHROME_VERSION = '88.0.4324.96-1'
+URL_TEMPLATE = (
+    'https://github.com/webnicer/chrome-downloads/raw/master/x64.deb/'
+    'google-chrome-stable_{}_amd64.deb'
+)
+CHROME_DEB_FILE = 'google-chrome.deb'
+
+
+def install_chrome(version):
+    """Install Chrome from the URL in URL_TEMPLATE.
+
+    Args:
+        version: str. The version of Chrome to install. This must be one
+            of the versions available from
+            github.com/webnicer/chrome-downloads.
+    """
+    _ = common.run_cmd(['sudo', 'apt-get', 'update'])
+    _ = common.run_cmd(['sudo', 'apt-get', 'install', 'libappindicator3-1'])
+    _ = common.run_cmd([
+        'curl', '-L', '-o', CHROME_DEB_FILE,
+        URL_TEMPLATE.format(version)])
+    _ = common.run_cmd([
+        'sudo', 'sed', '-i',
+        's|HERE/chrome\\"|HERE/chrome\\" --disable-setuid-sandbox|g',
+        '/opt/google/chrome/google-chrome'])
+    _ = common.run_cmd(['sudo', 'dpkg', '-i', CHROME_DEB_FILE])
+
+
+def get_chrome_version():
+    """Get the current version of Chrome.
+
+    Note that this only works on Linux systems. On macOS, for example,
+    the `google-chrome` command may not work.
+
+    Returns:
+        str. The version of Chrome we found.
+    """
+    output = common.run_cmd(['google-chrome', '--version'])
+    chrome_version = ''.join(re.findall(r'([0-9]|\.)', output))
+    return chrome_version
+
+
+def main():
+    """Install Chrome and check the correct version was installed."""
+    install_chrome(CHROME_VERSION)
+    found_version = get_chrome_version()
+    if not CHROME_VERSION.startswith(found_version):
+        raise RuntimeError(
+            (
+                'Chrome version {} should have been installed. '
+                'Version {} was found instead.'
+            ).format(CHROME_VERSION, found_version)
+        )
+    python_utils.PRINT(
+        'Chrome version {} installed.'.format(found_version))
+
+
+if __name__ == '__main__':  # pragma: no cover
+    main()


Test patch:
diff --git a/scripts/backend_test_shards.json b/scripts/backend_test_shards.json
--- a/scripts/backend_test_shards.json
+++ b/scripts/backend_test_shards.json
@@ -190,7 +190,8 @@
         "android_validation_constants_test",
         "core.domain.skill_fetchers_test",
         "scripts.check_e2e_tests_are_captured_in_ci_test",
-        "scripts.run_e2e_tests_test"
+        "scripts.run_e2e_tests_test",
+        "scripts.install_chrome_for_ci_test"
     ],
     "9": [
         "core.domain.user_jobs_one_off_test",
diff --git a/scripts/install_chrome_for_ci_test.py b/scripts/install_chrome_for_ci_test.py
new file mode 100644
--- /dev/null
+++ b/scripts/install_chrome_for_ci_test.py
@@ -0,0 +1,177 @@
+# Copyright 2021 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for scripts/install_chrome_for_ci.py."""
+
+from __future__ import absolute_import  # pylint: disable=import-only-modules
+from __future__ import unicode_literals  # pylint: disable=import-only-modules
+
+import subprocess
+
+from core.tests import test_utils
+import python_utils
+from scripts import common
+
+from scripts import install_chrome_for_ci
+
+
+URL = (
+    'https://github.com/webnicer/chrome-downloads/raw/master/x64.deb/'
+    'google-chrome-stable_88.0.4324.96-1_amd64.deb'
+)
+DOWNLOAD_VERSION = '88.0.4324.96-1'
+INSTALLED_VERSION = '88.0.4324.96'
+CHROME_DEB_FILE = 'google-chrome.deb'
+
+
+class InstallChromeTests(test_utils.GenericTestBase):
+
+    def test_success(self):
+        def mock_run_cmd(unused_tokens):
+            return ''
+
+        run_cmd_swap = self.swap_with_checks(
+            common, 'run_cmd', mock_run_cmd,
+            expected_args=[
+                (['sudo', 'apt-get', 'update'],),
+                (['sudo', 'apt-get', 'install', 'libappindicator3-1'],),
+                (['curl', '-L', '-o', CHROME_DEB_FILE, URL],),
+                ([
+                    'sudo', 'sed', '-i',
+                    's|HERE/chrome\\"|HERE/chrome\\" '
+                    '--disable-setuid-sandbox|g',
+                    '/opt/google/chrome/google-chrome'],),
+                (['sudo', 'dpkg', '-i', CHROME_DEB_FILE],),
+            ])
+
+        with run_cmd_swap:
+            install_chrome_for_ci.install_chrome(DOWNLOAD_VERSION)
+
+    def test_fail(self):
+        def mock_run_cmd(tokens):
+            command = ' '.join(tokens)
+            raise subprocess.CalledProcessError(1, command, '')
+
+        run_cmd_swap = self.swap_with_checks(
+            common, 'run_cmd', mock_run_cmd,
+            expected_args=[
+                (['sudo', 'apt-get', 'update'],),
+            ])
+
+        with run_cmd_swap:
+            with self.assertRaisesRegexp(
+                subprocess.CalledProcessError,
+                'Command \'sudo apt-get update\' returned non-zero exit '
+                'status 1'
+            ):
+                install_chrome_for_ci.install_chrome(DOWNLOAD_VERSION)
+
+
+class GetChromeVersionTests(test_utils.GenericTestBase):
+
+    def test_real_example(self):
+        def mock_run_cmd(unused_tokens):
+            return 'Google Chrome 88.0.4324.96 '
+
+        run_cmd_swap = self.swap_with_checks(
+            common, 'run_cmd', mock_run_cmd,
+            expected_args=[
+                (['google-chrome', '--version'],),
+            ])
+
+        with run_cmd_swap:
+            version = install_chrome_for_ci.get_chrome_version()
+            self.assertEqual(version, '88.0.4324.96')
+
+    def test_fails(self):
+        def mock_run_cmd(tokens):
+            command = ' '.join(tokens)
+            raise subprocess.CalledProcessError(1, command, '')
+
+        run_cmd_swap = self.swap_with_checks(
+            common, 'run_cmd', mock_run_cmd,
+            expected_args=[
+                (['google-chrome', '--version'],),
+            ])
+
+        with run_cmd_swap:
+            with self.assertRaisesRegexp(
+                subprocess.CalledProcessError,
+                'Command \'google-chrome --version\' returned non-zero exit '
+                'status 1'
+            ):
+                install_chrome_for_ci.get_chrome_version()
+
+
+class MainTests(test_utils.GenericTestBase):
+
+    def test_success(self):
+        def mock_install_chrome(unused_version):
+            return
+
+        def mock_get_chrome_version():
+            return INSTALLED_VERSION
+
+        def mock_print(unused_string):
+            return
+
+        install_chrome_swap = self.swap_with_checks(
+            install_chrome_for_ci, 'install_chrome',
+            mock_install_chrome,
+            expected_args=[
+                (DOWNLOAD_VERSION,),
+            ])
+        get_version_swap = self.swap_with_checks(
+            install_chrome_for_ci, 'get_chrome_version',
+            mock_get_chrome_version,
+            expected_args=[tuple()])
+        print_swap = self.swap_with_checks(
+            python_utils, 'PRINT', mock_print,
+            expected_args=[
+                (
+                    'Chrome version {} installed.'.format(
+                        INSTALLED_VERSION
+                    ),
+                )
+            ])
+
+        with install_chrome_swap, get_version_swap, print_swap:
+            install_chrome_for_ci.main()
+
+    def test_version_mismatch(self):
+        def mock_install_chrome(unused_version):
+            return
+
+        def mock_get_chrome_version():
+            return '123.0.12.45'
+
+        install_chrome_swap = self.swap_with_checks(
+            install_chrome_for_ci, 'install_chrome',
+            mock_install_chrome,
+            expected_args=[
+                (DOWNLOAD_VERSION,),
+            ])
+        get_version_swap = self.swap_with_checks(
+            install_chrome_for_ci, 'get_chrome_version',
+            mock_get_chrome_version,
+            expected_args=[tuple()])
+
+        with install_chrome_swap, get_version_swap:
+            with self.assertRaisesRegexp(
+                RuntimeError, (
+                    'Chrome version {} should have been installed. '
+                    'Version 123.0.12.45 was found instead.'
+                ).format(DOWNLOAD_VERSION)
+            ):
+                install_chrome_for_ci.main()