Repository: oppia/oppia

Issue:
Performance Testing Framework
This issue involves implementing  a performance testing framework for Oppia.

Refer to this doc for further info:
https://docs.google.com/document/d/1pNIVaFUvCJyUOxZWupGNbIP6dgWTfAbieAtxk1aQiJk/edit

Implementaion:
https://docs.google.com/document/d/1pNIVaFUvCJyUOxZWupGNbIP6dgWTfAbieAtxk1aQiJk/edit#bookmark=id.arl4i8nkbw1o

/cc @BenHenning @maitbayev 



Discussion:

Resolving patch:
diff --git a/scripts/pre_commit_linter.py b/scripts/pre_commit_linter.py
--- a/scripts/pre_commit_linter.py
+++ b/scripts/pre_commit_linter.py
@@ -119,6 +119,9 @@
         'google_appengine'),
     os.path.join(_PARENT_DIR, 'oppia_tools', 'webtest-1.4.2'),
     os.path.join(_PARENT_DIR, 'oppia_tools', 'numpy-1.6.1'),
+    os.path.join(_PARENT_DIR, 'oppia_tools', 'browsermob-proxy-0.7.1'),
+    os.path.join(_PARENT_DIR, 'oppia_tools', 'selenium-2.53.2'),
+    os.path.join(_PARENT_DIR, 'oppia_tools', 'xvfbwrapper-0.2.8'),
     os.path.join('third_party', 'gae-pipeline-1.9.17.0'),
     os.path.join('third_party', 'bleach-1.2.2'),
     os.path.join('third_party', 'gae-mapreduce-1.9.17.0'),


Test patch:
diff --git a/core/tests/gae_suite.py b/core/tests/gae_suite.py
--- a/core/tests/gae_suite.py
+++ b/core/tests/gae_suite.py
@@ -44,6 +44,9 @@
     os.path.join(
         OPPIA_TOOLS_DIR, 'google_appengine_1.9.19', 'google_appengine',
         'lib', 'webob_0_9'),
+    os.path.join(OPPIA_TOOLS_DIR, 'browsermob-proxy-0.7.1'),
+    os.path.join(OPPIA_TOOLS_DIR, 'selenium-2.53.2'),
+    os.path.join(OPPIA_TOOLS_DIR, 'xvfbwrapper-0.2.8'),
     CURR_DIR,
     os.path.join(THIRD_PARTY_DIR, 'bleach-1.2.2'),
     os.path.join(THIRD_PARTY_DIR, 'gae-cloud-storage-1.9.15.0'),
diff --git a/core/tests/performance_framework/__init__.py b/core/tests/performance_framework/__init__.py
new file mode 100644
diff --git a/core/tests/performance_framework/perf_domain.py b/core/tests/performance_framework/perf_domain.py
new file mode 100644
--- /dev/null
+++ b/core/tests/performance_framework/perf_domain.py
@@ -0,0 +1,146 @@
+# Copyright 2016 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains domain objects for storing page session stats as provided by a
+HTTP Archive file (also referred to as HAR).
+
+Selenium and Browsermob-proxy are used for capturing session information, such
+as load times and page size statistics. Timing statistics are retrieved directly
+from the browser console rather than the proxy server, as the latter is sluggish
+and gives inaccurate timings.
+"""
+
+import utils
+
+
+class PageSessionMetrics(object):
+    """Contains methods to process stats and provide performance metrics.
+
+    page_session_stats is a dictionary containing page load statistics from an
+    HTTP Archive.
+    (https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html)
+
+    page_session_timings is a dictionary containing metrics associated with page
+    loading and includes the following keys:
+
+        timing: maps to a dict containing the keys (all in milliseconds):
+            domInteractive: the Unix time the the parser finished its work on
+                            the main document
+            domLoading: the Unix time the DOM started loading
+            fetchStart: the Unix time the page began loading
+            loadEventEnd: the Unix time the page finished loading
+            requestStart: the Unix time the request started
+            responseStart: the Unix time the response started
+            responseEnd: the Unix time the request finished
+    """
+
+    TIMING_PROPERTIES = [
+        'loadEventEnd', 'fetchStart', 'domComplete', 'domInteractive',
+        'responseEnd', 'requestStart'
+    ]
+
+    def __init__(self, page_session_stats=None, page_session_timings=None):
+        self.page_session_stats = page_session_stats
+        self.page_session_timings = page_session_timings
+
+        self.page_load_timings = {}
+        if self.page_session_timings:
+            self.page_load_timings = page_session_timings['timing']
+
+        self._validate()
+
+    def _validate(self):
+        """Validates various properties of a PageSessionMetrics object."""
+        if not self.page_session_stats and not self.page_session_timings:
+            raise utils.ValidationError(
+                'Expected one of page_session_stats or page_session_timings '
+                'to be provided.')
+
+        if self.page_session_stats:
+            if 'log' not in self.page_session_stats:
+                raise utils.ValidationError(
+                    'Expected the page load stats to have a \'log\' entry')
+
+            if 'entries' not in self.page_session_stats['log']:
+                raise utils.ValidationError(
+                    'Expected the log entry of the page load stats to include '
+                    'an additional \'entries\' element')
+
+            for entry in self.page_session_stats['log']['entries']:
+                if '_error' in entry['response']:
+                    raise utils.ValidationError(
+                        'Expected a valid server response, found server '
+                        'not reachable.')
+
+            if self.get_request_count() == 0:
+                raise utils.ValidationError(
+                    'Expected the log entry of the page load stats to include '
+                    'a positive number of requests.')
+
+            if self.get_total_page_size_bytes() == 0:
+                raise utils.ValidationError(
+                    'Expected the total size of a page including all its '
+                    'resources to be positive.')
+
+        if self.page_session_timings:
+            for timing_prop in self.TIMING_PROPERTIES:
+                if timing_prop not in self.page_load_timings:
+                    raise utils.ValidationError(
+                        'Expected the timing entry of the page load timings to '
+                        'include %s property' % timing_prop)
+
+            if self.get_page_load_time_millisecs() < 0:
+                raise utils.ValidationError(
+                    'Expected the page load time to be positive.')
+            if self.get_dom_ready_time_millisecs() < 0:
+                raise utils.ValidationError(
+                    'Expected the dom ready time to be positive.')
+            if self.get_request_time_millisecs() < 0:
+                raise utils.ValidationError(
+                    'Expected the request time to be positive.')
+
+
+    def get_request_count(self):
+        """Returns the number of requests made prior to the page load
+        completing.
+        """
+        return len(self.page_session_stats['log']['entries'])
+
+    def get_total_page_size_bytes(self):
+        """Returns the total size of a page including all of its resources."""
+        total_size = 0
+
+        for entry in self.page_session_stats['log']['entries']:
+            total_size += int(entry['response']['bodySize'])
+
+        return total_size
+
+    def _get_duration_millisecs(self, event_end, event_initial):
+        # Timestamps are in milliseconds.
+        initial_timestamp = self.page_load_timings[event_initial]
+        end_timestamp = self.page_load_timings[event_end]
+
+        return end_timestamp - initial_timestamp
+
+    def get_page_load_time_millisecs(self):
+        """Returns the total page load time (in milliseconds)."""
+        return self._get_duration_millisecs('loadEventEnd', 'fetchStart')
+
+    def get_dom_ready_time_millisecs(self):
+        """Returns the total dom ready time (in milliseconds)."""
+        return self._get_duration_millisecs('domComplete', 'domInteractive')
+
+    def get_request_time_millisecs(self):
+        """Returns the total request time (in milliseconds)."""
+        return self._get_duration_millisecs('responseEnd', 'requestStart')
diff --git a/core/tests/performance_framework/perf_services.py b/core/tests/performance_framework/perf_services.py
new file mode 100644
--- /dev/null
+++ b/core/tests/performance_framework/perf_services.py
@@ -0,0 +1,226 @@
+# Copyright 2016 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains a utility for fetching performance data using Selenium and
+Browsermob-proxy.
+"""
+
+import os
+import time
+import urlparse
+from browsermobproxy import Server
+from selenium import webdriver
+
+# Used for headless testing i.e, making browser windows invisible to the user.
+from xvfbwrapper import Xvfb
+
+from core.tests.performance_framework import perf_domain
+
+CHROME_DRIVER_PATH = os.path.join(
+    '..', 'node_modules', 'protractor', 'selenium', 'chromedriver_2.21')
+
+BROWSERMOB_PROXY_PATH = os.path.join(
+    '..', 'oppia_tools', 'browsermob-proxy-2.1.1', 'bin', 'browsermob-proxy')
+
+
+class SeleniumPerformanceDataFetcher(object):
+    """Fetches performance data for locally served Oppia pages using Selenium
+    and Browsermob-proxy.
+
+    Selenium is used to programitically interact with a browser.
+    Browsermob-proxy is used to capture HTTP Archive (referred to as HAR) data.
+
+    The HTTP Archive format is a JSON-formatted archive file format used for
+    logging a web browser's interaction with a site. It contains detailed
+    performance data, including information about page loading and displaying
+    and per-resource statistics. Each entry contains the URL requested and
+    request and response headers. For additional details, please see:
+    https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html
+
+    The HTTP Archive data is retrieved using the proxy server by recording the
+    communication between the client and server.
+    """
+
+    # Chrome is the default browser that selenium will use for extracting
+    # performance stats.
+    DEFAULT_BROWSER_SOURCE = 'chrome'
+    SUPPORTED_BROWSER_SOURCES = ['chrome', 'firefox']
+
+    # Duration (in seconds) to wait for the complete page to load, otherwise
+    # XHR requests made post initial page load will not be recorded.
+    WAIT_DURATION_SECS = 3
+
+    def __init__(self, browser=DEFAULT_BROWSER_SOURCE):
+        if browser in self.SUPPORTED_BROWSER_SOURCES:
+            self.browser = browser
+        else:
+            error_msg = 'Unsupported browser specified: %s' % browser
+            raise ValueError(error_msg)
+
+    def get_page_metrics_for_url(self, page_url):
+        """Returns a PageSessionMetrics domain object for a given page URL.
+        """
+        with Xvfb() as _:
+            server, proxy = self._setup_proxy_server()
+            driver = self._setup_driver(proxy=proxy, use_proxy=True)
+
+            proxy.new_har(page_url, options={'captureHeaders': True})
+
+            driver.get(page_url)
+
+            self._wait_until_page_load_is_finished()
+
+        har_dict = proxy.har
+
+        self._stop_proxy_server(server)
+        self._stop_driver(driver)
+
+        return perf_domain.PageSessionMetrics(
+            page_session_stats=har_dict, page_session_timings=None)
+
+    def get_page_metrics_from_cached_session(self, page_url):
+        """Returns a PageSessionMetrics domain object for a given page URL
+        while simulating a cached session i.e, a return user.
+
+        Note: This method is stand-alone and does not require
+        `get_page_session_metrics_for_url` to be called before it.
+        """
+        with Xvfb() as _:
+            server, proxy = self._setup_proxy_server()
+            driver = self._setup_driver(proxy=proxy, use_proxy=True)
+
+            # Fetch the page once. This leads to caching of various resources.
+            driver.get(page_url)
+            self._wait_until_page_load_is_finished()
+
+            # Start recording har data for the next page fetch.
+            proxy.new_har(page_url, options={'captureHeaders': True})
+            driver.get(page_url)
+
+            self._wait_until_page_load_is_finished()
+
+        har_dict = proxy.har
+
+        self._stop_proxy_server(server)
+        self._stop_driver(driver)
+
+        return perf_domain.PageSessionMetrics(
+            page_session_stats=har_dict, page_session_timings=None)
+
+    def get_page_timings_for_url(self, page_url):
+        """Returns a PageSessionMetrics domain object initialized using
+        page load timings for a page URL.
+        """
+        with Xvfb() as _:
+            driver = self._setup_driver(proxy=None, use_proxy=False)
+
+            driver.get(page_url)
+            self._wait_until_page_load_is_finished()
+
+            page_session_timings = (
+                driver.execute_script("return window.performance"))
+
+        self._stop_driver(driver)
+
+        return perf_domain.PageSessionMetrics(
+            page_session_stats=None, page_session_timings=page_session_timings)
+
+    def get_page_timings_from_cached_session(self, page_url):
+        """Returns a PageSessionMetrics domain object initialized using page
+        load timings for a page URL while simulating a cached session i.e, a
+        return user.
+        """
+        with Xvfb() as _:
+            driver = self._setup_driver(proxy=None, use_proxy=False)
+
+            # Fetch the page once. This leads to caching of various resources.
+            driver.get(page_url)
+            self._wait_until_page_load_is_finished()
+
+            driver.get(page_url)
+            self._wait_until_page_load_is_finished()
+
+            page_session_timings = (
+                driver.execute_script("return window.performance"))
+
+        self._stop_driver(driver)
+
+        return perf_domain.PageSessionMetrics(
+            page_session_stats=None, page_session_timings=page_session_timings)
+
+    def _wait_until_page_load_is_finished(self):
+        # Waits for the complete page to load, otherwise XHR requests
+        # made post initial page load will not be recorded.
+        time.sleep(self.WAIT_DURATION_SECS)
+
+    def _setup_proxy_server(self, downstream_kbps=None, upstream_kbps=None,
+                            latency=None):
+        server = Server(BROWSERMOB_PROXY_PATH)
+        server.start()
+        proxy = server.create_proxy()
+
+        # The proxy server is pretty sluggish, setting the limits might not
+        # achieve the desired behavior.
+        proxy_options = {}
+
+        if downstream_kbps:
+            proxy_options['downstream_kbps'] = downstream_kbps
+
+        if upstream_kbps:
+            proxy_options['upstream_kbps'] = upstream_kbps
+
+        if latency:
+            proxy_options['latency'] = latency
+
+        if len(proxy_options.items()) > 0:
+            proxy.limits(proxy_options)
+
+        return server, proxy
+
+    def _setup_driver(self, proxy=None, use_proxy=False):
+        """Initializes a Selenium webdriver instance to programmatically
+        interact with a browser.
+        """
+        driver = None
+        if self.browser == 'chrome':
+            chrome_options = webdriver.ChromeOptions()
+            # Disable several subsystems which run network requests in the
+            # background. This helps reduce noise when measuring network
+            # performance.
+            chrome_options.add_argument("--disable-background-networking")
+
+            if use_proxy:
+                proxy_url = urlparse.urlparse(proxy.proxy).path
+                proxy_argument = "--proxy-server={0}".format(proxy_url)
+                chrome_options.add_argument(proxy_argument)
+
+            driver = webdriver.Chrome(
+                CHROME_DRIVER_PATH,
+                chrome_options=chrome_options)
+
+        elif self.browser == 'firefox':
+            firefox_profile = webdriver.FirefoxProfile()
+
+            if use_proxy:
+                firefox_profile.set_proxy(proxy.selenium_proxy())
+
+            driver = webdriver.Firefox(firefox_profile=firefox_profile)
+
+        return driver
+
+    def _stop_proxy_server(self, server):
+        server.stop()
+
+    def _stop_driver(self, driver):
+        driver.quit()
diff --git a/core/tests/performance_tests/__init__.py b/core/tests/performance_tests/__init__.py
new file mode 100644
diff --git a/core/tests/performance_tests/base.py b/core/tests/performance_tests/base.py
new file mode 100644
--- /dev/null
+++ b/core/tests/performance_tests/base.py
@@ -0,0 +1,44 @@
+# Copyright 2016 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common utilities for performance test classes"""
+
+import unittest
+
+from core.tests.performance_framework import perf_services
+
+
+class TestBase(unittest.TestCase):
+    """Base class for performance tests."""
+
+    def setUp(self):
+        self.data_fetcher = perf_services.SeleniumPerformanceDataFetcher(
+            browser='chrome')
+        self.page_metrics = None
+
+    def _record_page_metrics_for_url(self, page_url):
+        self.page_metrics = (
+            self.data_fetcher.get_page_metrics_for_url(page_url))
+
+    def _record_page_metrics_from_cached_session(self, page_url):
+        self.page_metrics = (
+            self.data_fetcher.get_page_metrics_from_cached_session(page_url))
+
+    def _record_page_timings_for_url(self, page_url):
+        self.page_metrics = (
+            self.data_fetcher.get_page_timings_for_url(page_url))
+
+    def _record_page_timings_from_cached_session(self, page_url):
+        self.page_metrics = (
+            self.data_fetcher.get_page_timings_from_cached_session(page_url))
diff --git a/core/tests/performance_tests/splash_test.py b/core/tests/performance_tests/splash_test.py
new file mode 100644
--- /dev/null
+++ b/core/tests/performance_tests/splash_test.py
@@ -0,0 +1,60 @@
+# Copyright 2016 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Performance tests for the splash page."""
+
+from core.tests.performance_tests import base
+
+
+class SplashPagePerformanceTest(base.TestBase):
+    """Performance tests for the splash page."""
+
+    SPLASH_URL = 'http://localhost:9501/splash'
+
+    def setUp(self):
+        super(SplashPagePerformanceTest, self).setUp()
+
+    def test_splash_page_has_less_than_10_megabytes_sent_to_the_client(self):
+        self._record_page_metrics_for_url(self.SPLASH_URL)
+
+        self.assertLessEqual(
+            self.page_metrics.get_total_page_size_bytes(), 10000000)
+
+    def test_splash_page_loads_under_10_seconds(self):
+        self._record_page_timings_for_url(self.SPLASH_URL)
+
+        self.assertLessEqual(
+            self.page_metrics.get_page_load_time_millisecs(), 10000)
+
+
+class SplashPagePerformanceForCachedStateTest(base.TestBase):
+    """Performance tests for the splash page for the cached state or
+    return user.
+    """
+    SPLASH_URL = 'http://localhost:9501/splash'
+
+    def setUp(self):
+        super(SplashPagePerformanceForCachedStateTest, self).setUp()
+
+    def test_splash_page_has_less_than_1_megabytes_sent_to_the_client(self):
+        self._record_page_metrics_from_cached_session(self.SPLASH_URL)
+
+        self.assertLessEqual(
+            self.page_metrics.get_total_page_size_bytes(), 1000000)
+
+    def test_splash_page_loads_under_3_seconds(self):
+        self._record_page_timings_from_cached_session(self.SPLASH_URL)
+
+        self.assertLessEqual(
+            self.page_metrics.get_page_load_time_millisecs(), 3000)
diff --git a/scripts/run_performance_tests.sh b/scripts/run_performance_tests.sh
new file mode 100644
--- /dev/null
+++ b/scripts/run_performance_tests.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# Copyright 2016 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##########################################################################
+
+# INSTRUCTIONS:
+#
+# Run this script from the oppia root folder:
+#   bash scripts/run_performance_tests.sh
+# 
+# The root folder MUST be named 'oppia'.
+
+function cleanup {
+  # Send a kill signal to the dev server.
+  kill `ps aux | grep "[Dd]ev_appserver.py --host=0.0.0.0 --port=9501" | awk '{print $2}'`
+
+  # Wait for the servers to go down; suppress "connection refused" error output
+  # from nc since that is exactly what we are expecting to happen.
+  while ( nc -vz localhost 9501 >/dev/null 2>&1 ); do sleep 1; done
+
+  echo Done!
+}
+
+if [ -z "$BASH_VERSION" ]
+then
+  echo ""
+  echo "  Please run me using bash: "
+  echo ""
+  echo "     bash $0"
+  echo ""
+  return 1
+fi
+
+set -e
+source $(dirname $0)/setup.sh || exit 1
+source $(dirname $0)/setup_gae.sh || exit 1
+
+export DEFAULT_SKIP_INSTALLING_THIRD_PARTY_LIBS=false
+export DEFAULT_RUN_MINIFIED_TESTS=false
+maybeInstallDependencies "$@"
+
+if ( nc -vz localhost 8181 ); then
+  echo ""
+  echo "  There is already a server running on localhost:8181."
+  echo "  Please terminate it before running the performance tests."
+  echo "  Exiting."
+  echo ""
+  exit 1
+fi
+  
+
+# Forces the cleanup function to run on exit.
+# Developers: note that at the end of this script, the cleanup() function at
+# the top of the file is run.
+trap cleanup EXIT
+
+# Change execute status of browsermob-proxy
+chmod 744 $TOOLS_DIR/browsermob-proxy-2.1.1/bin/browsermob-proxy
+
+# Start a demo server.
+($PYTHON_CMD $GOOGLE_APP_ENGINE_HOME/dev_appserver.py --host=0.0.0.0 --port=9501 --clear_datastore=yes --dev_appserver_log_level=critical --log_level=critical --skip_sdk_update_check=true .)&
+
+# Wait for the servers to come up.
+while ! nc -vz localhost 9501; do sleep 1; done
+
+$PYTHON_CMD scripts/backend_tests.py --test_target=core.tests.performance_tests.splash_test $@ 
+
+chmod 644 $TOOLS_DIR/browsermob-proxy-2.1.1/bin/browsermob-proxy
+rm bmp.log server.log