Repository: oppia/oppia

Issue:
Paralllelize the build process so that it takes less time.
The build process (running python/scripts.py) takes a long time. We would like to make it faster, and one way is by parallelizing the minification/hashes step. The aim of this issue is to do that.


Discussion:
Yup. This is really very important.

Resolving patch:
diff --git a/scripts/build.py b/scripts/build.py
--- a/scripts/build.py
+++ b/scripts/build.py
@@ -24,6 +24,8 @@
 import re
 import shutil
 import subprocess
+import threading
+import time
 
 ASSETS_SRC_DIR = os.path.join('assets', '')
 ASSETS_OUT_DIR = os.path.join('build', 'assets', '')
@@ -200,35 +202,10 @@ def process_html(source_path, target_path, file_hashes):
         filepath_with_hash = _insert_hash(filepath, file_hash)
         content = content.replace(filepath, filepath_with_hash)
     content = REMOVE_WS(' ', content)
-    ensure_directory_exists(target_path)
     d = open(target_path, 'w+')
     d.write(content)
 
 
-def process_css(source_path, target_path):
-    """Runs the given CSS file through a minifier and outputs it to target_path.
-
-    Args:
-        source_path: str. Absolute path to file to be minified.
-        target_path: str. Absolute path to location where to copy
-            the minified file.
-    """
-    ensure_directory_exists(target_path)
-    _minify(source_path, target_path)
-
-
-def process_js(source_path, target_path):
-    """Runs the given JS file through a minifier and outputs it to target_path.
-
-    Args:
-        source_path: str. Absolute path to file to be minified.
-        target_path: str. Absolute path to location where to copy
-            the minified file.
-    """
-    ensure_directory_exists(target_path)
-    _minify(source_path, target_path)
-
-
 def get_dependency_directory(dependency):
     """Get dependency directory from dependecy dictionary.
 
@@ -549,6 +526,38 @@ def save_hashes_as_json(target_filepath, file_hashes):
     file_hashes[relative_filepath] = file_hash
 
 
+def minify_func(source_path, target_path, file_hashes, filename):
+    if filename.endswith('.html'):
+        process_html(source_path, target_path, file_hashes)
+    elif filename.endswith('.css'):
+        _minify(source_path, target_path)
+    elif filename.endswith('.js'):
+        _minify(source_path, target_path)
+    else:
+        shutil.copyfile(source_path, target_path)
+
+
+def _execute_tasks(tasks, batch_size=24):
+    """Starts all tasks and checks the results.
+
+    Runs no more than 'batch_size' tasks at a time.
+    """
+    remaining_tasks = list(tasks)
+    currently_running_tasks = []
+
+    while remaining_tasks or currently_running_tasks:
+        if currently_running_tasks:
+            for task in list(currently_running_tasks):
+                if not task.is_alive():
+                    currently_running_tasks.remove(task)
+
+        while remaining_tasks and len(currently_running_tasks) < batch_size:
+            task = remaining_tasks.pop()
+            currently_running_tasks.append(task)
+            task.start()
+        time.sleep(1)
+
+
 def build_files(source, target, file_hashes):
     """Minifies all CSS and JS files, removes whitespace from HTML and
     interpolates paths in HTML to include hashes in source
@@ -567,6 +576,7 @@ def build_files(source, target, file_hashes):
     shutil.rmtree(target)
 
     for root, dirs, files in os.walk(os.path.join(os.getcwd(), source)):
+        tasks = []
         for directory in dirs:
             print 'Processing %s' % os.path.join(root, directory)
         for filename in files:
@@ -576,16 +586,16 @@ def build_files(source, target, file_hashes):
             if source not in source_path:
                 continue
             target_path = source_path.replace(source, target)
+            ensure_directory_exists(target_path)
 
-            if filename.endswith('.html'):
-                process_html(source_path, target_path, file_hashes)
-            elif filename.endswith('.css'):
-                process_css(source_path, target_path)
-            elif filename.endswith('.js'):
-                process_js(source_path, target_path)
-            else:
-                ensure_directory_exists(target_path)
-                shutil.copyfile(source_path, target_path)
+            task = threading.Thread(
+                target=minify_func,
+                args=(source_path, target_path, file_hashes, filename))
+            tasks.append(task)
+        try:
+            _execute_tasks(tasks)
+        except Exception as e:
+            print e
 
 
 def generate_build_directory():


Test patch: