Repository: bokeh/bokeh

Issue:
Unify bokehjs build system with extensions' compiler
The idea is to have bring fully featured build system to extensions' compiler, which currently is an ad-hoc approach, e.g. it doesn't allow to use npm packages. This work will also allow for generating custom bokeh bundles at runtime. This issue focuses only on technical aspects. User oriented development workflow will be established in future work.


Discussion:

Resolving patch:
diff --git a/_setup_support.py b/_setup_support.py
--- a/_setup_support.py
+++ b/_setup_support.py
@@ -4,6 +4,7 @@
 from __future__ import print_function
 
 import shutil
+from glob import glob
 from os.path import dirname, exists, join, realpath, relpath
 import os, re, subprocess, sys, time
 
@@ -378,16 +379,9 @@ def install_js():
     if exists(target_tslibdir):
         shutil.rmtree(target_tslibdir)
     if exists(TSLIB):
-        # keep in sync with bokehjs/src/compiler/compile.ts
-        lib = {
-            "lib.es5.d.ts",
-            "lib.dom.d.ts",
-            "lib.es2015.core.d.ts",
-            "lib.es2015.promise.d.ts",
-            "lib.es2015.symbol.d.ts",
-            "lib.es2015.iterable.d.ts",
-        }
-        shutil.copytree(TSLIB, target_tslibdir, ignore=lambda _, files: [ f for f in files if f not in lib ])
+        os.mkdir(target_tslibdir)
+        for lib_file in glob(join(TSLIB, "lib.*.d.ts")):
+            shutil.copy(lib_file, target_tslibdir)
 
 # -----------------------------------------------------------------------------
 # Helpers for collecting package data
diff --git a/bokeh/command/__init__.py b/bokeh/command/__init__.py
--- a/bokeh/command/__init__.py
+++ b/bokeh/command/__init__.py
@@ -8,6 +8,9 @@
 
 The following subcommands are available:
 
+build
+    Manage and build a bokeh extension
+
 html
     Create standalone HTML files for one or more applications
 
diff --git a/bokeh/command/subcommands/build.py b/bokeh/command/subcommands/build.py
new file mode 100644
--- /dev/null
+++ b/bokeh/command/subcommands/build.py
@@ -0,0 +1,81 @@
+#-----------------------------------------------------------------------------
+# Copyright (c) 2012 - 2019, Anaconda, Inc., and Bokeh Contributors.
+# All rights reserved.
+#
+# The full license is in the file LICENSE.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+'''
+
+'''
+
+#-----------------------------------------------------------------------------
+# Boilerplate
+#-----------------------------------------------------------------------------
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import logging
+log = logging.getLogger(__name__)
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Standard library imports
+from os.path import join
+
+# External imports
+
+# Bokeh imports
+from bokeh.settings import settings
+from bokeh.util.compiler import _run_nodejs
+
+from ..subcommand import Subcommand
+
+#-----------------------------------------------------------------------------
+# Globals and constants
+#-----------------------------------------------------------------------------
+
+__all__ = (
+    'Build',
+)
+
+#-----------------------------------------------------------------------------
+# Private API
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# General API
+#-----------------------------------------------------------------------------
+
+class Build(Subcommand):
+    '''
+
+    '''
+
+    name = "build"
+
+    help = "Manage and build a bokeh extension"
+
+    args = (
+        ("base_dir", dict(
+            metavar="BASE_DIR",
+            type=str,
+            nargs="?",
+            default=".",
+        )),
+    )
+
+    def invoke(self, args):
+        bokehjs_dir = settings.bokehjsdir()
+        print(bokehjs_dir)
+        compiler_script = join(bokehjs_dir, "js", "compiler.js")
+        output = _run_nodejs([compiler_script, "build", "--base-dir", args.base_dir, "--bokehjs-dir", bokehjs_dir])
+        print(output)
+
+#-----------------------------------------------------------------------------
+# Dev API
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------


Test patch:
diff --git a/bokeh/command/tests/test_bootstrap.py b/bokeh/command/tests/test_bootstrap.py
--- a/bokeh/command/tests/test_bootstrap.py
+++ b/bokeh/command/tests/test_bootstrap.py
@@ -60,7 +60,7 @@ def test_no_subcommand(capsys):
     with pytest.raises(SystemExit):
         main(["bokeh"])
     out, err = capsys.readouterr()
-    assert err == "ERROR: Must specify subcommand, one of: html, info, json, png, sampledata, secret, serve, static or svg\n"
+    assert err == "ERROR: Must specify subcommand, one of: build, html, info, json, png, sampledata, secret, serve, static or svg\n"
     assert out == ""
 
 def test_version(capsys):
diff --git a/bokeh/util/tests/test_compiler.py b/bokeh/util/tests/test_compiler.py
--- a/bokeh/util/tests/test_compiler.py
+++ b/bokeh/util/tests/test_compiler.py
@@ -52,32 +52,7 @@ def test_nodejs_compile_coffeescript():
 """, deps=["some/module"])
 
     assert buc.nodejs_compile("""(a, b) -> a + b +""", "coffeescript", "some.coffee") == \
-        dict(error=dict(
-            message="unexpected end of input",
-            text="some.coffee:unexpected end of input"))
-
-    assert buc.nodejs_compile("""some = require some/module'""", "coffeescript", "some.coffee") == \
-        dict(error=dict(
-            line=1,
-            column=27,
-            message="missing '",
-            text="some.coffee:1:27:missing '",
-            extract="some = require some/module'",
-            annotated="some.coffee:1:27:missing '\n  some = require some/module'\n                            ^"))
-
-    assert buc.nodejs_compile("""(a, b) -> a + b +""", "coffeescript", "some.coffee") == \
-        dict(error=dict(
-            message="unexpected end of input",
-            text="some.coffee:unexpected end of input"))
-
-    assert buc.nodejs_compile("""some = require some/module'""", "coffeescript", "some.coffee") == \
-        dict(error=dict(
-            line=1,
-            column=27,
-            message="missing '",
-            text="some.coffee:1:27:missing '",
-            extract="some = require some/module'",
-            annotated="some.coffee:1:27:missing '\n  some = require some/module'\n                            ^"))
+        dict(error="SyntaxError: unexpected end of input")
 
 def test_nodejs_compile_javascript():
     assert buc.nodejs_compile("""function f(a, b) { return a + b; };""", "javascript", "some.js") == \
@@ -104,13 +79,7 @@ def test_nodejs_compile_less():
         dict(code=""".bk-some-style{color:#800080}""")
 
     assert buc.nodejs_compile(""".bk-some-style color: green; }""", "less", "some.less") == \
-        dict(error=dict(
-            line=1,
-            column=21,
-            message="Unrecognised input",
-            text="some.less:1:21:Unrecognised input",
-            extract=".bk-some-style color: green; }",
-            annotated="some.less:1:21:Unrecognised input\n  .bk-some-style color: green; }"))
+        dict(error="ParseError: Unrecognised input in some.less on line 1, column 21:\n1 .bk-some-style color: green; }\n")
 
 def test_Implementation():
     obj = buc.Implementation()
diff --git a/bokehjs/make/tasks/test.ts b/bokehjs/make/tasks/test.ts
--- a/bokehjs/make/tasks/test.ts
+++ b/bokehjs/make/tasks/test.ts
@@ -2,11 +2,11 @@ import {spawn} from "child_process"
 import {argv} from "yargs"
 
 import {task, log, BuildError} from "../task"
-import {compileTypeScript} from "../compiler"
+import {compile_typescript} from "@compiler/compiler"
 import * as paths from "../paths"
 
 task("test:compile", async () => {
-  const success = compileTypeScript("./test/tsconfig.json", {log})
+  const success = compile_typescript("./test/tsconfig.json", {log})
 
   if (argv.emitError && !success)
     process.exit(1)
diff --git a/bokehjs/test/tsconfig.json b/bokehjs/test/tsconfig.json
--- a/bokehjs/test/tsconfig.json
+++ b/bokehjs/test/tsconfig.json
@@ -18,6 +18,7 @@
     "experimentalDecorators": true,
     "module": "commonjs",
     "moduleResolution": "node",
+    "resolveJsonModule": true,
     "target": "ES2015",
     "lib": ["es2015", "dom"],
     "types": ["node", "mocha"],