Repository: bokeh/bokeh

Issue:
tests scripts enhancements.
- [x] the test-matrix script needs to support dev build from the bokeh Binstar channel
- [x] the interactive-tester script needs to accept an specific location for examples (use-case, a conda env containing a dev build which is shipped with the Examples folder, so I want to pass this path: `/media/datos/miniconda3/envs/test34/Examples/bokeh`)

ping @Maggie-M 



Discussion:

Resolving patch:
diff --git a/scripts/interactive_tester.py b/scripts/interactive_tester.py
--- a/scripts/interactive_tester.py
+++ b/scripts/interactive_tester.py
@@ -1,3 +1,4 @@
+from __future__ import print_function
 import argparse
 import importlib
 import os
@@ -6,7 +7,7 @@
 import sys
 import textwrap
 import time
-
+import json
 
 # TODO:
 #       catch and log exceptions in examples files that fail to open
@@ -35,6 +36,7 @@
     '../../examples/charts/boxplot.py',
 ]
 
+SESSION_FILE = os.path.abspath("INTERACTIVE_TESTER_SESSION.json")
 
 def get_parser():
     """Create the parser that will be used to add arguments to the script.
@@ -46,7 +48,11 @@ def get_parser():
                     The --location option allows you to select a specific examples subdirectory to test all files in,
                     ignoring __init__.py
 
-                    Location arguments you can choose:
+                    Location arguments can be any valid path to a folder with the examples, like:
+
+                     -l /path/to/my/examplesyou can choose:
+
+                    or any of the pre-built keywords that point to the related examples:
                         - file
                         - notebook
                         - server
@@ -61,6 +67,10 @@ def get_parser():
                         help="don't save a log of any errors discovered")
     parser.add_argument('-l', '--location', action='store', default=False,
                         help="example directory in which you wish to test")
+    parser.add_argument('--reuse-session', action='store_true', dest='reuseSession', default=False,
+                        help="do not clean last session log and start from where you left")
+    parser.add_argument('--notebook-options', action='store', dest='notebookOptions', default="",
+                        help="options to be forwarded to ipython notebook to customize it's behaviour")
 
     return parser
 
@@ -80,7 +90,36 @@ def depend_check(dependency):
     return found
 
 
-def main(testing_ground=None):
+def save_session(session):
+    """
+    Save the session object to the SESSION_FILE
+
+    Args:
+        session(dict): dict with all the example files and results of each run
+    """
+    with open(SESSION_FILE, 'w') as res_file:
+        json.dump(session, res_file)
+
+
+def get_session():
+    """
+    Return last stored session
+    """
+    try:
+        with open(SESSION_FILE, 'r') as res_file:
+            return json.load(res_file)
+    except IOError:
+        return {}
+
+
+def clean_session():
+    """
+    Removes previous session file
+    """
+    if os.path.exists(SESSION_FILE):
+        os.remove(SESSION_FILE)
+
+def main(testing_ground=None, notebook_options=""):
     """
     Collect and run .py or .ipynb examples from a set list or given examples directory, ignoring __init__.py
     User input is collected to determine a properly or improperly displayed page
@@ -111,21 +150,40 @@ def main(testing_ground=None):
 
     Log = []
 
+    lastSession = get_session()
+
     for index, fileName in enumerate(TestFiles):
         if testing_ground:
             fileName = "%s/%s" % (testing_ground, fileName)
         try:
-            command = get_cmd(fileName)
-            opener(fileName, command)
 
-            if results.nolog:
-                # Don't display 'next file' message after opening final file in a dir
-                if index != len(TestFiles)-1:
-                    input("\nPress enter to open next file ")
+            if not fileName in lastSession:
+                lastSession[fileName] = "TESTING..."
+                save_session(lastSession)
+
+                command = get_cmd(fileName, notebook_options)
+                opener(fileName, command)
+
+                if results.nolog:
+                    # Don't display 'next file' message after opening final file in a dir
+                    if index != len(TestFiles)-1:
+                        input("\nPress enter to open next file ")
+                else:
+                    ErrorReport = test_status()
+                    if ErrorReport:
+                        Log.append("\n\n%s: \n %s" % (fileName, ErrorReport))
+
+                    lastSession[fileName] = ErrorReport
+                    save_session(lastSession)
             else:
-                ErrorReport = test_status()
-                if ErrorReport:
-                    Log.append("\n\n%s: \n %s" % (fileName, ErrorReport))
+                prevRes = lastSession[fileName]
+                if prevRes == "TESTING...":
+                    print("RESULT OF %s LAST RUN NOT REGISTERED!!" % fileName)
+                    ErrorReport = test_status()
+                    lastSession[fileName] = ErrorReport
+                    save_session(lastSession)
+                else:
+                    print("%s detected in last session: SKIPPING" % fileName)
 
         except (KeyboardInterrupt, EOFError):
             break
@@ -139,7 +197,7 @@ def main(testing_ground=None):
         logger(Log, log_name)
 
 
-def get_cmd(some_file):
+def get_cmd(some_file, notebook_options=""):
     """Determines how to open a file depending
     on whether it is a .py or a .ipynb file
     """
@@ -147,7 +205,7 @@ def get_cmd(some_file):
     if some_file.endswith('.py'):
         command = "python"
     elif some_file.endswith('.ipynb'):
-        command = "ipython notebook"
+        command = "ipython notebook %s" % notebook_options
 
     return command
 
@@ -209,6 +267,12 @@ def logger(error_array, name):
                     sys.exit(1)
 
             test_dir = DIRECTORIES[target]
+
+        elif os.path.exists(results.location):
+            # in case target is not one of the recognized keys and is a
+            # valid path we can run the examples in that folder
+            test_dir = results.location
+            print("Running examples in custom location:", test_dir)
         else:
             print("Test location '%s' not recognized.\nPlease type 'python interactive_tester.py -h' for a list of valid test directories."
                  % results.location)
@@ -220,4 +284,9 @@ def logger(error_array, name):
         print("Server examples require bokeh-server. Make sure you've typed 'bokeh-server' in another terminal tab.")
         time.sleep(5)
 
-    main(test_dir)
+    if not results.reuseSession:
+        print("cleaning previous session file...",)
+        clean_session()
+        print("OK")
+
+    main(test_dir, notebook_options=results.notebookOptions)


Test patch:
diff --git a/scripts/test_matrix.py b/scripts/test_matrix.py
--- a/scripts/test_matrix.py
+++ b/scripts/test_matrix.py
@@ -49,6 +49,8 @@ def get_parser():
                         help='Previous version of bokeh', required=True)
     parser.add_argument('-v', '--version', action='store', default=False,
                         help='Version of bokeh to test', required=True)
+    parser.add_argument('-c', '--channel', action='store', default=False,
+                        help='binstar channel', required=False)
     parser.add_argument('--keep', action='store_true', default=False,
                         help="Don't delete conda envs created by this script")
     parser.add_argument('--dry-run', action='store_true', default=False,
@@ -70,16 +72,17 @@ def cleaner(env_path):
 def conda_creator(env_name, pkgs):
     """Create a conda environment of a given name containing a given string of pkgs.
     """
-
-    call_wrapper("conda create --yes -n %s %s" % (env_name, pkgs), shell=True)
+    call_wrapper("conda create --yes -n %s %s %s" % (env_name, pkgs, OPTIONS["channel"]), shell=True)
 
 
 def bokeh_installer(env_name, install_string):
     """Activate an environment and run its install string to either install or update bokeh using
     conda or pip.
     """
-
-    command_string = 'source activate %s; %s' % (env_name, install_string)
+    if os.name == 'nt':
+        command_string = 'activate %s & %s' % (env_name, install_string)
+    else:
+        command_string = 'source activate %s; %s' % (env_name, install_string)
 
     result = call_wrapper(command_string, shell=True)
 
@@ -89,8 +92,10 @@ def bokeh_installer(env_name, install_string):
 def version_check(env_name, expected_ver):
     """Check a given environment's version of bokeh.
     """
-
-    command_string = 'source activate %s; python -c "import sys, bokeh; sys.exit(0 if bokeh.__version__ == %r else 1)"' % (env_name, expected_ver)
+    if os.name == 'nt':
+        command_string = 'activate %s & python -c "import sys, bokeh; sys.exit(0 if bokeh.__version__ == %r else 1)"' % (env_name, expected_ver)
+    else:
+        command_string = 'source activate %s; python -c "import sys, bokeh; sys.exit(0 if bokeh.__version__ == %r else 1)"' % (env_name, expected_ver)
 
     result = call_wrapper(command_string, shell=True)
 
@@ -105,7 +110,11 @@ def run_tests(env_name):
     test_failure = ''
     file_name  = "tmpfile.txt"
     tmpfile = open(file_name, "w+")
-    command_string = 'source activate %s; python -c "import nose, os, sys, bokeh; bokeh.test(exit=True)"' % env_name
+
+    if os.name == 'nt':
+        command_string = 'activate %s & python -c "import nose, os, sys, bokeh; bokeh.test(exit=True)"' % env_name
+    else:
+        command_string = 'source activate %s; python -c "import nose, os, sys, bokeh; bokeh.test(exit=True)"' % env_name
 
     result = call_wrapper(command_string, shell=True, stderr=tmpfile)
 
@@ -150,6 +159,10 @@ def logger(failure_list):
     parser = get_parser()
     ops = parser.parse_args()
     OPTIONS["dry-run"] = ops.dry_run
+    if ops.channel:
+        OPTIONS["channel"] = "-c %s" % ops.channel
+    else:
+        OPTIONS["channel"] = ""
     preversion = ops.previous
     current_version = ops.version