Repository: numba/numba

Issue:
Reload environment variables when compiling
User report:

> Hi how can I generate the Control Flow Graphs as described in the documentation? http://numba.pydata.org/numba-doc/0.19.1/developer/architecture.html#stage-1-analyze-bytecode
> 
> the documentation asks me set the enviornment variable NUMBA_DUMP_CFG as 1
> 
> so ran !export NUMBA_DUMP_CFG=1 in the ipython console but it doesn't seem to change anything



Discussion:

Resolving patch:
diff --git a/numba/compiler.py b/numba/compiler.py
--- a/numba/compiler.py
+++ b/numba/compiler.py
@@ -224,6 +224,9 @@ class Pipeline(object):
     """
     def __init__(self, typingctx, targetctx, library, args, return_type, flags,
                  locals):
+        # Make sure the environment is reloaded
+        config.reload_config()
+
         self.typingctx = typingctx
 
         subtargetoptions = {}
diff --git a/numba/config.py b/numba/config.py
--- a/numba/config.py
+++ b/numba/config.py
@@ -12,6 +12,8 @@
 IS_WIN32 = sys.platform.startswith('win32')
 MACHINE_BITS = tuple.__itemsize__ * 8
 IS_32BITS = MACHINE_BITS == 32
+# Python version in (major, minor) tuple
+PYVERSION = sys.version_info[:2]
 
 _cpu_name = ll.get_host_cpu_name()
 
@@ -20,106 +22,149 @@ class NumbaWarning(Warning):
     pass
 
 
-def _readenv(name, ctor, default):
-    try:
-        res = os.environ[name]
-        if res == '':
-            return default
-    except KeyError:
-        return default
+def _parse_cc(text):
+    """
+    Parse CUDA compute capability version string.
+    """
+    if not text:
+        return None
     else:
-        try:
-            return ctor(res)
-        except:
-            warnings.warn("environ %s defined but failed to parse '%s'" %
-                          (name, res), RuntimeWarning)
-            return default
+        m = re.match(r'(\d+)\.(\d+)', text)
+        if not m:
+            raise ValueError("NUMBA_FORCE_CUDA_CC must be specified as a "
+                             "string of \"major.minor\" where major "
+                             "and minor are decimals")
+        grp = m.groups()
+        return int(grp[0]), int(grp[1])
 
-# Print warnings to screen about function compilation
-#   0 = Numba warnings suppressed (default)
-#   1 = All Numba warnings shown
-WARNINGS = _readenv("NUMBA_WARNINGS", int, 0)
-if WARNINGS == 0:
-    warnings.simplefilter('ignore', NumbaWarning)
 
-# Debug flag to control compiler debug print
-DEBUG = _readenv("NUMBA_DEBUG", int, 0)
+class _EnvReloader(object):
+
+    def __init__(self):
+        self.reset()
 
-# JIT Debug flag to trigger IR instruction print
-DEBUG_JIT = _readenv("NUMBA_DEBUG_JIT", int, 0)
+    def reset(self):
+        self.old_environ = {}
+        self.update(force=True)
+
+    def update(self, force=False):
+        new_environ = {}
+        for name, value in os.environ.items():
+            if name.startswith('NUMBA_'):
+                new_environ[name] = value
+        # We update the config variables if at least one NUMBA environment
+        # variable was modified.  This lets the user modify values
+        # directly in the config module without having them when
+        # reload_config() is called by the compiler.
+        if force or self.old_environ != new_environ:
+            self.process_environ(new_environ)
+            # Store a copy
+            self.old_environ = dict(new_environ)
 
-# Enable debugging of front-end operation (up to and including IR generation)
-DEBUG_FRONTEND = _readenv("NUMBA_DEBUG_FRONTEND", int, 0)
+    def process_environ(self, environ):
+        for env_name, value in environ.items():
+            config_name = env_name[6:]
+            meth = getattr(self, 'parse_' + config_name)
+            globals()[config_name] = meth(value)
+        # Store a copy
+        self.old_environ = dict(os.environ)
 
-# Optimization level
-OPT = _readenv("NUMBA_OPT", int, 3)
+    def process_environ(self, environ):
+        def _readenv(name, ctor, default):
+            value = environ.get(name)
+            if value is None:
+                return default
+            try:
+                return ctor(value)
+            except Exception:
+                warnings.warn("environ %s defined but failed to parse '%s'" %
+                              (name, res), RuntimeWarning)
+                return default
 
-# Force dump of Python bytecode
-DUMP_BYTECODE = _readenv("NUMBA_DUMP_BYTECODE", int, DEBUG_FRONTEND)
+        # Print warnings to screen about function compilation
+        #   0 = Numba warnings suppressed (default)
+        #   1 = All Numba warnings shown
+        WARNINGS = _readenv("NUMBA_WARNINGS", int, 0)
+        if WARNINGS == 0:
+            warnings.simplefilter('ignore', NumbaWarning)
 
-# Force dump of control flow graph
-DUMP_CFG = _readenv("NUMBA_DUMP_CFG", int, DEBUG_FRONTEND)
+        # Debug flag to control compiler debug print
+        DEBUG = _readenv("NUMBA_DEBUG", int, 0)
 
-# Force dump of Numba IR
-DUMP_IR = _readenv("NUMBA_DUMP_IR", int, DEBUG_FRONTEND)
+        # JIT Debug flag to trigger IR instruction print
+        DEBUG_JIT = _readenv("NUMBA_DEBUG_JIT", int, 0)
 
-# Force dump of LLVM IR
-DUMP_LLVM = _readenv("NUMBA_DUMP_LLVM", int, DEBUG)
+        # Enable debugging of front-end operation (up to and including IR generation)
+        DEBUG_FRONTEND = _readenv("NUMBA_DEBUG_FRONTEND", int, 0)
 
-# Force dump of Function optimized LLVM IR
-DUMP_FUNC_OPT = _readenv("NUMBA_DUMP_FUNC_OPT", int, DEBUG)
+        # Optimization level
+        OPT = _readenv("NUMBA_OPT", int, 3)
 
-# Force dump of Optimized LLVM IR
-DUMP_OPTIMIZED = _readenv("NUMBA_DUMP_OPTIMIZED", int, DEBUG)
+        # Force dump of Python bytecode
+        DUMP_BYTECODE = _readenv("NUMBA_DUMP_BYTECODE", int, DEBUG_FRONTEND)
 
-# Force disable loop vectorize
-# Loop vectorizer is disabled on 32-bit win32 due to a bug (#649)
-LOOP_VECTORIZE = _readenv("NUMBA_LOOP_VECTORIZE", int,
-                          not (IS_WIN32 and IS_32BITS))
+        # Force dump of control flow graph
+        DUMP_CFG = _readenv("NUMBA_DUMP_CFG", int, DEBUG_FRONTEND)
 
-# Force dump of generated assembly
-DUMP_ASSEMBLY = _readenv("NUMBA_DUMP_ASSEMBLY", int, DEBUG)
+        # Force dump of Numba IR
+        DUMP_IR = _readenv("NUMBA_DUMP_IR", int, DEBUG_FRONTEND)
 
-# Force dump of type annotation
-ANNOTATE = _readenv("NUMBA_DUMP_ANNOTATION", int, 0)
+        # Force dump of LLVM IR
+        DUMP_LLVM = _readenv("NUMBA_DUMP_LLVM", int, DEBUG)
 
-# Dump type annotation in html format
-HTML = _readenv("NUMBA_DUMP_HTML", str, None)
+        # Force dump of Function optimized LLVM IR
+        DUMP_FUNC_OPT = _readenv("NUMBA_DUMP_FUNC_OPT", int, DEBUG)
 
-# Python version in (major, minor) tuple
-PYVERSION = sys.version_info[:2]
+        # Force dump of Optimized LLVM IR
+        DUMP_OPTIMIZED = _readenv("NUMBA_DUMP_OPTIMIZED", int, DEBUG)
 
-# Disable CUDA support
-DISABLE_CUDA = _readenv("NUMBA_DISABLE_CUDA", int, int(MACHINE_BITS==32))
+        # Force disable loop vectorize
+        # Loop vectorizer is disabled on 32-bit win32 due to a bug (#649)
+        LOOP_VECTORIZE = _readenv("NUMBA_LOOP_VECTORIZE", int,
+                                  not (IS_WIN32 and IS_32BITS))
 
-# Allow interpreter fallback so that Numba @jit decorator will never fail
-# Use for migrating from old numba (<0.12) which supported closure, and other
-# yet-to-be-supported features.
-COMPATIBILITY_MODE = _readenv("NUMBA_COMPATIBILITY_MODE", int, 0)
+        # Force dump of generated assembly
+        DUMP_ASSEMBLY = _readenv("NUMBA_DUMP_ASSEMBLY", int, DEBUG)
 
-# Force CUDA compute capability
-def _force_cc(text):
-    if not text:
-        return None
-    else:
-        m = re.match(r'(\d+)\.(\d+)', text)
-        if not m:
-            raise ValueError("NUMBA_FORCE_CUDA_CC must be specified as a "
-                             "string of \"major.minor\" where major "
-                             "and minor are decimals")
-        grp = m.groups()
-        return int(grp[0]), int(grp[1])
+        # Force dump of type annotation
+        ANNOTATE = _readenv("NUMBA_DUMP_ANNOTATION", int, 0)
+
+        # Dump type annotation in html format
+        HTML = _readenv("NUMBA_DUMP_HTML", str, None)
+
+        # Disable CUDA support
+        DISABLE_CUDA = _readenv("NUMBA_DISABLE_CUDA", int, int(MACHINE_BITS==32))
+
+        # Allow interpreter fallback so that Numba @jit decorator will never fail
+        # Use for migrating from old numba (<0.12) which supported closure, and other
+        # yet-to-be-supported features.
+        COMPATIBILITY_MODE = _readenv("NUMBA_COMPATIBILITY_MODE", int, 0)
+
+        # Force CUDA compute capability to a specific version
+        FORCE_CUDA_CC = _readenv("NUMBA_FORCE_CUDA_CC", _parse_cc, None)
+
+        # x86-64 specific
+        # Enable AVX on supported platforms where it won't degrade performance.
+        ENABLE_AVX = _readenv("NUMBA_ENABLE_AVX", int,
+                              _cpu_name not in ('corei7-avx', 'core-avx-i'))
+
+        # Disable jit for debugging
+        DISABLE_JIT = _readenv("NUMBA_DISABLE_JIT", int, 0)
+
+        # Enable CUDA simulator
+        ENABLE_CUDASIM = _readenv("NUMBA_ENABLE_CUDASIM", int, 0)
 
+        # Inject the configuration values into the module globals
+        for name, value in locals().items():
+            if name.isupper():
+                globals()[name] = value
 
-FORCE_CUDA_CC = _readenv("NUMBA_FORCE_CUDA_CC", _force_cc, None)
 
-# x86-64 specific
-# Enable AVX on supported platforms where it won't degrade performance.
-ENABLE_AVX = _readenv("NUMBA_ENABLE_AVX", int,
-                      _cpu_name not in ('corei7-avx', 'core-avx-i'))
+_env_reloader = _EnvReloader()
 
-# Disable jit for debugging
-DISABLE_JIT = _readenv("NUMBA_DISABLE_JIT", int, 0)
 
-# Enable CUDA simulator
-ENABLE_CUDASIM = _readenv("NUMBA_ENABLE_CUDASIM", int, 0)
+def reload_config():
+    """
+    Reload the configuration from environment variables, if necessary.
+    """
+    _env_reloader.update()


Test patch:
diff --git a/numba/tests/test_debug.py b/numba/tests/test_debug.py
--- a/numba/tests/test_debug.py
+++ b/numba/tests/test_debug.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, absolute_import
 
+import os
 import platform
 import textwrap
 
@@ -20,18 +21,18 @@ def simple_gen(x, y):
 
 class DebugTestBase(TestCase):
 
+    all_dumps = set(['bytecode', 'cfg', 'ir', 'llvm', 'func_opt_llvm',
+                     'optimized_llvm', 'assembly'])
+
     def assert_fails(self, *args, **kwargs):
         self.assertRaises(AssertionError, *args, **kwargs)
 
-    def check_debug_output(self, out, enabled_dumps):
-        all_dumps = dict.fromkeys(['bytecode', 'cfg', 'ir', 'llvm',
-                                   'func_opt_llvm', 'optimized_llvm',
-                                   'assembly'],
-                                  False)
-        for name in enabled_dumps:
-            assert name in all_dumps
-            all_dumps[name] = True
-        for name, enabled in sorted(all_dumps.items()):
+    def check_debug_output(self, out, dump_names):
+        enabled_dumps = dict.fromkeys(self.all_dumps, False)
+        for name in dump_names:
+            assert name in enabled_dumps
+            enabled_dumps[name] = True
+        for name, enabled in sorted(enabled_dumps.items()):
             check_meth = getattr(self, '_check_dump_%s' % name)
             if enabled:
                 check_meth(out)
@@ -66,7 +67,7 @@ def _check_dump_assembly(self, out):
             self.assertIn('xorl', out)
 
 
-class TestFunctionDebugOutput(DebugTestBase):
+class FunctionDebugTestBase(DebugTestBase):
 
     func_name = 'simple_nopython'
 
@@ -77,6 +78,9 @@ def compile_simple_nopython(self):
             self.assertPreciseEqual(cres.entry_point(2), 3)
         return out.getvalue()
 
+
+class TestFunctionDebugOutput(FunctionDebugTestBase):
+
     def test_dump_bytecode(self):
         with override_config('DUMP_BYTECODE', True):
             out = self.compile_simple_nopython()
@@ -137,5 +141,26 @@ def test_dump_ir_generator(self):
         self.assertIn(expected_gen_info, out)
 
 
+class TestEnvironmentOverride(FunctionDebugTestBase):
+    """
+    Test that environment variables are reloaded by Numba when modified.
+    """
+
+    def test_debug(self):
+        out = self.compile_simple_nopython()
+        self.assertFalse(out)
+        os.environ['NUMBA_DEBUG'] = '1'
+        try:
+            out = self.compile_simple_nopython()
+            # Note that all variables dependent on NUMBA_DEBUG are
+            # updated too.
+            self.check_debug_output(out, ['ir', 'llvm', 'func_opt_llvm',
+                                          'optimized_llvm', 'assembly'])
+        finally:
+            del os.environ['NUMBA_DEBUG']
+        out = self.compile_simple_nopython()
+        self.assertFalse(out)
+
+
 if __name__ == '__main__':
     unittest.main()