Repository: numba/numba

Issue:
Feature Request: add ability to run -03 twice
This bug first occurs in version 0.54 (0.49-0.53 are OK). This results in a performance decrease of a factor of 4 in a bit optimized summation and a factor of 2 in a naive implementation.

**Example**

```python
import numpy as np
import numba as nb
import time


f = np.random.rand(32,33,2000,64)
b = np.random.rand(32,64)

@nb.njit(fastmath=True)
def test_numba_1(f,b):
    fnew = np.zeros((f.shape[1],f.shape[2]))
    
    for i in range(f.shape[0]):
        for j in range(f.shape[1]):
            for k in range(f.shape[2]):
                for l in range(f.shape[3]):
                    fnew[j,k] += f[i,j,k,l] * b[i,l]
    return fnew

@nb.njit(fastmath=True)
def test_numba_2(f,b):
    fnew = np.zeros((f.shape[1],f.shape[2])) 
    
    for i in range(f.shape[0]):
        for j in range(f.shape[1]):
            for k in range(f.shape[2]):
                acc=fnew[j,k]
                for l in range(f.shape[3]):
                    acc += f[i,j,k,l] * b[i,l]
                fnew[j,k]=acc
    return fnew

%timeit test_numba_1(f,b)
%timeit test_numba_2(f,b)

#'0.53.1' Last version without bug
#108 ms ± 1.35 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
#53.9 ms ± 809 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

#'0.54.0' First version with bug
#216 ms ± 4.68 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
#200 ms ± 2.93 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

#'0.55.0'
#218 ms ± 5.36 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
#201 ms ± 1.73 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
```

Another question would be if, a rewrite pass is possible in Numba to optimize `test_numba_1` automatically to `test_numba_2`. Clang shows exactly the same behavior as Numba before version 0.54, therefore this likely has to be fixed before generating LLVM-IR.

```c
//clang v11
//clang -shared -O3 -march=native -ffast-math -o summation.dll summation.c
__declspec(dllexport) void eq_nb_1(size_t s0,size_t s1,size_t s2,size_t s3,double f[][s1][s2][s3],double b[][s3],double fnew[][s2])
{
  for (size_t i=0;i<s0;i++){
    for (size_t j=0;j<s1;j++){
      for (size_t k=0;k<s2;k++){
         for (size_t l=0;l<s3;l++){
           fnew[j][k]+=f[i][j][k][l]*b[i][l];
         }
      }
    }
  }
}

__declspec(dllexport) void eq_nb_2(size_t s0,size_t s1,size_t s2,size_t s3,double f[][s1][s2][s3],double b[][s3],double fnew[][s2])
{
  for (size_t i=0;i<s0;i++){
    for (size_t j=0;j<s1;j++){
      for (size_t k=0;k<s2;k++){
         double acc=fnew[j][k];
         for (size_t l=0;l<s3;l++){
           acc+=f[i][j][k][l]*b[i][l];
         }
         fnew[j][k]=acc;
      }
    }
  }
}
```
```python
import numpy as np
import ctypes

lib = ctypes.cdll.LoadLibrary("summation.dll")

def eq_nb_1(f,b):
    fnew = np.zeros((f.shape[1],f.shape[2]))
    s_0=ctypes.c_size_t(f.shape[0])
    s_1=ctypes.c_size_t(f.shape[1])
    s_2=ctypes.c_size_t(f.shape[2])
    s_3=ctypes.c_size_t(f.shape[3])
    
    f_p   =f.ctypes
    b_p   =b.ctypes
    fnew_p=fnew.ctypes
    
    lib.eq_nb_1(s_0,s_1,s_2,s_3,f_p,b_p,fnew_p)
    return fnew

def eq_nb_2(f,b):
    fnew = np.zeros((f.shape[1],f.shape[2]))
    s_0=ctypes.c_size_t(f.shape[0])
    s_1=ctypes.c_size_t(f.shape[1])
    s_2=ctypes.c_size_t(f.shape[2])
    s_3=ctypes.c_size_t(f.shape[3])
    
    f_p   =f.ctypes
    b_p   =b.ctypes
    fnew_p=fnew.ctypes
    
    lib.eq_nb_2(s_0,s_1,s_2,s_3,f_p,b_p,fnew_p)
    return fnew

%timeit eq_nb_1(f,b)
%timeit eq_nb_2(f,b)
#100 ms ± 311 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
#50.5 ms ± 1.08 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
```
 


Discussion:
Hi @max9111, thanks for the report. I can confirm there's a performance regression when moving from Numba 0.53 to newer versions. I'll mark this for discussion at the next triage meeting, which will happen next Tuesday.
IIRC 0.54 was the first version using LLVM 11 and there were some things to do with missed vectorization that Numba encountered when the upgrade was undertaken (which were fixed, but I guess this one was missed).

This diff recovers the 0.53 performance, but at the cost of increased compilation effort:
```diff
diff --git a/numba/core/codegen.py b/numba/core/codegen.py
--- a/numba/core/codegen.py
+++ b/numba/core/codegen.py
@@ -1189,7 +1189,7 @@ class CPUCodegen(Codegen):
         self._data_layout = str(self._target_data)
         self._mpm_cheap = self._module_pass_manager(loop_vectorize=False,
                                                     slp_vectorize=False,
-                                                    opt=0,
+                                                    opt=2,
                                                     cost="cheap")
         self._mpm_full = self._module_pass_manager()

```
which suggests it's still possible to do the expected optimisation and it's just a question of working out precisely what combination of passes that is!
This issue is marked as stale as it has had no activity in the past 30 days. Please close this issue if no further response or action is needed. Otherwise, please respond with any updates and confirm that this issue still needs to be addressed.
What appears to be the same performance bug has just been reported to https://stackoverflow.com/questions/75833328/why-do-these-two-different-ways-to-sum-a-2d-array-have-such-different-performanc . Is a fix planned for the next release?
This issue was raised at the Numba public meeting on 2023-03-28. The outcome is that the 0.58 release series will have support for an environment variable that if set will reinstate the previous behaviour (effectively running `-O3` twice).
I have updated the title of this Issue to reflect it's change in state. It is now a feature request scheduled for inclusion in Numba 0.58.

The feature request is: as a Numba user I want to be able to set the environment variable `NUMBA_RUN_O3_TWICE` which will effectively "run -O3 twice" and run the code path from the patch at: https://github.com/numba/numba/issues/8172#issuecomment-1160474583
@esc 
I have a general question:

**Why is that the new default?**

A real-world function without a severe performance drop (factor 1,5-5x) is hard to find. The improvement of compilation time is usually neglectable, it's often a few percent, with caching it doesn’t matter at all.
What is the reason for this? It absolutely makes no sense at all. Compilation times can be high in heavily NumPy - vectorized codes. If Numba is able to compile that at all (quite some features missing, but anyway extremely complicate to optimize this code) the perfomance benefit is quite often neglectable (no real usecase). Usually there is some simple (or maybe not as simple but doable)  rewrite necessary to get a good result in this case.
All this without reimplementation in (C/Fortran)+Wrapper+Dependencies, which is a different story in terms of effort. That's the beauty of Numba.

I really don't get the point to make this bug or “feature” as a default. To my opinion it is one of the most significant bugs in the last years. 

The patch from https://github.com/numba/numba/issues/8172#issuecomment-1160474583 slows down the fast variant in the reproducer from https://github.com/numba/numba/issues/9038. Without the patch, `dist_cityblock_2()` runs in:

```
2.34 µs ± 10.9 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each)
```

With the patch, it runs in:

```
4.05 µs ± 12.7 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each)
```

so applying the patch above does not look like it would universally improve performance. I can see why a user would want the option to use `opt=2` at their discretion though.
> The patch from [#8172 (comment)](https://github.com/numba/numba/issues/8172#issuecomment-1160474583) slows down the fast variant in the reproducer from #9038. Without the patch, `dist_cityblock_2()` runs in:
> 
> ```
> 2.34 µs ± 10.9 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each)
> ```
> 
> With the patch, it runs in:
> 
> ```
> 4.05 µs ± 12.7 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each)
> ```
> 
> so applying the patch above does not look like it would universally improve performance. I can see why a user would want the option to use `opt=2` at their discretion though.

Numba 0.57 seems to change the behavoir, on 0.56 I can't reproduce this behaviour:

    %timeit dist_cityblock_1(x, y)
    #0.56, windows installed via pip (default)
    #3.82 µs ± 1.45 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
    #0.56, windows installed via pip (opt=2)
    #2.02 µs ± 24.3 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

    #0.57, windows installed via pip (default)
    #3.76 µs ± 0.962 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
    #0.57, windows installed via pip (opt=2)
    #4.13 µs ± 61.4 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
    
    %timeit dist_cityblock_2(x, y)
    #0.56, windows installed via pip (default)
    #6.56 µs ± 4.09 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
    #0.56, windows installed via pip (opt=2)
    #2.02 µs ± 24.3 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

    #0.57, windows installed via pip (default)
    #2.09 µs ± 19.5 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
    #0.57, windows installed via pip (opt=2)
    #5.61 µs ± 2.13 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

The issue with the initial question seems to be fixed with 0.57, but opt=2 seems to be slower than the default now:

    %timeit test_numba_1(f,b)
    #0.56, windows installed via pip (default)
    #123 ms ± 290 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)
    #0.56, windows installed via pip (opt=2)
    #78.1 ms ± 182 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

    #0.57, windows installed via pip (default)
    #77.3 ms ± 499 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)
    #0.57, windows installed via pip (opt=2)
    #77.6 ms ± 491 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

    %timeit test_numba_2(f,b)
    #0.56, windows installed via pip (default)
    #120 ms ± 914 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)
    #0.56, windows installed via pip (opt=2)
    #36.2 ms ± 556 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)


    #0.57, windows installed via pip (default)
    #38 ms ± 1.04 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
    #0.57, windows installed via pip (opt=2)
    #71 ms ± 341 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

On other [examples](https://stackoverflow.com/a/76583728/4045774), there seems to be still a bit slower timings with 0.57 (default) to previous versions with (opt=2). 
It does look like 0.57.0 has sped up the problematic code at https://stackoverflow.com/questions/75833328/why-do-these-two-different-ways-to-sum-a-2d-array-have-such-different-performanc as well.  %timeit sum_array_slow(a) is now the same speed as %timeit sum_array_fast(a) more or less.  I had been expecting to have to wait for this 0.58 to see this speed up.
> Numba 0.57 seems to change the behavoir

Numba 0.57 went from LLVM 11 to 14, so that might explain a lot of changes - unfortunately it does make it hard to directly compare the two.

Resolving patch:
diff --git a/numba/core/codegen.py b/numba/core/codegen.py
--- a/numba/core/codegen.py
+++ b/numba/core/codegen.py
@@ -1187,10 +1187,30 @@ def _init(self, llvm_module):
         self._engine = JitEngine(engine)
         self._target_data = engine.target_data
         self._data_layout = str(self._target_data)
-        self._mpm_cheap = self._module_pass_manager(loop_vectorize=False,
+
+        if config.OPT.is_opt_max:
+            # If the OPT level is set to 'max' then the user is requesting that
+            # compilation time is traded for potential performance gain. This
+            # currently manifests as running the "cheap" pass at -O3
+            # optimisation level with loop-vectorization enabled. There's no
+            # guarantee that this will increase runtime performance, it may
+            # detriment it, this is here to give the user an easily accessible
+            # option to try.
+            loopvect = True
+            opt_level = 3
+        else:
+            # The default behaviour is to do an opt=0 pass to try and inline as
+            # much as possible with the cheapest cost of doing so. This is so
+            # that the ref-op pruner pass that runs after the cheap pass will
+            # have the largest possible scope for working on pruning references.
+            loopvect = False
+            opt_level = 0
+
+        self._mpm_cheap = self._module_pass_manager(loop_vectorize=loopvect,
                                                     slp_vectorize=False,
-                                                    opt=0,
+                                                    opt=opt_level,
                                                     cost="cheap")
+
         self._mpm_full = self._module_pass_manager()
 
         self._engine.set_object_cache(self._library_class._object_compiled_hook,
diff --git a/numba/core/config.py b/numba/core/config.py
--- a/numba/core/config.py
+++ b/numba/core/config.py
@@ -80,6 +80,49 @@ def _validate_captured_errors_style(style_str):
         return rendered_style
 
 
+class _OptLevel(int):
+    """This class holds the "optimisation level" set in `NUMBA_OPT`. As this env
+    var can be an int or a string, but is almost always interpreted as an int,
+    this class subclasses int so as to get the common behaviour but stores the
+    actual value as a `_raw_value` member. The value "max" is a special case
+    and the property `is_opt_max` can be queried to find if the optimisation
+    level (supplied value at construction time) is "max"."""
+
+    def __new__(cls, *args, **kwargs):
+        assert len(args) == 1
+        (value,) = args
+        _int_value = 3 if value == 'max' else int(value)
+        # the int ctor is always called with an appropriate integer value
+        new = super().__new__(cls, _int_value, **kwargs)
+        # raw value is max or int
+        new._raw_value = value if value == 'max' else _int_value
+        return new
+
+    @property
+    def is_opt_max(self):
+        """Returns True if the the optimisation level is "max" False
+        otherwise."""
+        return self._raw_value == "max"
+
+    def __repr__(self):
+        if isinstance(self._raw_value, str):
+            arg = f"'{self._raw_value}'"
+        else:
+            arg = self._raw_value
+        return f"_OptLevel({arg})"
+
+
+def _process_opt_level(opt_level):
+
+    if opt_level not in ('0', '1', '2', '3', 'max'):
+        msg = ("Environment variable `NUMBA_OPT` is set to an unsupported "
+               f"value '{opt_level}', supported values are 0, 1, 2, 3, and "
+               "'max'")
+        raise ValueError(msg)
+    else:
+        return _OptLevel(opt_level)
+
+
 class _EnvReloader(object):
 
     def __init__(self):
@@ -150,9 +193,11 @@ def _readenv(name, ctor, default):
                 return default() if callable(default) else default
             try:
                 return ctor(value)
-            except Exception:
-                warnings.warn("environ %s defined but failed to parse '%s'" %
-                              (name, value), RuntimeWarning)
+            except Exception as e:
+                warnings.warn(f"Environment variable '{name}' is defined but "
+                              f"its associated value '{value}' could not be "
+                              f"parsed.\nThe parse failed with exception: {e}.",
+                              RuntimeWarning)
                 return default
 
         def optional_str(x):
@@ -259,7 +304,7 @@ def optional_str(x):
                                 ("" if str(CPU_NAME).lower() == 'generic'
                                  else None))
         # Optimization level
-        OPT = _readenv("NUMBA_OPT", int, 3)
+        OPT = _readenv("NUMBA_OPT", _process_opt_level, _OptLevel(3))
 
         # Force dump of Python bytecode
         DUMP_BYTECODE = _readenv("NUMBA_DUMP_BYTECODE", int, DEBUG_FRONTEND)


Test patch:
diff --git a/numba/tests/test_config.py b/numba/tests/test_config.py
--- a/numba/tests/test_config.py
+++ b/numba/tests/test_config.py
@@ -2,6 +2,7 @@
 import tempfile
 from textwrap import dedent
 import unittest
+from unittest import mock
 from numba.tests.support import (TestCase, temp_directory, override_env_config,
                                  run_in_subprocess)
 from numba.core import config
@@ -113,11 +114,95 @@ def test_illegal_error_style_handling(self):
         code = ("from numba import njit\n@njit\ndef foo():\n\t"
                 f"print('{source_compiled}')\nfoo()")
         out, err = run_in_subprocess(dedent(code), env=new_env)
-        expected = ("environ NUMBA_CAPTURED_ERRORS defined but failed to parse "
-                    "\'not_a_known_style\'")
-        self.assertIn(expected, err.decode('utf-8'))
+        expected = ("Environment variable \'NUMBA_CAPTURED_ERRORS\' is defined "
+                    "but its associated value \'not_a_known_style\' could not "
+                    "be parsed.")
+        err_msg = err.decode('utf-8')
+        self.assertIn(expected, err_msg)
+        ex_expected = ("The parse failed with exception: Invalid style in "
+                       "NUMBA_CAPTURED_ERRORS: not_a_known_style")
+        self.assertIn(ex_expected, err_msg)
         self.assertIn(source_compiled, out.decode('utf-8'))
 
 
+class TestNumbaOptLevel(TestCase):
+    # Tests that the setting of NUMBA_OPT influences the "cheap" module pass.
+    # Spot checks NUMBA_OPT={'max', '3', '0'}
+
+    def check(self, expected, opt_value, raw_value):
+        # local imports for state-safety
+        from numba import config, njit
+
+        # check opt value and its raw_value
+        self.assertEqual(config.OPT, opt_value)
+        self.assertEqual(config.OPT._raw_value, raw_value)
+
+        # Patch the CPUCodegen to make capture calls to the
+        # `_module_pass_manager` through a `side_effect` function that asserts
+        # that the kwargs being passed are as expected per the "NUMBA_OPT"
+        # level. The `side_effect` function immediately raises with a knwon
+        # message to abort further stages compilation once the check is
+        # complete.
+        from numba.core.codegen import CPUCodegen
+        side_effect_message = "expected side effect"
+
+        def side_effect(*args, **kwargs):
+            self.assertEqual(kwargs, expected)
+            raise RuntimeError(side_effect_message)
+
+        with mock.patch.object(CPUCodegen, '_module_pass_manager',
+                               side_effect=side_effect):
+            with self.assertRaises(RuntimeError) as raises:
+                njit(lambda : ...)()
+
+            self.assertIn(side_effect_message, str(raises.exception))
+
+    @TestCase.run_test_in_subprocess(envvars={'NUMBA_OPT': 'max'})
+    def test_opt_max(self):
+        # NUMBA_OPT='max' should set opt to 3 and enable loop_vectorize
+        expected = {'loop_vectorize': True,
+                    'slp_vectorize': False,
+                    'opt': 3,
+                    'cost': 'cheap'}
+        self.check(expected, 3, 'max')
+
+    @TestCase.run_test_in_subprocess(envvars={'NUMBA_OPT': '3'})
+    def test_opt_3(self):
+        # NUMBA_OPT='3' should not impact opt or loop_vectorize
+        expected = {'loop_vectorize': False,
+                    'slp_vectorize': False,
+                    'opt': 0,
+                    'cost': 'cheap'}
+        self.check(expected, 3, 3)
+
+    @TestCase.run_test_in_subprocess(envvars={'NUMBA_OPT': '0'})
+    def test_opt_0(self):
+        # NUMBA_OPT='0' should not impact opt or loop_vectorize
+        expected = {'loop_vectorize': False,
+                    'slp_vectorize': False,
+                    'opt': 0,
+                    'cost': 'cheap'}
+        self.check(expected, 0, 0)
+
+    @TestCase.run_test_in_subprocess()
+    def test_opt_default(self):
+        # NUMBA_OPT is not set, the default should not impact opt or
+        # loop_vectorize
+        expected = {'loop_vectorize': False,
+                    'slp_vectorize': False,
+                    'opt': 0,
+                    'cost': 'cheap'}
+        self.check(expected, 3, 3)
+
+    @TestCase.run_test_in_subprocess(envvars={'NUMBA_OPT': 'invalid'})
+    def test_opt_invalid(self):
+        # NUMBA_OPT='invalid' should just proceed as default case
+        expected = {'loop_vectorize': False,
+                    'slp_vectorize': False,
+                    'opt': 0,
+                    'cost': 'cheap'}
+        self.check(expected, 3, 3)
+
+
 if __name__ == '__main__':
     unittest.main()