Repository: numba/numba

Issue:
Windows: `PermissionError` using `@njit(cache=True)` with multiprocessing
Writing cache for compiled functions from multiple processes may clash and thus raises `PermissionError` under Windows operation systems. See:

``` python
from multiprocessing import Pool
from numba import njit

@njit(cache=True)
def callee(i):
    return i

def caller(i):
    return callee(i)

def main():
    p = 2
    with Pool(p) as pool:
        sum(pool.imap(caller, range(p)))

if __name__ == "__main__":
    main()
```

```
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
  File "C:\Miniconda3\envs\numba\lib\site-packages\numba\caching.py", line 373, in save_overload
    data_name = overloads[key]
KeyError: ((int64,), ('x86_64-pc-windows-msvc', 'amdfam10', '-adx,-aes,-avx,-avx2,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512pf,-avx512vl,-bmi,-bmi2,+cmov,+cx16,-f16c,-fma,-fma4,-fsgsbase,-hle,+lzcnt,+mmx,-movbe,-pclmul,+popcnt,+prfchw,-rdrnd,-rdseed,-rtm,-sha,+sse,+sse2,+sse3,-sse4.1,-sse4.2,+sse4a,-ssse3,-tbm,-xop'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Miniconda3\envs\numba\lib\multiprocessing\pool.py", line 119, in worker
    result = (True, func(*args, **kwds))
  File "C:\numba\njit_cache_multiprocessing.py", line 10, in caller
    return callee(i)
  File "C:\Miniconda3\envs\numba\lib\site-packages\numba\dispatcher.py", line 281, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "C:\Miniconda3\envs\numba\lib\site-packages\numba\dispatcher.py", line 529, in compile
    self._cache.save_overload(sig, cres)
  File "C:\Miniconda3\envs\numba\lib\site-packages\numba\caching.py", line 382, in save_overload
    self._save_index(overloads)
  File "C:\Miniconda3\envs\numba\lib\site-packages\numba\caching.py", line 477, in _save_index
    f.write(data)
  File "C:\Miniconda3\envs\numba\lib\contextlib.py", line 66, in __exit__
    next(self.gen)
  File "C:\Miniconda3\envs\numba\lib\site-packages\numba\caching.py", line 428, in _open_for_write
    utils.file_replace(tmpname, filepath)
PermissionError: [WinError 5] Zugriff verweigert: 'C:\\numba\\__pycache__\\__mp_main__.callee-5.py34.nbi.tmp.2332' -> 'C:\\numba\\__pycache__\\__mp_main__.callee-5.py34.nbi'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "njit_cache_multiprocessing.py", line 18, in <module>
    main()
  File "njit_cache_multiprocessing.py", line 15, in main
    sum(pool.imap(caller, range(p)))
  File "C:\Miniconda3\envs\numba\lib\multiprocessing\pool.py", line 698, in next
    raise value
PermissionError: [WinError None] Zugriff verweigert: 'C:\\numba\\__pycache__\\__mp_main__.callee-5.py34.nbi.tmp.2332' -> 'C:\\numba\\__pycache__\\__mp_main__.callee-5.py34.nbi'
```



Discussion:
Thanks for the report! I can reproduce indeed.


Resolving patch:
diff --git a/numba/caching.py b/numba/caching.py
old mode 100644
new mode 100755
--- a/numba/caching.py
+++ b/numba/caching.py
@@ -344,6 +344,11 @@ def load_overload(self, sig, target_context):
         Load and recreate the cached CompileResult for the given signature,
         using the *target_context*.
         """
+        with self._guard_against_spurious_io_errors():
+            return self._load_overload(sig, target_context)
+        # None returned if the `with` block swallows an exception
+
+    def _load_overload(self, sig, target_context):
         if not self._enabled:
             return
         overloads = self._load_index()
@@ -361,6 +366,10 @@ def save_overload(self, sig, cres):
         """
         Save the CompileResult for the given signature in the cache.
         """
+        with self._guard_against_spurious_io_errors():
+            self._save_overload(sig, cres)
+
+    def _save_overload(self, sig, cres):
         if not self._enabled:
             return
         if not self._check_cachable(cres):
@@ -415,6 +424,20 @@ def _data_name(self, number):
     def _data_path(self, name):
         return os.path.join(self._cache_path, name)
 
+    @contextlib.contextmanager
+    def _guard_against_spurious_io_errors(self):
+        if os.name == 'nt':
+            # Guard against permission errors due to accessing the file
+            # from several processes (see #2028)
+            try:
+                yield
+            except EnvironmentError as e:
+                if e.errno != errno.EACCES:
+                    raise
+        else:
+            # No such conditions under non-Windows OSes
+            yield
+
     @contextlib.contextmanager
     def _open_for_write(self, filepath):
         """


Test patch:
diff --git a/numba/tests/cache_usecases.py b/numba/tests/cache_usecases.py
--- a/numba/tests/cache_usecases.py
+++ b/numba/tests/cache_usecases.py
@@ -16,6 +16,14 @@
 from numba.tests.support import TestCase, captured_stderr
 
 
+@jit(cache=True, nopython=True)
+def simple_usecase(x):
+    return x
+
+def simple_usecase_caller(x):
+    return simple_usecase(x)
+
+
 @jit(cache=True, nopython=True)
 def add_usecase(x, y):
     return x + y + Z
diff --git a/numba/tests/test_dispatcher.py b/numba/tests/test_dispatcher.py
--- a/numba/tests/test_dispatcher.py
+++ b/numba/tests/test_dispatcher.py
@@ -2,6 +2,7 @@
 
 import errno
 import imp
+import multiprocessing
 import os
 import shutil
 import stat
@@ -895,5 +896,30 @@ def execute_with_input():
         self.assertEqual(err.strip(), "cache hits = 1")
 
 
+class TestMultiprocessCache(BaseCacheTest):
+
+    # Nested multiprocessing.Pool raises AssertionError:
+    # "daemonic processes are not allowed to have children"
+    _numba_parallel_test_ = False
+
+    here = os.path.dirname(__file__)
+    usecases_file = os.path.join(here, "cache_usecases.py")
+    modname = "dispatcher_caching_test_fodder"
+
+    def test_multiprocessing(self):
+        # Check caching works from multiple processes at once (#2028)
+        mod = self.import_module()
+        # Calling a pure Python caller of the JIT-compiled function is
+        # necessary to reproduce the issue.
+        f = mod.simple_usecase_caller
+        n = 3
+        pool = multiprocessing.Pool(n)
+        try:
+            res = sum(pool.imap(f, range(n)))
+        finally:
+            pool.close()
+        self.assertEqual(res, n * (n - 1) // 2)
+
+
 if __name__ == '__main__':
     unittest.main()