Repository: celery/billiard

Issue:
multiple billiard.Value instances sharing the same state
**Problem description**

When using multiple instances of `billiard.Value` and changing one variable's value, the other one is changed as well. In the example below when I update the variable `value`, `done` is set to the same value as well. When replacing `billiard` with `multiprocessing` the example works as expected.

**Example**

See [below](#issuecomment-311695221) for an easier example.

```python
import ctypes
import time
from billiard import Pool, Value


def foo():
    for i in range(10):
        value.value = i
        time.sleep(1)

    done.value = True


def bar():
    while True:
        print(value.value, '\t', done.value)
        time.sleep(0.5)


if __name__ == '__main__':
    value = Value(ctypes.c_int, 0)
    done = Value(ctypes.c_bool, False)

    with Pool() as pool:
        pool.apply_async(bar)
        pool.apply(foo)
```

**Expected output**

```
0 	 False
0 	 False
1 	 False
1 	 False
1 	 False
2 	 False
3 	 False
3 	 False
4 	 False
4 	 False
5 	 False
5 	 False
6 	 False
6 	 False
7 	 False
7 	 False
8 	 False
8 	 False
9 	 False
9 	 False
9 	 True
```

**Actual output**

```
0 	 False
0 	 False
1 	 True
1 	 True
2 	 True
2 	 True
3 	 True
3 	 True
4 	 True
4 	 True
5 	 True
5 	 True
6 	 True
6 	 True
7 	 True
7 	 True
8 	 True
8 	 True
9 	 True
9 	 True
```

**Versions**

```shell
 % pip freeze | grep billiard  
billiard==3.5.0.2
 % python --version          
Python 3.6.0
 % uname -a
Darwin HOST 16.4.0 Darwin Kernel Version 16.4.0: Thu Dec 22 22:53:21 PST 2016; root:xnu-3789.41.3~3/RELEASE_X86_64 x86_64
 % sw_vers -productVersion 
10.12.3
```

Discussion:
Birne94: Python 2.7 tests (cpython und pypy) appear to fail for some unrelated issue, values might be broken on non Python 3.

`OSError: [Errno 17] File exists: '/tmp/pymp-JfdEz9/pym-2339-cp7Vsb'`

listingmirror: @Birne94 Any idea how this would interact with the "CELERY_WORKER_MAX_MEMORY_PER_CHILD" property? Like are internal properties set with a BufferedWrapper?

I just started using CELERY_WORKER_MAX_MEMORY_PER_CHILD, and I was confused what was going on. I run with 10 subprocesses. When any one of the 10 hit the limit, it seems like all 10 would get killed instead of just the one that used too much memory.  Randomly trying out your branch to see what it does...
thedrow: The tests fail for Python 2.7 which is a blocker for merging this PR.
The test `test_issue_229` was added in this PR so it means that something is wrong with the current implementation. We could branch on Python 2/3 if needed but this needs to be fixed.

@Birne94 Can you please take a look?

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index 9e6c09e..e2366da 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,16 @@
+3.5.0.3 - 2017-07-16
+--------------------
+
+- Adds Process._authkey alias to .authkey for 2.7 compat.
+- Remove superfluous else clause from max_memory_per_child_check.
+- Document and test all supported Python versions.
+- Extend 'Process' to be compatible with < Py3.5.
+- Use a properly initialized logger in pool.py error logging.
+- _trywaitkill can now kill a whole process group if the worker process declares itself as a group leader.
+- Fix cpython issue 14881 (See http://bugs.python.org/issue14881).
+- Fix for a crash on windows.
+- Fix messaging in case of worker exceeds max memory.
+
 3.5.0.2 - 2016-10-03
 --------------------
 
diff --git a/billiard/__init__.py b/billiard/__init__.py
index ada838a..9c7c92a 100644
--- a/billiard/__init__.py
+++ b/billiard/__init__.py
@@ -22,7 +22,7 @@ from __future__ import absolute_import
 import sys
 from . import context
 
-VERSION = (3, 5, 0, 2)
+VERSION = (3, 5, 0, 3)
 __version__ = '.'.join(map(str, VERSION[0:4])) + "".join(VERSION[4:])
 __author__ = 'R Oudkerk / Python Software Foundation'
 __author_email__ = 'python-dev@python.org'
diff --git a/billiard/heap.py b/billiard/heap.py
index 278bfe5..b7581ce 100644
--- a/billiard/heap.py
+++ b/billiard/heap.py
@@ -73,11 +73,12 @@ else:
             self.size = size
             self.fd = fd
             if fd == -1:
-                self.fd, name = tempfile.mkstemp(
-                    prefix='pym-%d-' % (os.getpid(), ),
-                    dir=util.get_temp_dir(),
-                )
                 if PY3:
+                    self.fd, name = tempfile.mkstemp(
+                        prefix='pym-%d-' % (os.getpid(),),
+                        dir=util.get_temp_dir(),
+                    )
+
                     os.unlink(name)
                     util.Finalize(self, os.close, (self.fd,))
                     with io.open(self.fd, 'wb', closefd=False) as f:
@@ -90,6 +91,10 @@ else:
                         f.write(b'\0' * (size % bs))
                         assert f.tell() == size
                 else:
+                    name = tempfile.mktemp(
+                        prefix='pym-%d-' % (os.getpid(),),
+                        dir=util.get_temp_dir(),
+                    )
                     self.fd = os.open(
                         name, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600,
                     )
diff --git a/billiard/pool.py b/billiard/pool.py
index 2734f2a..1bb9d10 100644
--- a/billiard/pool.py
+++ b/billiard/pool.py
@@ -47,7 +47,7 @@ from .five import Empty, Queue, range, values, reraise, monotonic
 from .util import Finalize, debug
 
 MAXMEM_USED_FMT = """\
-child process exiting after exceeding memory limit ({0}KiB / {0}KiB)
+child process exiting after exceeding memory limit ({0}KiB / {1}KiB)
 """
 
 PY3 = sys.version_info[0] == 3
diff --git a/billiard/sharedctypes.py b/billiard/sharedctypes.py
index 6334661..97675df 100644
--- a/billiard/sharedctypes.py
+++ b/billiard/sharedctypes.py
@@ -155,7 +155,7 @@ def rebuild_ctype(type_, wrapper, length):
         obj = type_.from_buffer(buf)
     else:
         obj = type_.from_address(wrapper.get_address())
-        obj._wrapper = wrapper
+    obj._wrapper = wrapper
     return obj
 
 #


Test patch:
diff --git a/t/unit/test_values.py b/t/unit/test_values.py
new file mode 100644
index 0000000..4b0bfc2
--- /dev/null
+++ b/t/unit/test_values.py
@@ -0,0 +1,77 @@
+from __future__ import absolute_import
+import pytest
+
+from billiard import Value, RawValue, Lock, Process
+
+
+class test_values:
+
+    codes_values = [
+        ('i', 4343, 24234),
+        ('d', 3.625, -4.25),
+        ('h', -232, 234),
+        ('c', 'x'.encode('latin'), 'y'.encode('latin'))
+        ]
+
+    def test_issue_229(self):
+        """Test fix for issue #229"""
+
+        a = Value('i', 0)
+        b = Value('i', 0)
+
+        a.value = 5
+        assert a.value == 5
+        assert b.value == 0
+
+    @classmethod
+    def _test(cls, values):
+        for sv, cv in zip(values, cls.codes_values):
+            sv.value = cv[2]
+
+    def test_value(self, raw=False):
+        if raw:
+            values = [RawValue(code, value)
+                      for code, value, _ in self.codes_values]
+        else:
+            values = [Value(code, value)
+                      for code, value, _ in self.codes_values]
+
+        for sv, cv in zip(values, self.codes_values):
+            assert sv.value == cv[1]
+
+        proc = Process(target=self._test, args=(values,))
+        proc.daemon = True
+        proc.start()
+        proc.join()
+
+        for sv, cv in zip(values, self.codes_values):
+            assert sv.value == cv[2]
+
+    def test_rawvalue(self):
+        self.test_value(raw=True)
+
+    def test_getobj_getlock(self):
+        val1 = Value('i', 5)
+        lock1 = val1.get_lock()
+        obj1 = val1.get_obj()
+
+        val2 = Value('i', 5, lock=None)
+        lock2 = val2.get_lock()
+        obj2 = val2.get_obj()
+
+        lock = Lock()
+        val3 = Value('i', 5, lock=lock)
+        lock3 = val3.get_lock()
+        obj3 = val3.get_obj()
+        assert lock == lock3
+
+        arr4 = Value('i', 5, lock=False)
+        assert not hasattr(arr4, 'get_lock')
+        assert not hasattr(arr4, 'get_obj')
+
+        with pytest.raises(AttributeError):
+            Value('i', 5, lock='navalue')
+
+        arr5 = RawValue('i', 5)
+        assert not hasattr(arr5, 'get_lock')
+        assert not hasattr(arr5, 'get_obj')