Repository: celery/celery

Issue:
LimitedSet does not perform as expected - tests fail
## Checklist

- Celery Version v3.1.25
- [ X] I have verified that the issue exists against the `master` branch of Celery.

## Steps to reproduce
Run unit tests

## Expected behavior
That they'd pass :wink:

## Actual behavior
Why they all pass except the ones related to LimitedSet

The results appear to be the absolute invert of what is expected; have a look; here is the output:
```
__________________________________________________ test_LimitedSet.test_add ___________________________________________________

self = <celery.tests.utils.test_datastructures.test_LimitedSet testMethod=test_add>

    def test_add(self):
        if sys.platform == 'win32':
            raise SkipTest('Not working properly on Windows')
        s = LimitedSet(maxlen=2)
        s.add('foo')
        s.add('bar')
        for n in 'foo', 'bar':
            self.assertIn(n, s)
        s.add('baz')
        for n in 'bar', 'baz':
>           self.assertIn(n, s)
E           AssertionError: 'bar' not found in LimitedSet(2)

celery/tests/utils/test_datastructures.py:185: AssertionError
__________________________________________________ test_LimitedSet.test_iter __________________________________________________

self = <celery.tests.utils.test_datastructures.test_LimitedSet testMethod=test_iter>

    def test_iter(self):
        if sys.platform == 'win32':
            raise SkipTest('Not working on Windows')
        s = LimitedSet(maxlen=3)
        items = ['foo', 'bar', 'baz', 'xaz']
        for item in items:
            s.add(item)
        l = list(iter(s))
        for item in items[1:]:
>           self.assertIn(item, l)
E           AssertionError: 'bar' not found in ['baz', 'foo', 'xaz']

celery/tests/utils/test_datastructures.py:242: AssertionError
_________________________________________________ test_LimitedSet.test_update _________________________________________________

self = <celery.tests.utils.test_datastructures.test_LimitedSet testMethod=test_update>

    def test_update(self):
        s1 = LimitedSet(maxlen=2)
        s1.add('foo')
        s1.add('bar')
    
        s2 = LimitedSet(maxlen=2)
        s2.update(s1)
        self.assertItemsEqual(list(s2), ['foo', 'bar'])
    
        s2.update(['bla'])
>       self.assertItemsEqual(list(s2), ['bla', 'bar'])

celery/tests/utils/test_datastructures.py:280: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
celery/tests/case.py:367: in assertItemsEqual
    return self.assertSequenceEqual(expected, actual, msg=msg)
E   AssertionError: Sequences differ: ['bla', 'foo'] != ['bar', 'bla']
E   
E   First differing element 0:
E   bla
E   bar
E   
E   - ['bla', 'foo']
E   + ['bar', 'bla']
```
Just to give you more details on my environment (in case it's related):

* I'm using CentOS 7.x putting my on Python 2.7.5
* billiard v3.3.0.20
* kombu v3.0.37
* pyparsing v1.5.6
* pytest v3.0.6
* dateutil v2.2-1
* anyjson v0.3.3
* amqp v2.1.4
* pytz v2016.10
* uuid v1.30



Discussion:

Resolving patch:
diff --git a/celery/utils/collections.py b/celery/utils/collections.py
--- a/celery/utils/collections.py
+++ b/celery/utils/collections.py
@@ -3,7 +3,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import sys
-import time
+from celery.five import monotonic
 
 from collections import (
     Callable, Mapping, MutableMapping, MutableSet, Sequence,
@@ -526,7 +526,7 @@ class LimitedSet(object):
         False
         >>> len(s)  # maxlen is reached
         50000
-        >>> s.purge(now=time.time() + 7200)  # clock + 2 hours
+        >>> s.purge(now=monotonic() + 7200)  # clock + 2 hours
         >>> len(s)  # now only minlen items are cached
         4000
         >>>> 57000 in s  # even this item is gone now
@@ -573,7 +573,7 @@ def clear(self):
     def add(self, item, now=None):
         # type: (Any, float) -> None
         """Add a new item, or reset the expiry time of an existing item."""
-        now = now or time.time()
+        now = now or monotonic()
         if item in self._data:
             self.discard(item)
         entry = (now, item)
@@ -624,7 +624,7 @@ def purge(self, now=None):
             now (float): Time of purging -- by default right now.
                 This can be useful for unit testing.
         """
-        now = now or time.time()
+        now = now or monotonic()
         now = now() if isinstance(now, Callable) else now
         if self.maxlen:
             while len(self._data) > self.maxlen:


Test patch:
diff --git a/t/unit/utils/test_collections.py b/t/unit/utils/test_collections.py
--- a/t/unit/utils/test_collections.py
+++ b/t/unit/utils/test_collections.py
@@ -5,7 +5,7 @@
 
 from collections import Mapping
 from itertools import count
-from time import time
+from celery.five import monotonic
 
 from case import skip
 from billiard.einfo import ExceptionInfo
@@ -198,21 +198,21 @@ def test_purge(self):
         s = LimitedSet(maxlen=10, expires=1)
         [s.add(i) for i in range(10)]
         s.maxlen = 2
-        s.purge(now=time() + 100)
+        s.purge(now=monotonic() + 100)
         assert len(s) == 0
 
         # not expired
         s = LimitedSet(maxlen=None, expires=1)
         [s.add(i) for i in range(10)]
         s.maxlen = 2
-        s.purge(now=lambda: time() - 100)
+        s.purge(now=lambda: monotonic() - 100)
         assert len(s) == 2
 
         # expired -> minsize
         s = LimitedSet(maxlen=10, minlen=10, expires=1)
         [s.add(i) for i in range(20)]
         s.minlen = 3
-        s.purge(now=time() + 3)
+        s.purge(now=monotonic() + 3)
         assert s.minlen == len(s)
         assert len(s._heap) <= s.maxlen * (
             100. + s.max_heap_percent_overload) / 100
@@ -293,8 +293,6 @@ def test_update(self):
 
     def test_iterable_and_ordering(self):
         s = LimitedSet(maxlen=35, expires=None)
-        # we use a custom clock here, as time.time() does not have enough
-        # precision when called quickly (can return the same value twice).
         clock = count(1)
         for i in reversed(range(15)):
             s.add(i, now=next(clock))