Repository: redis/redis-py

Issue:
how to use bitfield command?
how to use bitfield command? Its good to have help doc to explain how to use all redis commands.


Discussion:
Hello @arulmuruganas 

The `BITFIELD` command is indeed not supported by the current version. This should be rectified.
So here's the thing - [`BITFIELD`](https://redis.io/commands/bitfield) is tricky to get right because of its versatility.

@nicois provided the following snippet with a sane and simple approach:

```python
from enum import IntEnum as IntEnum

class BitField(IntEnum):
    GET = 0
    SET = 1
    INCRBY = 2
    OVERFLOW = 3

class Overflow(IntEnum):
    WRAP = 0
    SAT = 1
    FAIL = 2

def bitfield_get(key, type_val, offset):
    print('in {}, args {}'.format(__name__, [key, type_val, offset]))
    return 0

def bitfield_set(key, type_val, offset, value):
    print('in {}, args {}'.format(__name__, [key, type_val, offset, value]))
    return 1

def bitfield_incrby(key, type_val, offset, incr):
    print('in {}, args {}'.format(__name__, [key, type_val, offset, incr]))

def bitfield_overflow(key, op):
    print('in {}, args {}'.format(__name__, [key, op]))


bit_func = [bitfield_get, bitfield_set, bitfield_incrby, bitfield_overflow]

def bit_field(key, *args):
    res = []
    for	i in args:
        bitfield_op = i[0]
        if bitfield_op > BitField.OVERFLOW:
            raise Exception('Not a defined OP')
        res.append(bit_func[bitfield_op](key, *i[1:]))
    return res

print(bit_field('key', [BitField.SET, 0, 1, 2], [BitField.INCRBY, 0, 1, 2]))
```

However. I was thinking that would, perhaps (although I tend to overcomplicate an am borderline psychotic), be neater and more consistent to provide a pipeline-like API. Something like:

```python
with redis.BitField('key') as b:
    b.get('u14', '#2')
    b.overflow(...)
    b.set(...)
    ...
    replies = b.execute()
```

2 cents? /cc @andymccurdy @RoeyPrat @grokzen @coleifer :)
I like the context manager approach you suggest. It seems pretty trivial to implement as well.
@andymccurdy - trivial for you, challenging for others. Challenge accepted - please provide guidance with #1054 :)
For what its worth, the context-manager seems unnecessary and could be accomplished using the usual "method-chaining" approach. In other words, I'm not sure what value the "context" adds and in fact it makes things a bit confusing. Do you have to call .execute() from within the context? Can you call it from outside the managed context? If so, what does the context "mean"?
In other words, the context manager seems arbitrary because you can accomplish all that using regular-old method chaining or a stateful instance that aggregates the method calls internally.

You still gotta call `.execute()`, and the fact that you can do this from either within or without the context seems confusing.
For what it's worth, here's a prototype I've been messing with:

```python

class BitField(Container):
    def incrby(self, fmt, offset, increment, overflow=None):
        bfo = BitFieldOperation(self.database, self.key)
        return bfo.incrby(fmt, offset, increment, overflow)

    def get(self, fmt, offset):
        bfo = BitFieldOperation(self.database, self.key)
        return bfo.get(fmt, offset)

    def set(self, fmt, offset, value):
        bfo = BitFieldOperation(self.database, self.key)
        return bfo.set(fmt, offset, value)


class BitFieldOperation(object):
    def __init__(self, database, key):
        self.database = database
        self.key = key
        self.operations = []
        self._last_overflow = None  # Default is "WRAP".

    def incrby(self, fmt, offset, increment, overflow=None):
        if overflow is not None and overflow != self._last_overflow:
            self._last_overflow = overflow
            self.operations.append(('OVERFLOW', overflow))

        self.operations.append(('INCRBY', fmt, offset, increment))
        return self

    def get(self, fmt, offset):
        self.operations.append(('GET', fmt, offset))
        return self

    def set(self, fmt, offset, value):
        self.operations.append(('SET', fmt, offset, value))
        return self

    @property
    def command(self):
        return reduce(operator.add, self.operations, ('BITFIELD', self.key))

    def execute(self):
        return self.database.execute_command(*self.command)

    def __iter__(self):
        return iter(self.execute())
```

Usage:

```python
    def test_simple_operations(self):
        resp = (self.bf
                .set('u8', 8, 255)
                .get('u8', 0)
                .get('u4', 8)  # 1111
                .get('u4', 12)  # 1111
                .get('u4', 13)  # 1110
                .execute())
        self.assertEqual(resp, [0, 0, 15, 15, 14])

        resp = (self.bf
                .set('u8', 4, 1)  # 00ff -> 001f (returns old val, 0x0f)
                .get('u16', 0)  # 001f (00011111)
                .set('u16', 0, 0))  # 001f -> 0000
        self.assertEqual(list(resp), [15, 31, 31])

```
~~Note: upon reading the docs I think I've misunderstood how to use OVERFLOW -- so feel free to disregard the bits about how overflow is handled.~~

Edit: fixed code snippet.
I also think a slice notation is a natural way to get and set values...using the above APIs I provided, you can add:

```python
    def __getitem__(self, item):
        if not isinstance(item, slice):
            raise ValueError('Must use a slice.')
        if item.stop is None or item.stop < 0:
            raise ValueError('slice must have a non-negative upper-bound')
        start = item.start or 0
        return self.get('u%s' % (item.stop - start), start).execute()[0]

    def __setitem__(self, item, value):
        if not isinstance(item, slice):
            raise ValueError('Must use a slice.')
        if item.stop is None or item.stop < 0:
            raise ValueError('slice must have a non-negative upper-bound')
        start = item.start or 0
        nbits = item.stop - start
        if value >= (1 << nbits):
            raise ValueError('value exceeds width specified by slice')
        self.set('u%s' % nbits, start, value).execute()
```

Then:

```python
        resp = self.bf.set('u8', 0, 166).execute()  # 10100110

        self.assertEqual(self.bf[:8], 166)
        self.assertEqual(self.bf[:4], 10)  # 1010
        self.assertEqual(self.bf[4:8], 6)   # 0110
        self.assertEqual(self.bf[2:6], 9) # 1001
        self.assertEqual(self.bf[6:10], 8) # 10?? -> 1000
        self.assertEqual(self.bf[8:16], 0)  # Undefined, defaults to zero.

        self.assertRaises(ValueError, lambda: self.bf[1])
        self.assertRaises(ValueError, lambda: self.bf[1:])

        self.bf[:8] = 89  # 01011001
        self.assertEqual(self.bf[:8], 89)
        def overflow():
            self.bf[:8] = 256
        self.assertRaises(ValueError, overflow)
        self.bf[:8] = 255
        self.assertEqual(self.bf[:8], 255)
```
Cleaned-up: https://github.com/coleifer/walrus/blob/617d2efafd3f2021054c228c2e1a1bfcf15f575b/walrus/containers.py#L1448-L1625
@coleifer This approach is similar to what I was thinking. Good point on not needing it to be a context manager. I like the idea of using slice notation as shorthand, but it seems like all slice-based access would operate on unsigned or signed values without the ability to mix the two based on the offset. Maybe this is OK?

Are you interested in adding the `BitFieldOperation` class to redis-py?


^ Me too.

Mebbe also have SignedBitField just for slices?
I guess you could also take signed/unsigned as an __init__ argument and all slice operations are based on that. Do people treat some offsets as signed and others as unsigned within the same key?
I really don't know - bitfield is one of Redis' best kept secrets despite my many attempts at changing that :)

It really depends on the use case. If used for packed counters, then no and unsigned makes perfect sense. If used for some struct-like compaction, the contrary I would say.
> but it seems like all slice-based access would operate on unsigned or signed values without the ability to mix the two based on the offset

What I've done in the past is, admittedly gross, but to use the "step" parameter of the slice for a 3rd parameter. I think using unsigned by default makes sense because it's very easy translate between the bytes and the value -- just straight-up powers of 2. When you add a sign bit it seems a lot more complicated...plus how many people are trying to store negative integers in a bitfield? Not to mention that using sign bits makes the offsets a whole lot more important.

I'd be happy for you to take as much or as little of the code / APIs I've shared. Assuming this type of logic gets merged into redis-py I'll certainly remove as much duplicate code as possible from the "walrus" project.
Great, thanks. I'll work on this tonight.

Resolving patch:
diff --git a/redis/client.py b/redis/client.py
--- a/redis/client.py
+++ b/redis/client.py
@@ -1005,6 +1005,13 @@ def bitcount(self, key, start=None, end=None):
             raise RedisError("Both start and end must be specified")
         return self.execute_command('BITCOUNT', *params)
 
+    def bitfield(self, key, default_overflow=None):
+        """
+        Return a BitFieldOperation instance to conveniently construct one or
+        more bitfield operations on ``key``.
+        """
+        return BitFieldOperation(self, key, default_overflow=default_overflow)
+
     def bitop(self, operation, dest, *keys):
         """
         Perform a bitwise operation using ``operation`` between ``keys`` and
@@ -3448,3 +3455,100 @@ def __call__(self, keys=[], args=[], client=None):
             # Overwrite the sha just in case there was a discrepancy.
             self.sha = client.script_load(self.script)
             return client.evalsha(self.sha, len(keys), *args)
+
+
+class BitFieldOperation(object):
+    """
+    Command builder for BITFIELD commands.
+    """
+    def __init__(self, client, key, default_overflow=None):
+        self.client = client
+        self.key = key
+        self._default_overflow = default_overflow
+        self.reset()
+
+    def reset(self):
+        """
+        Reset the state of the instance to when it was constructed
+        """
+        self.operations = []
+        self._last_overflow = 'WRAP'
+        self.overflow(self._default_overflow or self._last_overflow)
+
+    def overflow(self, overflow):
+        """
+        Update the overflow algorithm of successive INCRBY operations
+        :param overflow: Overflow algorithm, one of WRAP, SAT, FAIL. See the
+            Redis docs for descriptions of these algorithmsself.
+        :returns: a :py:class:`BitFieldOperation` instance.
+        """
+        overflow = overflow.upper()
+        if overflow != self._last_overflow:
+            self._last_overflow = overflow
+            self.operations.append(('OVERFLOW', overflow))
+        return self
+
+    def incrby(self, fmt, offset, increment, overflow=None):
+        """
+        Increment a bitfield by a given amount.
+        :param fmt: format-string for the bitfield being updated, e.g. 'u8'
+            for an unsigned 8-bit integer.
+        :param offset: offset (in number of bits). If prefixed with a
+            '#', this is an offset multiplier, e.g. given the arguments
+            fmt='u8', offset='#2', the offset will be 16.
+        :param int increment: value to increment the bitfield by.
+        :param str overflow: overflow algorithm. Defaults to WRAP, but other
+            acceptable values are SAT and FAIL. See the Redis docs for
+            descriptions of these algorithms.
+        :returns: a :py:class:`BitFieldOperation` instance.
+        """
+        if overflow is not None:
+            self.overflow(overflow)
+
+        self.operations.append(('INCRBY', fmt, offset, increment))
+        return self
+
+    def get(self, fmt, offset):
+        """
+        Get the value of a given bitfield.
+        :param fmt: format-string for the bitfield being read, e.g. 'u8' for
+            an unsigned 8-bit integer.
+        :param offset: offset (in number of bits). If prefixed with a
+            '#', this is an offset multiplier, e.g. given the arguments
+            fmt='u8', offset='#2', the offset will be 16.
+        :returns: a :py:class:`BitFieldOperation` instance.
+        """
+        self.operations.append(('GET', fmt, offset))
+        return self
+
+    def set(self, fmt, offset, value):
+        """
+        Set the value of a given bitfield.
+        :param fmt: format-string for the bitfield being read, e.g. 'u8' for
+            an unsigned 8-bit integer.
+        :param offset: offset (in number of bits). If prefixed with a
+            '#', this is an offset multiplier, e.g. given the arguments
+            fmt='u8', offset='#2', the offset will be 16.
+        :param int value: value to set at the given position.
+        :returns: a :py:class:`BitFieldOperation` instance.
+        """
+        self.operations.append(('SET', fmt, offset, value))
+        return self
+
+    @property
+    def command(self):
+        cmd = ['BITFIELD', self.key]
+        for ops in self.operations:
+            cmd.extend(ops)
+        return cmd
+
+    def execute(self):
+        """
+        Execute the operation(s) in a single BITFIELD command. The return value
+        is a list of values corresponding to each operation. If the client
+        used to create this instance was a pipeline, the list of values
+        will be present within the pipeline's execute.
+        """
+        command = self.command
+        self.reset()
+        return self.client.execute_command(*command)


Test patch:
diff --git a/tests/test_commands.py b/tests/test_commands.py
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -2115,6 +2115,68 @@ def test_xtrim(self, r):
         # 1 message is trimmed
         assert r.xtrim(stream, 3, approximate=False) == 1
 
+    def test_bitfield_operations(self, r):
+        # comments show affected bits
+        bf = r.bitfield('a')
+        resp = (bf
+                .set('u8', 8, 255)     # 00000000 11111111
+                .get('u8', 0)          # 00000000
+                .get('u4', 8)                   # 1111
+                .get('u4', 12)                      # 1111
+                .get('u4', 13)                       # 111 0
+                .execute())
+        assert resp == [0, 0, 15, 15, 14]
+
+        # .set() returns the previous value...
+        resp = (bf
+                .set('u8', 4, 1)           # 0000 0001
+                .get('u16', 0)         # 00000000 00011111
+                .set('u16', 0, 0)      # 00000000 00000000
+                .execute())
+        assert resp == [15, 31, 31]
+
+        # incrby adds to the value
+        resp = (bf
+                .incrby('u8', 8, 254)  # 00000000 11111110
+                .incrby('u8', 8, 1)    # 00000000 11111111
+                .get('u16', 0)         # 00000000 11111111
+                .execute())
+        assert resp == [254, 255, 255]
+
+        # Verify overflow protection works as a method:
+        r.delete('a')
+        resp = (bf
+                .set('u8', 8, 254)     # 00000000 11111110
+                .overflow('fail')
+                .incrby('u8', 8, 2)    # incrby 2 would overflow, None returned
+                .incrby('u8', 8, 1)    # 00000000 11111111
+                .incrby('u8', 8, 1)    # incrby 1 would overflow, None returned
+                .get('u16', 0)         # 00000000 11111111
+                .execute())
+        assert resp == [0, None, 255, None, 255]
+
+        # Verify overflow protection works as arg to incrby:
+        r.delete('a')
+        resp = (bf
+                .set('u8', 8, 255)           # 00000000 11111111
+                .incrby('u8', 8, 1)          # 00000000 00000000  wrap default
+                .set('u8', 8, 255)           # 00000000 11111111
+                .incrby('u8', 8, 1, 'FAIL')  # 00000000 11111111  fail
+                .incrby('u8', 8, 1)          # 00000000 11111111  still fail
+                .get('u16', 0)               # 00000000 11111111
+                .execute())
+        assert resp == [0, 0, 0, None, None, 255]
+
+        # test default default_overflow
+        r.delete('a')
+        bf = r.bitfield('a', default_overflow='FAIL')
+        resp = (bf
+                .set('u8', 8, 255)     # 00000000 11111111
+                .incrby('u8', 8, 1)    # 00000000 11111111  fail default
+                .get('u16', 0)         # 00000000 11111111
+                .execute())
+        assert resp == [0, None, 255]
+
 
 class TestStrictCommands(object):
     def test_strict_zadd(self, sr):
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -252,3 +252,20 @@ def test_exec_error_in_no_transaction_pipeline_unicode_command(self, r):
             assert unicode(ex.value).startswith(expected)
 
         assert r[key] == b'1'
+
+    def test_pipeline_with_bitfield(self, r):
+        with r.pipeline() as pipe:
+            pipe.set('a', '1')
+            bf = pipe.bitfield('b')
+            pipe2 = (bf
+                     .set('u8', 8, 255)
+                     .get('u8', 0)
+                     .get('u4', 8)  # 1111
+                     .get('u4', 12)  # 1111
+                     .get('u4', 13)  # 1110
+                     .execute())
+            pipe.get('a')
+            response = pipe.execute()
+
+            assert pipe == pipe2
+            assert response == [True, [0, 0, 15, 15, 14], b'1']