Repository: bmoscon/orderbook (language: python)

Issue:
Methods to return data as list
Provide following methods to export data:

1. prices() - returns list of prices for each level (would be called on .bids or .asks)
2. to_list() - like to dict, but returns a list of pairs (price, size) for each level in the side (would be called on .bids or .asks)
3. ob.to_list() - like above, but would be a dict {bids: [(price, size)..., asks: [....]}


Discussion / hints:
Hi, i want to known this feature any progress?
@bmoscon  Looking forward to your reply
@lijiachang i havent started this yet

Fix patch:
diff --git a/orderbook/sorteddict.c b/orderbook/sorteddict.c
index 666787f..8cdf015 100644
--- a/orderbook/sorteddict.c
+++ b/orderbook/sorteddict.c
@@ -339,6 +339,66 @@ PyObject* SortedDict_todict(SortedDict *self, PyObject *unused, PyObject *kwargs
 }
 
 
+PyObject* SortedDict_tolist(SortedDict *self, PyObject *args, PyObject *kwargs)
+{
+    static char* keywords[] = {"n_levels", NULL};
+    
+    int n_levels = -1;
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", keywords, &n_levels)) {
+        return NULL;
+    }
+
+    int data_len = PyDict_Size(self->data);
+    // Set n_levels to data_len if n_levels is the default value
+    if (n_levels == -1 | n_levels > data_len) {
+        n_levels = data_len;
+    }
+
+    if (EXPECT(PyErr_Occurred() != NULL, 0)) {
+        return NULL;
+    }
+
+    if (EXPECT(update_keys(self), 0)) {
+        return NULL;
+    }
+
+    PyObject *ret = PyList_New(n_levels);
+    if (EXPECT(!ret, 0)) {
+        return NULL;
+    }
+
+    for (int i = 0; i < n_levels; ++i) {
+        // new reference
+        PyObject *key = PySequence_GetItem(self->keys, i);
+        if (EXPECT(!key, 0)) {
+            return NULL;
+        }
+
+        // borrowed reference
+        PyObject *value = PyDict_GetItem(self->data, key);
+        if (EXPECT(!value, 0)) {
+            Py_DECREF(key);
+            return value;
+        }
+
+        // Build tuple of (i.e., key, value)
+        PyObject *tuple_entry = PyTuple_New(2);
+        if (EXPECT(!tuple_entry, 0)) {
+            Py_DECREF(key);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(tuple_entry, 0, key);
+        Py_INCREF(value);
+        PyTuple_SET_ITEM(tuple_entry, 1, value);
+
+        // Add tuple to list
+        PyList_SET_ITEM(ret, i, tuple_entry);
+    }
+    
+    return ret;
+}
+
+
 PyObject* SortedDict_truncate(SortedDict *self, PyObject *Py_UNUSED(ignored))
 {
     if (self->depth) {
diff --git a/orderbook/sorteddict.h b/orderbook/sorteddict.h
index 754ef73..c81d3c5 100644
--- a/orderbook/sorteddict.h
+++ b/orderbook/sorteddict.h
@@ -42,6 +42,7 @@ int SortedDict_init(SortedDict *self, PyObject *args, PyObject *kwds);
 PyObject* SortedDict_keys(SortedDict *self, PyObject *Py_UNUSED(ignored));
 PyObject* SortedDict_index(SortedDict *self, PyObject *index);
 PyObject* SortedDict_todict(SortedDict *self, PyObject *unused, PyObject *kwargs);
+PyObject* SortedDict_tolist(SortedDict *self, PyObject *args, PyObject *kwargs);
 PyObject* SortedDict_truncate(SortedDict *self, PyObject *Py_UNUSED(ignored));
 
 Py_ssize_t SortedDict_len(const SortedDict *self);
@@ -68,6 +69,7 @@ static PyMethodDef SortedDict_methods[] = {
     {"index", (PyCFunction) SortedDict_index, METH_O, "return a key, value tuple at index N"},
     {"truncate", (PyCFunction) SortedDict_truncate, METH_NOARGS, "truncate to length max_depth"},
     {"to_dict", (PyCFunction) SortedDict_todict, METH_VARARGS | METH_KEYWORDS, "return a python dictionary, sorted by keys"},
+    {"to_list", (PyCFunction) SortedDict_tolist, METH_VARARGS | METH_KEYWORDS, "return a list of key, value tuple with length specified in input argument. if no argument is passed or the argument is larger than length of self->data, return items with length of self->data."},
     {NULL}
 };
 


Test patch:
diff --git a/perf/performance_test.py b/perf/performance_test.py
index 95f0b47..477b1be 100644
--- a/perf/performance_test.py
+++ b/perf/performance_test.py
@@ -25,7 +25,7 @@ def wrapper(*args, **kwargs):
         startt = time.time()
         ret = f(*args, **kwargs)
         total_time = time.time() - startt
-        print("Time:", total_time)
+        print(f"Time: {total_time * 1000:.6f} ms")
         return ret
     return wrapper
 
@@ -85,6 +85,8 @@ def random_data_test(size):
             values.append(random.uniform(-100000.0, 100000.0))
         values = set(values)
 
+    print("===== Write Performance =====")
+
     @profile
     def test_ordered(dictionary):
         for v in values:
@@ -120,6 +122,62 @@ def test_unordered(unordered):
     print(f"Python dict (non sorted) with {size} entries")
     test_unordered(raw_python)
 
+    print("===== Read Performance =====")
+    @profile
+    def access_top10_c_index(c_dictionary):
+        for idx in range(10):
+            a = c_dictionary.index(idx)
+
+    @profile
+    def access_top10_c_keys(c_dictionary):
+        for idx, key in enumerate(c_dictionary.keys()):
+            if idx >= 10:
+                break
+            a = c_dictionary[key]
+
+    @profile
+    def access_top10_c_tolist(c_dictionary):
+        a = c_dictionary.to_list(10)
+
+    @profile
+    def access_top10_c_todict(c_dictionary):
+        d = c_dictionary.to_dict()
+        for idx, key in enumerate(d):
+            a = d[key]
+            if idx >= 10:
+                break
+
+    @profile
+    def access_top10_iter(dictionary):
+        for idx, key in enumerate(dictionary):
+            a = dictionary[key]
+            if idx >= 10:
+                break
+
+    print(f"C lib with {size} entries (access)")
+    
+    print(f"- index impl ", end="")
+    access_top10_c_index(asc)
+    
+    print(f"- todict impl ", end="")
+    access_top10_c_todict(asc)
+
+    print(f"- keys impl ", end="")
+    access_top10_c_keys(asc)
+
+    print(f"- iter impl (incorrect when called multiple times) ", end="")
+    access_top10_iter(asc)
+
+    print(f"- tolist impl ", end="")
+    access_top10_c_tolist(asc)
+    
+    print(f"Orderbook SortedDict Python lib with {size} entries (access)")
+    access_top10_iter(python_sd)
+    
+    print(f"Python dict (non sorted) with {size} entries (access)")
+    access_top10_iter(raw_python)
+
+
 
 def random_data_performance():
     for size in (10, 100, 200, 400, 500, 1000, 2000, 10000, 100000, 200000, 500000):
diff --git a/tests/test_sorteddict.py b/tests/test_sorteddict.py
index eb8b3be..8fff670 100644
--- a/tests/test_sorteddict.py
+++ b/tests/test_sorteddict.py
@@ -140,6 +140,81 @@ def test_to_dict():
             d[key] < previous
         previous = d[key]
 
+def test_to_list():
+    random.seed()
+    values = []
+    asc = SortedDict(ordering='ASC')
+    desc = SortedDict(ordering='DESC')
+
+    for _ in range(2000):
+        values.append(random.uniform(0.0, 100000.0))
+    values = set(values)
+
+    for v in values:
+        asc[v] = str(v)
+        desc[v] = str(v)
+
+    lst = asc.to_list()
+    _keys = list(list(zip(*lst))[0])
+    assert _keys == list(asc.keys())
+    assert sorted(_keys) == _keys
+    previous = None
+    for key, val in lst:
+        assert val == asc[key]
+        val = float(val)
+        if previous:
+            assert val > previous
+        previous = val
+
+    lst = desc.to_list()
+    _keys = list(list(zip(*lst))[0])
+    assert _keys == list(desc.keys())
+    assert list(reversed(sorted(_keys))) == _keys
+    previous = None
+    for key, val in lst:
+        assert val == desc[key]
+        val = float(val)
+        if previous:
+            assert val < previous
+        previous = val
+
+def test_to_list_specified_length():
+    random.seed()
+    values = []
+    asc = SortedDict(ordering='ASC')
+    desc = SortedDict(ordering='DESC')
+
+    for _ in range(2000):
+        values.append(random.uniform(0.0, 100000.0))
+    values = set(values)
+
+    for v in values:
+        asc[v] = str(v)
+        desc[v] = str(v)
+
+    lst = asc.to_list(20)
+    _keys = list(list(zip(*lst))[0])
+    assert _keys == list(asc.keys())[:20]
+    assert sorted(_keys) == _keys
+    previous = None
+    for key, val in lst:
+        assert val == asc[key]
+        val = float(val)
+        if previous:
+            assert val > previous
+        previous = val
+
+    lst = desc.to_list(20)
+    _keys = list(list(zip(*lst))[0])
+    assert _keys == list(desc.keys())[:20]
+    assert list(reversed(sorted(_keys))) == _keys
+    previous = None
+    for key, val in lst:
+        assert val == desc[key]
+        val = float(val)
+        if previous:
+            assert val < previous
+        previous = val
 
 def test_init_from_dict():
     with pytest.raises(TypeError):