Repository: liampauling/flumine
Pull request #345 (2020-12-05 09:16:15)

history and version update


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 12b7c01..332624e 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,18 @@
 Release History
 ---------------
 
+1.14.13 (2020-12-05)
++++++++++++++++++++
+
+**Improvements**
+
+- Backtest market catalogue middleware example (@lunswor)
+- #344 Initial work on improving calls when subscribed to 5k+ markets
+
+**Bug Fixes**
+
+- #342 market/limit on close order size remaining bug
+
 1.14.12 (2020-11-28)
 +++++++++++++++++++
 
diff --git a/docs/markets.md b/docs/markets.md
index cd75221..1c8a278 100644
--- a/docs/markets.md
+++ b/docs/markets.md
@@ -6,6 +6,10 @@ Within markets you have market objects which contains current up to date market
 
 ### Class variables
 
+- `flumine` Framework
+- `market_id` MarketBook id
+- `closed` Closed bool
+- `date_time_closed` Closed datetime
 - `market_book` Latest MarketBook object
 - `market_catalogue` Latest MarketCatalogue object
 - `context` Market context, store market specific context e.g. simulated data store
@@ -36,3 +40,19 @@ For example backtesting uses [simulated middleware](https://github.com/liampauli
 
 !!! note
     Middleware will be executed in the order it is added and before the strategies are processed.
+
+Please see below for the example middleware class if you wish to create your own:
+
+```python
+from flumine.markets.middleware import Middleware
+
+class CustomMiddleware(Middleware):
+    def __call__(self, market) -> None:
+        pass  # called on each MarketBook update
+
+    def add_market(self, market) -> None:
+        print("market {0} added".format(market.market_id))
+
+    def remove_market(self, market) -> None:
+        print("market {0} removed".format(market.market_id))
+```
diff --git a/examples/middlewares/marketcatalogue.py b/examples/middlewares/marketcatalogue.py
new file mode 100644
index 0000000..4b3b6f4
--- /dev/null
+++ b/examples/middlewares/marketcatalogue.py
@@ -0,0 +1,19 @@
+import json
+import os
+
+from betfairlightweight.resources import MarketCatalogue
+from flumine.markets.middleware import Middleware
+
+MARKET_CATALOGUE_PATH = ""  # update to correct path
+
+
+# Will read and parse the market_catalogue file and add to the market object when backtesting
+# Usage framework.add_middleware(MarketCatalogueMiddleware())
+class MarketCatalogueMiddleware(Middleware):
+    def __call__(self, market) -> None:
+        catalogue_file_path = os.path.join(MARKET_CATALOGUE_PATH, market.market_id)
+        if os.path.exists(catalogue_file_path):
+            with open(catalogue_file_path, "r") as r:
+                data = r.read()
+                catalogue_json_data = json.loads(data)
+                market.market_catalogue = MarketCatalogue(**catalogue_json_data)
diff --git a/flumine/__version__.py b/flumine/__version__.py
index 4101cdd..4889a40 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.14.12"
+__version__ = "1.14.13"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/baseflumine.py b/flumine/baseflumine.py
index 073e5bf..0d03dcc 100644
--- a/flumine/baseflumine.py
+++ b/flumine/baseflumine.py
@@ -154,12 +154,15 @@ class BaseFlumine:
 
     def _process_market_orders(self) -> None:
         for market in self.markets:
-            if market.market_book:
-                bet_delay = market.market_book.bet_delay
-            else:
-                bet_delay = None
-            for order_package in market.blotter.process_orders(self.client, bet_delay):
-                self.handler_queue.put(order_package)
+            if market.blotter.pending_orders:
+                if market.market_book:
+                    bet_delay = market.market_book.bet_delay
+                else:
+                    bet_delay = None
+                for order_package in market.blotter.process_orders(
+                    self.client, bet_delay
+                ):
+                    self.handler_queue.put(order_package)
 
     def _process_order_package(self, order_package) -> None:
         """Validate trading controls and
diff --git a/flumine/markets/blotter.py b/flumine/markets/blotter.py
index 4ee8587..f605e46 100644
--- a/flumine/markets/blotter.py
+++ b/flumine/markets/blotter.py
@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Tuple
+from typing import Iterable
 
 from ..order.ordertype import OrderTypes
 from ..utils import chunks, calculate_unmatched_exposure, calculate_matched_exposure
@@ -73,15 +73,26 @@ class Blotter:
         orders.clear()
         return packages
 
+    @property
+    def pending_orders(self) -> bool:
+        return any(
+            (
+                self.pending_place,
+                self.pending_cancel,
+                self.pending_update,
+                self.pending_replace,
+            )
+        )
+
     @property
     def live_orders(self):
         return iter(self._live_orders)
 
     @property
-    def has_live_orders(self):
+    def has_live_orders(self) -> bool:
         return bool(self._live_orders)
 
-    def process_closed_market(self, market_book):
+    def process_closed_market(self, market_book) -> None:
         for order in self:
             for runner in market_book.runners:
                 if (order.selection_id, order.handicap) == (
diff --git a/flumine/order/order.py b/flumine/order/order.py
index 27ca471..36a95f7 100644
--- a/flumine/order/order.py
+++ b/flumine/order/order.py
@@ -390,10 +390,15 @@ class BetfairOrder(BaseOrder):
         try:
             return self.current_order.size_remaining or 0.0
         except AttributeError:
-            if self.size_matched > 0:  # placeResponse does not include sizeRemaining
-                return round(self.order_type.size - self.size_matched, 2)
+            if self.order_type.ORDER_TYPE == OrderTypes.LIMIT:
+                if (
+                    self.size_matched > 0
+                ):  # placeResponse does not include sizeRemaining
+                    return round(self.order_type.size - self.size_matched, 2)
+                else:
+                    return self.order_type.size
             else:
-                return self.order_type.size
+                return self.order_type.liability
 
     @property
     def size_cancelled(self) -> float:


Test patch:
diff --git a/tests/test_baseflumine.py b/tests/test_baseflumine.py
index f95a891..9d92fa9 100644
--- a/tests/test_baseflumine.py
+++ b/tests/test_baseflumine.py
@@ -99,12 +99,16 @@ class BaseFlumineTest(unittest.TestCase):
 
     def test__process_market_orders(self):
         mock_market = mock.Mock()
+        mock_market.blotter.pending_orders = True
         mock_market.blotter.process_orders.return_value = [1, 2, 3]
-        self.base_flumine.markets = [mock_market]
+        mock_market_two = mock.Mock()
+        mock_market_two.blotter.pending_orders = False
+        self.base_flumine.markets = [mock_market, mock_market_two]
         self.base_flumine._process_market_orders()
         mock_market.blotter.process_orders.assert_called_with(
             self.mock_client, mock_market.market_book.bet_delay
         )
+        mock_market_two.blotter.process_orders.assert_not_called()
 
     def test__process_order_package(self):
         mock_trading_control = mock.Mock()
diff --git a/tests/test_blotter.py b/tests/test_blotter.py
index 74d1ad7..dd2fea5 100644
--- a/tests/test_blotter.py
+++ b/tests/test_blotter.py
@@ -71,6 +71,11 @@ class BlotterTest(unittest.TestCase):
         )
         self.assertEqual(mock_orders, [])
 
+    def test_pending_orders(self):
+        self.assertFalse(self.blotter.pending_orders)
+        self.blotter.pending_update = [1]
+        self.assertTrue(self.blotter.pending_orders)
+
     def test_live_orders(self):
         self.assertEqual(list(self.blotter.live_orders), [])
         mock_order = mock.Mock(complete=False)
diff --git a/tests/test_order.py b/tests/test_order.py
index b2f96be..5c6b0e9 100644
--- a/tests/test_order.py
+++ b/tests/test_order.py
@@ -423,6 +423,7 @@ class BetfairOrderTest(unittest.TestCase):
         self.assertEqual(self.order.size_matched, mock_current_order.size_matched)
 
     def test_size_remaining(self):
+        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
         self.mock_order_type.size = 0
         self.assertEqual(self.order.size_remaining, 0)
         self.mock_order_type.size = 10
@@ -431,6 +432,7 @@ class BetfairOrderTest(unittest.TestCase):
         self.assertEqual(self.order.size_remaining, mock_current_order.size_remaining)
 
     def test_size_remaining_missing(self):
+        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
         self.mock_order_type.size = 2.51
         self.assertEqual(self.order.size_remaining, 2.51)
 
@@ -439,10 +441,17 @@ class BetfairOrderTest(unittest.TestCase):
         new_callable=mock.PropertyMock,
     )
     def test_size_remaining_missing_partial_match(self, mock_size_matched):
+        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
         mock_size_matched.return_value = 2
         self.mock_order_type.size = 10
         self.assertEqual(self.order.size_remaining, 8)
 
+    def test_size_remaining_market_on_close(self):
+        self.mock_order_type.ORDER_TYPE = OrderTypes.MARKET_ON_CLOSE
+        self.mock_order_type.size = ValueError
+        self.order.responses.current_order = None
+        self.assertEqual(self.order.size_remaining, self.mock_order_type.liability)
+
     def test_size_cancelled(self):
         self.assertEqual(self.order.size_cancelled, 0)
         mock_current_order = mock.Mock(size_cancelled=10)