Repository: liampauling/flumine
Pull request #544 (2022-01-05 11:28:16)

Passing market book object avoids need to pass individual parameters when calculating market exposure.
Breaking change as two integer parameters are replaced by a market_book instance.


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 13ba630..e7e2f0c 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,21 +3,6 @@
 Release History
 ---------------
 
-1.21.0 (2022-01-06)
-+++++++++++++++++++
-
-**Improvements**
-
-- #528 smart matching on passive orders
-
-**Bug Fixes**
-
-- #528 simulation processing on in flight requests
-
-**Libraries**
-
-- black upgraded to 21.12b0
-
 1.20.13 (2021-12-03)
 +++++++++++++++++++
 
diff --git a/docs/advanced.md b/docs/advanced.md
index 5cdf800..b070094 100644
--- a/docs/advanced.md
+++ b/docs/advanced.md
@@ -112,10 +112,6 @@ logger.setLevel(logging.INFO)
 
 Updated to True when backtesting or paper trading
 
-#### simulated_strategy_isolation
-
-Defaults to True to match orders per strategy, when False prevents double counting of passive liquidity on all orders regardless of strategy.
-
 #### instance_id
 
 Store server id or similar (e.g. AWS ec2 instanceId)
diff --git a/docs/markets.md b/docs/markets.md
index 3f18ae4..fb2cbd1 100644
--- a/docs/markets.md
+++ b/docs/markets.md
@@ -59,7 +59,7 @@ The blotter is a simple and fast class to hold all orders for a particular marke
 - `strategy_orders(strategy)` Returns all orders related to a strategy
 - `strategy_selection_orders(strategy, selection_id, handicap)` Returns all orders related to a strategy selection
 - `selection_exposure(strategy, lookup)` Returns strategy/selection exposure
-- `market_exposure(strategy, num_runners, num_winners)` Returns strategy/market exposure (default num_winners=1)
+- `market_exposure(strategy, market_book)` Returns strategy/market exposure
 
 ### Properties
 
diff --git a/examples/middleware/marketcatalogue.py b/examples/middleware/marketcatalogue.py
index df01eb3..1fbab2a 100644
--- a/examples/middleware/marketcatalogue.py
+++ b/examples/middleware/marketcatalogue.py
@@ -8,7 +8,7 @@ MARKET_CATALOGUE_PATH = ""  # update to correct path
 
 
 # Will read and parse the market_catalogue file and add to the market object when backtesting
-# Usage framework.add_market_middleware(MarketCatalogueMiddleware())
+# Usage framework.add_middleware(MarketCatalogueMiddleware())
 class MarketCatalogueMiddleware(Middleware):
     def add_market(self, market) -> None:
         catalogue_file_path = os.path.join(MARKET_CATALOGUE_PATH, market.market_id)
diff --git a/flumine/__version__.py b/flumine/__version__.py
index d642a99..0e4c53a 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.21.0"
+__version__ = "1.20.13"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/config.py b/flumine/config.py
index ad22ae7..ffedfbc 100644
--- a/flumine/config.py
+++ b/flumine/config.py
@@ -2,7 +2,6 @@ import os
 import socket
 
 simulated = False
-simulated_strategy_isolation = True
 
 instance_id = None  # instance id (e.g. AWS ec2 instanceId)
 
diff --git a/flumine/markets/blotter.py b/flumine/markets/blotter.py
index 867f0fe..00add47 100644
--- a/flumine/markets/blotter.py
+++ b/flumine/markets/blotter.py
@@ -110,7 +110,7 @@ class Blotter:
 
     """ position """
 
-    def market_exposure(self, strategy, num_runners, num_winners: int = 1) -> float:
+    def market_exposure(self, strategy, market_book) -> float:
         """Returns worst-case exposure for market, which is the maximum potential loss (negative),
         arising from the worst race outcome, or the minimum potential profit (positive).
         """
@@ -125,8 +125,8 @@ class Blotter:
         differences = [
             wpp["worst_possible_profit_on_win"] - wpp["worst_possible_profit_on_lose"]
             for wpp in worst_possible_profits
-        ] + (num_runners - len(runners)) * [0]
-        worst_differences = sorted(differences)[:num_winners]
+        ] + (market_book.number_of_active_runners - len(runners)) * [0]
+        worst_differences = sorted(differences)[: market_book.number_of_winners]
         return sum(worst_possible_profits_on_loses) + sum(worst_differences)
 
     def selection_exposure(self, strategy, lookup: tuple) -> float:
diff --git a/flumine/markets/middleware.py b/flumine/markets/middleware.py
index 85fee17..3420544 100644
--- a/flumine/markets/middleware.py
+++ b/flumine/markets/middleware.py
@@ -4,7 +4,6 @@ from betfairlightweight.resources.bettingresources import RunnerBook
 
 from ..order.order import OrderStatus, OrderTypes
 from ..utils import wap
-from .. import config
 
 logger = logging.getLogger(__name__)
 
@@ -181,81 +180,14 @@ class SimulatedMiddleware(Middleware):
         price_adjusted = round(price * (1 - (adjustment_factor / 100)), 2)
         return max(price_adjusted, 1.01)  # min: 1.01
 
-    def _process_simulated_orders(self, market, market_analytics: dict) -> None:
-        """
-        #538 smart matching
-          - isolation per order
-            Potential double counting of passive liquidity, old logic no longer implemented
-          - isolation per strategy (default)
-            Prevent double counting of passive liquidity per strategy
-          - isolation per instance
-            Prevent double counting of passive liquidity on all orders regardless of strategy (interaction across strategies)
-        """
-        # isolation per strategy (default)
-        if config.simulated_strategy_isolation:
-            for strategy, orders in market.blotter._strategy_orders.items():
-                live_orders = [
-                    o
-                    for o in orders
-                    if o.status
-                    in [
-                        OrderStatus.EXECUTABLE,
-                        OrderStatus.CANCELLING,
-                        OrderStatus.UPDATING,
-                        OrderStatus.REPLACING,
-                    ]
-                    and o.simulated
-                ]
-                if live_orders:
-                    _lookup = {k: v.traded.copy() for k, v in market_analytics.items()}
-                    live_orders_sorted = self._sort_orders(live_orders)
-                    for order in live_orders_sorted:
-                        runner_traded = _lookup[(order.selection_id, order.handicap)]
-                        order.simulated(market.market_book, runner_traded)
-        else:  # isolation per instance
-            live_orders = list(market.blotter.live_orders)
-            if live_orders:
-                _lookup = {k: v.traded.copy() for k, v in market_analytics.items()}
-                live_orders_sorted = self._sort_orders(live_orders)
-                for order in live_orders_sorted:
-                    if (
-                        order.status
-                        in [
-                            OrderStatus.EXECUTABLE,
-                            OrderStatus.CANCELLING,
-                            OrderStatus.UPDATING,
-                            OrderStatus.REPLACING,
-                        ]
-                        and order.simulated
-                    ):
-                        runner_traded = _lookup[(order.selection_id, order.handicap)]
-                        order.simulated(market.market_book, runner_traded)
-
     @staticmethod
-    def _sort_orders(orders: list) -> list:
-        # order by betId (default), side (Lay,Back) and then price
-        lay_orders = sorted(
-            [
-                o
-                for o in orders
-                if o.side == "LAY"
-                and o.order_type.ORDER_TYPE != OrderTypes.MARKET_ON_CLOSE
-            ],
-            key=lambda x: -x.order_type.price,
-        )
-        back_orders = sorted(
-            [
-                o
-                for o in orders
-                if o.side == "BACK"
-                and o.order_type.ORDER_TYPE != OrderTypes.MARKET_ON_CLOSE
-            ],
-            key=lambda x: x.order_type.price,
-        )
-        moc = [
-            o for o in orders if o.order_type.ORDER_TYPE == OrderTypes.MARKET_ON_CLOSE
-        ]
-        return lay_orders + back_orders + moc
+    def _process_simulated_orders(market, market_analytics: dict) -> None:
+        for order in market.blotter.live_orders:
+            if order.status == OrderStatus.EXECUTABLE and order.simulated:
+                runner_analytics = market_analytics[
+                    (order.selection_id, order.handicap)
+                ]
+                order.simulated(market.market_book, runner_analytics)
 
     @staticmethod
     def _process_runner(


Test patch:
diff --git a/flumine/backtest/simulated.py b/flumine/backtest/simulated.py
index dbd48e3..83313bd 100644
--- a/flumine/backtest/simulated.py
+++ b/flumine/backtest/simulated.py
@@ -33,7 +33,7 @@ class Simulated:
         self._piq = 0.0
         self._bsp_reconciled = False
 
-    def __call__(self, market_book: MarketBook, traded: dict) -> None:
+    def __call__(self, market_book: MarketBook, runner_analytics) -> None:
         # simulates order matching
         if self._bsp_reconciled is False and market_book.bsp_reconciled:
             if self.take_sp:
@@ -52,8 +52,10 @@ class Simulated:
                         return
 
             # todo estimated piq cancellations
-            if traded:
-                self._process_traded(market_book.publish_time_epoch, traded)
+            if runner_analytics.traded:
+                self._process_traded(
+                    market_book.publish_time_epoch, runner_analytics.traded
+                )
 
     def place(
         self, order_package, market_book: MarketBook, instruction: dict, bet_id: int
@@ -92,10 +94,11 @@ class Simulated:
                 error_code="RUNNER_REMOVED",
             )
         if self.order.order_type.ORDER_TYPE == OrderTypes.LIMIT:
+            available_to_back = get_price(runner.ex.available_to_back, 0) or 1.01
+            available_to_lay = get_price(runner.ex.available_to_lay, 0) or 1000
             price = self.order.order_type.price
             size = self.order.order_type.size
             if self.order.side == "BACK":
-                available_to_back = get_price(runner.ex.available_to_back, 0) or 1.01
                 if (
                     not order_package.client.best_price_execution
                     and available_to_back > price
@@ -116,7 +119,6 @@ class Simulated:
                     return self._create_place_response(bet_id)
                 available = runner.ex.available_to_lay
             else:
-                available_to_lay = get_price(runner.ex.available_to_lay, 0) or 1000
                 if (
                     not order_package.client.best_price_execution
                     and available_to_lay < price
@@ -329,18 +331,14 @@ class Simulated:
                     )
                 )
             if side == "BACK" and traded_price >= price:
-                matched = self._calculate_process_traded(publish_time, traded_size)
-                if matched:
-                    traded[traded_price] = traded_size - matched
+                self._calculate_process_traded(publish_time, traded_size)
             elif side == "LAY" and traded_price <= price:
-                matched = self._calculate_process_traded(publish_time, traded_size)
-                if matched:
-                    traded[traded_price] = traded_size - matched
-
-    def _calculate_process_traded(self, publish_time: int, traded_size: float) -> float:
-        _traded_size = traded_size / 2
-        if self._piq - _traded_size < 0:
-            size = _traded_size - self._piq
+                self._calculate_process_traded(publish_time, traded_size)
+
+    def _calculate_process_traded(self, publish_time: int, traded_size: float) -> None:
+        traded_size = traded_size / 2
+        if self._piq - traded_size < 0:
+            size = traded_size - self._piq
             size = round(min(self.size_remaining, size), 2)
             if size:
                 self._update_matched(
@@ -348,18 +346,15 @@ class Simulated:
                         publish_time,
                         self.order.order_type.price,
                         size,
-                    ]
+                    ]  # todo takes the worst price, i.e what was asked
                 )
-            _matched = (self._piq + size) * 2
             self._piq = 0
-            return _matched
         else:
-            self._piq -= _traded_size
+            self._piq -= traded_size
             if logger.isEnabledFor(logging.DEBUG):
                 logger.debug(
                     "Simulated order {0} PIQ: {1}".format(self.order.id, self._piq)
                 )
-            return traded_size
 
     @property
     def take_sp(self) -> bool:
diff --git a/requirements-test.txt b/requirements-test.txt
index a1d84ae..d71d913 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,5 +1,5 @@
 # Tests & Linting
-black==21.12b0
+black==21.11b1
 coverage
 
 # Documentation
diff --git a/tests/test_blotter.py b/tests/test_blotter.py
index 0a52661..ba3553b 100644
--- a/tests/test_blotter.py
+++ b/tests/test_blotter.py
@@ -481,18 +481,23 @@ class BlotterTest(unittest.TestCase):
                 order_type=order[6],
                 complete=order[7],
             )
+
+        mock_market_book = mock.Mock(number_of_active_runners=6, number_of_winners=1)
         self.assertEqual(  # single winner
-            self.blotter.market_exposure(mock_strategy, 6), -20.2
+            self.blotter.market_exposure(mock_strategy, mock_market_book), -20.2
         )
+        mock_market_book = mock.Mock(number_of_active_runners=6, number_of_winners=2)
         self.assertEqual(  # muliple winners
-            self.blotter.market_exposure(mock_strategy, 6, 2), -24.6
+            self.blotter.market_exposure(mock_strategy, mock_market_book), -24.6
         )
-        self.assertEqual(  # num winmers > num runners traded
-            self.blotter.market_exposure(mock_strategy, 20, 7), -28.6
+        mock_market_book = mock.Mock(number_of_active_runners=20, number_of_winners=7)
+        self.assertEqual(  # num winners > num runners traded
+            self.blotter.market_exposure(mock_strategy, mock_market_book), -28.6
         )
 
     def test_greened_market_position(self):
         mock_strategy = mock.Mock()
+        mock_market_book = mock.Mock(number_of_active_runners=6, number_of_winners=1)
         mock_trade = mock.Mock(strategy=mock_strategy)
         orders = [
             # (order_id, selection_id, side, average_price_matched, size_matched, size_remaining, order_type, complete)
@@ -515,7 +520,7 @@ class BlotterTest(unittest.TestCase):
                 complete=order[7],
             )
         self.assertEqual(  # single winner
-            self.blotter.market_exposure(mock_strategy, 6), 0.5
+            self.blotter.market_exposure(mock_strategy, mock_market_book), 0.5
         )
 
     def test_complete_order(self):
diff --git a/tests/test_config.py b/tests/test_config.py
index a992cf2..0c5cfd6 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -6,7 +6,6 @@ from flumine import config
 class ConfigTest(unittest.TestCase):
     def test_init(self):
         self.assertFalse(config.simulated)
-        self.assertTrue(config.simulated_strategy_isolation)
         self.assertIsInstance(config.customer_strategy_ref, str)
         self.assertIsInstance(config.process_id, int)
         self.assertIsNone(config.current_time)
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 21d64c0..619d857 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -270,43 +270,12 @@ class SimulatedMiddlewareTest(unittest.TestCase):
         self.assertEqual(self.middleware._calculate_reduction_factor(10, 75.18), 2.48)
         self.assertEqual(self.middleware._calculate_reduction_factor(1.01, 75.18), 1.01)
 
-    @mock.patch("flumine.markets.middleware.config")
-    def test__process_simulated_orders_strategy_isolation(self, mock_config):
-        mock_config.simulated_strategy_isolation = True
+    def test__process_simulated_orders(self):
         mock_market_book = mock.Mock()
         mock_market = mock.Mock()
         mock_order = mock.Mock(
-            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE, side="LAY"
+            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE
         )
-        mock_order.order_type.price = 1.02
-        mock_order.order_type.ORDER_TYPE = OrderTypes.LIMIT
-        mock_order_two = mock.Mock(
-            selection_id=123, handicap=1, status=OrderStatus.PENDING
-        )
-        mock_order_three = mock.Mock(
-            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE, simulated=False
-        )
-        mock_market.blotter._strategy_orders = {
-            "test": [mock_order, mock_order_two, mock_order_three]
-        }
-        mock_market_analytics = {
-            (mock_order.selection_id, mock_order.handicap): mock.Mock(traded={1: 2})
-        }
-        mock_market.market_book = mock_market_book
-        self.middleware._process_simulated_orders(mock_market, mock_market_analytics)
-        mock_order.simulated.assert_called_with(mock_market_book, {1: 2})
-        mock_order_two.simulated.assert_not_called()
-
-    @mock.patch("flumine.markets.middleware.config")
-    def test__process_simulated_orders(self, mock_config):
-        mock_config.simulated_strategy_isolation = False
-        mock_market_book = mock.Mock()
-        mock_market = mock.Mock()
-        mock_order = mock.Mock(
-            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE, side="LAY"
-        )
-        mock_order.order_type.price = 1.02
-        mock_order.order_type.ORDER_TYPE = OrderTypes.LIMIT
         mock_order_two = mock.Mock(
             selection_id=123, handicap=1, status=OrderStatus.PENDING
         )
@@ -318,52 +287,12 @@ class SimulatedMiddlewareTest(unittest.TestCase):
             mock_order_two,
             mock_order_three,
         ]
-        mock_market_analytics = {
-            (mock_order.selection_id, mock_order.handicap): mock.Mock(traded={1: 2})
-        }
+        mock_market_analytics = {(mock_order.selection_id, mock_order.handicap): "test"}
         mock_market.market_book = mock_market_book
         self.middleware._process_simulated_orders(mock_market, mock_market_analytics)
-        mock_order.simulated.assert_called_with(mock_market_book, {1: 2})
+        mock_order.simulated.assert_called_with(mock_market_book, "test")
         mock_order_two.simulated.assert_not_called()
 
-    def test__sort_orders(self):
-        order_one = mock.Mock(side="LAY", bet_id=1)
-        order_one.order_type.price = 1.01
-        order_two = mock.Mock(side="LAY", bet_id=2)
-        order_two.order_type.price = 1.02
-        order_three = mock.Mock(side="LAY", bet_id=3)
-        order_three.order_type.price = 1.01
-        order_four = mock.Mock(side="BACK", bet_id=4)
-        order_four.order_type.price = 1.2
-        order_five = mock.Mock(side="BACK", bet_id=5)
-        order_five.order_type.price = 1.2
-        order_six = mock.Mock(side="BACK", bet_id=6)
-        order_six.order_type.price = 1.19
-        order_seven = mock.Mock(side="BACK", bet_id=6)
-        order_seven.order_type.price = "ERROR"
-        order_seven.order_type.ORDER_TYPE = OrderTypes.MARKET_ON_CLOSE
-        orders = [
-            order_one,
-            order_two,
-            order_three,
-            order_four,
-            order_five,
-            order_six,
-            order_seven,
-        ]
-        self.assertEqual(
-            self.middleware._sort_orders(orders),
-            [
-                order_two,
-                order_one,
-                order_three,
-                order_six,
-                order_four,
-                order_five,
-                order_seven,
-            ],
-        )
-
     @mock.patch("flumine.markets.middleware.RunnerAnalytics")
     def test__process_runner(self, mock_runner_analytics):
         market_analytics = {}
diff --git a/tests/test_simulated.py b/tests/test_simulated.py
index 842e3b8..2a87142 100644
--- a/tests/test_simulated.py
+++ b/tests/test_simulated.py
@@ -39,12 +39,13 @@ class SimulatedTest(unittest.TestCase):
     def test_call_process_traded(
         self, mock__process_sp, mock__process_traded, _, mock__get_runner
     ):
-        mock_market_book = mock.Mock(bsp_reconciled=False)
-        traded = {1: 2}
-        self.simulated(mock_market_book, traded)
+        mock_market_book = mock.Mock()
+        mock_market_book.bsp_reconciled = False
+        mock_runner_analytics = mock.Mock()
+        self.simulated(mock_market_book, mock_runner_analytics)
         mock__process_sp.assert_not_called()
         mock__process_traded.assert_called_with(
-            mock_market_book.publish_time_epoch, traded
+            mock_market_book.publish_time_epoch, mock_runner_analytics.traded
         )
 
     @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
@@ -604,9 +605,9 @@ class SimulatedTest(unittest.TestCase):
         mock__calculate_process_traded.assert_not_called()
 
     def test__calculate_process_traded(self):
-        self.assertEqual(self.simulated._calculate_process_traded(1234582, 2.00), 2)
-        self.assertEqual(self.simulated._calculate_process_traded(1234583, 2.00), 2)
-        self.assertEqual(self.simulated._calculate_process_traded(1234584, 2.00), 0)
+        self.simulated._calculate_process_traded(1234582, 2.00)
+        self.simulated._calculate_process_traded(1234583, 2.00)
+        self.simulated._calculate_process_traded(1234584, 2.00)
         self.assertEqual(
             self.simulated.matched, [[1234582, 12, 1.00], [1234583, 12, 1.00]]
         )
@@ -614,19 +615,12 @@ class SimulatedTest(unittest.TestCase):
 
     def test__calculate_process_traded_piq(self):
         self.simulated._piq = 2.00
-        self.assertEqual(self.simulated._calculate_process_traded(1234585, 4.00), 4)
+        self.simulated._calculate_process_traded(1234585, 4.00)
         self.assertEqual(self.simulated.matched, [])
         self.assertEqual(self.simulated._piq, 0)
-        self.assertEqual(self.simulated._calculate_process_traded(1234586, 4.00), 4)
+        self.simulated._calculate_process_traded(1234586, 4.00)
         self.assertEqual(self.simulated.matched, [[1234586, 12, 2.00]])
         self.assertEqual(self.simulated._piq, 0)
-        self.assertEqual(self.simulated._calculate_process_traded(1234586, 4.00), 0)
-
-    def test__calculate_process_traded_piq_match(self):
-        self.simulated._piq = 4.00
-        self.assertEqual(self.simulated._calculate_process_traded(1234585, 20.00), 12)
-        self.assertEqual(self.simulated.matched, [[1234585, 12, 2.0]])
-        self.assertEqual(self.simulated._piq, 0)
 
     def test_take_sp(self):
         self.assertFalse(self.simulated.take_sp)