Repository: liampauling/flumine
Pull request #465 (2021-07-10 22:04:03)

If a MOC bet has been reconciled and a NR is declared, we need to adj…
If a BSP bet has been reconciled, and the NR is declared, we also need to adjust the size of the order.


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 3839ad2..d868ca5 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,29 +3,6 @@
 Release History
 ---------------
 
-1.19.7 (2021-07-12)
-+++++++++++++++++++
-
-**Improvements**
-
--
-**Bug Fixes**
-
-- #454 SP nr size adjustment (@jsphon)
-
-1.19.6 (2021-07-09)
-+++++++++++++++++++
-
-**Improvements**
-
-- #452 transaction force parameter (@flxbe)
-- market `date_time_created` added
-
-**Bug Fixes**
-
-- #454 SP nr adjustments (@jsphon)
-- Handle missing mc from historic files (@mlabour)
-
 1.19.5 (2021-07-05)
 +++++++++++++++++++
 
diff --git a/docs/controls.md b/docs/controls.md
index 0d08991..4715053 100644
--- a/docs/controls.md
+++ b/docs/controls.md
@@ -13,17 +13,6 @@ Before placing an order flumine will check the client and trading controls, this
 - `OrderValidation`: Checks order is valid (size/odds)
 - `StrategyExposure`: Checks order does not invalidate `strategy.validate_order`, `strategy.max_order_exposure` or `strategy.max_selection_exposure`
 
-### Skipping Controls
-
-Sometimes it is desirable to skip the controls, for example when canceling an open order even if the transaction count has already reached the betfair transaction limit. This can be done by passing `force=True` when placing or changing an order:
-
-```py
-market.place_order(order, force=True)
-transaction.place_order(order, force=True)
-```
-
-This works for markets and transactions and is supported by the operations `place_order`, `cancel_order`, `update_order`, and `replace_order`.
-
 ## Logging Controls
 
 Custom logging is available using the `LoggingControl` class, the [base class](https://github.com/liampauling/flumine/blob/master/flumine/controls/loggingcontrols.py#L12) creates debug logs and can be used as follows:
diff --git a/flumine/__version__.py b/flumine/__version__.py
index fe9ae70..76c3946 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.19.7"
+__version__ = "1.19.5"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/execution/transaction.py b/flumine/execution/transaction.py
index 93ced92..01bb323 100644
--- a/flumine/execution/transaction.py
+++ b/flumine/execution/transaction.py
@@ -41,17 +41,9 @@ class Transaction:
         self._pending_replace = []  # list of (<Order>, market_version)
 
     def place_order(
-        self,
-        order,
-        market_version: int = None,
-        execute: bool = True,
-        force: bool = False,
+        self, order, market_version: int = None, execute: bool = True
     ) -> bool:
-        if (
-            execute
-            and not force
-            and self._validate_controls(order, OrderPackageType.PLACE) is False
-        ):
+        if execute and self._validate_controls(order, OrderPackageType.PLACE) is False:
             return False
         # place
         order.place(self.market.market_book.publish_time)
@@ -77,13 +69,8 @@ class Transaction:
             self._pending_orders = True
         return True
 
-    def cancel_order(
-        self, order, size_reduction: float = None, force: bool = False
-    ) -> bool:
-        if (
-            not force
-            and self._validate_controls(order, OrderPackageType.CANCEL) is False
-        ):
+    def cancel_order(self, order, size_reduction: float = None) -> bool:
+        if self._validate_controls(order, OrderPackageType.CANCEL) is False:
             return False
         # cancel
         order.cancel(size_reduction)
@@ -91,13 +78,8 @@ class Transaction:
         self._pending_orders = True
         return True
 
-    def update_order(
-        self, order, new_persistence_type: str, force: bool = False
-    ) -> bool:
-        if (
-            not force
-            and self._validate_controls(order, OrderPackageType.UPDATE) is False
-        ):
+    def update_order(self, order, new_persistence_type: str) -> bool:
+        if self._validate_controls(order, OrderPackageType.UPDATE) is False:
             return False
         # update
         order.update(new_persistence_type)
@@ -106,12 +88,9 @@ class Transaction:
         return True
 
     def replace_order(
-        self, order, new_price: float, market_version: int = None, force: bool = False
+        self, order, new_price: float, market_version: int = None
     ) -> bool:
-        if (
-            not force
-            and self._validate_controls(order, OrderPackageType.REPLACE) is False
-        ):
+        if self._validate_controls(order, OrderPackageType.REPLACE) is False:
             return False
         # replace
         order.replace(new_price)
diff --git a/flumine/markets/market.py b/flumine/markets/market.py
index a99c6e7..f966936 100644
--- a/flumine/markets/market.py
+++ b/flumine/markets/market.py
@@ -29,7 +29,6 @@ class Market:
         self.flumine = flumine
         self.market_id = market_id
         self.closed = False
-        self.date_time_created = datetime.datetime.utcnow()
         self.date_time_closed = None
         self.market_book = market_book
         self.market_catalogue = market_catalogue
@@ -76,28 +75,23 @@ class Market:
         order,
         market_version: int = None,
         execute: bool = True,
-        force: bool = False,
     ) -> bool:
         with self.transaction() as t:
-            return t.place_order(order, market_version, execute, force)
+            return t.place_order(order, market_version, execute)
 
-    def cancel_order(
-        self, order, size_reduction: float = None, force: bool = False
-    ) -> bool:
+    def cancel_order(self, order, size_reduction: float = None) -> bool:
         with self.transaction() as t:
-            return t.cancel_order(order, size_reduction, force)
+            return t.cancel_order(order, size_reduction)
 
-    def update_order(
-        self, order, new_persistence_type: str, force: bool = False
-    ) -> bool:
+    def update_order(self, order, new_persistence_type: str) -> bool:
         with self.transaction() as t:
-            return t.update_order(order, new_persistence_type, force)
+            return t.update_order(order, new_persistence_type)
 
     def replace_order(
-        self, order, new_price: float, market_version: int = None, force: bool = False
+        self, order, new_price: float, market_version: int = None
     ) -> bool:
         with self.transaction() as t:
-            return t.replace_order(order, new_price, market_version, force)
+            return t.replace_order(order, new_price, market_version)
 
     @property
     def event(self) -> dict:
diff --git a/flumine/markets/middleware.py b/flumine/markets/middleware.py
index bafe57c..38c5e85 100644
--- a/flumine/markets/middleware.py
+++ b/flumine/markets/middleware.py
@@ -117,6 +117,13 @@ class SimulatedMiddleware(Middleware):
                                 / (100 - runner_adjustment_factor)
                             )
                             order.order_type.liability *= multiplier
+                            if order.average_price_matched:
+                                # We will get here if the NR is declared inplay
+                                order.current_order.size_matched = round(
+                                    order.order_type.liability
+                                    / (order.average_price_matched - 1),
+                                    2,
+                                )
                             logger.warning(
                                 "WIN MARKET_ON_CLOSE Order adjusted due to non runner {0}".format(
                                     order.selection_id
@@ -126,6 +133,13 @@ class SimulatedMiddleware(Middleware):
                         elif market.market_type in {"PLACE", "OTHER_PLACE"}:
                             multiplier = (100 - removal_adjustment_factor) * 0.01
                             order.order_type.liability *= multiplier
+                            if order.average_price_matched:
+                                # We will get here if the NR is declared inplay
+                                order.current_order.size_matched = round(
+                                    order.order_type.liability
+                                    / (order.average_price_matched - 1),
+                                    2,
+                                )
                             logger.warning(
                                 "PLACE MARKET_ON_CLOSE Order adjusted due to non runner {0}".format(
                                     order.selection_id
diff --git a/flumine/streams/historicalstream.py b/flumine/streams/historicalstream.py
index d3f0d65..04fdaed 100644
--- a/flumine/streams/historicalstream.py
+++ b/flumine/streams/historicalstream.py
@@ -24,8 +24,6 @@ class FlumineMarketStream(MarketStream):
 
     def _process(self, data: list, publish_time: int) -> bool:
         for market_book in data:
-            if "id" not in market_book:
-                continue
             market_id = market_book["id"]
             full_image = market_book.get("img", False)
             market_book_cache = self._caches.get(market_id)
diff --git a/flumine/utils.py b/flumine/utils.py
index 53fb9cf..4c14c0b 100644
--- a/flumine/utils.py
+++ b/flumine/utils.py
@@ -47,8 +47,6 @@ def get_file_md(file_dir: str, value: str) -> Optional[str]:
     with open(file_dir, "r") as f:
         first_line = f.readline()
         update = json.loads(first_line)
-    if "mc" not in update or not isinstance(update["mc"], list) or not update["mc"]:
-        return None
     md = update["mc"][0].get("marketDefinition", {})
     return md.get(value)
 


Test patch:
diff --git a/tests/test_markets.py b/tests/test_markets.py
index e9bcc01..cb3fd3a 100644
--- a/tests/test_markets.py
+++ b/tests/test_markets.py
@@ -109,16 +109,13 @@ class MarketTest(unittest.TestCase):
         self.assertEqual(self.market.flumine, self.mock_flumine)
         self.assertEqual(self.market.market_id, "1.234")
         self.assertFalse(self.market.closed)
-        self.assertIsNotNone(self.market.date_time_created)
+        self.assertFalse(self.market.orders_cleared)
+        self.assertFalse(self.market.market_cleared)
         self.assertIsNone(self.market.date_time_closed)
         self.assertEqual(self.market.market_book, self.mock_market_book)
         self.assertEqual(self.market.market_catalogue, self.mock_market_catalogue)
         self.assertTrue(self.market.update_market_catalogue)
-        self.assertFalse(self.market.orders_cleared)
-        self.assertFalse(self.market.market_cleared)
         self.assertEqual(self.market.context, {"simulated": {}})
-        self.assertIsNotNone(self.market.blotter)
-        self.assertEqual(self.market._transaction_id, 0)
 
     def test_call(self):
         mock_market_book = mock.Mock()
@@ -152,29 +149,29 @@ class MarketTest(unittest.TestCase):
     def test_place_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.place_order(mock_order, 2, False, force=True))
-        mock_transaction.place_order.assert_called_with(mock_order, 2, False, True)
+        self.assertTrue(self.market.place_order(mock_order, 2, False))
+        mock_transaction.place_order.assert_called_with(mock_order, 2, False)
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_cancel_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.cancel_order(mock_order, 2.02, force=True))
-        mock_transaction.cancel_order.assert_called_with(mock_order, 2.02, True)
+        self.assertTrue(self.market.cancel_order(mock_order, 2.02))
+        mock_transaction.cancel_order.assert_called_with(mock_order, 2.02)
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_update_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.update_order(mock_order, "test", force=True))
-        mock_transaction.update_order.assert_called_with(mock_order, "test", True)
+        self.assertTrue(self.market.update_order(mock_order, "test"))
+        mock_transaction.update_order.assert_called_with(mock_order, "test")
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_replace_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.replace_order(mock_order, 2, False, force=True))
-        mock_transaction.replace_order.assert_called_with(mock_order, 2, False, True)
+        self.assertTrue(self.market.replace_order(mock_order, 2, False))
+        mock_transaction.replace_order.assert_called_with(mock_order, 2, False)
 
     def test_event(self):
         mock_market_catalogue = mock.Mock()
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 4cc68bd..3fcc046 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -140,7 +140,13 @@ class SimulatedMiddlewareTest(unittest.TestCase):
     def test__process_runner_removal_sp_win(self):
         order_type = MarketOnCloseOrder(liability=200)
         mock_order = mock.Mock(
-            selection_id=1234, handicap=0, order_type=order_type, info={}, side="LAY"
+            selection_id=1234,
+            handicap=0,
+            order_type=order_type,
+            info={},
+            side="LAY",
+            current_order=mock.Mock(size_matched=0),
+            average_price_matched=None,
         )
 
         mock_market_book = mock.Mock()
@@ -156,10 +162,67 @@ class SimulatedMiddlewareTest(unittest.TestCase):
         # defined in the example here: https://github.com/liampauling/flumine/issues/454
         self.assertEqual(mock_order.order_type.liability, 75)
 
+    def test__process_runner_removal_sp_win_inplay(self):
+        order_type = MarketOnCloseOrder(liability=200)
+        mock_order = mock.Mock(
+            selection_id=1234,
+            handicap=0,
+            order_type=order_type,
+            info={},
+            side="LAY",
+            current_order=mock.Mock(size_matched=0),
+            average_price_matched=10,
+        )
+
+        mock_market_book = mock.Mock()
+        mock_market_book.runners = [
+            mock.Mock(selection_id=1234, handicap=0, adjustment_factor=20)
+        ]
+        mock_market = mock.Mock(
+            market_type="WIN", blotter=[mock_order], market_book=mock_market_book
+        )
+        self.middleware._process_runner_removal(mock_market, 12345, 0, 50)
+
+        # The liability of £200 is adjusted by the multiplier of 37.5%, which s
+        # defined in the example here: https://github.com/liampauling/flumine/issues/454
+        self.assertEqual(mock_order.order_type.liability, 75)
+        # Size matched should be 75 / (10.0-1.0) \approx 8.33
+        self.assertEqual(8.33, mock_order.current_order.size_matched)
+
     def test__process_runner_removal_sp_place(self):
         order_type = MarketOnCloseOrder(liability=200)
         mock_order = mock.Mock(
-            selection_id=1234, handicap=0, order_type=order_type, info={}, side="LAY"
+            selection_id=1234,
+            handicap=0,
+            order_type=order_type,
+            info={},
+            side="LAY",
+            current_order=mock.Mock(size_matched=0),
+            average_price_matched=None,
+        )
+
+        mock_market_book = mock.Mock()
+        mock_market_book.runners = [
+            mock.Mock(selection_id=1234, handicap=0, adjustment_factor=20)
+        ]
+        mock_market = mock.Mock(
+            market_type="PLACE", blotter=[mock_order], market_book=mock_market_book
+        )
+        self.middleware._process_runner_removal(mock_market, 12345, 0, 50)
+
+        # The liability of £200 is reduced by the non runner's adjustment factor of 50%
+        self.assertEqual(mock_order.order_type.liability, 100)
+
+    def test__process_runner_removal_sp_place_inplay(self):
+        order_type = MarketOnCloseOrder(liability=200)
+        mock_order = mock.Mock(
+            selection_id=1234,
+            handicap=0,
+            order_type=order_type,
+            info={},
+            side="LAY",
+            current_order=mock.Mock(size_matched=0),
+            average_price_matched=10.0,
         )
 
         mock_market_book = mock.Mock()
@@ -173,6 +236,8 @@ class SimulatedMiddlewareTest(unittest.TestCase):
 
         # The liability of £200 is reduced by the non runner's adjustment factor of 50%
         self.assertEqual(mock_order.order_type.liability, 100)
+        # Size matched should be 100 / (10.0-1.0) \approx 11.11
+        self.assertEqual(11.11, mock_order.current_order.size_matched)
 
     def test__process_streaming_update(self):
         mock_market_book = mock.Mock(
diff --git a/tests/test_transaction.py b/tests/test_transaction.py
index 1ba3e5c..f9181fd 100644
--- a/tests/test_transaction.py
+++ b/tests/test_transaction.py
@@ -110,23 +110,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_place = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch("flumine.execution.transaction.get_market_notes")
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    @mock.patch("flumine.execution.transaction.events")
-    def test_force_place_order(
-        self, mock_events, mock__validate_controls, mock_get_market_notes
-    ):
-        self.transaction.market.blotter = mock.MagicMock()
-        self.transaction.market.blotter.has_trade.return_value = False
-        mock_order = mock.Mock(id="123", lookup=(1, 2, 3))
-        self.assertTrue(self.transaction.place_order(mock_order, force=True))
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_place = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -150,18 +133,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_cancel = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    def test_force_cancel_order(self, mock__validate_controls):
-        mock_order = mock.Mock()
-        self.assertTrue(self.transaction.cancel_order(mock_order, 0.01, force=True))
-        mock_order.cancel.assert_called_with(0.01)
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_cancel = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -185,20 +156,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_update = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    def test_force_update_order(self, mock__validate_controls):
-        mock_order = mock.Mock()
-        self.assertTrue(
-            self.transaction.update_order(mock_order, "PERSIST", force=True)
-        )
-        mock_order.update.assert_called_with("PERSIST")
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_update = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -222,20 +179,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_replace = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    def test_force_replace_order(self, mock__validate_controls):
-        mock_order = mock.Mock()
-        self.assertTrue(
-            self.transaction.replace_order(mock_order, 1.01, 321, force=True)
-        )
-        mock_order.replace.assert_called_with(1.01)
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_replace = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch("flumine.execution.transaction.Transaction._create_order_package")
     def test_execute(self, mock__create_order_package):
         self.transaction._pending_orders = True