Repository: liampauling/flumine
Pull request #462 (2021-07-08 17:09:54)

Allow for historical data files to contain data that are not market book data


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 9d47d29..d868ca5 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,18 +3,6 @@
 Release History
 ---------------
 
-1.19.6 (2021-07-09)
-+++++++++++++++++++
-
-**Improvements**
-
-- #452 transaction force parameter (@flxbe)
-
-**Bug Fixes**
-
-- #454 SP nr adjustments (@jsphon)
-- Handle missing mc from historic files (@mlabour)
-
 1.19.5 (2021-07-05)
 +++++++++++++++++++
 
diff --git a/docs/controls.md b/docs/controls.md
index 0d08991..4715053 100644
--- a/docs/controls.md
+++ b/docs/controls.md
@@ -13,17 +13,6 @@ Before placing an order flumine will check the client and trading controls, this
 - `OrderValidation`: Checks order is valid (size/odds)
 - `StrategyExposure`: Checks order does not invalidate `strategy.validate_order`, `strategy.max_order_exposure` or `strategy.max_selection_exposure`
 
-### Skipping Controls
-
-Sometimes it is desirable to skip the controls, for example when canceling an open order even if the transaction count has already reached the betfair transaction limit. This can be done by passing `force=True` when placing or changing an order:
-
-```py
-market.place_order(order, force=True)
-transaction.place_order(order, force=True)
-```
-
-This works for markets and transactions and is supported by the operations `place_order`, `cancel_order`, `update_order`, and `replace_order`.
-
 ## Logging Controls
 
 Custom logging is available using the `LoggingControl` class, the [base class](https://github.com/liampauling/flumine/blob/master/flumine/controls/loggingcontrols.py#L12) creates debug logs and can be used as follows:
diff --git a/flumine/__version__.py b/flumine/__version__.py
index b622618..76c3946 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.19.6"
+__version__ = "1.19.5"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/execution/transaction.py b/flumine/execution/transaction.py
index 93ced92..01bb323 100644
--- a/flumine/execution/transaction.py
+++ b/flumine/execution/transaction.py
@@ -41,17 +41,9 @@ class Transaction:
         self._pending_replace = []  # list of (<Order>, market_version)
 
     def place_order(
-        self,
-        order,
-        market_version: int = None,
-        execute: bool = True,
-        force: bool = False,
+        self, order, market_version: int = None, execute: bool = True
     ) -> bool:
-        if (
-            execute
-            and not force
-            and self._validate_controls(order, OrderPackageType.PLACE) is False
-        ):
+        if execute and self._validate_controls(order, OrderPackageType.PLACE) is False:
             return False
         # place
         order.place(self.market.market_book.publish_time)
@@ -77,13 +69,8 @@ class Transaction:
             self._pending_orders = True
         return True
 
-    def cancel_order(
-        self, order, size_reduction: float = None, force: bool = False
-    ) -> bool:
-        if (
-            not force
-            and self._validate_controls(order, OrderPackageType.CANCEL) is False
-        ):
+    def cancel_order(self, order, size_reduction: float = None) -> bool:
+        if self._validate_controls(order, OrderPackageType.CANCEL) is False:
             return False
         # cancel
         order.cancel(size_reduction)
@@ -91,13 +78,8 @@ class Transaction:
         self._pending_orders = True
         return True
 
-    def update_order(
-        self, order, new_persistence_type: str, force: bool = False
-    ) -> bool:
-        if (
-            not force
-            and self._validate_controls(order, OrderPackageType.UPDATE) is False
-        ):
+    def update_order(self, order, new_persistence_type: str) -> bool:
+        if self._validate_controls(order, OrderPackageType.UPDATE) is False:
             return False
         # update
         order.update(new_persistence_type)
@@ -106,12 +88,9 @@ class Transaction:
         return True
 
     def replace_order(
-        self, order, new_price: float, market_version: int = None, force: bool = False
+        self, order, new_price: float, market_version: int = None
     ) -> bool:
-        if (
-            not force
-            and self._validate_controls(order, OrderPackageType.REPLACE) is False
-        ):
+        if self._validate_controls(order, OrderPackageType.REPLACE) is False:
             return False
         # replace
         order.replace(new_price)
diff --git a/flumine/markets/market.py b/flumine/markets/market.py
index 560b9f0..f966936 100644
--- a/flumine/markets/market.py
+++ b/flumine/markets/market.py
@@ -75,28 +75,23 @@ class Market:
         order,
         market_version: int = None,
         execute: bool = True,
-        force: bool = False,
     ) -> bool:
         with self.transaction() as t:
-            return t.place_order(order, market_version, execute, force)
+            return t.place_order(order, market_version, execute)
 
-    def cancel_order(
-        self, order, size_reduction: float = None, force: bool = False
-    ) -> bool:
+    def cancel_order(self, order, size_reduction: float = None) -> bool:
         with self.transaction() as t:
-            return t.cancel_order(order, size_reduction, force)
+            return t.cancel_order(order, size_reduction)
 
-    def update_order(
-        self, order, new_persistence_type: str, force: bool = False
-    ) -> bool:
+    def update_order(self, order, new_persistence_type: str) -> bool:
         with self.transaction() as t:
-            return t.update_order(order, new_persistence_type, force)
+            return t.update_order(order, new_persistence_type)
 
     def replace_order(
-        self, order, new_price: float, market_version: int = None, force: bool = False
+        self, order, new_price: float, market_version: int = None
     ) -> bool:
         with self.transaction() as t:
-            return t.replace_order(order, new_price, market_version, force)
+            return t.replace_order(order, new_price, market_version)
 
     @property
     def event(self) -> dict:
diff --git a/flumine/markets/middleware.py b/flumine/markets/middleware.py
index bafe57c..61eff19 100644
--- a/flumine/markets/middleware.py
+++ b/flumine/markets/middleware.py
@@ -99,40 +99,7 @@ class SimulatedMiddleware(Middleware):
                         extra=order.info,
                     )
                 else:
-                    # TODO: "Where an SP lay bet in a win market has a maximum odds limit specified,..."
                     if (
-                        order.order_type.ORDER_TYPE == OrderTypes.MARKET_ON_CLOSE
-                    ) and order.side == "LAY":
-                        if market.market_type == "WIN":
-                            runner = [
-                                x
-                                for x in market.market_book.runners
-                                if x.selection_id == order.selection_id
-                                and x.handicap == order.handicap
-                            ][0]
-                            runner_adjustment_factor = runner.adjustment_factor
-                            # See https://github.com/liampauling/flumine/issues/454
-                            multiplier = 1 - (
-                                removal_adjustment_factor
-                                / (100 - runner_adjustment_factor)
-                            )
-                            order.order_type.liability *= multiplier
-                            logger.warning(
-                                "WIN MARKET_ON_CLOSE Order adjusted due to non runner {0}".format(
-                                    order.selection_id
-                                ),
-                                extra=order.info,
-                            )
-                        elif market.market_type in {"PLACE", "OTHER_PLACE"}:
-                            multiplier = (100 - removal_adjustment_factor) * 0.01
-                            order.order_type.liability *= multiplier
-                            logger.warning(
-                                "PLACE MARKET_ON_CLOSE Order adjusted due to non runner {0}".format(
-                                    order.selection_id
-                                ),
-                                extra=order.info,
-                            )
-                    elif (
                         removal_adjustment_factor
                         and removal_adjustment_factor >= WIN_MINIMUM_ADJUSTMENT_FACTOR
                     ):
@@ -147,7 +114,7 @@ class SimulatedMiddleware(Middleware):
                         logger.warning(
                             "Order adjusted due to non runner {0}".format(
                                 order.selection_id
-                            ),
+                            ).format(order.selection_id),
                             extra=order.info,
                         )
 
diff --git a/flumine/streams/historicalstream.py b/flumine/streams/historicalstream.py
index 04fdaed..d3f0d65 100644
--- a/flumine/streams/historicalstream.py
+++ b/flumine/streams/historicalstream.py
@@ -24,6 +24,8 @@ class FlumineMarketStream(MarketStream):
 
     def _process(self, data: list, publish_time: int) -> bool:
         for market_book in data:
+            if "id" not in market_book:
+                continue
             market_id = market_book["id"]
             full_image = market_book.get("img", False)
             market_book_cache = self._caches.get(market_id)
diff --git a/flumine/utils.py b/flumine/utils.py
index 4c14c0b..53fb9cf 100644
--- a/flumine/utils.py
+++ b/flumine/utils.py
@@ -47,6 +47,8 @@ def get_file_md(file_dir: str, value: str) -> Optional[str]:
     with open(file_dir, "r") as f:
         first_line = f.readline()
         update = json.loads(first_line)
+    if "mc" not in update or not isinstance(update["mc"], list) or not update["mc"]:
+        return None
     md = update["mc"][0].get("marketDefinition", {})
     return md.get(value)
 


Test patch:
diff --git a/tests/test_markets.py b/tests/test_markets.py
index 512abee..cb3fd3a 100644
--- a/tests/test_markets.py
+++ b/tests/test_markets.py
@@ -149,29 +149,29 @@ class MarketTest(unittest.TestCase):
     def test_place_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.place_order(mock_order, 2, False, force=True))
-        mock_transaction.place_order.assert_called_with(mock_order, 2, False, True)
+        self.assertTrue(self.market.place_order(mock_order, 2, False))
+        mock_transaction.place_order.assert_called_with(mock_order, 2, False)
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_cancel_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.cancel_order(mock_order, 2.02, force=True))
-        mock_transaction.cancel_order.assert_called_with(mock_order, 2.02, True)
+        self.assertTrue(self.market.cancel_order(mock_order, 2.02))
+        mock_transaction.cancel_order.assert_called_with(mock_order, 2.02)
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_update_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.update_order(mock_order, "test", force=True))
-        mock_transaction.update_order.assert_called_with(mock_order, "test", True)
+        self.assertTrue(self.market.update_order(mock_order, "test"))
+        mock_transaction.update_order.assert_called_with(mock_order, "test")
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_replace_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.replace_order(mock_order, 2, False, force=True))
-        mock_transaction.replace_order.assert_called_with(mock_order, 2, False, True)
+        self.assertTrue(self.market.replace_order(mock_order, 2, False))
+        mock_transaction.replace_order.assert_called_with(mock_order, 2, False)
 
     def test_event(self):
         mock_market_catalogue = mock.Mock()
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 4cc68bd..5feebd9 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -10,7 +10,6 @@ from flumine.markets.middleware import (
     WIN_MINIMUM_ADJUSTMENT_FACTOR,
     PLACE_MINIMUM_ADJUSTMENT_FACTOR,
 )
-from flumine.order.ordertype import MarketOnCloseOrder
 
 
 class MiddlewareTest(unittest.TestCase):
@@ -137,43 +136,6 @@ class SimulatedMiddlewareTest(unittest.TestCase):
         self.middleware._process_runner_removal(mock_market, 12345, 0, None)
         self.assertEqual(mock_order.simulated.matched, [[123, 8.6, 10]])
 
-    def test__process_runner_removal_sp_win(self):
-        order_type = MarketOnCloseOrder(liability=200)
-        mock_order = mock.Mock(
-            selection_id=1234, handicap=0, order_type=order_type, info={}, side="LAY"
-        )
-
-        mock_market_book = mock.Mock()
-        mock_market_book.runners = [
-            mock.Mock(selection_id=1234, handicap=0, adjustment_factor=20)
-        ]
-        mock_market = mock.Mock(
-            market_type="WIN", blotter=[mock_order], market_book=mock_market_book
-        )
-        self.middleware._process_runner_removal(mock_market, 12345, 0, 50)
-
-        # The liability of £200 is adjusted by the multiplier of 37.5%, which s
-        # defined in the example here: https://github.com/liampauling/flumine/issues/454
-        self.assertEqual(mock_order.order_type.liability, 75)
-
-    def test__process_runner_removal_sp_place(self):
-        order_type = MarketOnCloseOrder(liability=200)
-        mock_order = mock.Mock(
-            selection_id=1234, handicap=0, order_type=order_type, info={}, side="LAY"
-        )
-
-        mock_market_book = mock.Mock()
-        mock_market_book.runners = [
-            mock.Mock(selection_id=1234, handicap=0, adjustment_factor=20)
-        ]
-        mock_market = mock.Mock(
-            market_type="PLACE", blotter=[mock_order], market_book=mock_market_book
-        )
-        self.middleware._process_runner_removal(mock_market, 12345, 0, 50)
-
-        # The liability of £200 is reduced by the non runner's adjustment factor of 50%
-        self.assertEqual(mock_order.order_type.liability, 100)
-
     def test__process_streaming_update(self):
         mock_market_book = mock.Mock(
             streaming_update={"img": True, "rc": [{"id": 3}, {"id": 4}]},
diff --git a/tests/test_transaction.py b/tests/test_transaction.py
index 1ba3e5c..f9181fd 100644
--- a/tests/test_transaction.py
+++ b/tests/test_transaction.py
@@ -110,23 +110,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_place = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch("flumine.execution.transaction.get_market_notes")
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    @mock.patch("flumine.execution.transaction.events")
-    def test_force_place_order(
-        self, mock_events, mock__validate_controls, mock_get_market_notes
-    ):
-        self.transaction.market.blotter = mock.MagicMock()
-        self.transaction.market.blotter.has_trade.return_value = False
-        mock_order = mock.Mock(id="123", lookup=(1, 2, 3))
-        self.assertTrue(self.transaction.place_order(mock_order, force=True))
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_place = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -150,18 +133,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_cancel = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    def test_force_cancel_order(self, mock__validate_controls):
-        mock_order = mock.Mock()
-        self.assertTrue(self.transaction.cancel_order(mock_order, 0.01, force=True))
-        mock_order.cancel.assert_called_with(0.01)
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_cancel = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -185,20 +156,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_update = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    def test_force_update_order(self, mock__validate_controls):
-        mock_order = mock.Mock()
-        self.assertTrue(
-            self.transaction.update_order(mock_order, "PERSIST", force=True)
-        )
-        mock_order.update.assert_called_with("PERSIST")
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_update = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -222,20 +179,6 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_replace = []
         self.assertFalse(self.transaction._pending_orders)
 
-    @mock.patch(
-        "flumine.execution.transaction.Transaction._validate_controls",
-        return_value=False,
-    )
-    def test_force_replace_order(self, mock__validate_controls):
-        mock_order = mock.Mock()
-        self.assertTrue(
-            self.transaction.replace_order(mock_order, 1.01, 321, force=True)
-        )
-        mock_order.replace.assert_called_with(1.01)
-        mock__validate_controls.assert_not_called()
-        self.transaction._pending_replace = [(mock_order, None)]
-        self.assertTrue(self.transaction._pending_orders)
-
     @mock.patch("flumine.execution.transaction.Transaction._create_order_package")
     def test_execute(self, mock__create_order_package):
         self.transaction._pending_orders = True