Repository: liampauling/flumine
Pull request #463 (2021-07-09 08:45:35)

history and version update


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index d868ca5..9188f6f 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,19 @@
 Release History
 ---------------
 
+1.19.6 (2021-07-09)
++++++++++++++++++++
+
+**Improvements**
+
+- #452 transaction force parameter (@flxbe)
+- market `date_time_created` added
+
+**Bug Fixes**
+
+- #454 SP nr adjustments (@jsphon)
+- Handle missing mc from historic files (@mlabour)
+
 1.19.5 (2021-07-05)
 +++++++++++++++++++
 
diff --git a/docs/controls.md b/docs/controls.md
index 4715053..0d08991 100644
--- a/docs/controls.md
+++ b/docs/controls.md
@@ -13,6 +13,17 @@ Before placing an order flumine will check the client and trading controls, this
 - `OrderValidation`: Checks order is valid (size/odds)
 - `StrategyExposure`: Checks order does not invalidate `strategy.validate_order`, `strategy.max_order_exposure` or `strategy.max_selection_exposure`
 
+### Skipping Controls
+
+Sometimes it is desirable to skip the controls, for example when canceling an open order even if the transaction count has already reached the betfair transaction limit. This can be done by passing `force=True` when placing or changing an order:
+
+```py
+market.place_order(order, force=True)
+transaction.place_order(order, force=True)
+```
+
+This works for markets and transactions and is supported by the operations `place_order`, `cancel_order`, `update_order`, and `replace_order`.
+
 ## Logging Controls
 
 Custom logging is available using the `LoggingControl` class, the [base class](https://github.com/liampauling/flumine/blob/master/flumine/controls/loggingcontrols.py#L12) creates debug logs and can be used as follows:
diff --git a/flumine/__version__.py b/flumine/__version__.py
index 76c3946..b622618 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.19.5"
+__version__ = "1.19.6"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/execution/transaction.py b/flumine/execution/transaction.py
index 01bb323..93ced92 100644
--- a/flumine/execution/transaction.py
+++ b/flumine/execution/transaction.py
@@ -41,9 +41,17 @@ class Transaction:
         self._pending_replace = []  # list of (<Order>, market_version)
 
     def place_order(
-        self, order, market_version: int = None, execute: bool = True
+        self,
+        order,
+        market_version: int = None,
+        execute: bool = True,
+        force: bool = False,
     ) -> bool:
-        if execute and self._validate_controls(order, OrderPackageType.PLACE) is False:
+        if (
+            execute
+            and not force
+            and self._validate_controls(order, OrderPackageType.PLACE) is False
+        ):
             return False
         # place
         order.place(self.market.market_book.publish_time)
@@ -69,8 +77,13 @@ class Transaction:
             self._pending_orders = True
         return True
 
-    def cancel_order(self, order, size_reduction: float = None) -> bool:
-        if self._validate_controls(order, OrderPackageType.CANCEL) is False:
+    def cancel_order(
+        self, order, size_reduction: float = None, force: bool = False
+    ) -> bool:
+        if (
+            not force
+            and self._validate_controls(order, OrderPackageType.CANCEL) is False
+        ):
             return False
         # cancel
         order.cancel(size_reduction)
@@ -78,8 +91,13 @@ class Transaction:
         self._pending_orders = True
         return True
 
-    def update_order(self, order, new_persistence_type: str) -> bool:
-        if self._validate_controls(order, OrderPackageType.UPDATE) is False:
+    def update_order(
+        self, order, new_persistence_type: str, force: bool = False
+    ) -> bool:
+        if (
+            not force
+            and self._validate_controls(order, OrderPackageType.UPDATE) is False
+        ):
             return False
         # update
         order.update(new_persistence_type)
@@ -88,9 +106,12 @@ class Transaction:
         return True
 
     def replace_order(
-        self, order, new_price: float, market_version: int = None
+        self, order, new_price: float, market_version: int = None, force: bool = False
     ) -> bool:
-        if self._validate_controls(order, OrderPackageType.REPLACE) is False:
+        if (
+            not force
+            and self._validate_controls(order, OrderPackageType.REPLACE) is False
+        ):
             return False
         # replace
         order.replace(new_price)
diff --git a/flumine/markets/market.py b/flumine/markets/market.py
index f966936..a99c6e7 100644
--- a/flumine/markets/market.py
+++ b/flumine/markets/market.py
@@ -29,6 +29,7 @@ class Market:
         self.flumine = flumine
         self.market_id = market_id
         self.closed = False
+        self.date_time_created = datetime.datetime.utcnow()
         self.date_time_closed = None
         self.market_book = market_book
         self.market_catalogue = market_catalogue
@@ -75,23 +76,28 @@ class Market:
         order,
         market_version: int = None,
         execute: bool = True,
+        force: bool = False,
     ) -> bool:
         with self.transaction() as t:
-            return t.place_order(order, market_version, execute)
+            return t.place_order(order, market_version, execute, force)
 
-    def cancel_order(self, order, size_reduction: float = None) -> bool:
+    def cancel_order(
+        self, order, size_reduction: float = None, force: bool = False
+    ) -> bool:
         with self.transaction() as t:
-            return t.cancel_order(order, size_reduction)
+            return t.cancel_order(order, size_reduction, force)
 
-    def update_order(self, order, new_persistence_type: str) -> bool:
+    def update_order(
+        self, order, new_persistence_type: str, force: bool = False
+    ) -> bool:
         with self.transaction() as t:
-            return t.update_order(order, new_persistence_type)
+            return t.update_order(order, new_persistence_type, force)
 
     def replace_order(
-        self, order, new_price: float, market_version: int = None
+        self, order, new_price: float, market_version: int = None, force: bool = False
     ) -> bool:
         with self.transaction() as t:
-            return t.replace_order(order, new_price, market_version)
+            return t.replace_order(order, new_price, market_version, force)
 
     @property
     def event(self) -> dict:
diff --git a/flumine/markets/middleware.py b/flumine/markets/middleware.py
index 61eff19..bafe57c 100644
--- a/flumine/markets/middleware.py
+++ b/flumine/markets/middleware.py
@@ -99,7 +99,40 @@ class SimulatedMiddleware(Middleware):
                         extra=order.info,
                     )
                 else:
+                    # TODO: "Where an SP lay bet in a win market has a maximum odds limit specified,..."
                     if (
+                        order.order_type.ORDER_TYPE == OrderTypes.MARKET_ON_CLOSE
+                    ) and order.side == "LAY":
+                        if market.market_type == "WIN":
+                            runner = [
+                                x
+                                for x in market.market_book.runners
+                                if x.selection_id == order.selection_id
+                                and x.handicap == order.handicap
+                            ][0]
+                            runner_adjustment_factor = runner.adjustment_factor
+                            # See https://github.com/liampauling/flumine/issues/454
+                            multiplier = 1 - (
+                                removal_adjustment_factor
+                                / (100 - runner_adjustment_factor)
+                            )
+                            order.order_type.liability *= multiplier
+                            logger.warning(
+                                "WIN MARKET_ON_CLOSE Order adjusted due to non runner {0}".format(
+                                    order.selection_id
+                                ),
+                                extra=order.info,
+                            )
+                        elif market.market_type in {"PLACE", "OTHER_PLACE"}:
+                            multiplier = (100 - removal_adjustment_factor) * 0.01
+                            order.order_type.liability *= multiplier
+                            logger.warning(
+                                "PLACE MARKET_ON_CLOSE Order adjusted due to non runner {0}".format(
+                                    order.selection_id
+                                ),
+                                extra=order.info,
+                            )
+                    elif (
                         removal_adjustment_factor
                         and removal_adjustment_factor >= WIN_MINIMUM_ADJUSTMENT_FACTOR
                     ):
@@ -114,7 +147,7 @@ class SimulatedMiddleware(Middleware):
                         logger.warning(
                             "Order adjusted due to non runner {0}".format(
                                 order.selection_id
-                            ).format(order.selection_id),
+                            ),
                             extra=order.info,
                         )
 
diff --git a/flumine/streams/historicalstream.py b/flumine/streams/historicalstream.py
index 04fdaed..d3f0d65 100644
--- a/flumine/streams/historicalstream.py
+++ b/flumine/streams/historicalstream.py
@@ -24,6 +24,8 @@ class FlumineMarketStream(MarketStream):
 
     def _process(self, data: list, publish_time: int) -> bool:
         for market_book in data:
+            if "id" not in market_book:
+                continue
             market_id = market_book["id"]
             full_image = market_book.get("img", False)
             market_book_cache = self._caches.get(market_id)
diff --git a/flumine/utils.py b/flumine/utils.py
index 4c14c0b..53fb9cf 100644
--- a/flumine/utils.py
+++ b/flumine/utils.py
@@ -47,6 +47,8 @@ def get_file_md(file_dir: str, value: str) -> Optional[str]:
     with open(file_dir, "r") as f:
         first_line = f.readline()
         update = json.loads(first_line)
+    if "mc" not in update or not isinstance(update["mc"], list) or not update["mc"]:
+        return None
     md = update["mc"][0].get("marketDefinition", {})
     return md.get(value)
 


Test patch:
diff --git a/tests/test_markets.py b/tests/test_markets.py
index cb3fd3a..e9bcc01 100644
--- a/tests/test_markets.py
+++ b/tests/test_markets.py
@@ -109,13 +109,16 @@ class MarketTest(unittest.TestCase):
         self.assertEqual(self.market.flumine, self.mock_flumine)
         self.assertEqual(self.market.market_id, "1.234")
         self.assertFalse(self.market.closed)
-        self.assertFalse(self.market.orders_cleared)
-        self.assertFalse(self.market.market_cleared)
+        self.assertIsNotNone(self.market.date_time_created)
         self.assertIsNone(self.market.date_time_closed)
         self.assertEqual(self.market.market_book, self.mock_market_book)
         self.assertEqual(self.market.market_catalogue, self.mock_market_catalogue)
         self.assertTrue(self.market.update_market_catalogue)
+        self.assertFalse(self.market.orders_cleared)
+        self.assertFalse(self.market.market_cleared)
         self.assertEqual(self.market.context, {"simulated": {}})
+        self.assertIsNotNone(self.market.blotter)
+        self.assertEqual(self.market._transaction_id, 0)
 
     def test_call(self):
         mock_market_book = mock.Mock()
@@ -149,29 +152,29 @@ class MarketTest(unittest.TestCase):
     def test_place_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.place_order(mock_order, 2, False))
-        mock_transaction.place_order.assert_called_with(mock_order, 2, False)
+        self.assertTrue(self.market.place_order(mock_order, 2, False, force=True))
+        mock_transaction.place_order.assert_called_with(mock_order, 2, False, True)
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_cancel_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.cancel_order(mock_order, 2.02))
-        mock_transaction.cancel_order.assert_called_with(mock_order, 2.02)
+        self.assertTrue(self.market.cancel_order(mock_order, 2.02, force=True))
+        mock_transaction.cancel_order.assert_called_with(mock_order, 2.02, True)
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_update_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.update_order(mock_order, "test"))
-        mock_transaction.update_order.assert_called_with(mock_order, "test")
+        self.assertTrue(self.market.update_order(mock_order, "test", force=True))
+        mock_transaction.update_order.assert_called_with(mock_order, "test", True)
 
     @mock.patch("flumine.markets.market.Market.transaction")
     def test_replace_order(self, mock_transaction):
         mock_transaction.return_value.__enter__.return_value = mock_transaction
         mock_order = mock.Mock()
-        self.assertTrue(self.market.replace_order(mock_order, 2, False))
-        mock_transaction.replace_order.assert_called_with(mock_order, 2, False)
+        self.assertTrue(self.market.replace_order(mock_order, 2, False, force=True))
+        mock_transaction.replace_order.assert_called_with(mock_order, 2, False, True)
 
     def test_event(self):
         mock_market_catalogue = mock.Mock()
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 5feebd9..4cc68bd 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -10,6 +10,7 @@ from flumine.markets.middleware import (
     WIN_MINIMUM_ADJUSTMENT_FACTOR,
     PLACE_MINIMUM_ADJUSTMENT_FACTOR,
 )
+from flumine.order.ordertype import MarketOnCloseOrder
 
 
 class MiddlewareTest(unittest.TestCase):
@@ -136,6 +137,43 @@ class SimulatedMiddlewareTest(unittest.TestCase):
         self.middleware._process_runner_removal(mock_market, 12345, 0, None)
         self.assertEqual(mock_order.simulated.matched, [[123, 8.6, 10]])
 
+    def test__process_runner_removal_sp_win(self):
+        order_type = MarketOnCloseOrder(liability=200)
+        mock_order = mock.Mock(
+            selection_id=1234, handicap=0, order_type=order_type, info={}, side="LAY"
+        )
+
+        mock_market_book = mock.Mock()
+        mock_market_book.runners = [
+            mock.Mock(selection_id=1234, handicap=0, adjustment_factor=20)
+        ]
+        mock_market = mock.Mock(
+            market_type="WIN", blotter=[mock_order], market_book=mock_market_book
+        )
+        self.middleware._process_runner_removal(mock_market, 12345, 0, 50)
+
+        # The liability of £200 is adjusted by the multiplier of 37.5%, which s
+        # defined in the example here: https://github.com/liampauling/flumine/issues/454
+        self.assertEqual(mock_order.order_type.liability, 75)
+
+    def test__process_runner_removal_sp_place(self):
+        order_type = MarketOnCloseOrder(liability=200)
+        mock_order = mock.Mock(
+            selection_id=1234, handicap=0, order_type=order_type, info={}, side="LAY"
+        )
+
+        mock_market_book = mock.Mock()
+        mock_market_book.runners = [
+            mock.Mock(selection_id=1234, handicap=0, adjustment_factor=20)
+        ]
+        mock_market = mock.Mock(
+            market_type="PLACE", blotter=[mock_order], market_book=mock_market_book
+        )
+        self.middleware._process_runner_removal(mock_market, 12345, 0, 50)
+
+        # The liability of £200 is reduced by the non runner's adjustment factor of 50%
+        self.assertEqual(mock_order.order_type.liability, 100)
+
     def test__process_streaming_update(self):
         mock_market_book = mock.Mock(
             streaming_update={"img": True, "rc": [{"id": 3}, {"id": 4}]},
diff --git a/tests/test_transaction.py b/tests/test_transaction.py
index f9181fd..1ba3e5c 100644
--- a/tests/test_transaction.py
+++ b/tests/test_transaction.py
@@ -110,6 +110,23 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_place = []
         self.assertFalse(self.transaction._pending_orders)
 
+    @mock.patch("flumine.execution.transaction.get_market_notes")
+    @mock.patch(
+        "flumine.execution.transaction.Transaction._validate_controls",
+        return_value=False,
+    )
+    @mock.patch("flumine.execution.transaction.events")
+    def test_force_place_order(
+        self, mock_events, mock__validate_controls, mock_get_market_notes
+    ):
+        self.transaction.market.blotter = mock.MagicMock()
+        self.transaction.market.blotter.has_trade.return_value = False
+        mock_order = mock.Mock(id="123", lookup=(1, 2, 3))
+        self.assertTrue(self.transaction.place_order(mock_order, force=True))
+        mock__validate_controls.assert_not_called()
+        self.transaction._pending_place = [(mock_order, None)]
+        self.assertTrue(self.transaction._pending_orders)
+
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -133,6 +150,18 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_cancel = []
         self.assertFalse(self.transaction._pending_orders)
 
+    @mock.patch(
+        "flumine.execution.transaction.Transaction._validate_controls",
+        return_value=False,
+    )
+    def test_force_cancel_order(self, mock__validate_controls):
+        mock_order = mock.Mock()
+        self.assertTrue(self.transaction.cancel_order(mock_order, 0.01, force=True))
+        mock_order.cancel.assert_called_with(0.01)
+        mock__validate_controls.assert_not_called()
+        self.transaction._pending_cancel = [(mock_order, None)]
+        self.assertTrue(self.transaction._pending_orders)
+
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -156,6 +185,20 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_update = []
         self.assertFalse(self.transaction._pending_orders)
 
+    @mock.patch(
+        "flumine.execution.transaction.Transaction._validate_controls",
+        return_value=False,
+    )
+    def test_force_update_order(self, mock__validate_controls):
+        mock_order = mock.Mock()
+        self.assertTrue(
+            self.transaction.update_order(mock_order, "PERSIST", force=True)
+        )
+        mock_order.update.assert_called_with("PERSIST")
+        mock__validate_controls.assert_not_called()
+        self.transaction._pending_update = [(mock_order, None)]
+        self.assertTrue(self.transaction._pending_orders)
+
     @mock.patch(
         "flumine.execution.transaction.Transaction._validate_controls",
         return_value=True,
@@ -179,6 +222,20 @@ class TransactionTest(unittest.TestCase):
         self.transaction._pending_replace = []
         self.assertFalse(self.transaction._pending_orders)
 
+    @mock.patch(
+        "flumine.execution.transaction.Transaction._validate_controls",
+        return_value=False,
+    )
+    def test_force_replace_order(self, mock__validate_controls):
+        mock_order = mock.Mock()
+        self.assertTrue(
+            self.transaction.replace_order(mock_order, 1.01, 321, force=True)
+        )
+        mock_order.replace.assert_called_with(1.01)
+        mock__validate_controls.assert_not_called()
+        self.transaction._pending_replace = [(mock_order, None)]
+        self.assertTrue(self.transaction._pending_orders)
+
     @mock.patch("flumine.execution.transaction.Transaction._create_order_package")
     def test_execute(self, mock__create_order_package):
         self.transaction._pending_orders = True