Repository: liampauling/flumine
Pull request #367 (2021-01-13 16:42:40)

#192 correctly lapse limit orders when market version is bumped durin…
…g a 'material change' as per docs:

 `However, to prevent falsely blocking bets we keep track of the last
  material change (which we define as one performed under suspension**)
  and will only accept bets placed with that version or later.`


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 74bec06..da0f330 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,17 +3,6 @@
 Release History
 ---------------
 
-1.15.4 (2021-01-18)
-+++++++++++++++++++
-
-**Improvements**
-
-- Restrict catalogue requests to market version update
-
-**Bug Fixes**
-
-- #192 correctly lapse limit orders
-
 1.15.3 (2021-01-11)
 +++++++++++++++++++
 
diff --git a/flumine/__version__.py b/flumine/__version__.py
index 469f00e..ee306f4 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.15.4"
+__version__ = "1.15.3"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/baseflumine.py b/flumine/baseflumine.py
index 09cbb4f..3c5be77 100644
--- a/flumine/baseflumine.py
+++ b/flumine/baseflumine.py
@@ -228,7 +228,6 @@ class BaseFlumine:
                     )
                 else:
                     market.market_catalogue = market_catalogue
-                market.update_market_catalogue = False
 
     def _process_current_orders(self, event: events.CurrentOrdersEvent) -> None:
         process_current_orders(self.markets, self.strategies, event)  # update state
diff --git a/flumine/markets/market.py b/flumine/markets/market.py
index 036c78d..5675019 100644
--- a/flumine/markets/market.py
+++ b/flumine/markets/market.py
@@ -24,13 +24,10 @@ class Market:
         self.date_time_closed = None
         self.market_book = market_book
         self.market_catalogue = market_catalogue
-        self.update_market_catalogue = True
         self.context = {"simulated": {}}  # data store (raceCard / scores etc)
         self.blotter = Blotter(market_id)
 
     def __call__(self, market_book: MarketBook):
-        if market_book.version != self.market_book.version:
-            self.update_market_catalogue = True
         self.market_book = market_book
 
     def open_market(self) -> None:
diff --git a/flumine/markets/middleware.py b/flumine/markets/middleware.py
index a73c780..7d76c1e 100644
--- a/flumine/markets/middleware.py
+++ b/flumine/markets/middleware.py
@@ -99,10 +99,6 @@ class SimulatedMiddleware(Middleware):
                         extra=order.info,
                     )
                 else:
-                    if order.status == OrderStatus.EXECUTABLE:
-                        # todo cancel if not PERSIST
-                        # todo does a market version bump occur if withdrawal is below the limit?
-                        pass
                     if (
                         removal_adjustment_factor
                         and removal_adjustment_factor >= WIN_MINIMUM_ADJUSTMENT_FACTOR
diff --git a/flumine/worker.py b/flumine/worker.py
index ae591c6..892d786 100644
--- a/flumine/worker.py
+++ b/flumine/worker.py
@@ -116,12 +116,8 @@ def keep_alive(context: dict, flumine) -> None:
 
 def poll_market_catalogue(context: dict, flumine) -> None:
     client = flumine.client
-    markets = [
-        m.market_id
-        for m in list(flumine.markets.markets.values())
-        if m.update_market_catalogue
-    ]
-    for market_ids in chunks(markets, 25):
+    live_markets = list(flumine.markets.markets.keys())
+    for market_ids in chunks(live_markets, 25):
         try:
             market_catalogues = client.betting_client.betting.list_market_catalogue(
                 filter=filters.market_filter(market_ids=market_ids),


Test patch:
diff --git a/flumine/backtest/simulated.py b/flumine/backtest/simulated.py
index 2c4af90..50a7814 100644
--- a/flumine/backtest/simulated.py
+++ b/flumine/backtest/simulated.py
@@ -29,12 +29,12 @@ class Simulated:
         self.size_cancelled = 0.0
         self.size_lapsed = 0.0
         self.size_voided = 0.0
+        self.market_version = None  # version at place so we can lapse if needed
         self._piq = 0.0
         self._bsp_reconciled = False
 
-    def __call__(self, market_book: MarketBook, runner_analytics):
+    def __call__(self, market_book: MarketBook, runner_analytics) -> None:
         # simulates order matching
-        # todo handle limit lapsing
         runner = self._get_runner(market_book)
         if (
             self._bsp_reconciled is False
@@ -46,6 +46,13 @@ class Simulated:
         elif (
             self.order.order_type.ORDER_TYPE == OrderTypes.LIMIT and self.size_remaining
         ):
+            if market_book.version != self.market_version:
+                self.market_version = market_book.version  # update for next time
+                if market_book.status == "SUSPENDED":  # Material change
+                    if self.order.order_type.persistence_type == "LAPSE":
+                        self.size_lapsed += self.size_remaining
+                        return
+
             # todo estimated piq cancellations
             self._process_traded(
                 market_book.publish_time_epoch, runner_analytics.traded
@@ -56,7 +63,9 @@ class Simulated:
     ) -> SimulatedPlaceResponse:
         # simulates placeOrder request->matching->response
         # todo instruction/fillkill/timeInForce etc
-        # todo check marketVersion
+        # todo check marketVersion or reject entire package?
+
+        self.market_version = market_book.version
         if self.order.order_type.ORDER_TYPE == OrderTypes.LIMIT:
             runner = self._get_runner(market_book)
 
diff --git a/tests/test_baseflumine.py b/tests/test_baseflumine.py
index 55c922b..9d92fa9 100644
--- a/tests/test_baseflumine.py
+++ b/tests/test_baseflumine.py
@@ -208,7 +208,7 @@ class BaseFlumineTest(unittest.TestCase):
 
     @mock.patch("flumine.baseflumine.events")
     @mock.patch("flumine.baseflumine.BaseFlumine.log_control")
-    def test__process_market_catalogues(self, mock_log_control, mock_events):
+    def test__process_market_catalogue(self, mock_log_control, mock_events):
         mock_market = mock.Mock()
         mock_market.market_catalogue = None
         mock_markets = mock.Mock()
@@ -221,7 +221,6 @@ class BaseFlumineTest(unittest.TestCase):
         self.base_flumine._process_market_catalogues(mock_event)
         self.assertEqual(mock_market.market_catalogue, mock_market_catalogue)
         mock_log_control.assert_called_with(mock_events.MarketEvent(mock_market))
-        self.assertFalse(mock_market.update_market_catalogue)
 
     def test__process_current_orders(self):
         mock_event = mock.Mock()
diff --git a/tests/test_markets.py b/tests/test_markets.py
index d15211f..a55f85e 100644
--- a/tests/test_markets.py
+++ b/tests/test_markets.py
@@ -112,14 +112,12 @@ class MarketTest(unittest.TestCase):
         self.assertIsNone(self.market.date_time_closed)
         self.assertEqual(self.market.market_book, self.mock_market_book)
         self.assertEqual(self.market.market_catalogue, self.mock_market_catalogue)
-        self.assertTrue(self.market.update_market_catalogue)
         self.assertEqual(self.market.context, {"simulated": {}})
 
     def test_call(self):
         mock_market_book = mock.Mock()
         self.market(mock_market_book)
         self.assertEqual(self.market.market_book, mock_market_book)
-        self.assertTrue(self.market.update_market_catalogue)
 
     def test_open_market(self):
         self.market.open_market()
diff --git a/tests/test_simulated.py b/tests/test_simulated.py
index 44b7614..8e457b6 100644
--- a/tests/test_simulated.py
+++ b/tests/test_simulated.py
@@ -28,13 +28,32 @@ class SimulatedTest(unittest.TestCase):
         self.assertEqual(self.simulated.size_lapsed, 0)
         self.assertEqual(self.simulated.size_voided, 0)
         self.assertEqual(self.simulated._piq, 0)
+        self.assertIsNone(self.simulated.market_version)
         self.assertFalse(self.simulated._bsp_reconciled)
 
     @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
     @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
     @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
     @mock.patch("flumine.backtest.simulated.Simulated._process_sp")
-    def test_call(self, mock__process_sp, mock__process_traded, _, mock__get_runner):
+    def test_call_process_traded(
+        self, mock__process_sp, mock__process_traded, _, mock__get_runner
+    ):
+        mock_market_book = mock.Mock()
+        mock_market_book.bsp_reconciled = False
+        mock_runner_analytics = mock.Mock()
+        self.simulated(mock_market_book, mock_runner_analytics)
+        mock__process_sp.assert_not_called()
+        mock__process_traded.assert_called_with(
+            mock_market_book.publish_time_epoch, mock_runner_analytics.traded
+        )
+
+    @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
+    @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
+    @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
+    @mock.patch("flumine.backtest.simulated.Simulated._process_sp")
+    def test_call_process_sp(
+        self, mock__process_sp, mock__process_traded, _, mock__get_runner
+    ):
         mock_market_book = mock.Mock()
         mock_market_book.bsp_reconciled = True
         mock_runner_analytics = mock.Mock()
@@ -44,6 +63,25 @@ class SimulatedTest(unittest.TestCase):
         )
         mock__process_traded.assert_not_called()
 
+    @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
+    @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
+    @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
+    @mock.patch("flumine.backtest.simulated.Simulated._process_sp")
+    def test_call_market_version(
+        self, mock__process_sp, mock__process_traded, _, mock__get_runner
+    ):
+        self.simulated.market_version = 123
+        self.simulated.order.order_type.persistence_type = "LAPSE"
+        mock_market_book = mock.Mock(
+            bsp_reconciled=False, version=124, status="SUSPENDED"
+        )
+        mock_runner_analytics = mock.Mock()
+        self.simulated(mock_market_book, mock_runner_analytics)
+        self.assertEqual(self.simulated.size_lapsed, 2.0)
+        self.assertEqual(self.simulated.size_remaining, 0.0)
+        mock__process_sp.assert_not_called()
+        mock__process_traded.assert_not_called()
+
     @mock.patch("flumine.backtest.simulated.Simulated._get_runner")
     @mock.patch("flumine.backtest.simulated.Simulated.take_sp", return_value=True)
     @mock.patch("flumine.backtest.simulated.Simulated._process_traded")
@@ -89,6 +127,7 @@ class SimulatedTest(unittest.TestCase):
         mock_runner.ex.available_to_lay = [{"price": 13, "size": 120}]
         mock__get_runner.return_value = mock_runner
         resp = self.simulated.place(mock_client, mock_market_book, {}, 1)
+        self.assertEqual(self.simulated.market_version, mock_market_book.version)
         self.assertEqual(resp.average_price_matched, 12)
         self.assertEqual(resp.size_matched, 2)
         self.assertEqual(
diff --git a/tests/test_worker.py b/tests/test_worker.py
index 6fe26d7..ed26e10 100644
--- a/tests/test_worker.py
+++ b/tests/test_worker.py
@@ -91,16 +91,11 @@ class WorkersTest(unittest.TestCase):
     def test_poll_market_catalogue(self, mock_events):
         mock_context = mock.Mock()
         mock_flumine = mock.Mock()
-        mock_market_one = mock.Mock(market_id="1.234", update_market_catalogue=True)
-        mock_market_two = mock.Mock(market_id="5.678", update_market_catalogue=False)
-        mock_flumine.markets.markets = {
-            "1.234": mock_market_one,
-            "5.678": mock_market_two,
-        }
+        mock_flumine.markets.markets = {"1.234": None, "5.678": None}
 
         worker.poll_market_catalogue(mock_context, mock_flumine)
         mock_flumine.client.betting_client.betting.list_market_catalogue.assert_called_with(
-            filter={"marketIds": ["1.234"]},
+            filter={"marketIds": list(mock_flumine.markets.markets.keys())},
             market_projection=[
                 "COMPETITION",
                 "EVENT",