Repository: liampauling/flumine
Pull request #410 (2021-04-12 08:26:16)

MarketRecorder example clean up fix where txt file deletion is attemp…
…ted multiple times


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index ff207c5..77f1cf9 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,18 @@
 Release History
 ---------------
 
+1.18.2 (2021-04-12)
++++++++++++++++++++
+
+**Improvements**
+
+- #217 error handling improvements
+
+**Bug Fixes**
+
+- Blotter live orders mutable bugfix
+- MarketRecorder example clean up fix
+
 1.18.1 (2021-04-08)
 +++++++++++++++++++
 
@@ -23,7 +35,7 @@ Release History
 - #404 Multi market processing (concurrent event market processing)
 - strategy order placement depreciated (breaking change as per warning version 1.17.0)
 - strategy.log_validation_failures depreciated (breaking change as per warning version 1.17.7)
-- Cleared order added to order object on closure (@arch4672)
+- #185 Cleared order added to order object on closure (@arch4672)
 - gzip market catalogue data in marketrecorder.py example
 
 **Bug Fixes**
diff --git a/examples/strategies/marketrecorder.py b/examples/strategies/marketrecorder.py
index 94e380e..c22f1a1 100644
--- a/examples/strategies/marketrecorder.py
+++ b/examples/strategies/marketrecorder.py
@@ -63,6 +63,8 @@ class MarketRecorder(BaseStrategy):
                 )
             else:
                 return
+        else:
+            self._loaded_markets.append(market_id)
         logger.info("Closing market %s" % market_id)
 
         file_dir = os.path.join(self.local_dir, self.recorder_id, market_id)
@@ -94,8 +96,6 @@ class MarketRecorder(BaseStrategy):
         # clean up
         self._clean_up()
 
-        self._loaded_markets.append(market_id)
-
     def _compress_file(self, file_dir: str) -> str:
         """compresses txt file into filename.gz"""
         compressed_file_dir = "{0}.gz".format(file_dir)
@@ -114,23 +114,23 @@ class MarketRecorder(BaseStrategy):
         directory = os.path.join(self.local_dir, self.recorder_id)
         for file in os.listdir(directory):
             if file.endswith(".gz"):
-                file_stats = os.stat(os.path.join(directory, file))
+                gz_path = os.path.join(directory, file)
+                file_stats = os.stat(gz_path)
                 seconds_since = time.time() - file_stats.st_mtime
                 if seconds_since > self._market_expiration:
-                    txt_path = os.path.join(directory, file.split(".gz")[0])
-                    gz_path = os.path.join(directory, file)
-                    if self._remove_file:
-                        logger.info(
-                            "Removing: %s, age: %ss"
-                            % (txt_path, round(seconds_since, 2))
-                        )
-                        os.remove(txt_path)
                     if self._remove_gz_file:
                         logger.info(
                             "Removing: %s, age: %ss"
                             % (gz_path, round(seconds_since, 2))
                         )
                         os.remove(gz_path)
+                    txt_path = os.path.join(directory, file.split(".gz")[0])
+                    if os.path.exists(txt_path) and self._remove_file:
+                        logger.info(
+                            "Removing: %s, age: %ss"
+                            % (txt_path, round(seconds_since, 2))
+                        )
+                        os.remove(txt_path)
 
     @staticmethod
     def _create_metadata(market_definition: dict) -> dict:
diff --git a/flumine/__version__.py b/flumine/__version__.py
index e591975..0a28fcb 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.18.1"
+__version__ = "1.18.2"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/baseflumine.py b/flumine/baseflumine.py
index 7369163..0beb9e5 100644
--- a/flumine/baseflumine.py
+++ b/flumine/baseflumine.py
@@ -19,6 +19,7 @@ from .order.process import process_current_orders
 from .controls.clientcontrols import BaseControl, MaxTransactionCount
 from .controls.tradingcontrols import OrderValidation, StrategyExposure
 from .controls.loggingcontrols import LoggingControl
+from .exceptions import FlumineException
 from . import config, utils
 
 logger = logging.getLogger(__name__)
@@ -143,11 +144,13 @@ class BaseFlumine:
 
             # process middleware
             for middleware in self._market_middleware:
-                middleware(market)  # todo err handling?
+                utils.call_middleware_error_handling(middleware, market)
 
             for strategy in self.strategies:
-                if utils.call_check_market(strategy.check_market, market, market_book):
-                    utils.call_process_market_book(
+                if utils.call_strategy_error_handling(
+                    strategy.check_market, market, market_book
+                ):
+                    utils.call_strategy_error_handling(
                         strategy.process_market_book, market, market_book
                     )
 
@@ -215,17 +218,29 @@ class BaseFlumine:
             if market.closed is False:
                 for strategy in self.strategies:
                     strategy_orders = market.blotter.strategy_orders(strategy)
-                    strategy.process_orders(market, strategy_orders)
+                    utils.call_process_orders_error_handling(
+                        strategy, market, strategy_orders
+                    )
 
     def _process_custom_event(self, event: events.CustomEvent) -> None:
         try:
             event.callback(self, event)
+        except FlumineException as e:
+            logger.error(
+                "FlumineException error {0} in _process_custom_event {1}".format(
+                    e, event.callback
+                ),
+                exc_info=True,
+            )
         except Exception as e:
             logger.exception(
                 "Unknown error {0} in _process_custom_event {1}".format(
                     e, event.callback
-                )
+                ),
+                exc_info=True,
             )
+            if config.raise_errors:
+                raise
 
     def _process_close_market(self, event: events.CloseMarketEvent) -> None:
         market_book = event.event
diff --git a/flumine/markets/blotter.py b/flumine/markets/blotter.py
index 51651b9..43838ee 100644
--- a/flumine/markets/blotter.py
+++ b/flumine/markets/blotter.py
@@ -50,8 +50,8 @@ class Blotter:
         return self._strategy_selection_orders[(strategy, selection_id, handicap)]
 
     @property
-    def live_orders(self):
-        return iter(self._live_orders)
+    def live_orders(self) -> Iterable:
+        return iter(list(self._live_orders))
 
     @property
     def has_live_orders(self) -> bool:
diff --git a/flumine/utils.py b/flumine/utils.py
index 59a94f6..cd0833d 100644
--- a/flumine/utils.py
+++ b/flumine/utils.py
@@ -191,21 +191,19 @@ def wap(matched: list) -> Tuple[float, float]:
         return round(b, 2), round(a / b, 2)
 
 
-def call_check_market(
-    strategy_check_market: Callable, market, market_book: MarketBook
-) -> bool:
+def call_strategy_error_handling(
+    func: Callable, market, market_book: MarketBook
+) -> Optional[bool]:
     try:
-        return strategy_check_market(market, market_book)
+        return func(market, market_book)
     except FlumineException as e:
         logger.error(
-            "FlumineException %s in strategy_check_market %s %s"
-            % (e, strategy_check_market, market.market_id),
+            "FlumineException %s in %s (%s)" % (e, func.__name__, market.market_id),
             exc_info=True,
         )
     except Exception as e:
         logger.critical(
-            "Unknown error %s in strategy_check_market %s %s"
-            % (e, strategy_check_market, market.market_id),
+            "Unknown error %s in %s (%s)" % (e, func.__name__, market.market_id),
             exc_info=True,
         )
         if config.raise_errors:
@@ -213,21 +211,34 @@ def call_check_market(
     return False
 
 
-def call_process_market_book(
-    strategy_process_market_book: Callable, market, market_book: MarketBook
-) -> None:
+def call_middleware_error_handling(middleware, market) -> None:
     try:
-        strategy_process_market_book(market, market_book)
+        middleware(market)
     except FlumineException as e:
         logger.error(
-            "FlumineException %s in strategy_process_market_book %s %s"
-            % (e, strategy_process_market_book, market.market_id),
+            "FlumineException %s in %s (%s)" % (e, middleware, market.market_id),
             exc_info=True,
         )
     except Exception as e:
         logger.critical(
-            "Unknown error %s in strategy_process_market_book %s %s"
-            % (e, strategy_process_market_book, market.market_id),
+            "Unknown error %s in %s (%s)" % (e, middleware, market.market_id),
+            exc_info=True,
+        )
+        if config.raise_errors:
+            raise
+
+
+def call_process_orders_error_handling(strategy, market, strategy_orders: list) -> None:
+    try:
+        strategy.process_orders(market, strategy_orders)
+    except FlumineException as e:
+        logger.error(
+            "FlumineException %s in %s (%s)" % (e, strategy, market.market_id),
+            exc_info=True,
+        )
+    except Exception as e:
+        logger.critical(
+            "Unknown error %s in %s (%s)" % (e, strategy, market.market_id),
             exc_info=True,
         )
         if config.raise_errors:


Test patch:
diff --git a/flumine/backtest/backtest.py b/flumine/backtest/backtest.py
index 8bdaf0d..8b6f023 100644
--- a/flumine/backtest/backtest.py
+++ b/flumine/backtest/backtest.py
@@ -133,14 +133,16 @@ class FlumineBacktest(BaseFlumine):
 
             # process middleware
             for middleware in self._market_middleware:
-                middleware(market)  # todo err handling?
+                utils.call_middleware_error_handling(middleware, market)
 
             # process current orders
             self._process_backtest_orders(market)
 
             for strategy in self.strategies:
-                if utils.call_check_market(strategy.check_market, market, market_book):
-                    utils.call_process_market_book(
+                if utils.call_strategy_error_handling(
+                    strategy.check_market, market, market_book
+                ):
+                    utils.call_strategy_error_handling(
                         strategy.process_market_book, market, market_book
                     )
 
@@ -154,16 +156,13 @@ class FlumineBacktest(BaseFlumine):
         orders through strategies
         """
         blotter = market.blotter
-        completed_orders = []
         for order in blotter.live_orders:
             process.process_current_order(order)
             if order.trade.status == TradeStatus.COMPLETE:
-                completed_orders.append(order)
-        for order in completed_orders:
-            blotter.complete_order(order)
+                blotter.complete_order(order)
         for strategy in self.strategies:
             strategy_orders = blotter.strategy_orders(strategy)
-            strategy.process_orders(market, strategy_orders)
+            utils.call_process_orders_error_handling(strategy, market, strategy_orders)
 
     def _check_pending_packages(self, market_id: str) -> None:
         processed = []
diff --git a/tests/test_baseflumine.py b/tests/test_baseflumine.py
index 74c3440..4dd4a61 100644
--- a/tests/test_baseflumine.py
+++ b/tests/test_baseflumine.py
@@ -1,7 +1,7 @@
 import unittest
 from unittest import mock
 
-from flumine.baseflumine import BaseFlumine
+from flumine.baseflumine import BaseFlumine, FlumineException
 
 
 class BaseFlumineTest(unittest.TestCase):
@@ -190,6 +190,32 @@ class BaseFlumineTest(unittest.TestCase):
         self.base_flumine._process_custom_event(mock_event)
         mock_event.callback.assert_called_with(self.base_flumine, mock_event)
 
+    def test__process_custom_event_flu_error(self):
+        mock_market = mock.Mock()
+        self.base_flumine.markets = [mock_market]
+        mock_event = mock.Mock()
+        mock_event.callback.side_effect = FlumineException()
+        self.base_flumine._process_custom_event(mock_event)
+        mock_event.callback.assert_called_with(self.base_flumine, mock_event)
+
+    def test__process_custom_event_error(self):
+        mock_market = mock.Mock()
+        self.base_flumine.markets = [mock_market]
+        mock_event = mock.Mock()
+        mock_event.callback.side_effect = ValueError()
+        self.base_flumine._process_custom_event(mock_event)
+        mock_event.callback.assert_called_with(self.base_flumine, mock_event)
+
+    @mock.patch("flumine.baseflumine.config")
+    def test__process_custom_event_error_raise(self, mock_config):
+        mock_config.raise_errors = True
+        mock_market = mock.Mock()
+        self.base_flumine.markets = [mock_market]
+        mock_event = mock.Mock()
+        mock_event.callback.side_effect = ValueError()
+        with self.assertRaises(ValueError):
+            self.base_flumine._process_custom_event(mock_event)
+
     @mock.patch("flumine.baseflumine.BaseFlumine.info")
     @mock.patch("flumine.baseflumine.BaseFlumine.log_control")
     def test__process_close_market(self, mock_log_control, mock_info):
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 28f6ca3..f1578dc 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -187,69 +187,105 @@ class UtilsTest(unittest.TestCase):
         self.assertEqual(utils.wap([]), (0, 0))
         self.assertEqual(utils.wap([(123456789, 1.5, 0)]), (0, 0))
 
-    def test_call_check_market(self):
+    def test_call_strategy_error_handling(self):
         mock_strategy_check = mock.Mock()
         mock_market = mock.Mock()
         mock_market_book = mock.Mock()
-        utils.call_check_market(mock_strategy_check, mock_market, mock_market_book)
+        utils.call_strategy_error_handling(
+            mock_strategy_check, mock_market, mock_market_book
+        )
         mock_strategy_check.assert_called_with(mock_market, mock_market_book)
 
-    def test_call_check_market_flumine_error(self):
-        mock_strategy_check = mock.Mock(side_effect=FlumineException)
+    def test_call_strategy_error_handling_flumine_error(self):
+        mock_strategy_check = mock.MagicMock(side_effect=FlumineException)
+        mock_strategy_check.__name__ = "mock_strategy_check"
         mock_market = mock.Mock()
         mock_market_book = mock.Mock()
         self.assertFalse(
-            utils.call_check_market(mock_strategy_check, mock_market, mock_market_book)
+            utils.call_strategy_error_handling(
+                mock_strategy_check, mock_market, mock_market_book
+            )
         )
         mock_strategy_check.assert_called_with(mock_market, mock_market_book)
 
-    def test_call_check_market_error(self):
-        mock_strategy_check = mock.Mock(side_effect=ValueError)
+    def test_call_strategy_error_handling_error(self):
+        mock_strategy_check = mock.MagicMock(side_effect=ValueError)
+        mock_strategy_check.__name__ = "mock_strategy_check"
         mock_market = mock.Mock()
         mock_market_book = mock.Mock()
         self.assertFalse(
-            utils.call_check_market(mock_strategy_check, mock_market, mock_market_book)
+            utils.call_strategy_error_handling(
+                mock_strategy_check, mock_market, mock_market_book
+            )
         )
         mock_strategy_check.assert_called_with(mock_market, mock_market_book)
 
     @mock.patch("flumine.utils.config")
-    def test_call_check_market_raise(self, mock_config):
+    def test_call_strategy_error_handling_raise(self, mock_config):
         mock_config.raise_errors = True
-        mock_strategy_check = mock.Mock(side_effect=ValueError)
+        mock_strategy_check = mock.MagicMock(side_effect=ValueError)
+        mock_strategy_check.__name__ = "mock_strategy_check"
         mock_market = mock.Mock()
         mock_market_book = mock.Mock()
         with self.assertRaises(ValueError):
-            utils.call_check_market(mock_strategy_check, mock_market, mock_market_book)
+            utils.call_strategy_error_handling(
+                mock_strategy_check, mock_market, mock_market_book
+            )
+
+    def test_call_middleware_error_handling(self):
+        mock_middlware = mock.Mock()
+        mock_market = mock.Mock()
+        utils.call_middleware_error_handling(mock_middlware, mock_market)
+        mock_middlware.assert_called_with(mock_market)
+
+    def test_call_middleware_error_handling_flumine_error(self):
+        mock_middlware = mock.MagicMock(side_effect=FlumineException)
+        mock_market = mock.Mock()
+        utils.call_middleware_error_handling(mock_middlware, mock_market)
+        mock_middlware.assert_called_with(mock_market)
 
-    def test_call_process_market_book(self):
+    def test_call_middleware_error_handling_error(self):
+        mock_middlware = mock.MagicMock(side_effect=ValueError)
+        mock_market = mock.Mock()
+        utils.call_middleware_error_handling(mock_middlware, mock_market)
+        mock_middlware.assert_called_with(mock_market)
+
+    @mock.patch("flumine.utils.config")
+    def test_call_middleware_error_handling_raise(self, mock_config):
+        mock_config.raise_errors = True
+        mock_middlware = mock.MagicMock(side_effect=ValueError)
+        mock_market = mock.Mock()
+        with self.assertRaises(ValueError):
+            utils.call_middleware_error_handling(mock_middlware, mock_market)
+
+    def test_call_process_orders_error_handling(self):
         mock_strategy = mock.Mock()
         mock_market = mock.Mock()
-        mock_market_book = mock.Mock()
-        utils.call_process_market_book(mock_strategy, mock_market, mock_market_book)
-        mock_strategy.assert_called_with(mock_market, mock_market_book)
+        utils.call_process_orders_error_handling(mock_strategy, mock_market, [])
+        mock_strategy.process_orders.assert_called_with(mock_market, [])
 
-    def test_call_process_market_book_flumine_error(self):
-        mock_strategy = mock.Mock(side_effect=FlumineException)
+    def test_call_process_orders_error_handling_flumine_error(self):
+        mock_strategy = mock.MagicMock()
+        mock_strategy.process_orders.side_effect = FlumineException
         mock_market = mock.Mock()
-        mock_market_book = mock.Mock()
-        utils.call_process_market_book(mock_strategy, mock_market, mock_market_book)
-        mock_strategy.assert_called_with(mock_market, mock_market_book)
+        utils.call_process_orders_error_handling(mock_strategy, mock_market, [])
+        mock_strategy.process_orders.assert_called_with(mock_market, [])
 
-    def test_call_process_market_book_error(self):
-        mock_strategy = mock.Mock(side_effect=ZeroDivisionError)
+    def test_call_process_orders_error_handling_error(self):
+        mock_strategy = mock.MagicMock()
+        mock_strategy.process_orders.side_effect = ValueError
         mock_market = mock.Mock()
-        mock_market_book = mock.Mock()
-        utils.call_process_market_book(mock_strategy, mock_market, mock_market_book)
-        mock_strategy.assert_called_with(mock_market, mock_market_book)
+        utils.call_process_orders_error_handling(mock_strategy, mock_market, [])
+        mock_strategy.process_orders.assert_called_with(mock_market, [])
 
     @mock.patch("flumine.utils.config")
-    def test_call_process_market_book_raise(self, mock_config):
+    def test_call_process_orders_error_handling_raise(self, mock_config):
         mock_config.raise_errors = True
-        mock_strategy = mock.Mock(side_effect=ZeroDivisionError)
+        mock_strategy = mock.MagicMock()
+        mock_strategy.process_orders.side_effect = ValueError
         mock_market = mock.Mock()
-        mock_market_book = mock.Mock()
-        with self.assertRaises(ZeroDivisionError):
-            utils.call_process_market_book(mock_strategy, mock_market, mock_market_book)
+        with self.assertRaises(ValueError):
+            utils.call_process_orders_error_handling(mock_strategy, mock_market, [])
 
     def test_get_runner_book(self):
         mock_market_book = mock.Mock()