Repository: liampauling/flumine
Pull request #164 (2020-05-18 12:43:09)

Task/middleware refactor


Patch:
diff --git a/flumine/baseflumine.py b/flumine/baseflumine.py
index f64b26b..4ee16a0 100644
--- a/flumine/baseflumine.py
+++ b/flumine/baseflumine.py
@@ -41,6 +41,7 @@ class BaseFlumine:
 
         # all markets
         self.markets = Markets()
+        self._market_middleware = []
 
         # all strategies
         self.strategies = Strategies()
@@ -109,6 +110,10 @@ class BaseFlumine:
             if not market:
                 market = self._add_live_market(market_id, market_book)
 
+            # process middleware
+            for middleware in self._market_middleware:
+                middleware(market)  # todo err handling?
+
             # process market
             market(market_book)
 
diff --git a/flumine/markets/market.py b/flumine/markets/market.py
index f62c884..7eed373 100644
--- a/flumine/markets/market.py
+++ b/flumine/markets/market.py
@@ -3,8 +3,6 @@ import logging
 from betfairlightweight.resources.bettingresources import MarketBook, MarketCatalogue
 
 from .blotter import Blotter
-from .middleware import SimulatedMiddleware
-from .. import config
 from ..order.order import OrderStatus
 from ..events import events
 
@@ -24,22 +22,15 @@ class Market:
         self.closed = False
         self.market_book = market_book
         self.market_catalogue = market_catalogue
-        self._simulated_middleware = SimulatedMiddleware()
-        self._middleware = []
+        self.context = {"simulated": {}}  # data store (raceCard / scores etc)
         self.blotter = Blotter(self)
 
-        if config.simulated:
-            self._middleware.append(self._simulated_middleware)
-
     def __call__(self, market_book: MarketBook):
         self.market_book = market_book
-        # process middleware
-        for middleware in self._middleware:
-            middleware(self.market_catalogue, self.market_book)  # todo error handling?
         # process simulated orders
         for order in self.blotter:
             if order.simulated and order.status == OrderStatus.EXECUTABLE:
-                runner_analytics = self._simulated_middleware.runners.get(
+                runner_analytics = self.context["simulated"].get(
                     (order.selection_id, order.handicap)
                 )
                 order.simulated(self.market_book, runner_analytics)
diff --git a/flumine/markets/middleware.py b/flumine/markets/middleware.py
index 88cf218..e225562 100644
--- a/flumine/markets/middleware.py
+++ b/flumine/markets/middleware.py
@@ -1,21 +1,12 @@
-import datetime
 import logging
-from betfairlightweight.resources.bettingresources import (
-    MarketBook,
-    MarketCatalogue,
-    RunnerBook,
-)
-
-from .blotter import Blotter
-from .. import config
+from collections import defaultdict
+from betfairlightweight.resources.bettingresources import MarketBook, RunnerBook
 
 logger = logging.getLogger(__name__)
 
 
 class Middleware:
-    def __call__(
-        self, market_catalogue: MarketCatalogue, market_book: MarketBook
-    ) -> None:
+    def __call__(self, market, market_book: MarketBook) -> None:
         raise NotImplementedError
 
 
@@ -28,20 +19,21 @@ class SimulatedMiddleware:
     """
 
     def __init__(self):
-        self.runners = {}  # {(selectionId, handicap): RunnerAnalytics}
+        # {marketId: {(selectionId, handicap): RunnerAnalytics}}
+        self.markets = defaultdict(dict)
 
-    def __call__(
-        self, market_catalogue: MarketCatalogue, market_book: MarketBook
-    ) -> None:
+    def __call__(self, market, market_book: MarketBook) -> None:
+        market_analytics = self.markets[market.market_id]
         for runner in market_book.runners:
             if runner.status == "ACTIVE":
-                self._process_runner(runner)
+                self._process_runner(market_analytics, runner)
+        market.context["simulated"] = market_analytics
 
-    def _process_runner(self, runner: RunnerBook) -> None:
+    def _process_runner(self, market_analytics: dict, runner: RunnerBook) -> None:
         try:
-            runner_analytics = self.runners[(runner.selection_id, runner.handicap)]
+            runner_analytics = market_analytics[(runner.selection_id, runner.handicap)]
         except KeyError:
-            runner_analytics = self.runners[
+            runner_analytics = market_analytics[
                 (runner.selection_id, runner.handicap)
             ] = RunnerAnalytics(runner)
         runner_analytics(runner)
@@ -56,6 +48,7 @@ class RunnerAnalytics:
     def __call__(self, runner: RunnerBook):
         self.traded = self._calculate_traded(runner)
         self._traded_volume = runner.ex.traded_volume
+        self._runner = runner
 
     def _calculate_traded(self, runner: RunnerBook) -> dict:
         if self._traded_volume == {}:


Test patch:
diff --git a/flumine/backtest/backtest.py b/flumine/backtest/backtest.py
index 0322198..5360e78 100644
--- a/flumine/backtest/backtest.py
+++ b/flumine/backtest/backtest.py
@@ -10,6 +10,7 @@ from .utils import PendingPackages
 from ..markets.market import Market
 from ..order.orderpackage import OrderPackageType
 from ..order import process
+from ..markets.middleware import SimulatedMiddleware
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +27,7 @@ class FlumineBacktest(BaseFlumine):
     def __init__(self, client):
         super(FlumineBacktest, self).__init__(client)
         self._pending_packages = PendingPackages()
+        self._market_middleware.append(SimulatedMiddleware())
 
     def run(self) -> None:
         if self.client.EXCHANGE != ExchangeType.SIMULATED:
@@ -60,6 +62,7 @@ class FlumineBacktest(BaseFlumine):
             self._unpatch_datetime()
 
     def _process_market_books(self, event: events.MarketBookEvent) -> None:
+        # todo DRY!
         for market_book in event.event:
             market_id = market_book.market_id
             config.current_time = market_book.publish_time
@@ -76,6 +79,10 @@ class FlumineBacktest(BaseFlumine):
                 market = self._add_live_market(market_id, market_book)
                 self.log_control(events.MarketEvent(market))
 
+            # process middleware
+            for middleware in self._market_middleware:
+                middleware(market, market_book)  # todo err handling?
+
             # process market
             market(market_book)
 
diff --git a/tests/test_baseflumine.py b/tests/test_baseflumine.py
index 2e184e3..3242a2d 100644
--- a/tests/test_baseflumine.py
+++ b/tests/test_baseflumine.py
@@ -13,6 +13,7 @@ class BaseFlumineTest(unittest.TestCase):
         self.assertFalse(self.base_flumine.BACKTEST)
         self.assertEqual(self.base_flumine.client, self.mock_client)
         self.assertFalse(self.base_flumine._running)
+        self.assertEqual(self.base_flumine._market_middleware, [])
         self.assertEqual(self.base_flumine._logging_controls, [])
         self.assertEqual(len(self.base_flumine._trading_controls), 2)
         self.assertEqual(self.base_flumine._workers, [])
diff --git a/tests/test_markets.py b/tests/test_markets.py
index c416d1a..5cda272 100644
--- a/tests/test_markets.py
+++ b/tests/test_markets.py
@@ -103,13 +103,14 @@ class MarketTest(unittest.TestCase):
         self.assertFalse(self.market.closed)
         self.assertEqual(self.market.market_book, self.mock_market_book)
         self.assertEqual(self.market.market_catalogue, self.mock_market_catalogue)
-        self.assertEqual(self.market._middleware, [])
+        self.assertEqual(self.market.context, {"simulated": {}})
 
     def test_call(self):
-        mock_middleware = mock.Mock()
-        self.market._middleware = [mock_middleware]
         mock_order = mock.Mock()
         mock_order.status = OrderStatus.EXECUTABLE
+        self.market.context["simulated"] = {
+            (mock_order.selection_id, mock_order.handicap): "test"
+        }
         mock_order_two = mock.Mock()
         mock_order_two.status = OrderStatus.PENDING
         mock_order_three = mock.Mock()
@@ -119,8 +120,7 @@ class MarketTest(unittest.TestCase):
         mock_market_book = mock.Mock()
         self.market(mock_market_book)
         self.assertEqual(self.market.market_book, mock_market_book)
-        mock_middleware.assert_called_with(self.mock_market_catalogue, mock_market_book)
-        mock_order.simulated.assert_called_with(mock_market_book, None)
+        mock_order.simulated.assert_called_with(mock_market_book, "test")
         mock_order_two.simulated.assert_not_called()
 
     def test_open_market(self):
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 9388ad9..ab95438 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -18,25 +18,27 @@ class SimulatedMiddlewareTest(unittest.TestCase):
         self.middleware = SimulatedMiddleware()
 
     def test_init(self):
-        self.assertEqual(self.middleware.runners, {})
+        self.assertEqual(self.middleware.markets, {})
 
     @mock.patch("flumine.markets.middleware.SimulatedMiddleware._process_runner")
     def test_call(self, mock__process_runner):
-        mock_market_catalogue = mock.Mock()
+        mock_market = mock.Mock(context={})
         mock_market_book = mock.Mock()
         mock_runner = mock.Mock()
         mock_runner.status = "ACTIVE"
         mock_market_book.runners = [mock_runner]
-        self.middleware(mock_market_catalogue, mock_market_book)
-        mock__process_runner.assert_called_with(mock_runner)
+        self.middleware(mock_market, mock_market_book)
+        mock__process_runner.assert_called_with({}, mock_runner)
+        self.assertEqual(mock_market.context, {"simulated": {}})
 
     @mock.patch("flumine.markets.middleware.RunnerAnalytics")
     def test__process_runner(self, mock_runner_analytics):
+        market_analytics = {}
         mock_runner = mock.Mock()
-        self.middleware._process_runner(mock_runner)
-        self.assertEqual(len(self.middleware.runners), 1)
-        self.middleware._process_runner(mock_runner)
-        self.assertEqual(len(self.middleware.runners), 1)
+        self.middleware._process_runner(market_analytics, mock_runner)
+        self.assertEqual(len(market_analytics), 1)
+        self.middleware._process_runner(market_analytics, mock_runner)
+        self.assertEqual(len(market_analytics), 1)
         mock_runner_analytics.assert_called_with(mock_runner)
 
 
@@ -59,6 +61,7 @@ class RunnerAnalyticsTest(unittest.TestCase):
             self.runner_analytics._traded_volume, mock_runner.ex.traded_volume
         )
         self.assertEqual(self.runner_analytics.traded, mock__calculate_traded())
+        self.assertEqual(self.runner_analytics._runner, mock_runner)
 
     def test__calculate_traded_dict_empty(self):
         mock_runner = mock.Mock()