Repository: liampauling/flumine
Pull request #407 (2021-04-08 09:54:33)

`market_type` filter enabled when backtesting markets
docs update
temp logging cleanup


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index d9b60b3..ff207c5 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,18 @@
 Release History
 ---------------
 
+1.18.1 (2021-04-08)
++++++++++++++++++++
+
+**Improvements**
+
+- `market_type` filter enabled when backtesting markets
+- Remove temp logging
+
+**Bug Fixes**
+
+- Validate logging typo
+
 1.18.0 (2021-04-07)
 +++++++++++++++++++
 
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 8f90b3e..91dc77c 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -140,19 +140,9 @@ flumine.add_strategy(strategy)
 Flumine can be used to paper trade strategies live using the following code:
 
 ```python
-import betfairlightweight
-from flumine import Flumine, clients
+from flumine import clients
 
-trading = betfairlightweight.APIClient("username")
 client = clients.BetfairClient(trading, paper_trade=True)
-framework = Flumine(client=client)
-
-strategy = ExampleStrategy(
-    market_filter={"markets": ["/tmp/marketdata/1.170212754"]}
-)
-framework.add_strategy(strategy)
-
-framework.run()
 ```
 
 Market data will be recieved as per live but any orders will use Simulated execution and Simulated order polling to replicate live trading.
@@ -180,6 +170,8 @@ framework.run()
 
 Note the use of market filter to pass the file directories.
 
+### Listener kwargs
+
 Sometimes a subset of the market lifetime is required, this can be optimised by limiting the number of updates to process resulting in faster backtesting:
 
 ```python
@@ -202,8 +194,7 @@ It is also possible to process events with multiple markets such as win/place in
 
 ```python
 strategy = ExampleStrategy(
-    market_filter={"markets": [..], "event_processing": True},
-    context={"stake": 2},
+    market_filter={"markets": [..], "event_processing": True}
 )
 ```
 
@@ -213,6 +204,16 @@ The `Market` object contains a helper method for accessing other event linked ma
 place_market = market.event["PLACE"]
 ```
 
+### Market Type Filter
+
+When backtesting you can filter markets to be processed by using the `market_type` filter as per live:
+
+```python
+strategy = ExampleStrategy(
+    market_filter={"markets": [..], "market_type": ["MATCH_ODDS"]}
+)
+```
+
 ### Simulation
 
 Backtesting uses the `SimulatedExecution` execution class and tries to accurately simulate matching with the following:
@@ -224,7 +225,8 @@ Backtesting uses the `SimulatedExecution` execution class and tries to accuratel
 - Order lapse and reduction on runner removal
 - BSP
 
-Limitations:
+Limitations #192:
 
 - Queue cancellations
 - Double counting of liquidity
+- Currency fluctuations
diff --git a/flumine/__version__.py b/flumine/__version__.py
index 2126860..e591975 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.18.0"
+__version__ = "1.18.1"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/execution/baseexecution.py b/flumine/execution/baseexecution.py
index a7fc681..53ecf8b 100644
--- a/flumine/execution/baseexecution.py
+++ b/flumine/execution/baseexecution.py
@@ -39,9 +39,6 @@ class BaseExecution:
             func = self.execute_replace
         else:
             raise NotImplementedError()
-        # temp latency logging to find time
-        _latency_before_submit = order_package.elapsed_seconds
-        _t = time.time()
         self._thread_pool.submit(func, order_package, http_session)
         logger.info(
             "Thread pool submit",
@@ -49,12 +46,10 @@ class BaseExecution:
                 "trading_function": func.__name__,
                 "session": http_session,
                 "latency": round(order_package.elapsed_seconds, 4),
-                "latency_before_submit": round(_latency_before_submit, 4),
                 "order_package": order_package.info,
                 "thread_pool": {
                     "num_threads": len(self._thread_pool._threads),
                     "work_queue_size": self._thread_pool._work_queue.qsize(),
-                    "submit_latency": round(time.time() - _t, 4),
                 },
             },
         )
diff --git a/flumine/strategy/strategy.py b/flumine/strategy/strategy.py
index 3323050..4925bd4 100644
--- a/flumine/strategy/strategy.py
+++ b/flumine/strategy/strategy.py
@@ -1,5 +1,4 @@
 import logging
-import warnings
 from typing import Type, Iterator
 from betfairlightweight import filters
 from betfairlightweight.resources import MarketBook
@@ -144,7 +143,7 @@ class BaseStrategy:
         # validate context
         if runner_context.trade_count >= self.max_trade_count:
             order.violation_msg = "strategy.validate_order failed: trade_count ({0}) >= max_trade_count ({1})".format(
-                runner_context.trade_count, self.max_live_trade_count
+                runner_context.trade_count, self.max_trade_count
             )
             return False
         elif runner_context.live_trade_count >= self.max_live_trade_count:
diff --git a/flumine/streams/streams.py b/flumine/streams/streams.py
index 5767bfa..9cd92e0 100644
--- a/flumine/streams/streams.py
+++ b/flumine/streams/streams.py
@@ -22,6 +22,7 @@ class Streams:
     def __call__(self, strategy: BaseStrategy) -> None:
         if self.flumine.BACKTEST:
             markets = strategy.market_filter.get("markets")
+            market_types = strategy.market_filter.get("market_types")
             event_processing = strategy.market_filter.get("event_processing", False)
             events = strategy.market_filter.get("events")
             listener_kwargs = strategy.market_filter.get("listener_kwargs", {})
@@ -37,17 +38,19 @@ class Streams:
                 )
             elif markets:
                 for market in markets:
-                    stream = self.add_historical_stream(
-                        strategy, market, event_processing, **listener_kwargs
-                    )
-                    strategy.streams.append(stream)
-                    strategy.historic_stream_ids.append(stream.stream_id)
+                    market_type = get_file_md(market, "marketType")
+                    if market_types and market_type and market_type not in market_types:
+                        logger.warning(
+                            "Skipping market %s (%s) for strategy %s"
+                            % (market, market_type, strategy)
+                        )
+                    else:
+                        stream = self.add_historical_stream(
+                            strategy, market, event_processing, **listener_kwargs
+                        )
+                        strategy.streams.append(stream)
+                        strategy.historic_stream_ids.append(stream.stream_id)
             elif events:
-                for event in events:
-                    # todo?
-                    # for market in event dir
-                    # create historical stream with event_processing=True
-                    pass
                 raise NotImplementedError()
         else:
             stream = self.add_stream(strategy)


Test patch:
diff --git a/tests/test_streams.py b/tests/test_streams.py
index e2dd83e..aff68d8 100644
--- a/tests/test_streams.py
+++ b/tests/test_streams.py
@@ -34,8 +34,9 @@ class StreamsTest(unittest.TestCase):
         mock_add_stream.assert_called_with(mock_strategy)
         self.assertEqual(len(mock_strategy.streams), 1)
 
+    @mock.patch("flumine.streams.streams.get_file_md")
     @mock.patch("flumine.streams.streams.Streams.add_historical_stream")
-    def test_call_backtest_markets(self, mock_add_historical_stream):
+    def test_call_backtest_markets(self, mock_add_historical_stream, mock_get_file_md):
         self.mock_flumine.BACKTEST = True
         mock_strategy = mock.Mock(
             streams=[],
@@ -55,6 +56,33 @@ class StreamsTest(unittest.TestCase):
         )
         self.assertEqual(len(mock_strategy.streams), 1)
         self.assertEqual(len(mock_strategy.historic_stream_ids), 1)
+        mock_get_file_md.assert_called_with(
+            "dubs of the mad skint and british", "marketType"
+        )
+
+    @mock.patch("flumine.streams.streams.get_file_md", return_value="PLACE")
+    @mock.patch("flumine.streams.streams.Streams.add_historical_stream")
+    def test_call_backtest_markets_type(
+        self, mock_add_historical_stream, mock_get_file_md
+    ):
+        self.mock_flumine.BACKTEST = True
+        mock_strategy = mock.Mock(
+            streams=[],
+            historic_stream_ids=[],
+            market_filter={
+                "markets": ["dubs of the mad skint and british"],
+                "listener_kwargs": {"canary_yellow": True},
+                "market_types": ["WIN"],
+            },
+        )
+        self.streams(mock_strategy)
+
+        mock_add_historical_stream.assert_not_called()
+        self.assertEqual(len(mock_strategy.streams), 0)
+        self.assertEqual(len(mock_strategy.historic_stream_ids), 0)
+        mock_get_file_md.assert_called_with(
+            "dubs of the mad skint and british", "marketType"
+        )
 
     @mock.patch("flumine.streams.streams.Streams.add_historical_stream")
     def test_call_backtest_events(self, mock_add_historical_stream):