Repository: liampauling/flumine
Pull request #387 (2021-03-05 15:15:37)

Task/strategy context imp


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 19055e8..a1dba32 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,21 @@
 Release History
 ---------------
 
+1.17.7 (2021-03-05)
++++++++++++++++++++
+
+**Improvements**
+
+- strategy.log_validation_failures marked for depreciation and logging pushed up to trading control
+- strategy.multi_order_trades var added to allow multiple orders to be placed under a single trade
+- RunnerContext trades made public
+- Docs cleanup and unused trade vars removed
+- config.max_workers renamed to max_execution_workers (*breaking change)
+
+**Bug Fixes**
+
+- Prevent double counting of trades if place called more than once
+
 1.17.6 (2021-03-05)
 +++++++++++++++++++
 
diff --git a/docs/advanced.md b/docs/advanced.md
index 30d414a..f012b55 100644
--- a/docs/advanced.md
+++ b/docs/advanced.md
@@ -40,10 +40,11 @@ The Flumine class can be adapted by overriding the following functions:
 - `client` Strategy client, half implemented when flumine will be migrated to multi clients
 - `max_trade_count` Max total number of trades per runner
 - `max_live_trade_count` Max live (with executable orders) trades per runner
+- `multi_order_trades` Allow multiple live orders per trade
 
 ### Functions
 
-The following functions can be overridden dependant on the strategy:
+The following functions can be overridden dependent on the strategy:
 
 `add()` Function called when strategy is added to framework
 
@@ -90,12 +91,12 @@ Before placing an order flumine will check the client and trading controls, this
 
 ### Client Controls
 
-- `MaxOrderCount`: Checks order count is not over betfair transaction limit (5000 per hour) 
+- `MaxTransactionCount`: Checks transaction count is not over betfair transaction limit (5000 per hour) 
 
 ### Trading Controls
 
 - `OrderValidation`: Checks order is valid (size/odds)
-- `StrategyExposure`: Checks order does not go over `strategy.max_order_exposure` and `strategy.max_selection_exposure`
+- `StrategyExposure`: Checks order does not invalidate `strategy.validate_order`, `strategy.max_order_exposure` or `strategy.max_selection_exposure`
 
 ## Logging Controls
 
@@ -116,8 +117,8 @@ framework.add_logging_control(control)
 
 By default flumine adds the following workers:
  
-- `keep_alive`: runs every 1200s to make sure the client is either logged in or kept alive
-- `poll_account_balance`: runs every 60s to poll account balance endpoint
+- `keep_alive`: runs every 1200s (or session_timeout/2) to make sure the client is either logged in or kept alive
+- `poll_account_balance`: runs every 120s to poll account balance endpoint
 - `poll_market_catalogue`: runs every 60s to poll listMarketCatalogue endpoint
 - `poll_cleared_orders`: runs when closed market is added to `flumine.cleared_market_queue`
 
@@ -190,6 +191,6 @@ Used for backtesting
 
 Raises errors on strategy functions, see [Error Handling](/advanced/#error-handling)
 
-### max_workers
+### max_execution_workers
 
 Max number of workers in execution thread pool
diff --git a/docs/trades.md b/docs/trades.md
index 8daf1a8..fe79ab8 100644
--- a/docs/trades.md
+++ b/docs/trades.md
@@ -31,9 +31,6 @@ trade.orders  # [<BetfairOrder>]
 - `handicap` Runner handicap
 - `strategy` Strategy object
 - `notes` Trade notes, used to store market / trigger info for later analysis
-- `fill_kill` Not implemented
-- `offset` Not implemented
-- `green` Not implemented
 - `place_reset_seconds` Seconds to wait since `runner_context.reset` before allowing another order
 - `reset_seconds` Seconds to wait since `runner_context.place` before allowing another order
 
diff --git a/flumine/__version__.py b/flumine/__version__.py
index 9cef1e8..f1ca466 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.17.6"
+__version__ = "1.17.7"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/baseflumine.py b/flumine/baseflumine.py
index d4daa63..cad54ba 100644
--- a/flumine/baseflumine.py
+++ b/flumine/baseflumine.py
@@ -57,8 +57,10 @@ class BaseFlumine:
         self.streams.add_client(client)
 
         # order execution class
-        self.simulated_execution = SimulatedExecution(self, config.max_workers)
-        self.betfair_execution = BetfairExecution(self, config.max_workers)
+        self.simulated_execution = SimulatedExecution(
+            self, config.max_execution_workers
+        )
+        self.betfair_execution = BetfairExecution(self, config.max_execution_workers)
 
         # logging controls (e.g. database logger)
         self._logging_controls = []
diff --git a/flumine/config.py b/flumine/config.py
index 6da8a8a..dbc7a91 100644
--- a/flumine/config.py
+++ b/flumine/config.py
@@ -15,4 +15,4 @@ current_time = None  # used for backtesting
 
 raise_errors = False  # used for call_check_market / call_process_market_book
 
-max_workers = 32  # max number of workers in execution thread pool
+max_execution_workers = 32  # max number of workers in execution thread pool
diff --git a/flumine/controls/tradingcontrols.py b/flumine/controls/tradingcontrols.py
index dcbefe9..b32ebd3 100644
--- a/flumine/controls/tradingcontrols.py
+++ b/flumine/controls/tradingcontrols.py
@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 class OrderValidation(BaseControl):
 
     """
-    Checks order price and size is valid for
+    Validates order price and size is valid for
     exchange.
     """
 
@@ -99,8 +99,13 @@ class OrderValidation(BaseControl):
 class StrategyExposure(BaseControl):
 
     """
-    Checks exposure does not exceed strategy
-    max exposure.
+    Validates:
+        - `strategy.validate_order` function
+        - `strategy.max_order_exposure` is not violated if order is executed
+        - `strategy.max_selection_exposure` is not violated if order is executed
+
+    Exposure calculation includes pending,
+    executable and execution complete orders.
     """
 
     NAME = "STRATEGY_EXPOSURE"
@@ -110,7 +115,7 @@ class StrategyExposure(BaseControl):
             # strategy.validate_order
             runner_context = order.trade.strategy.get_runner_context(*order.lookup)
             if order.trade.strategy.validate_order(runner_context, order) is False:
-                return self._on_error(order, "strategy.validate_order failure")
+                return self._on_error(order, order.violation_msg)
 
         if package_type in (
             OrderPackageType.PLACE,
diff --git a/flumine/order/order.py b/flumine/order/order.py
index 972e202..30fd1a0 100644
--- a/flumine/order/order.py
+++ b/flumine/order/order.py
@@ -62,6 +62,7 @@ class BaseOrder:
         self.status = None
         self.status_log = []
         self.violation_msg = None
+        self.context = {}  # store order specific notes/triggers
 
         self.bet_id = None
         self.update_data = {}  # stores cancel/update/replace data
diff --git a/flumine/order/trade.py b/flumine/order/trade.py
index 1e65c2f..8cbd3de 100644
--- a/flumine/order/trade.py
+++ b/flumine/order/trade.py
@@ -29,13 +29,10 @@ class Trade:
         handicap: float,
         strategy: BaseStrategy,
         notes: collections.OrderedDict = None,
-        fill_kill=None,
-        offset=None,
-        green=None,
         place_reset_seconds: float = 0.0,  # seconds to wait since `runner_context.reset` before allowing another order
         reset_seconds: float = 0.0,  # seconds to wait since `runner_context.place` before allowing another order
     ):
-        self.id = uuid.uuid1()
+        self.id = str(uuid.uuid1())
         self.market_id = market_id
         self.selection_id = selection_id
         self.handicap = handicap
@@ -44,9 +41,6 @@ class Trade:
             notes if notes else collections.OrderedDict()
         )  # trade notes (e.g. triggers/market state)
         self.market_notes = None  # back,lay,lpt
-        self.fill_kill = fill_kill  # todo
-        self.offset = offset  # todo
-        self.green = green  # todo
         self.place_reset_seconds = place_reset_seconds
         self.reset_seconds = reset_seconds
         self.orders = []  # all orders linked to trade
@@ -158,7 +152,7 @@ class Trade:
     @property
     def info(self) -> dict:
         return {
-            "id": str(self.id),
+            "id": self.id,
             "strategy": str(self.strategy),
             "place_reset_seconds": self.place_reset_seconds,
             "reset_seconds": self.reset_seconds,
diff --git a/flumine/strategy/runnercontext.py b/flumine/strategy/runnercontext.py
index 854e39e..722135c 100644
--- a/flumine/strategy/runnercontext.py
+++ b/flumine/strategy/runnercontext.py
@@ -13,39 +13,41 @@ class RunnerContext:
         self.invested = False
         self.datetime_last_placed = None
         self.datetime_last_reset = None
-        self._trades = []
-        self._live_trades = []
+        self.trades = []
+        self.live_trades = []
 
     def place(self, trade_id) -> None:
         self.invested = True
         self.datetime_last_placed = datetime.datetime.utcnow()
-        self._trades.append(trade_id)
-        self._live_trades.append(trade_id)
+        if trade_id not in self.trades:
+            self.trades.append(trade_id)
+        if trade_id not in self.live_trades:
+            self.live_trades.append(trade_id)
 
     def reset(self, trade_id) -> None:
         self.datetime_last_reset = datetime.datetime.utcnow()
         try:
-            self._live_trades.remove(trade_id)
+            self.live_trades.remove(trade_id)
         except ValueError:
             logger.warning(
-                "Trade '%s' not present in RunnerContext live trades on reset"
+                "Trade '%s' not present in RunnerContext live_trades on reset"
                 % trade_id
             )
 
     @property
     def executable_orders(self) -> bool:
-        if self._live_trades:
+        if self.live_trades:
             return True
         else:
             return False
 
     @property
     def trade_count(self) -> int:
-        return len(self._trades)
+        return len(self.trades)
 
     @property
     def live_trade_count(self) -> int:
-        return len(self._live_trades)
+        return len(self.live_trades)
 
     @property
     def placed_elapsed_seconds(self) -> Optional[float]:
diff --git a/flumine/strategy/strategy.py b/flumine/strategy/strategy.py
index 019aa6c..987ab76 100644
--- a/flumine/strategy/strategy.py
+++ b/flumine/strategy/strategy.py
@@ -13,7 +13,6 @@ from ..utils import create_cheap_hash
 logger = logging.getLogger(__name__)
 
 STRATEGY_NAME_HASH_LENGTH = 13
-
 DEFAULT_MARKET_DATA_FILTER = filters.streaming_market_data_filter(
     fields=[
         "EX_ALL_OFFERS",
@@ -28,6 +27,17 @@ DEFAULT_MARKET_DATA_FILTER = filters.streaming_market_data_filter(
 
 
 class BaseStrategy:
+
+    """
+    Strategy object to process MarketBook data
+    from streams, order placement and handling
+    logic to be added where required. Only
+    MarketBooks from provided filter and data
+    filter are processed.
+    Runner context available to store current
+    live trades.
+    """
+
     def __init__(
         self,
         market_filter: dict,
@@ -40,22 +50,25 @@ class BaseStrategy:
         max_selection_exposure: float = 100,
         max_order_exposure: float = 10,
         client: BaseClient = None,
-        max_trade_count: int = 1e6,  # max total number of trades per runner
-        max_live_trade_count: int = 1,  # max live (with executable orders) trades per runner
+        max_trade_count: int = 1e6,
+        max_live_trade_count: int = 1,
+        multi_order_trades: bool = False,
         log_validation_failures: bool = False,
     ):
         """
-        Processes data from streams.
-
         :param market_filter: Streaming market filter
         :param market_data_filter: Streaming market data filter
         :param streaming_timeout: Streaming timeout in seconds, will call snap() on cache
         :param conflate_ms: Streaming conflation
-        :param stream_class: Can be Market or Data
-        :param name: Strategy name
-        :param context: Dictionary holding additional vars
+        :param stream_class: Can be Market or Data (raw)
+        :param name: Strategy name (will default to class name)
+        :param context: Dictionary holding additional user specific vars
         :param max_selection_exposure: Max exposure per selection
         :param max_order_exposure: Max exposure per order
+        :param client: flumine client used for order placement
+        :param max_trade_count: max total number of trades per runner
+        :param max_live_trade_count: max live (with executable orders) trades per runner
+        :param multi_order_trades: allow multiple live orders per trade
         :param log_validation_failures: determines whether or not to log validation failures
         """
         self.market_filter = market_filter
@@ -70,11 +83,16 @@ class BaseStrategy:
         self.client = client
         self.max_trade_count = max_trade_count
         self.max_live_trade_count = max_live_trade_count
+        self.multi_order_trades = multi_order_trades
 
         self._invested = {}  # {(marketId, selectionId, handicap): RunnerContext}
         self.streams = []  # list of streams strategy is subscribed
         self.historic_stream_ids = []
-        self.log_validation_failures = log_validation_failures
+        if log_validation_failures:
+            warnings.warn(
+                "strategy.log_validation_failures is deprecated and will be removed from v1.18 onwards (all validation failures are now logged in the trading control)",
+                PendingDeprecationWarning,
+            )
         # cache
         self.name_hash = create_cheap_hash(self.name, STRATEGY_NAME_HASH_LENGTH)
 
@@ -177,43 +195,38 @@ class BaseStrategy:
         return market.replace_order(order, new_price, market_version)
 
     def validate_order(self, runner_context: RunnerContext, order) -> bool:
+        # allow multiple orders per trade
+        if self.multi_order_trades:
+            if order.trade.id in runner_context.live_trades:
+                return True
         # validate context
         if runner_context.trade_count >= self.max_trade_count:
-            if self.log_validation_failures:
-                logger.warning(
-                    "validate_order failed: trade_count (%s) >= max_trade_count (%s)",
-                    runner_context.trade_count,
-                    self.max_live_trade_count,
-                )
+            order.violation_msg = "strategy.validate_order failed: trade_count ({0}) >= max_trade_count ({1})".format(
+                runner_context.trade_count, self.max_live_trade_count
+            )
             return False
         elif runner_context.live_trade_count >= self.max_live_trade_count:
-            if self.log_validation_failures:
-                logger.warning(
-                    "validate_order failed: live_trade_count (%s) >= max_live_trade_count (%s)"
-                    % (runner_context.live_trade_count, self.max_live_trade_count)
-                )
+            order.violation_msg = "strategy.validate_order failed: live_trade_count ({0}) >= max_live_trade_count ({1})".format(
+                runner_context.live_trade_count, self.max_live_trade_count
+            )
             return False
         elif (
             runner_context.placed_elapsed_seconds
             and runner_context.placed_elapsed_seconds < order.trade.place_reset_seconds
         ):
-            if self.log_validation_failures:
-                logger.warning(
-                    "validate_order failed: placed_elapsed_seconds (%s) < place_reset_seconds (%s)",
-                    runner_context.placed_elapsed_seconds,
-                    order.trade.place_reset_seconds,
-                )
+            order.violation_msg = "strategy.validate_order failed: placed_elapsed_seconds ({0}) < place_reset_seconds ({1})".format(
+                runner_context.placed_elapsed_seconds,
+                order.trade.place_reset_seconds,
+            )
             return False
         elif (
             runner_context.reset_elapsed_seconds
             and runner_context.reset_elapsed_seconds < order.trade.reset_seconds
         ):
-            if self.log_validation_failures:
-                logger.warning(
-                    "validate_order failed: reset_elapsed_seconds (%s) < reset_seconds (%s)",
-                    runner_context.reset_elapsed_seconds,
-                    order.trade.reset_seconds,
-                )
+            order.violation_msg = "strategy.validate_order failed: reset_elapsed_seconds ({0}) < reset_seconds ({1})".format(
+                runner_context.reset_elapsed_seconds,
+                order.trade.reset_seconds,
+            )
             return False
         else:
             return True


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
index 899a1e6..472bf32 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -10,4 +10,4 @@ class ConfigTest(unittest.TestCase):
         self.assertIsInstance(config.process_id, int)
         self.assertIsNone(config.current_time)
         self.assertFalse(config.raise_errors)
-        self.assertEqual(config.max_workers, 32)
+        self.assertEqual(config.max_execution_workers, 32)
diff --git a/tests/test_order.py b/tests/test_order.py
index c6aa74f..2e976b0 100644
--- a/tests/test_order.py
+++ b/tests/test_order.py
@@ -37,6 +37,7 @@ class BaseOrderTest(unittest.TestCase):
         self.assertIsNone(self.order.status)
         self.assertEqual(self.order.status_log, [])
         self.assertIsNone(self.order.violation_msg)
+        self.assertEqual(self.order.context, {})
         self.assertIsNone(self.order.bet_id)
         self.assertIsNone(self.order.EXCHANGE)
         self.assertEqual(self.order.update_data, {})
diff --git a/tests/test_runnercontext.py b/tests/test_runnercontext.py
index 7e952e1..85ae669 100644
--- a/tests/test_runnercontext.py
+++ b/tests/test_runnercontext.py
@@ -7,6 +7,7 @@ from flumine.strategy.runnercontext import RunnerContext
 
 class RunnerContextTest(unittest.TestCase):
     def setUp(self):
+        self.id_ = str(uuid.uuid1())
         self.context = RunnerContext(12345)
 
     def test_init(self):
@@ -14,43 +15,41 @@ class RunnerContextTest(unittest.TestCase):
         self.assertFalse(self.context.invested)
         self.assertIsNone(self.context.datetime_last_placed)
         self.assertIsNone(self.context.datetime_last_reset)
-        self.assertEqual(self.context._trades, [])
-        self.assertEqual(self.context._live_trades, [])
+        self.assertEqual(self.context.trades, [])
+        self.assertEqual(self.context.live_trades, [])
 
     def test_place(self):
-        id_ = uuid.uuid1()
-        self.context.place(id_)
+        self.context.place(self.id_)
+        self.context.place(self.id_)  # append is ignored
         self.assertTrue(self.context.invested)
         self.assertIsNotNone(self.context.datetime_last_placed)
-        self.assertEqual(self.context._trades, [id_])
-        self.assertEqual(self.context._live_trades, [id_])
+        self.assertEqual(self.context.trades, [self.id_])
+        self.assertEqual(self.context.live_trades, [self.id_])
 
     def test_reset(self):
-        id_ = uuid.uuid1()
-        self.context._live_trades = [id_]
-        self.context.reset(id_)
+        self.context.live_trades = [self.id_]
+        self.context.reset(self.id_)
         self.assertIsNotNone(self.context.datetime_last_reset)
-        self.assertEqual(self.context._live_trades, [])
+        self.assertEqual(self.context.live_trades, [])
 
     def test_reset_error(self):
-        id_ = uuid.uuid1()
-        self.context.reset(id_)
+        self.context.reset(self.id_)
         self.assertIsNotNone(self.context.datetime_last_reset)
-        self.assertEqual(self.context._live_trades, [])
+        self.assertEqual(self.context.live_trades, [])
 
     def test_executable_orders(self):
         self.assertFalse(self.context.executable_orders)
-        self.context._live_trades = ["test"]
+        self.context.live_trades = [self.id_]
         self.assertTrue(self.context.executable_orders)
 
     def test_trade_count(self):
         self.assertEqual(self.context.trade_count, 0)
-        self.context._trades = ["test"]
+        self.context.trades = [self.id_]
         self.assertEqual(self.context.trade_count, 1)
 
     def test_live_trade_count(self):
         self.assertEqual(self.context.live_trade_count, 0)
-        self.context._live_trades = ["test"]
+        self.context.live_trades = [self.id_]
         self.assertEqual(self.context.live_trade_count, 1)
 
     def test_placed_elapsed_seconds(self):
diff --git a/tests/test_strategy.py b/tests/test_strategy.py
index 2e88b0b..e4a87fd 100644
--- a/tests/test_strategy.py
+++ b/tests/test_strategy.py
@@ -53,6 +53,8 @@ class BaseStrategyTest(unittest.TestCase):
             client=self.mock_client,
             max_trade_count=3,
             max_live_trade_count=4,
+            multi_order_trades=False,
+            log_validation_failures=True,
         )
 
     def test_init(self):
@@ -71,6 +73,22 @@ class BaseStrategyTest(unittest.TestCase):
         self.assertEqual(self.strategy.streams, [])
         self.assertEqual(self.strategy.historic_stream_ids, [])
         self.assertEqual(self.strategy.name_hash, "a94a8fe5ccb19")
+        self.assertFalse(self.strategy.multi_order_trades)
+        self.assertEqual(strategy.STRATEGY_NAME_HASH_LENGTH, 13)
+        self.assertEqual(
+            strategy.DEFAULT_MARKET_DATA_FILTER,
+            {
+                "fields": [
+                    "EX_ALL_OFFERS",
+                    "EX_TRADED",
+                    "EX_TRADED_VOL",
+                    "EX_LTP",
+                    "EX_MARKET_DEF",
+                    "SP_TRADED",
+                    "SP_PROJECTED",
+                ]
+            },
+        )
 
     def test_check_market_no_subscribed(self):
         mock_market = mock.Mock()
@@ -201,6 +219,20 @@ class BaseStrategyTest(unittest.TestCase):
         mock_order.trade.reset_seconds = 0.5
         self.assertFalse(self.strategy.validate_order(runner_context, mock_order))
 
+    def test_validate_order_multi(self):
+        mock_order = mock.Mock()
+        mock_order.trade.id = "test"
+        runner_context = mock.Mock(
+            trade_count=10,
+            live_trade_count=10,
+            live_trades=["test"],
+            placed_elapsed_seconds=None,
+            reset_elapsed_seconds=None,
+        )
+        self.assertFalse(self.strategy.validate_order(runner_context, mock_order))
+        self.strategy.multi_order_trades = True
+        self.assertTrue(self.strategy.validate_order(runner_context, mock_order))
+
     def test_executable_orders(self):
         mock_context = mock.Mock(executable_orders=True)
         self.strategy._invested = {("2", 456, 1): mock_context}
diff --git a/tests/test_trade.py b/tests/test_trade.py
index 51234c7..28c2935 100644
--- a/tests/test_trade.py
+++ b/tests/test_trade.py
@@ -11,9 +11,6 @@ class TradeTest(unittest.TestCase):
         logging.disable(logging.CRITICAL)
         mock_client = mock.Mock(paper_trade=False)
         self.mock_strategy = mock.Mock(client=mock_client)
-        self.mock_fill_kill = mock.Mock()
-        self.mock_offset = mock.Mock()
-        self.mock_green = mock.Mock()
         self.notes = collections.OrderedDict({"trigger": 123})
         self.trade = Trade(
             "1.234",
@@ -21,9 +18,6 @@ class TradeTest(unittest.TestCase):
             1.0,
             self.mock_strategy,
             self.notes,
-            self.mock_fill_kill,
-            self.mock_offset,
-            self.mock_green,
             12,
             34,
         )
@@ -34,9 +28,6 @@ class TradeTest(unittest.TestCase):
         self.assertEqual(self.trade.handicap, 1.0)
         self.assertEqual(self.trade.strategy, self.mock_strategy)
         self.assertEqual(self.trade.notes, self.notes)
-        self.assertEqual(self.trade.fill_kill, self.mock_fill_kill)
-        self.assertEqual(self.trade.offset, self.mock_offset)
-        self.assertEqual(self.trade.green, self.mock_green)
         self.assertEqual(self.trade.status_log, [])
         self.assertEqual(self.trade.orders, [])
         self.assertEqual(self.trade.offset_orders, [])
diff --git a/tests/test_tradingcontrols.py b/tests/test_tradingcontrols.py
index 8ddb182..0690e28 100644
--- a/tests/test_tradingcontrols.py
+++ b/tests/test_tradingcontrols.py
@@ -296,7 +296,7 @@ class TestStrategyExposure(unittest.TestCase):
         self.trading_control._validate(mock_order, OrderPackageType.PLACE)
         mock_on_error.assert_called_with(
             mock_order,
-            "strategy.validate_order failure",
+            mock_order.violation_msg,
         )
 
     @mock.patch("flumine.controls.tradingcontrols.StrategyExposure._on_error")