Repository: liampauling/flumine
Pull request #307 (2020-11-02 09:47:27)

betfairlightweight upgraded to 2.10.0 (api release 10/11/20)


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index e985ddb..553251c 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,19 @@
 Release History
 ---------------
 
+1.14.2 (2020-11-02)
++++++++++++++++++++
+
+**Improvements**
+
+- _process_cleared_orders called on market closure when backtesting / paper trading
+- size reduction handling added to simulated execution on cancel
+- Add py3.9 actions test
+
+**Libraries**
+
+- betfairlightweight upgraded to 2.10.0 (exchange stream api release 10/11/20)
+
 1.14.1 (2020-10-29)
 +++++++++++++++++++
 
diff --git a/flumine/__version__.py b/flumine/__version__.py
index d8848ae..9299449 100644
--- a/flumine/__version__.py
+++ b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.14.1"
+__version__ = "1.14.2"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/baseflumine.py b/flumine/baseflumine.py
index 555fe83..ba6d359 100644
--- a/flumine/baseflumine.py
+++ b/flumine/baseflumine.py
@@ -21,7 +21,6 @@ from .controls.tradingcontrols import OrderValidation, StrategyExposure
 from .controls.loggingcontrols import LoggingControl
 from . import config, utils
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -268,7 +267,10 @@ class BaseFlumine:
             if stream_id in strategy.stream_ids:
                 strategy.process_closed_market(market, event.event)
 
-        self.cleared_market_queue.put(market_id)
+        if self.BACKTEST or self.client.paper_trade:
+            self._process_cleared_orders(events.ClearedOrdersEvent(market))
+        else:
+            self.cleared_market_queue.put(market_id)
         self.log_control(event)
         logger.info("Market closed", extra={"market_id": market_id, **self.info})
 
diff --git a/flumine/execution/simulatedexecution.py b/flumine/execution/simulatedexecution.py
index 8fe3bd6..221aaff 100644
--- a/flumine/execution/simulatedexecution.py
+++ b/flumine/execution/simulatedexecution.py
@@ -65,7 +65,10 @@ class SimulatedExecution(BaseExecution):
                     order, simulated_response, order_package.package_type
                 )
                 if simulated_response.status == "SUCCESS":
-                    order.execution_complete()
+                    if order.size_cancelled == order.size_remaining:
+                        order.execution_complete()
+                    else:
+                        order.executable()
                 elif simulated_response.status == "FAILURE":
                     order.executable()
 
diff --git a/requirements.txt b/requirements.txt
index 9a938fb..2eae7bb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-betfairlightweight==2.9.2
+betfairlightweight==2.10.0
 tenacity==5.0.3
 python-json-logger==2.0.1
 requests
diff --git a/setup.py b/setup.py
index fd4dee6..d066f0d 100644
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,13 @@ setup(
     name=about["__title__"],
     version=about["__version__"],
     packages=find_packages(
-        exclude=["examples", "examples.strategies", "examples.workers"]
+        exclude=[
+            "examples",
+            "examples.strategies",
+            "examples.workers",
+            "examples.middleware",
+            "examples.controls",
+        ]
     ),
     package_dir={"flumine": "flumine"},
     install_requires=INSTALL_REQUIRES,


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index afb3ea5..2b3e4ba 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/examples/controls/__init__.py b/examples/controls/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/examples/controls/backtestloggingcontrol.py b/examples/controls/backtestloggingcontrol.py
new file mode 100644
index 0000000..a79eae8
--- /dev/null
+++ b/examples/controls/backtestloggingcontrol.py
@@ -0,0 +1,70 @@
+import csv
+import logging
+from flumine.controls.loggingcontrols import LoggingControl
+
+logger = logging.getLogger(__name__)
+
+FIELDNAMES = [
+    "bet_id",
+    "strategy_name",
+    "market_id",
+    "selection_id",
+    "trade_id",
+    "date_time_placed",
+    "price",
+    "price_matched",
+    "size",
+    "size_matched",
+    "profit",
+    "side",
+    "elapsed_seconds_executable",
+    "order_status",
+    "market_note",
+    "notes",
+]
+
+
+class BacktestLoggingControl(LoggingControl):
+    NAME = "BACKTEST_LOGGING_CONTROL"
+
+    def __init__(self, *args, **kwargs):
+        super(BacktestLoggingControl, self).__init__(*args, **kwargs)
+        self._setup()
+
+    def _setup(self):
+        with open("orders.txt", "w") as m:
+            csv_writer = csv.DictWriter(m, delimiter=",", fieldnames=FIELDNAMES)
+            csv_writer.writeheader()
+
+    def _process_cleared_orders_meta(self, event):
+        orders = event.event
+        with open("orders.txt", "a") as m:
+            for order in orders:
+                try:
+                    order_data = {
+                        "bet_id": order.bet_id,
+                        "strategy_name": order.trade.strategy,
+                        "market_id": order.market_id,
+                        "selection_id": order.selection_id,
+                        "trade_id": order.trade.id,
+                        "date_time_placed": order.responses.date_time_placed,
+                        "price": order.order_type.price,
+                        "price_matched": order.average_price_matched,
+                        "size": order.order_type.size,
+                        "size_matched": order.size_matched,
+                        "profit": order.simulated.profit,
+                        "side": order.side,
+                        "elapsed_seconds_executable": order.elapsed_seconds_executable,
+                        "order_status": order.status.value,
+                        "market_note": order.trade.market_notes,
+                        "notes": order.trade.notes_str,
+                    }
+                    csv_writer = csv.DictWriter(m, delimiter=",", fieldnames=FIELDNAMES)
+                    csv_writer.writerow(order_data)
+                except Exception as e:
+                    logger.error(
+                        "_process_cleared_orders_meta: %s" % e,
+                        extra={"order": order, "error": e},
+                    )
+
+        logger.info("Orders updated", extra={"order_count": len(orders)})
diff --git a/flumine/backtest/simulated.py b/flumine/backtest/simulated.py
index dfdf2ed..eff8941 100644
--- a/flumine/backtest/simulated.py
+++ b/flumine/backtest/simulated.py
@@ -132,7 +132,10 @@ class Simulated:
     def cancel(self) -> SimulatedCancelResponse:
         # simulates cancelOrder request->cancel->response
         if self.order.order_type.ORDER_TYPE == OrderTypes.LIMIT:
-            self.size_cancelled = self.size_remaining
+            _size_reduction = (
+                self.order.update_data.get("size_reduction") or self.size_remaining
+            )  # todo handle when size_reduction provided greater than size_remaining?
+            self.size_cancelled = _size_reduction
             return SimulatedCancelResponse(
                 status="SUCCESS",  # todo handle errors
                 size_cancelled=self.size_cancelled,
diff --git a/tests/test_baseflumine.py b/tests/test_baseflumine.py
index a7004f4..05d98a6 100644
--- a/tests/test_baseflumine.py
+++ b/tests/test_baseflumine.py
@@ -314,9 +314,13 @@ class BaseFlumineTest(unittest.TestCase):
 
         self.assertEqual(len(self.base_flumine.markets._markets), 3)
 
+    @mock.patch("flumine.baseflumine.events")
+    @mock.patch("flumine.baseflumine.BaseFlumine._process_cleared_orders")
     @mock.patch("flumine.baseflumine.BaseFlumine.info")
     @mock.patch("flumine.baseflumine.BaseFlumine.log_control")
-    def test__process_close_market_closed_paper(self, mock_log_control, mock_info):
+    def test__process_close_market_closed_paper(
+        self, mock_log_control, mock_info, mock__process_cleared_orders, mock_events
+    ):
         self.base_flumine.client.paper_trade = True
         mock_strategy = mock.Mock()
         mock_strategy.stream_ids = [1, 2, 3]
@@ -338,6 +342,9 @@ class BaseFlumineTest(unittest.TestCase):
         mock_event.event = mock_market_book
         self.base_flumine._process_close_market(mock_event)
         self.assertEqual(len(self.base_flumine.markets._markets), 4)
+        mock__process_cleared_orders.assert_called_with(
+            mock_events.ClearedOrdersEvent()
+        )
 
     @mock.patch("flumine.baseflumine.events")
     @mock.patch("flumine.baseflumine.BaseFlumine.log_control")
diff --git a/tests/test_execution.py b/tests/test_execution.py
index 214a712..c79213c 100644
--- a/tests/test_execution.py
+++ b/tests/test_execution.py
@@ -886,7 +886,7 @@ class SimulatedExecutionTest(unittest.TestCase):
 
     @mock.patch("flumine.execution.simulatedexecution.SimulatedExecution._order_logger")
     def test_execute_cancel(self, mock__order_logger):
-        mock_order = mock.Mock()
+        mock_order = mock.Mock(size_cancelled=2, size_remaining=2)
         mock_order.trade.__enter__ = mock.Mock()
         mock_order.trade.__exit__ = mock.Mock()
         mock_order_package = mock.Mock()
@@ -904,6 +904,26 @@ class SimulatedExecutionTest(unittest.TestCase):
         mock_order.trade.__enter__.assert_called_with()
         mock_order.trade.__exit__.assert_called_with(None, None, None)
 
+    @mock.patch("flumine.execution.simulatedexecution.SimulatedExecution._order_logger")
+    def test_execute_cancel_executable(self, mock__order_logger):
+        mock_order = mock.Mock(size_cancelled=2, size_remaining=3)
+        mock_order.trade.__enter__ = mock.Mock()
+        mock_order.trade.__exit__ = mock.Mock()
+        mock_order_package = mock.Mock()
+        mock_order_package.client.paper_trade = False
+        mock_order_package.__iter__ = mock.Mock(return_value=iter([mock_order]))
+        mock_sim_resp = mock.Mock()
+        mock_sim_resp.status = "SUCCESS"
+        mock_order.simulated.cancel.return_value = mock_sim_resp
+        self.execution.execute_cancel(mock_order_package, None)
+        mock_order.simulated.cancel.assert_called_with()
+        mock__order_logger.assert_called_with(
+            mock_order, mock_sim_resp, mock_order_package.package_type
+        )
+        mock_order.executable.assert_called_with()
+        mock_order.trade.__enter__.assert_called_with()
+        mock_order.trade.__exit__.assert_called_with(None, None, None)
+
     @mock.patch("flumine.execution.simulatedexecution.SimulatedExecution._order_logger")
     def test_execute_cancel_failure(self, mock__order_logger):
         mock_order = mock.Mock()
diff --git a/tests/test_simulated.py b/tests/test_simulated.py
index 47f064a..20d7d47 100644
--- a/tests/test_simulated.py
+++ b/tests/test_simulated.py
@@ -208,11 +208,19 @@ class SimulatedTest(unittest.TestCase):
         self.assertEqual(resp.error_code, "dubs of the mad skint and british")
 
     def test_cancel(self):
+        self.simulated.order.update_data = {}
         resp = self.simulated.cancel()
         self.assertEqual(self.simulated.size_cancelled, 2.0)
         self.assertEqual(resp.status, "SUCCESS")
         self.assertEqual(resp.size_cancelled, 2.0)
 
+    def test_cancel_reduction(self):
+        self.simulated.order.update_data = {"size_reduction": 0.50}
+        resp = self.simulated.cancel()
+        self.assertEqual(self.simulated.size_cancelled, 0.50)
+        self.assertEqual(resp.status, "SUCCESS")
+        self.assertEqual(resp.size_cancelled, 0.50)
+
     def test_cancel_else(self):
         self.simulated.order.order_type.ORDER_TYPE = OrderTypes.MARKET_ON_CLOSE
         resp = self.simulated.cancel()