Repository: vnpy/vnpy

Issue:
行情记录模块性能提升
在行情记录模块中，每一条数据都需要进行一次数据库访问（一个DB transaction)。数据库每秒进行的transaction数目非常有限，当记录标的太多（比如全市场行情记录）时，必然会造成队列拥塞


>     def run(self):
>         """"""
>         while self.active:
>             try:
>                 task = self.queue.get(timeout=1)
>                 task_type, data = task
> 
>                 if task_type == "tick":
>                     database_manager.save_tick_data([data])
>                 elif task_type == "bar":
>                     database_manager.save_bar_data([data])
> 
>             except Empty:
>                 continue
> 
>             except Exception:
>                 self.active = False
> 
>                 info = sys.exc_info()
>                 event = Event(EVENT_RECORDER_EXCEPTION, info)
>                 self.event_engine.put(event)
> 

可以每次从队列中先取出所有待记录数据，一次性把队列中已有数据写入数据库。这样可以大大降低数据库压力和记录延迟。



Discussion:
如果有人感兴趣，我可以提供实现

Resolving patch:
diff --git a/vnpy/app/data_recorder/engine.py b/vnpy/app/data_recorder/engine.py
--- a/vnpy/app/data_recorder/engine.py
+++ b/vnpy/app/data_recorder/engine.py
@@ -1,24 +1,24 @@
 """"""
 
 import sys
-from threading import Thread
-from queue import Queue, Empty
+import time
 from copy import copy
+from queue import Queue, Empty
+from threading import Thread
 
+from vnpy.app.spread_trading.base import EVENT_SPREAD_DATA, SpreadData
 from vnpy.event import Event, EventEngine
-from vnpy.trader.engine import BaseEngine, MainEngine
 from vnpy.trader.constant import Exchange
+from vnpy.trader.database import database_manager
+from vnpy.trader.engine import BaseEngine, MainEngine
+from vnpy.trader.event import EVENT_TICK, EVENT_CONTRACT
 from vnpy.trader.object import (
     SubscribeRequest,
     TickData,
     BarData,
     ContractData
 )
-from vnpy.trader.event import EVENT_TICK, EVENT_CONTRACT
 from vnpy.trader.utility import load_json, save_json, BarGenerator
-from vnpy.trader.database import database_manager
-from vnpy.app.spread_trading.base import EVENT_SPREAD_DATA, SpreadData
-
 
 APP_NAME = "DataRecorder"
 
@@ -36,9 +36,13 @@ def __init__(self, main_engine: MainEngine, event_engine: EventEngine):
         super().__init__(main_engine, event_engine, APP_NAME)
 
         self.queue = Queue()
+        self.last_commit_time = None
         self.thread = Thread(target=self.run)
         self.active = False
 
+        self.batch_size = None
+        self.commit_delay = None
+
         self.tick_recordings = {}
         self.bar_recordings = {}
         self.bar_generators = {}
@@ -53,10 +57,14 @@ def load_setting(self):
         setting = load_json(self.setting_filename)
         self.tick_recordings = setting.get("tick", {})
         self.bar_recordings = setting.get("bar", {})
+        self.batch_size = setting.get("batch_size", 10000)
+        self.commit_delay = setting.get("commit_delay", 30)
 
     def save_setting(self):
         """"""
         setting = {
+            "batch_size": self.batch_size,
+            "commit_delay": self.commit_delay,
             "tick": self.tick_recordings,
             "bar": self.bar_recordings
         }
@@ -65,21 +73,59 @@ def save_setting(self):
     def run(self):
         """"""
         while self.active:
+
             try:
-                task = self.queue.get(timeout=1)
-                task_type, data = task
+                items = []
 
-                if task_type == "tick":
-                    database_manager.save_tick_data([data])
-                elif task_type == "bar":
-                    database_manager.save_bar_data([data])
+                if self.last_commit_time is None:
+                    elapse_time = None
+                else:
+                    elapse_time = self.last_commit_time + self.commit_delay
 
-            except Empty:
-                continue
+                while len(items) < self.batch_size:
+                    # First, retrieve all pending items in the queue, up to `self.batch_size` items
+                    for _ in range(len(items), self.batch_size):
+                        try:
+                            item = self.queue.get_nowait()
+                            items.append(item)
+                        except Empty:
+                            break
 
-            except Exception:
-                self.active = False
+                    if len(items) >= self.batch_size:
+                        break
+
+                    # Second, we wait until `self.__last_commit_time + self.commit_delay`
+                    ts = time.time()
+
+                    if elapse_time is None or ts >= elapse_time:
+                        break
+
+                    try:
+                        item = self.queue.get(timeout=max(0., elapse_time - ts))
+
+                        # We do get something, try to get more
+                        items.append(item)
+                        continue
+                    except Empty:
+                        # No more item until the commit time, let's continue to write them back to DB
+                        break
 
+                self.last_commit_time = time.time()
+
+                ticks = [data for task_type, data in items if task_type == 'tick']
+                bars = [data for task_type, data in items if task_type == 'bar']
+
+                if len(ticks) > 0:
+                    database_manager.save_tick_data(ticks)
+
+                if len(bars) > 0:
+                    database_manager.save_bar_data(bars)
+
+            except:
+                import traceback
+                traceback.print_exc()
+
+                self.active = False
                 info = sys.exc_info()
                 event = Event(EVENT_RECORDER_EXCEPTION, info)
                 self.event_engine.put(event)
@@ -230,6 +276,8 @@ def put_event(self):
         bar_symbols.sort()
 
         data = {
+            "batch_size": self.batch_size,
+            "commit_delay": self.commit_delay,
             "tick": tick_symbols,
             "bar": bar_symbols
         }
diff --git a/vnpy/app/data_recorder/ui/widget.py b/vnpy/app/data_recorder/ui/widget.py
--- a/vnpy/app/data_recorder/ui/widget.py
+++ b/vnpy/app/data_recorder/ui/widget.py
@@ -64,6 +64,12 @@ def init_ui(self):
         remove_tick_button = QtWidgets.QPushButton("移除")
         remove_tick_button.clicked.connect(self.remove_tick_recording)
 
+        self.batch_size_edit = QtWidgets.QLineEdit()
+        self.commit_delay_edit = QtWidgets.QLineEdit()
+
+        self.batch_size_edit.editingFinished.connect(self.update_batch_size)
+        self.commit_delay_edit.editingFinished.connect(self.update_commit_delay)
+
         self.bar_recording_edit = QtWidgets.QTextEdit()
         self.bar_recording_edit.setReadOnly(True)
 
@@ -82,6 +88,13 @@ def init_ui(self):
         grid.addWidget(add_tick_button, 1, 1)
         grid.addWidget(remove_tick_button, 1, 2)
 
+        grid.addWidget(QtWidgets.QLabel("批量写入"), 0, 4)
+        grid.addWidget(self.batch_size_edit, 0, 5)
+        grid.addWidget(QtWidgets.QLabel("写入间隔"), 1, 4)
+        grid.addWidget(self.commit_delay_edit, 1, 5)
+
+
+
         hbox = QtWidgets.QHBoxLayout()
         hbox.addWidget(QtWidgets.QLabel("本地代码"))
         hbox.addWidget(self.symbol_line)
@@ -125,6 +138,9 @@ def process_update_event(self, event: Event):
         """"""
         data = event.data
 
+        self.batch_size_edit.setText(str(data["batch_size"]))
+        self.commit_delay_edit.setText(str(data["commit_delay"]))
+
         self.bar_recording_edit.clear()
         bar_text = "\n".join(data["bar"])
         self.bar_recording_edit.setText(bar_text)
@@ -165,3 +181,20 @@ def remove_tick_recording(self):
         """"""
         vt_symbol = self.symbol_line.text()
         self.recorder_engine.remove_tick_recording(vt_symbol)
+
+    def update_batch_size(self):
+        try:
+            self.recorder_engine.batch_size = int(self.batch_size_edit.text())
+            self.recorder_engine.save_setting()
+            self.recorder_engine.put_event()
+        except ValueError:
+            pass
+
+    def update_commit_delay(self):
+        try:
+            self.recorder_engine.commit_delay = int(self.commit_delay_edit.text())
+            self.recorder_engine.save_setting()
+            self.recorder_engine.put_event()
+        except ValueError:
+            pass
+


Test patch: