Repository: Ma-r-co/savoia

Issue:
Poor performance of HistoricCSVDataFeeder
Poor performance of HistoricCSVDataFeeder
 - Refine the usage of pandas

Discussion:

Patch:
diff --git a/src/savoia/datafeed/datafeed.py b/src/savoia/datafeed/datafeed.py
index 1fd53f0..942ca41 100644
--- a/src/savoia/datafeed/datafeed.py
+++ b/src/savoia/datafeed/datafeed.py
@@ -11,7 +11,7 @@ from savoia.event.event import Event, TickEvent
 from savoia.types.types import Pair
 
 from logging import getLogger, Logger
-from typing import List, Dict, Iterator
+from typing import List, Iterator, Tuple
 from queue import Queue
 from abc import ABCMeta, abstractmethod
 import time
@@ -57,7 +57,7 @@ class HistoricCSVDataFeeder(DataFeeder):
     pairs: List[Pair]
     feed_q: 'Queue[Event]'
     csv_dir: str
-    pair_frames: Dict[Pair, pd.DataFrame]
+    pair_frames: List[str]
     file_dates: List[str]
     cur_date_idx: int
     cur_date_pairs: pd.DataFrame
@@ -82,7 +82,6 @@ class HistoricCSVDataFeeder(DataFeeder):
         self.pairs = pairs
         self.feed_q = feed_q
         self.csv_dir = csv_dir
-        self.pair_frames = {}
         self.file_dates = self._list_all_file_dates()
         self.continue_backtest = True
         self.cur_date_idx = 0
@@ -111,7 +110,7 @@ class HistoricCSVDataFeeder(DataFeeder):
         return de_dup_csv
 
     def _open_convert_csv_files_for_day(self, date_str: str) \
-            -> Iterator[pd.DataFrame]:
+            -> Iterator[Tuple[str, str, str, str]]:
         """
         Opens the CSV files from the data directory, converting
         them into pandas DataFrames within a pairs dictionary.
@@ -121,20 +120,40 @@ class HistoricCSVDataFeeder(DataFeeder):
         ordered, allowing tick data events to be added to the queue
         in a chronological fashion.
         """
+        # for p in self.pairs:
+        #    pair_path = os.path.join(self.csv_dir, '%s_%s.csv' % (p, date_str))
+        #     self.logger.info("start read: %s", str(pair_path))
+        #     with open(pair_path, 'r') as f:
+
+        #     self.pair_frames[p] = pd.read_csv(
+        #         pair_path,
+        #         header=0,
+        #         index_col=0,
+        #         parse_dates=["Time"],
+        #         dayfirst=True,
+        #         names=("Time", "Ask", "Bid", "AskVolume", "BidVolume")
+        #     )
+        #     self.pair_frames[p]["Pair"] = p
+        #     self.logger.info("end read: %s", str(pair_path))
+        # return pd.concat(self.pair_frames.values()).sort_index().iterrows()
+        self.pair_frames = []
         for p in self.pairs:
             pair_path = os.path.join(self.csv_dir, '%s_%s.csv' % (p, date_str))
             self.logger.info("start read: %s", str(pair_path))
-            self.pair_frames[p] = pd.io.parsers.read_csv(
-                pair_path,
-                header=0,
-                index_col=0,
-                parse_dates=["Time"],
-                dayfirst=True,
-                names=("Time", "Ask", "Bid", "AskVolume", "BidVolume")
-            )
+            with open(pair_path, 'r') as f:
+                f.__next__()
+                for line in f:
+                    self.pair_frames.append(line + f',{p}')
             self.logger.info("end read: %s", str(pair_path))
-            self.pair_frames[p]["Pair"] = p
-        return pd.concat(self.pair_frames.values()).sort_index().iterrows()
+        self.logger.info('start sort')
+        self.pair_frames.sort()
+
+        def _gen() -> Iterator[Tuple[str, str, str, str]]:
+            for row in self.pair_frames:
+                date, ask, bid, ask_volume, bid_volume, pair = row.split(',')
+                yield date, ask, bid, pair
+
+        return _gen()
 
     def _update_csv_for_day(self) -> bool:
         try:
@@ -148,19 +167,19 @@ class HistoricCSVDataFeeder(DataFeeder):
 
     def _stream_next_tick(self) -> None:
         try:
-            index, row = next(self.cur_date_pairs)
+            date, ask, bid, pair = next(self.cur_date_pairs)
         except StopIteration:
             # End of the current days data
             if self._update_csv_for_day():
-                index, row = next(self.cur_date_pairs)
+                date, ask, bid, pair = next(self.cur_date_pairs)
             else:  # End of the data
                 self.continue_backtest = False
                 return
-        pair = row["Pair"]
-        bid = Decimal(str(row["Bid"])).quantize(DECIMAL_PLACES)
-        ask = Decimal(str(row["Ask"])).quantize(DECIMAL_PLACES)
+        date = pd.Timestamp(date)
+        bid = Decimal(bid).quantize(DECIMAL_PLACES)
+        ask = Decimal(ask).quantize(DECIMAL_PLACES)
 
-        tev = TickEvent(pair, index, bid, ask)
+        tev = TickEvent(pair, date, bid, ask)
         self.feed_q.put(tev)
 
     def run(self) -> None:
diff --git a/src/savoia/performance/performance.py b/src/savoia/performance/performance.py
index 49b1d68..a5dfa8b 100644
--- a/src/savoia/performance/performance.py
+++ b/src/savoia/performance/performance.py
@@ -26,7 +26,7 @@ def create_drawdowns(pnl: pd.Series) -> Tuple[pd.Series, float, int]:
 
     # Loop over the index range
     for t in range(1, len(idx)):
-        hwm.append(max(hwm[t - 1], pnl.ix[t]))
-        drawdown.ix[t] = (hwm[t] - pnl.ix[t])
-        duration.ix[t] = (0 if drawdown.ix[t] == 0 else duration.ix[t - 1] + 1)
+        hwm.append(max(hwm[t - 1], pnl.iloc[t]))
+        drawdown.iloc[t] = (hwm[t] - pnl.iloc[t])
+        duration.iloc[t] = (0 if drawdown.iloc[t] == 0 else duration.iloc[t - 1] + 1)
     return drawdown, drawdown.max(), duration.max()
diff --git a/src/savoia/portfolio/portfolio.py b/src/savoia/portfolio/portfolio.py
index 1d9ad1a..c7b668f 100644
--- a/src/savoia/portfolio/portfolio.py
+++ b/src/savoia/portfolio/portfolio.py
@@ -54,7 +54,7 @@ class Portfolio(object):
     def _create_equity_file(self) -> TextIO:
         filename: str = "backtest.csv"
         out_file: TextIO = open(os.path.join(OUTPUT_RESULTS_DIR, filename), "w")
-        header: str = "Timestamp,Balance"
+        header: str = "Timestamp,Equity"
         for pair in self.ticker.pairs:
             header += ",%s" % pair
         header += "\n"
@@ -102,7 +102,7 @@ class Portfolio(object):
         if self.isBacktest:
             out_line = f'{event.time}, {self.equity}'
             for pair in self.ticker.pairs:
-                out_line += ",{self.positions[pair].upl}"
+                out_line += f",{self.positions[pair].upl}"
             out_line += "\n"
             self.backtest_file.write(out_line)
 


Test patch:
diff --git a/tests/datafeed/datafeed_test.py b/tests/datafeed/datafeed_test.py
index 083bb66..847de37 100644
--- a/tests/datafeed/datafeed_test.py
+++ b/tests/datafeed/datafeed_test.py
@@ -54,18 +54,13 @@ def test_open_convert_csv_files_for_day(
     frame including all the ticks of a specified date.
     """
     ticker, df = setupDataFeeder
-    expected_frame = pd.io.parsers.read_csv(
+    expected_frame = open(
         os.path.join(df.csv_dir, 'expected_frame_%s.csv' % (date)),
-        header=0,
-        index_col=0,
-        parse_dates=["Time"],
-        dayfirst=True,
-        names=("Time", "Ask", "Bid", "AskVolume", "BidVolume", "Pair")
+        'r'
     )
     actual_frame = df._open_convert_csv_files_for_day(date)
-    for (a0, a1), (e0, e1) in zip(actual_frame, expected_frame.iterrows()):
-        assert a0 == e0
-        pd.testing.assert_series_equal(a1, e1)
+    for a0, e0 in zip(actual_frame, expected_frame):
+        assert a0 == tuple(e0[:-1].split(','))
 
 
 def test_update_csv_for_day(setupDataFeeder: Tuple[Ticker, DataFeeder]) -> None:
@@ -75,18 +70,12 @@ def test_update_csv_for_day(setupDataFeeder: Tuple[Ticker, DataFeeder]) -> None:
     ticker, df = setupDataFeeder
     assert df._update_csv_for_day() is True
     
-    expected_frame = pd.io.parsers.read_csv(
+    expected_frame = open(
         os.path.join(df.csv_dir, 'expected_frame_%s.csv' % (20140102)),
-        header=0,
-        index_col=0,
-        parse_dates=["Time"],
-        dayfirst=True,
-        names=("Time", "Ask", "Bid", "AskVolume", "BidVolume", "Pair")
+        'r'
     )
-    for (a0, a1), (e0, e1) in zip(df.cur_date_pairs,
-                                  expected_frame.iterrows()):
-        assert a0 == e0
-        pd.testing.assert_series_equal(a1, e1)
+    for a0, e0 in zip(df.cur_date_pairs, expected_frame):
+        assert a0 == tuple(e0[:-1].split(','))
 
     assert df._update_csv_for_day() is False
 
@@ -117,4 +106,4 @@ def test_run(setupDataFeeder: Tuple[Ticker, DataFeeder]) -> None:
     ticker, df = setupDataFeeder
     df.run()
     assert df.continue_backtest is False
-    assert df.feed_q.qsize() == 10
+    assert df.feed_q.qsize() == 11
diff --git a/tests/datafeed/expected_frame_20140101.csv b/tests/datafeed/expected_frame_20140101.csv
index 3cdb2e6..38e794a 100644
--- a/tests/datafeed/expected_frame_20140101.csv
+++ b/tests/datafeed/expected_frame_20140101.csv
@@ -1,6 +1,5 @@
-Time,Ask,Bid,AskVolume,BidVolume,Pair
-01.01.2014 00:02:24.967,1.50054,1.49854,2.46,2.2,GBPUSD
-01.01.2014 00:03:24.967,120.005,119.985,2.46,2.2,USDJPY
-01.01.2014 00:04:42.625,1.49979,1.49779,1.12,2.73,GBPUSD
-01.01.2014 00:06:42.625,119.998,119.978,1.12,2.73,USDJPY
-01.01.2014 00:07:18.417,1.50256,1.50056,1.04,2.94,GBPUSD
\ No newline at end of file
+01.01.2014 00:02:24.967,1.50054,1.49854,GBPUSD
+01.01.2014 00:03:24.967,120.005,119.985,USDJPY
+01.01.2014 00:04:42.625,1.49979,1.49779,GBPUSD
+01.01.2014 00:06:42.625,119.998,119.978,USDJPY
+01.01.2014 00:07:18.417,1.50256,1.50056,GBPUSD
diff --git a/tests/datafeed/expected_frame_20140101_old.csv b/tests/datafeed/expected_frame_20140101_old.csv
new file mode 100644
index 0000000..3cdb2e6
--- /dev/null
+++ b/tests/datafeed/expected_frame_20140101_old.csv
@@ -0,0 +1,6 @@
+Time,Ask,Bid,AskVolume,BidVolume,Pair
+01.01.2014 00:02:24.967,1.50054,1.49854,2.46,2.2,GBPUSD
+01.01.2014 00:03:24.967,120.005,119.985,2.46,2.2,USDJPY
+01.01.2014 00:04:42.625,1.49979,1.49779,1.12,2.73,GBPUSD
+01.01.2014 00:06:42.625,119.998,119.978,1.12,2.73,USDJPY
+01.01.2014 00:07:18.417,1.50256,1.50056,1.04,2.94,GBPUSD
\ No newline at end of file
diff --git a/tests/datafeed/expected_frame_20140102.csv b/tests/datafeed/expected_frame_20140102.csv
index 68e3163..3743930 100644
--- a/tests/datafeed/expected_frame_20140102.csv
+++ b/tests/datafeed/expected_frame_20140102.csv
@@ -1,6 +1,5 @@
-Time,Ask,Bid,AskVolume,BidVolume,Pair
-02.01.2014 00:02:24.967,1.50054,1.49854,2.46,2.2,GBPUSD
-02.01.2014 00:03:24.967,120.005,119.985,2.46,2.2,USDJPY
-02.01.2014 00:04:42.625,1.49979,1.49779,1.12,2.73,GBPUSD
-02.01.2014 00:06:42.625,119.998,119.978,1.12,2.73,USDJPY
-02.01.2014 00:07:18.417,1.50256,1.50056,1.04,2.94,GBPUSD
\ No newline at end of file
+02.01.2014 00:02:24.967,1.50054,1.49854,GBPUSD
+02.01.2014 00:03:24.967,120.005,119.985,USDJPY
+02.01.2014 00:04:42.625,1.49979,1.49779,GBPUSD
+02.01.2014 00:06:42.625,119.998,119.978,USDJPY
+02.01.2014 00:07:18.417,1.50256,1.50056,GBPUSD
diff --git a/tests/datafeed/expected_frame_20140102_old.csv b/tests/datafeed/expected_frame_20140102_old.csv
new file mode 100644
index 0000000..68e3163
--- /dev/null
+++ b/tests/datafeed/expected_frame_20140102_old.csv
@@ -0,0 +1,6 @@
+Time,Ask,Bid,AskVolume,BidVolume,Pair
+02.01.2014 00:02:24.967,1.50054,1.49854,2.46,2.2,GBPUSD
+02.01.2014 00:03:24.967,120.005,119.985,2.46,2.2,USDJPY
+02.01.2014 00:04:42.625,1.49979,1.49779,1.12,2.73,GBPUSD
+02.01.2014 00:06:42.625,119.998,119.978,1.12,2.73,USDJPY
+02.01.2014 00:07:18.417,1.50256,1.50056,1.04,2.94,GBPUSD
\ No newline at end of file
diff --git a/tests/engine/engine_test.py b/tests/engine/engine_test.py
index 9011228..a74e11e 100644
--- a/tests/engine/engine_test.py
+++ b/tests/engine/engine_test.py
@@ -7,53 +7,60 @@ import logging.config
 import os
 import json
 
+import pytest
 
-def setup_logging() -> None:
-    """Setup logging configuration"""
-    path = os.path.join('/Users/makoto/Pywork/savoia/src/savoia/config',
-        'logging.json')
-    if os.path.exists(path):
-        with open(path, 'rt') as f:
-            config = json.load(f)
-            logging.config.dictConfig(config)
-    else:
-        raise FileNotFoundError('Not exist : %s' % path)
-
-
-datafeed: datafeed_params = {
-    'module_name': 'HistoricCSVDataFeeder',
-    'params': {
-        'csv_dir': CSV_DATA_DIR,
+
+@pytest.mark.skip()
+def test_engine_run() -> None:
+    def setup_logging() -> None:
+        """Setup logging configuration"""
+        path = os.path.join('/Users/makoto/Pywork/savoia/src/savoia/config',
+            'logging.json')
+        if os.path.exists(path):
+            with open(path, 'rt') as f:
+                config = json.load(f)
+                logging.config.dictConfig(config)
+        else:
+            raise FileNotFoundError('Not exist : %s' % path)
+
+    datafeed: datafeed_params = {
+        'module_name': 'HistoricCSVDataFeeder',
+        'params': {
+            'csv_dir': CSV_DATA_DIR,
+        }
+    }
+
+    execution: execution_params = {
+        'module_name': 'SimulatedExecution',
+        'params': {
+            'heartbeat': 0
+        }
+    }
+
+    strategy: strategy_params = {
+        'module_name': 'DummyStrategy',
+        'params': {}
     }
-}
 
-execution: execution_params = {
-    'module_name': 'SimulatedExecution',
-    'params': {
-        'heartbeat': 0
+    engine: engine_params = {
+        'pairs': ['GBPUSD', 'USDJPY'],
+        'home_currency': 'JPY',
+        'equity': Decimal(10 ** 6),
+        'isBacktest': True,
+        'max_iters': 10 ** 7,
+        'heart_beat': 0
     }
-}
-
-strategy: strategy_params = {
-    'module_name': 'DummyStrategy',
-    'params': {}
-}
-
-engine: engine_params = {
-    'pairs': ['GBPUSD', 'USDJPY'],
-    'home_currency': 'JPY',
-    'equity': Decimal(10 ** 6),
-    'isBacktest': True,
-    'max_iters': 10 ** 7,
-    'heart_beat': 0
-}
-
-eg = Engine(
-    engine=engine,
-    datafeed=datafeed,
-    execution=execution,
-    strategy=strategy
-)
-
-setup_logging()
-eg.run()
+
+    eg = Engine(
+        engine=engine,
+        datafeed=datafeed,
+        execution=execution,
+        strategy=strategy
+    )
+
+    setup_logging()
+    eg.run()
+
+
+if __name__ == '__main__':
+    test_engine_run()
diff --git a/tests/portfolio/portfolio_test.py b/tests/portfolio/portfolio_test.py
index 05939df..12f1982 100644
--- a/tests/portfolio/portfolio_test.py
+++ b/tests/portfolio/portfolio_test.py
@@ -64,7 +64,7 @@ def test_create_equity_file(port: Portfolio) -> None:
 
     out_file.close()
     with open(filepath, "r") as f:
-        assert f.read() == "Timestamp,Balance,GBPUSD,USDJPY\n"
+        assert f.read() == "Timestamp,Equity,GBPUSD,USDJPY\n"
 
 
 # ================================================================