Repository: quantopian/zipline

Issue:
BUG: data.history with frequency of "1d" does not work for panel minute data 
With minute panel data now supported thanks to 9103516 I tried a simple example algo which fetches daily data through data.history.
However, this gives me the following error:

_data_portal.py, line 1094, in _get_daily_window_for_sids_
**AttributeError: 'DataPortal' object has no attribute '_equity_history_loader'**

Below the code to reproduce this issue:

``` python
from zipline import run_algorithm
from datetime import datetime
from pytz import UTC
from zipline.data.loader import load_from_yahoo
from zipline.api import order, symbol, record, sid
import pandas as pd
import numpy as np
from zipline.utils.calendars import get_calendar

# Taken from zipline/tests/test_algorithm
def create_minute_df_for_asset(trading_calendar,
                               start_dt,
                               end_dt,
                               interval=1,
                               start_val=1,
                               minute_blacklist=None):

    asset_minutes = trading_calendar.minutes_for_sessions_in_range(
        start_dt, end_dt
    )
    minutes_count = len(asset_minutes)
    minutes_arr = np.array(range(start_val, start_val + minutes_count))

    df = pd.DataFrame(
        {
            "open": minutes_arr + 1,
            "high": minutes_arr + 2,
            "low": minutes_arr - 1,
            "close": minutes_arr,
            "volume": 100 * minutes_arr,
        },
        index=asset_minutes,
    )

    if interval > 1:
        counter = 0
        while counter < len(minutes_arr):
            df[counter:(counter + interval - 1)] = 0
            counter += interval

    if minute_blacklist is not None:
        for minute in minute_blacklist:
            df.loc[minute] = 0

    return df

def initialize(context):
    context.security = sid(1)

def handle_data(context, data):
    price_history = data.history(context.security, "price", 200, "1d")
    order(sid(1), 10)

if __name__ == '__main__':
    start = datetime(2009, 1, 1, tzinfo=UTC)
    end = datetime(2016, 1, 1, tzinfo=UTC)
    capital_base = 100000

    trading_calendar = get_calendar('NYSE')
    dt_transform = trading_calendar.minute_to_session_label

    data = create_minute_df_for_asset(get_calendar('NYSE'), datetime(1999, 1, 4), datetime(2016, 7, 29))
    dfs = {1 : data }
    panel = pd.Panel(dfs)

    results = run_algorithm(
                  start,
                  end,
                  initialize,
                  capital_base,
                  handle_data=handle_data,
                  data=panel,
                  data_frequency='minute',
                  bundle=None,
                  default_extension=False,
                  )
```

Best,
Mario



Discussion:
@nathanwolfe this is happening because we're now only passing a minutely reader if the user is running minutely, so daily-frequency history calls don't work.  We either need to have the minutely panel reader able to do daily aggregation on the fly, or we need to take the input data, aggregate it once, and then wrap that up in a daily bar reader so that we can still service daily data queries.

@ssanderson: Which method do you think would be better? Personally I think aggregating it once would be better, since people using Panel data probably pass in data for the few equities that they are interested in, so there would be less loss in doing the aggregation for equities that don't have `data.history` called on them.

@nathanwolfe I think upfront aggregation makes sense here.

The real win from a performance perspective that we don't end up re-doing the aggregation over any given period more than once. (If we did it on the fly, then we'd end up calculating OHLCV aggregations on the same day over and over if you called history() once per day.)  The challenge is that we need to make sure we're using the user's calendar to bucket minutes into days.  The naive implementation of just taking the date of each day only works for NYSE.

cc @jbredeche I think to support this correctly we need a fast vectorized `minute_to_session_label` that can take an array/datetimeindex of minutes and convert to their session labels.

PR: https://github.com/quantopian/zipline/pull/1407

Just the fast `minute_to_session_label` for large DatetimeIndexes of minutes is done so far.


Resolving patch:
diff --git a/zipline/algorithm.py b/zipline/algorithm.py
--- a/zipline/algorithm.py
+++ b/zipline/algorithm.py
@@ -38,6 +38,7 @@
 from zipline._protocol import handle_non_market_minutes
 from zipline.assets.synthetic import make_simple_equity_info
 from zipline.data.data_portal import DataPortal
+from zipline.data.resample import minute_panel_to_session_panel
 from zipline.data.us_equity_pricing import PanelBarReader
 from zipline.errors import (
     AttachPipelineAfterInitialize,
@@ -677,21 +678,33 @@ def run(self, data=None, overwrite_sim_params=True):
                     )
                 )
 
-                if self.sim_params.data_frequency == 'daily':
-                    equity_reader_arg = 'equity_daily_reader'
-                elif self.sim_params.data_frequency == 'minute':
-                    equity_reader_arg = 'equity_minute_reader'
                 equity_reader = PanelBarReader(
                     self.trading_calendar,
                     copy_panel,
                     self.sim_params.data_frequency,
                 )
+                if self.sim_params.data_frequency == 'daily':
+                    equity_readers = {
+                        'equity_daily_reader': equity_reader,
+                    }
+                elif self.sim_params.data_frequency == 'minute':
+                    equity_readers = {
+                        'equity_minute_reader': equity_reader,
+                        'equity_daily_reader': PanelBarReader(
+                            self.trading_calendar,
+                            minute_panel_to_session_panel(
+                                copy_panel,
+                                self.trading_calendar,
+                            ),
+                            'daily',
+                        ),
+                    }
 
                 self.data_portal = DataPortal(
                     self.asset_finder,
                     self.trading_calendar,
                     first_trading_day=equity_reader.first_trading_day,
-                    **{equity_reader_arg: equity_reader}
+                    **equity_readers
                 )
 
         # Force a reset of the performance tracker, in case
diff --git a/zipline/data/resample.py b/zipline/data/resample.py
--- a/zipline/data/resample.py
+++ b/zipline/data/resample.py
@@ -65,6 +65,59 @@ def minute_frame_to_session_frame(minute_frame, calendar):
     return minute_frame.groupby(calendar.minute_to_session_label).agg(how)
 
 
+def minute_panel_to_session_panel(minute_panel, calendar):
+    """
+    Resample a Panel with minute data into a daily data Panel accepted by
+    PanelBarReader.
+
+    Parameters
+    ----------
+    minute_panel : pd.Panel
+        A pricing data Panel with axes:
+          * items: assets
+          * major_axis: minute pd.DatetimeIndex
+          * minor_axis: labels ['open', 'high', 'low', 'close', 'volume']
+    calendar : zipline.utils.calendars.trading_calendar.TradingCalendar
+        A TradingCalendar on which session labels to resample from minute
+        to session.
+    """
+    def get_first_valid(df):
+        """
+        Return a Series with the first non-nan value in each column.
+
+        On an all-nan column, this function gives nan.
+        """
+        # Indices of first non-nan values
+        idxs = pd.notnull(df).idxmax()
+
+        # Expand indices to one-hot boolean columns
+        indexer = pd.get_dummies(idxs).T.astype(bool)
+
+        # Preserve all indices' values, while others are masked to nan.
+        # Then `.max()` selects the non-nan value in each column.
+        return df.loc[indexer.index][indexer].max()
+
+    def sum_or_nan(df):
+        """
+        Return `df.sum()`, but nan where the whole column is nan.
+        """
+        return df.sum().mask(pd.isnull(df).all(axis=0))
+
+    def aggregator(session_panel):
+        return pd.DataFrame({
+            'open': get_first_valid(session_panel.loc[:, :, 'open']),
+            'close': get_first_valid(session_panel.loc[:, ::-1, 'close']),
+            'high': session_panel.loc[:, :, 'high'].max(),
+            'low': session_panel.loc[:, :, 'low'].min(),
+            'volume': sum_or_nan(session_panel.loc[:, :, 'volume']),
+        }).T
+
+    session_index = calendar.minute_index_to_session_labels(
+        minute_panel.major_axis
+    )
+    return minute_panel.groupby(session_index).agg(aggregator)
+
+
 def minute_to_session(column, close_locs, data, out):
     """
     Resample an array with minute data into an array with session data.


Test patch:
diff --git a/tests/data/test_resample.py b/tests/data/test_resample.py
--- a/tests/data/test_resample.py
+++ b/tests/data/test_resample.py
@@ -23,6 +23,7 @@
 
 from zipline.data.resample import (
     minute_frame_to_session_frame,
+    minute_panel_to_session_panel,
     DailyHistoryAggregator,
     MinuteResampleSessionBarReader,
     ReindexMinuteBarReader,
@@ -563,6 +564,21 @@ def test_minute_to_session(self):
                                 result.values,
                                 err_msg='sid={0}'.format(sid))
 
+    def test_minute_panel_to_session_panel(self):
+        minute_panel = pd.Panel(
+            {sid: self.equity_frames[sid]
+             for sid in self.ASSET_FINDER_EQUITY_SIDS}
+        )
+        expected = pd.Panel(
+            {sid: EXPECTED_SESSIONS[sid]
+             for sid in self.ASSET_FINDER_EQUITY_SIDS}
+        )
+        result = minute_panel_to_session_panel(
+            minute_panel,
+            self.nyse_calendar
+        )
+        assert_almost_equal(expected.values, result.values)
+
 
 class TestResampleSessionBars(WithBcolzFutureMinuteBarReader,
                               ZiplineTestCase):
diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py
--- a/tests/test_algorithm.py
+++ b/tests/test_algorithm.py
@@ -4813,6 +4813,16 @@ def analyze(context, results):
 
 class TestPanelData(WithTradingEnvironment, ZiplineTestCase):
 
+    def create_panel(self, sids, trading_calendar, start_dt, end_dt,
+                     create_df_for_asset, prev_close_column=False):
+        dfs = {}
+        for sid in sids:
+            dfs[sid] = create_df_for_asset(trading_calendar,
+                                           start_dt, end_dt, interval=sid)
+            if prev_close_column:
+                dfs[sid]['prev_close'] = dfs[sid]['close'].shift(1)
+        return pd.Panel(dfs)
+
     @parameterized.expand([
         ('daily',
          pd.Timestamp('2015-12-23', tz='UTC'),
@@ -4838,12 +4848,8 @@ def dt_transform(dt):
                                  data_frequency)
 
         sids = range(1, 3)
-        dfs = {}
-        for sid in sids:
-            dfs[sid] = create_df_for_asset(trading_calendar,
-                                           start_dt, end_dt, interval=sid)
-            dfs[sid]['prev_close'] = dfs[sid]['close'].shift(1)
-        panel = pd.Panel(dfs)
+        panel = self.create_panel(sids, trading_calendar, start_dt, end_dt,
+                                  create_df_for_asset, prev_close_column=True)
 
         price_record = pd.Panel(items=sids,
                                 major_axis=panel.major_axis,
@@ -4851,9 +4857,7 @@ def dt_transform(dt):
 
         def initialize(algo):
             algo.first_bar = True
-            algo.equities = []
-            for sid in sids:
-                algo.equities.append(algo.sid(sid))
+            algo.equities = [algo.sid(sid) for sid in sids]
 
         def handle_data(algo, data):
             price_record.loc[:, dt_transform(algo.get_datetime()),
@@ -4899,3 +4903,47 @@ def check_panels():
                 environ={'ZIPLINE_ROOT': root},
             )
             check_panels()
+
+    def test_minute_panel_daily_history(self):
+        sids = range(1, 3)
+        trading_calendar = get_calendar('NYSE')
+        start_dt = pd.Timestamp('2015-12-23', tz='UTC')
+        end_dt = pd.Timestamp('2015-12-30', tz='UTC')
+
+        panel = self.create_panel(
+            sids,
+            trading_calendar,
+            start_dt,
+            end_dt,
+            create_minute_df_for_asset,
+        )
+
+        def check_open_price(algo, data):
+            if algo.first_day:
+                algo.first_day = False
+            else:
+                np.testing.assert_array_equal(
+                    algo.last_open,
+                    data.history(
+                        algo.equities,
+                        'open',
+                        2,
+                        '1d',
+                    ).iloc[0]
+                )
+            algo.last_open = data.current(algo.equities, 'open')
+
+        def initialize(algo):
+            algo.first_day = True
+            algo.equities = [algo.sid(sid) for sid in sids]
+
+            algo.schedule_function(
+                check_open_price,
+                date_rules.every_day(),
+                time_rules.market_open(),
+            )
+
+        with tmp_trading_env(load=self.make_load_function()) as env:
+            trading_algo = TradingAlgorithm(initialize=initialize,
+                                            env=env)
+            trading_algo.run(data=panel)