fix return in total portfolio card

appComponents.py

@@ -11,11 +11,16 @@ import param
 import styling
 import description
 import plotly.graph_objs as go
+import utils
 # import warnings
 pn.extension('mathjax')
 pn.extension('plotly')
 
+
 class TotalReturnCard(Viewer):
+    '''
+    summary on the portfolio performance vs benchmark performance
+    '''
 
     start_date = param.Parameter()
     end_date = param.Parameter()
@@ -33,10 +38,11 @@ class TotalReturnCard(Viewer):
         # Calculate the risk, tracking error, active return
 
         # get the result from  entry with max time
-        most_recent_row = self.result.loc[self.result.time.idxmax()]
-        active_return = most_recent_row.active_return
+        most_recent_row = self.portfolio_df.loc[self.portfolio_df.period.idxmax(
+        )]
+        active_return = most_recent_row.cum_return_p - most_recent_row.cum_return_b
         tracking_error = most_recent_row.tracking_error
-        total_return = most_recent_row.weighted_return_p
+        total_return = most_recent_row.cum_return_p
         mkt_cap = most_recent_row.cash
         risk = most_recent_row.risk
 
@@ -151,64 +157,16 @@ class TotalReturnCard(Viewer):
 
         return report
 
-    def _create_result_df(self, analytic_b, analytic_p):
-        '''
-        calculate weighted return, tracking error, risk for the whole portfolio 
-        '''
-        return_b_df = processing.calculate_weighted_return(
-            analytic_b, self.start_date, self.end_date)
-        return_p_df = processing.calculate_weighted_return(
-            analytic_p, self.start_date, self.end_date)
-
-        # weighted pct
-        processing.calculate_weighted_pct(return_b_df)
-        processing.calculate_weighted_pct(return_p_df)
-
-        # not needed but to accomendate post processing
-        return_b_df['in_benchmark'] = True
-        return_p_df['in_portfolio'] = False
-        merged_df = pd.merge(return_b_df, return_p_df, on=[
-                             'ticker', 'time'], how='outer', suffixes=('_b', '_p'))
-        processing.post_process_merged_analytic_df(merged_df)
-
-        # fill emtpy weighted_return with 0
-        # merged_df['weighted_return_b'] = merged_df['weighted_return_b'].fillna(0)
-        # merged_df['weighted_return_p'] = merged_df['weighted_return_p'].fillna(0)
-
-        # aggregate on date
-        result = merged_df.groupby('time').aggregate({'weighted_return_p': 'sum',
-                                                      'weighted_return_b': 'sum',
-                                                      "cash": 'sum',
-                                                      'weighted_pct_p': 'sum',
-                                                      'weighted_pct_b': 'sum',
-                                                      })
-        # active return
-        result['active_return'] = result.weighted_return_p - \
-            result.weighted_return_b
-
-        result.sort_values('time', inplace=True)
-        # tracking error
-        result['tracking_error'] = result['active_return'].rolling(
-            len(result), min_periods=1).std() * np.sqrt(252)
-
-        # risk std of pct
-        result['risk'] = result['weighted_pct_b'].rolling(
-            len(result), min_periods=1).std() * np.sqrt(252)
-
-        # result.time = result.index
-        result.reset_index(inplace=True)
-        return result
-
     def create_plot(self):
 
-        fig = px.line(self.result, y=[
-                      'weighted_return_p', 'weighted_return_b'])
+        fig = px.line(self.portfolio_df, x='period', y=[
+                      'cum_return_p', 'cum_return_b'])
         fig.update_traces(mode="lines+markers",
                           marker=dict(size=5), line=dict(width=2))
         fig.update_layout(styling.plot_layout)
         colname_to_name = {
-            'weighted_return_p': 'Portfolio回报',
-            'weighted_return_b': 'benchmark回报'
+            'cum_return_p': 'Portfolio回报',
+            'cum_return_b': 'benchmark回报'
         }
         fig.for_each_trace(lambda t: t.update(name=colname_to_name.get(t.name, t.name),
                                               legendgroup=colname_to_name.get(
@@ -219,9 +177,16 @@ class TotalReturnCard(Viewer):
         # fig.layout.autosize = True
         return fig.to_dict()
 
+    def create_portfolio_df(self):
+        clip_p = utils.clip_df(self.start_date, self.end_date, self.p_stock_df)
+        clip_b = utils.clip_df(self.start_date, self.end_date, self.b_stock_df)
+        return processing.get_portfolio_anlaysis(analytic_b=clip_b,
+                                                 analytic_p=clip_p)
+
     @param.depends('start_date', 'end_date', 'b_stock_df', 'p_stock_df', watch=True)
     def update(self):
-        self.result = self._create_result_df(self.p_stock_df, self.b_stock_df)
+
+        self.portfolio_df = self.create_portfolio_df()
         fig = self.create_plot()
         report = self.create_report()
         self.report.object = report
@@ -239,7 +204,7 @@ class TotalReturnCard(Viewer):
         )
         self.start_date = self._date_range.value_start
         self.end_date = self._date_range.value_end
-        self.result = self._create_result_df(b_stock_df, p_stock_df)
+        self.portfolio_df = self.create_portfolio_df()
         self.plot_pane = pn.pane.Plotly(
             self.create_plot(), sizing_mode='stretch_width')
 
@@ -290,7 +255,7 @@ class DrawDownCard(Viewer):
             'cash': 'sum',
             'pnl': 'sum',
         })
-        
+
         # calcualte cum pnl
         agg_df['cum_pnl'] = agg_df['pnl'].cumsum()
 
@@ -716,21 +681,21 @@ class TopHeader(Viewer):
 
     def _process(self):
         '''calculate accumulative pnl, total return and Max Drawdown on return'''
-        
+
         # return
         result_df = processing.calculate_weighted_return(self.eval_df)
-        
+
         # merge by date
         agg_df = result_df.groupby('time').aggregate({
-                    'weighted_return': 'sum',
-                    'cash': 'sum',
+            'weighted_return': 'sum',
+            'cash': 'sum',
                     'pnl': 'sum',
-                })
+        })
         agg_df.reset_index(inplace=True)
 
         # accumulative pnl
         agg_df['cum_pnl'] = agg_df['pnl'].cumsum()
-        
+
         # calcualte drawdown
         result = processing.calculate_draw_down_on(agg_df)
         max_draw_down = result.drawn_down.min()

index_page.py

@@ -7,10 +7,8 @@ import db_operation as db
 pn.extension('mathjax')
 
 pn.extension('plotly')
-pn.extension('tabulator')
-db_url = 'sqlite:///instance/local.db'
-engine = create_engine(db_url)
 
+pn.extension('tabulator')
 
 analytic_p = db.get_portfolio_analytic_df()
 analytic_b = db.get_benchmark_analytic_df()
@@ -47,6 +45,8 @@ else:
 
 template = pn.template.FastListTemplate(
     title="Portfolio一览",
+    side_bar_width=200,
+    collapsed_sidebar = True,
     # sidebar=[freq, phase],
 )
 template.sidebar.append(SideNavBar())

processing.py

@@ -335,9 +335,9 @@ def calculate_log_return(df: pd.DataFrame):
     grouped = inter_df.groupby('ticker')
     inter_df['prev_w'] = grouped['weight'].shift(1)
     inter_df['prev_close'] = grouped['close'].shift(1)
+    inter_df['weighted_log_return'] = np.log(
+        (inter_df['close'] / inter_df['prev_close']) * inter_df['prev_w'])
     inter_df['log_return'] = np.log(inter_df['close'] / inter_df['prev_close'])
-    inter_df['weighted_log_return'] = inter_df['log_return'] * \
-        inter_df['prev_w']
     # patch
     df['log_return'] = inter_df['log_return']
     df['weighted_log_return'] = inter_df['weighted_log_return']
@@ -426,10 +426,10 @@ def create_analytic_df(price_df, profile_df):
     # daily stock price use begin of the date, need to convert profile_df day to begin of the date
     profile_df['time'] = profile_df['time'].map(
         lambda x: datetime(x.year, x.month, x.day))
-    
+
     # make every time entry the same dimension
     uni_profile_df = _uniformize_time_series(profile_df)
-    
+
     # TODO handle rename column here
     df = price_df.merge(uni_profile_df, on=['ticker', 'time'], how='outer')
     df.sort_values(by=['ticker', 'time'], inplace=True)
@@ -464,8 +464,6 @@ def calculate_attributes_between_dates(start, end, calculated_p_stock, calculate
     b_ranged_df = calculated_b_stock[(calculated_b_stock.date >= start) & (
         calculated_b_stock.date <= end)]
 
-    # return and weight of portfolio
-    p_start_df = p_ranged_df[p_ranged_df.date == p_ranged_df.date.min()]
     p_end_df = p_ranged_df[p_ranged_df.date == p_ranged_df.date.max()]
     p_concat = pd.concat([p_start_df, p_end_df])
     # pct is unweighted return
@@ -812,12 +810,56 @@ def calculate_draw_down_on(df, key='weighted_return'):
 
         return df
 
-# def calculate_accumulative_pnl(df):
-#     '''
-#     calculate accumulative pnl on analytic df
-#     '''
-#     df = df.sort_values(by=['time'])
-#     df['accumulative_pnl'] = df.groupby('ticker')['pnl'].rolling(
 
-#     )
-#     return df
+def _daily_return(df: pd.DataFrame):
+    '''
+    patch df with daily return
+    helper function for get_portfolio_anlaysis
+    '''
+    prev_ws = df.groupby('ticker')['weight'].shift(1)
+    df['return'] = df.pct * prev_ws
+
+
+def _agg_on_day(df: pd.DataFrame):
+    df['period'] = df.time.dt.to_period('D')
+    on_column = {'return': 'sum'}
+    if 'cash' in df.columns:
+        on_column['cash'] = 'sum'
+    if 'pnl' in df.columns:
+        on_column['pnl'] = 'sum'
+
+    agg_df = df.groupby('period').agg(on_column)
+    return agg_df.reset_index()
+
+
+def get_portfolio_anlaysis(analytic_p, analytic_b):
+    '''
+    return df contain daily pnl, daily return, accumulative return
+    risk and tracking error of portfolio and benchmark
+    '''
+    
+    # daily return(weighted pct)
+    _daily_return(analytic_p)
+    _daily_return(analytic_b)
+
+    # aggregate to daily
+    agg_p = _agg_on_day(analytic_p)
+    agg_b = _agg_on_day(analytic_b)
+
+    # accumulative return
+    agg_p['cum_return'] = (agg_p['return']+1).cumprod() - 1
+    agg_b['cum_return'] = (agg_b['return']+1).cumprod() - 1
+
+    # merge
+    merged_df = pd.merge(
+        agg_p, agg_b, on=['period'], how='outer', suffixes=('_p', '_b'))
+    merged_df.sort_values('period', inplace=True)
+
+    # risk, using population deviation
+    merged_df['risk'] = merged_df['return_p'].expanding(min_periods=1).std()
+
+    # tracking error
+    merged_df['tracking_error'] = (
+        merged_df['return_p'] - merged_df['return_b']).expanding(min_periods=1).std()
+
+    return merged_df

utils.py

@@ -8,6 +8,13 @@ from sqlalchemy import create_engine
 db_url = 'sqlite:///instance/local.db'
 
 
+def clip_df(start, end, df: pd.DataFrame, on='time'):
+    '''
+    return a copy of df between start and end date inclusive
+    '''
+    return df[df.time.between(start, end, inclusive='both')].copy()
+
+
 def time_in_beijing(strip_time_zone=True):
     '''
     return current time in Beijing as datetime object
@@ -70,14 +77,14 @@ def create_stocks_entry_from_excel(byte_string):
     '''
     uploaded_df = None
     with io.BytesIO(byte_string) as f:
-        uploaded_df = pd.read_excel(f)
+        uploaded_df = pd.read_excel(f, index_col=None)
 
     # throw exception if doesn't have required columns
     if not set(['证券代码', '持仓数量', '平均建仓成本', 'time_stamp']).issubset(uploaded_df.columns):
         raise Exception('Missing required columns')
     # print(uploaded_df)
     # uploaded_df = pd.read_excel()
-    uploaded_df.drop(columns='Unnamed: 0', inplace=True)
+    # uploaded_df.drop(columns='Unnamed: 0', inplace=True)
     # Define the regular expression pattern to match the string endings
     pattern = r'\.(sz|sh)$'
     # Define the replacement strings for each match group
@@ -153,7 +160,7 @@ def create_html_report(result: list[tuple]):
         title: str, title to display
         value: any, value to display
         type: str, used to format value
-    
+
     Returns
     -------
     html: str