move files to normal tracking except .db

.gitattributes

@@ -33,10 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-*.md filter=lfs diff=lfs merge=lfs -text
-*.gitignore filter=lfs diff=lfs merge=lfs -text
-/Dockerfile filter=lfs diff=lfs merge=lfs -text
-*.py filter=lfs diff=lfs merge=lfs -text
-*.ipynb filter=lfs diff=lfs merge=lfs -text
-*.txt filter=lfs diff=lfs merge=lfs -text
 *.db filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,3 +1,16 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a9c00611604e6f4573bccfadf332fabb75d29da697418d25490513ab1a3efbfe
-size 478
+FROM python:3.11
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+RUN python3 -m pip install --no-cache-dir --upgrade pip
+RUN python3 -m pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . /code
+
+CMD ["panel", "serve", "/code/portfolioEditingPage.py", "--address", "0.0.0.0", "--port", "7860", "--allow-websocket-origin", "lamonkey-portfolio-management.hf.space"]
+
+RUN mkdir /.cache
+RUN chmod 777 /.cache
+RUN mkdir .chroma
+RUN chmod 777 .chroma

README.md

@@ -1,3 +1,9 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:81ce2114bb00e93ca40ab89241d28276a2a84fde1ca34e9d9667f8e78105ad37
-size 206
+---
+title: Portfolio Management App
+emoji: 📈
+colorFrom: gray
+colorTo: green
+sdk: docker
+pinned: false
+---
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

api.py

@@ -1,3 +1,346 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:36359baa0769e9a965b43b0302a6784bd27cc8bc486a1a2f40099bc89fdd9875
-size 11874
+
+'''
+contain method for api call to jqdatasdk
+'''
+from dotenv import load_dotenv
+from datetime import datetime, timedelta
+import jqdatasdk as jq
+import pandas as pd
+from typing import List, Optional
+from sqlalchemy import create_engine
+import table_schema as ts
+import os
+db_url = 'sqlite:///local.db'
+load_dotenv()
+user_name = os.environ.get('JQDATA_USER')
+password = os.environ.get('JQDATA_PASSWORD')
+
+
+def auth_api(func):
+    """
+    decorator for function require jqdatasdk api
+    """
+    def wrapper(*args, **kwargs):
+
+        if (not jq.is_auth()):
+            jq.auth(user_name, password)
+
+        result = func(*args, **kwargs)
+        return result
+
+    return wrapper
+
+
+def aggregate_sector(input: str) -> Optional[str]:
+    '''
+    mapping from sector to aggregated sector retur None if not found
+    this handling is for spotting undefined sector in current mapping 
+    later 
+
+    Return: str -- aggregated sector
+            None if no mapping
+    '''
+    mapping = {
+        '电气设备I': '工业',
+        '建筑装饰I': '工业',
+        '交通运输I': '工业',
+        '机械设备I': '工业',
+        '国防军工I': '工业',
+        '综合I': '工业',
+        '电子I': '信息与通信',
+        '计算机I': '信息与通信',
+        '通信I': '信息与通信',
+        '传媒I': '信息与通信',
+        '纺织服装I': '消费',
+        '家用电器I': '消费',
+        '汽车I': '消费',
+        '休闲服务I': '消费',
+        '商业贸易I': '消费',
+        '食品饮料I': '消费',
+        '美容护理I': '消费',
+        '农林牧渔I': '消费',
+        '钢铁I': '原料与能源',
+        '建筑材料I': '原料与能源',
+        '有色金属I': '原料与能源',
+        '化工I': '原料与能源',
+        '轻工制造I': '原料与能源',
+        '煤炭I': '原料与能源',
+        '石油石化I': '原料与能源',
+        '采掘I': '原料与能源',
+        '医药生物I': '医药卫生',
+        '公用事业I': '公用事业',
+        '环保I': '公用事业',
+        '房地产I': '金融与地产',
+        '银行I': '金融与地产',
+        '非银金融I': '金融与地产'
+    }
+    # return the first mapping found
+    sectors = input.split(" ")
+    maped_name = "其他"
+    for sector in sectors:
+        maped_name = mapping.get(sector, None)
+        if maped_name is not None:
+            return maped_name
+
+    return maped_name
+
+
+@auth_api
+def get_all_stock_info() -> tuple[pd.DataFrame, List[str]]:
+    '''
+    return all stock information
+
+    Return
+    ------
+    tuple: tuple(pd.DataFrame, List[str])
+        DataFrame -- display_name | name | start_date | end_date | type
+    '''
+    error = []
+    try:
+        df = jq.get_all_securities()
+        df['ticker'] = df.index
+        df.reset_index(drop=True, inplace=True)
+        # df.reset_index(inplace=True)
+        return df, error
+    except Exception as e:
+        error.append(f'get_all_stock_info\n{e}')
+        return None, error
+
+
+@auth_api
+def add_detail_to_stocks(df: pd.DataFrame) -> List[str]:
+    """
+    add display_name, name, sector, and aggregate sector to each stock if not exist already
+    return a list of error message
+
+    Args: pd.DataFrame
+    ticker | date | weight | sector | aggregate_sector | display_name | name
+
+    Returns: List[str], error messages
+    """
+    error = []
+    df[['sector', 'aggregate_sector']] = df.groupby(
+        'ticker')[['sector', 'aggregate_sector']].ffill()
+    df[['display_name', 'name']] = df.groupby(
+        'ticker')[['display_name', 'name']].ffill()
+    not_have_sector = list(
+        df[df['aggregate_sector'].isnull()]['ticker'].unique())
+    not_have_name = list(df[df['name'].isnull()]['ticker'].unique())
+    # sector and aggregate sector
+    if len(not_have_sector) != 0:
+        try:
+            sectors = jq.get_industry(security=not_have_sector)
+            df['sector'] = df.apply(lambda x: x.sector if not pd.isna(x.sector)
+                                    else " ".join(value['industry_name']
+                                                  for value in sectors[x.ticker].values()), axis=1)
+            df['aggregate_sector'] = df.apply(
+                lambda x: x.aggregate_sector if not pd.isna(x.aggregate_sector)
+                else aggregate_sector(x.sector), axis=1
+            )
+        except Exception as e:
+            error.append(f'Error on creaet_sector_information\n{ticker}\n{e}')
+
+    # display_name and name
+    if len(not_have_name) != 0:
+        try:
+            for ticker in not_have_name:
+                detail = jq.get_security_info(ticker)
+                df.loc[df.ticker.isin(not_have_name)
+                       ]['display_name'] = detail.display_name
+                df.loc[df.ticker.isin(not_have_name)]['name'] = detail.name
+        except Exception as e:
+            error.append(f'Error on get display_name and name\n{ticker}\n{e}')
+
+    return error
+
+
+@auth_api
+def update_portfolio_profile(stocks: List[dict], current_p: pd.DataFrame = None) -> tuple[pd.DataFrame, List[str]]:
+    """create or update a portfolio profile,
+    return a time series of profile
+
+    Parameters
+    ----------
+    stocks : List[{ticker: Str, shares: float, date:datetime}]
+
+        update profile with a list of stock information
+
+    current_p : pd.DataFrame, optional
+        current portfolio profile, default is None
+
+    Returns
+    -------
+    updated_profile : pd.DataFrame
+        ticker | date | weight | sector | aggregate_sector | display_name | name
+
+    error : List[str]
+        a list of error message
+    """
+
+    error = []
+    profile_df = pd.DataFrame(stocks)
+    profile_df['sector'] = None
+    profile_df['aggregate_sector'] = None
+    
+    # add display_name
+    try:
+        with create_engine(db_url).connect() as conn:
+            info_df = pd.read_sql_table(ts.STOCKS_DETAILS_TABLE, conn)
+            profile_df = pd.merge(
+                profile_df, info_df[['display_name', 'ticker', 'name', 'aggregate_sector', ]], on='ticker', how='left')
+    except Exception as e:
+        error.append(f'create_portfolio \n{e}')
+
+    # get sector information
+    incoming_error = add_detail_to_stocks(profile_df)
+    error.extend(incoming_error)
+
+    # concate to existing profile if exist
+    if current_p is not None:
+        profile_df = pd.concat([profile_df, current_p], ignore_index=True)
+        profile_df.drop_duplicates(
+            subset=['ticker', 'date'], keep='last', inplace=True)
+        profile_df.reset_index(drop=True, inplace=True)
+
+    return profile_df, error
+
+
+@auth_api
+def get_all_stocks_detail():
+    '''get df contain all stock display_name, name, sector, aggregate_sector'''
+    detail_df = jq.get_all_securities()
+    detail_df['ticker'] = detail_df.index
+    detail_df.reset_index(drop=True, inplace=True)
+    industry_info = jq.get_industry(detail_df.ticker.to_list())
+    detail_df['sector'] = detail_df.apply(lambda x: " ".join(
+        value['industry_name']for value in industry_info[x.ticker].values()), axis=1)
+    detail_df['aggregate_sector'] = detail_df.apply(
+        lambda x: aggregate_sector(x.sector), axis=1)
+    return detail_df
+
+
+@auth_api
+def get_api_usage():
+    return jq.get_query_count()
+
+
+@auth_api
+def get_stocks_price(profile: pd.DataFrame, start_date: datetime, end_date: datetime, frequency='daily') -> tuple[pd.DataFrame, List[str]]:
+    """
+    Return a dataframe contain stock price between period of time for price in a portfolio profile
+
+    Arguments:
+        profile {pd.DataFrame} -- ticker | date | weight | sector | aggregate_sector | display_name | name
+        start_date {datetime} -- start date of the period include start date
+        end_date {datetime} -- end date of the period include end date
+        frequency {str} -- resolution of the price, default is daily
+
+    Returns: Tuple(pd.DataFrame, List[str])
+        pd.DataFrame -- ticker date open close high low volumn money
+        error_message {list} -- a list of error message
+    """
+    error_message = []
+    start_str = start_date.strftime('%Y-%m-%d')
+    end_str = end_date.strftime('%Y-%m-%d')
+    if profile.date.min() < start_date:
+        # hanlde benchmark doesn't have weight on the exact date
+        start_str = profile.date.min().strftime('%Y-%m-%d')
+
+    ticker = profile['ticker'].to_list()
+    try:
+
+        data = jq.get_price(ticker, start_date=start_str,
+                            end_date=end_str, frequency=frequency)
+        data.rename(columns={'time': 'date', 'code': "ticker"}, inplace=True)
+        return data, error_message
+    except Exception as e:
+        error_message.append(f'Error when fetching {ticker} \n {e}')
+        return None, error_message
+
+
+@auth_api
+def fetch_stocks_price(**params):
+    '''request list of stock price from start_date to end_date with frequency or count'''
+    stocks_df = jq.get_price(**params)
+    stocks_df.rename(columns={'code': 'ticker'}, inplace=True)
+    return stocks_df
+
+
+@auth_api
+def update_benchmark_profile(start_date: datetime,
+                             end_date: datetime,
+                             benchmark="000905.XSHG",
+                             delta_time=timedelta(days=7),
+                             profile: pd.DataFrame = None
+                             ) -> tuple[pd.DataFrame, List[str]]:
+    """
+    update benchmark profile with available new update between start_date and end_date,
+    if no profile is given, create a new profile for that duration
+    the minimum period is 1 day
+
+    return an updated dataframe if new update exist
+
+    Returns:
+        pd.DateFrame -- ticker | date | weight | sector | aggregate_sector | display_name | name
+        date| weight | display_name | actual_data(the date in the api database) | ticker
+    """
+    error_message = []
+    results = []
+    while start_date < end_date:
+        try:
+            date_str = start_date.strftime('%Y-%m-%d')
+            result = jq.get_index_weights(benchmark, date=date_str)
+            results.append(result)
+
+        except Exception as e:
+            error_message.append(f'Error when fetching {benchmark}\n\
+                                update on {date_str} is missing\n\
+                                {e}')
+
+        start_date += delta_time
+    # inlcude end date
+    try:
+        date_str = end_date.strftime('%Y-%m-%d')
+        result = jq.get_index_weights(benchmark, date=date_str)
+        results.append(result)
+    except Exception as e:
+        error_message.append(f'Error when fetching {benchmark}\n\
+                                update on {date_str} is missing\n\
+                                {e}')
+    # no update
+    if len(results) == 0:
+        return profile, error_message
+
+    # concate all result
+    update_df = pd.concat(results)
+    update_df['ticker'] = update_df.index
+    update_df['sector'] = None
+    update_df['aggregate_sector'] = None
+    update_df.reset_index(drop=True, inplace=True)
+    update_df['date'] = pd.to_datetime(update_df['date'])
+    # add display_name
+    try:
+        with create_engine(db_url).connect() as conn:
+            info_df = pd.read_sql('all_stock_info', conn)
+            update_df = pd.merge(
+                update_df, info_df[['ticker', 'name']], on='ticker', how='left')
+    except Exception as e:
+        # if all_stock_info not exist then create a name column manually
+        update_df['name'] = None
+        error_message.append(f'create_portfolio \n{e}')
+
+    # combine with existing profile if given
+    if profile is not None:
+        update_df = pd.concat([profile, update_df])
+
+    # remove duplicate result
+    update_df.drop_duplicates(
+        subset=['ticker', 'date'], keep='last', inplace=True)
+
+    # update deail
+    incoming_error = add_detail_to_stocks(update_df)
+    error_message.extend(incoming_error)
+
+    return update_df, error_message
+
+# get_all_stocks_detail()

apiMonitorPage.py

@@ -1,3 +1,40 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f131cfcd1af6e8c49821cd5e777890ada6e774ec6c8a98213700d75c85d819fc
-size 1173
+import panel as pn
+import pandas as pd
+import numpy as np
+from streamz import Stream
+from pipeline import stock_price_stream
+stream = Stream()
+stock_price_stream
+pn.extension('tabulator')
+pn.extension('vega')
+
+stream_df = pd.DataFrame(columns=['time', 'ticker', 'open', 'close', 'high', 'low',
+                                  'volume', 'money', 'in_portfolio', 'in_benchmark', 'aggregate_sector', 'display_name'])
+
+stream_table = pn.widgets.Tabulator(
+    stream_df, layout='fit_columns', width=1200, height=1200)
+# stream_table
+
+
+def stream_data(stream_df):
+    print('updating stream!!!')
+    # stream_df = pd.DataFrame(np.random.randn(5, 5), columns=list('ABCDE'))
+    stream_table.stream(stream_df, follow=True)
+
+
+def create_new_stream():
+    stream_df = pd.DataFrame(np.random.randn(5, 5), columns=list('ABCDE'))
+    stock_price_stream.emit(stream_df)
+# pn.state.add_periodic_callback(create_new_stream, period=1000, count=100)
+
+
+stock_price_stream.sink(stream_data)
+template = pn.template.FastListTemplate(
+    title='api monitor')
+# stock_price_stream.sink(print)
+template.main.extend(
+    [stream_table]
+)
+# )
+# stock_price_stream.sink(print)
+template.servable()

api_data_samples.ipynb

@@ -1,3 +1,887 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ad238008ccf71420f361d8cf659820229f2ed95fe71e2d22cd59e27c931a93b9
-size 31348
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy import create_engine\n",
+    "import pandas as pd\n",
+    "from datetime import timedelta\n",
+    "import jqdatasdk as jq\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load existing portfolio"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check if database need update\n",
+    "def fetch_data_for_table():\n",
+    "    '''\n",
+    "    return None if no update needed\n",
+    "    else return (starttime, endtime, frequency)\n",
+    "    '''\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Need to fetch stock price periodically from jqdatasdk, also need to check if portfolio updated, but this case can be handled seperately \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# fetch stock price from api\n",
+    "def fetch_stock_price():\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "auth success \n"
+     ]
+    }
+   ],
+   "source": [
+    "# TODO auth, remove later\n",
+    "user_name = \"13126862272\"\n",
+    "password = \"862272\"\n",
+    "jq.auth(user_name, password)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 09:31:00</th>\n",
+       "      <td>32.54</td>\n",
+       "      <td>31.73</td>\n",
+       "      <td>32.54</td>\n",
+       "      <td>31.72</td>\n",
+       "      <td>1205715.0</td>\n",
+       "      <td>38788451.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 09:32:00</th>\n",
+       "      <td>31.68</td>\n",
+       "      <td>32.00</td>\n",
+       "      <td>32.00</td>\n",
+       "      <td>31.39</td>\n",
+       "      <td>1084879.0</td>\n",
+       "      <td>34358448.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 09:33:00</th>\n",
+       "      <td>31.91</td>\n",
+       "      <td>31.62</td>\n",
+       "      <td>31.91</td>\n",
+       "      <td>31.61</td>\n",
+       "      <td>811350.0</td>\n",
+       "      <td>25785154.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 09:34:00</th>\n",
+       "      <td>31.62</td>\n",
+       "      <td>32.07</td>\n",
+       "      <td>32.07</td>\n",
+       "      <td>31.62</td>\n",
+       "      <td>760611.0</td>\n",
+       "      <td>24135183.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 09:35:00</th>\n",
+       "      <td>32.11</td>\n",
+       "      <td>31.98</td>\n",
+       "      <td>32.16</td>\n",
+       "      <td>31.98</td>\n",
+       "      <td>655989.0</td>\n",
+       "      <td>21046238.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 11:26:00</th>\n",
+       "      <td>32.25</td>\n",
+       "      <td>32.29</td>\n",
+       "      <td>32.29</td>\n",
+       "      <td>32.20</td>\n",
+       "      <td>118071.0</td>\n",
+       "      <td>3807211.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 11:27:00</th>\n",
+       "      <td>32.29</td>\n",
+       "      <td>32.30</td>\n",
+       "      <td>32.30</td>\n",
+       "      <td>32.21</td>\n",
+       "      <td>114251.0</td>\n",
+       "      <td>3686602.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 11:28:00</th>\n",
+       "      <td>32.26</td>\n",
+       "      <td>32.24</td>\n",
+       "      <td>32.30</td>\n",
+       "      <td>32.21</td>\n",
+       "      <td>60077.0</td>\n",
+       "      <td>1938244.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 11:29:00</th>\n",
+       "      <td>32.25</td>\n",
+       "      <td>32.28</td>\n",
+       "      <td>32.30</td>\n",
+       "      <td>32.22</td>\n",
+       "      <td>65634.0</td>\n",
+       "      <td>2117204.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-01-05 11:30:00</th>\n",
+       "      <td>32.23</td>\n",
+       "      <td>32.30</td>\n",
+       "      <td>32.30</td>\n",
+       "      <td>32.23</td>\n",
+       "      <td>49052.0</td>\n",
+       "      <td>1582947.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>120 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                      open  close   high    low     volume       money\n",
+       "2021-01-05 09:31:00  32.54  31.73  32.54  31.72  1205715.0  38788451.0\n",
+       "2021-01-05 09:32:00  31.68  32.00  32.00  31.39  1084879.0  34358448.0\n",
+       "2021-01-05 09:33:00  31.91  31.62  31.91  31.61   811350.0  25785154.0\n",
+       "2021-01-05 09:34:00  31.62  32.07  32.07  31.62   760611.0  24135183.0\n",
+       "2021-01-05 09:35:00  32.11  31.98  32.16  31.98   655989.0  21046238.0\n",
+       "...                    ...    ...    ...    ...        ...         ...\n",
+       "2021-01-05 11:26:00  32.25  32.29  32.29  32.20   118071.0   3807211.0\n",
+       "2021-01-05 11:27:00  32.29  32.30  32.30  32.21   114251.0   3686602.0\n",
+       "2021-01-05 11:28:00  32.26  32.24  32.30  32.21    60077.0   1938244.0\n",
+       "2021-01-05 11:29:00  32.25  32.28  32.30  32.22    65634.0   2117204.0\n",
+       "2021-01-05 11:30:00  32.23  32.30  32.30  32.23    49052.0   1582947.0\n",
+       "\n",
+       "[120 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## exam return type of each api \n",
+    "\n",
+    "# range of stock price in different resolution\n",
+    "single_stock_df = jq.get_price('002709.XSHE',start_date='2021-01-05 9:00:00', end_date='2021-01-05 12:00:00', frequency='1m')\n",
+    "single_stock_df\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>time</th>\n",
+       "      <th>code</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05 09:31:00</td>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>9.23</td>\n",
+       "      <td>9.16</td>\n",
+       "      <td>9.23</td>\n",
+       "      <td>9.08</td>\n",
+       "      <td>1417593.0</td>\n",
+       "      <td>13000397.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2021-01-05 09:32:00</td>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>9.17</td>\n",
+       "      <td>9.17</td>\n",
+       "      <td>9.17</td>\n",
+       "      <td>9.15</td>\n",
+       "      <td>394595.0</td>\n",
+       "      <td>3613283.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2021-01-05 09:33:00</td>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>9.16</td>\n",
+       "      <td>9.15</td>\n",
+       "      <td>9.16</td>\n",
+       "      <td>9.13</td>\n",
+       "      <td>492224.0</td>\n",
+       "      <td>4501667.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2021-01-05 09:34:00</td>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>9.14</td>\n",
+       "      <td>9.21</td>\n",
+       "      <td>9.22</td>\n",
+       "      <td>9.14</td>\n",
+       "      <td>451578.0</td>\n",
+       "      <td>4152355.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2021-01-05 09:35:00</td>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>9.20</td>\n",
+       "      <td>9.20</td>\n",
+       "      <td>9.22</td>\n",
+       "      <td>9.20</td>\n",
+       "      <td>503040.0</td>\n",
+       "      <td>4631781.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>715</th>\n",
+       "      <td>2021-01-05 11:26:00</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>279118.0</td>\n",
+       "      <td>1639550.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>716</th>\n",
+       "      <td>2021-01-05 11:27:00</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>253667.0</td>\n",
+       "      <td>1490048.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>717</th>\n",
+       "      <td>2021-01-05 11:28:00</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>137293.0</td>\n",
+       "      <td>806465.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>718</th>\n",
+       "      <td>2021-01-05 11:29:00</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>218351.0</td>\n",
+       "      <td>1282600.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>719</th>\n",
+       "      <td>2021-01-05 11:30:00</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>248941.0</td>\n",
+       "      <td>1462285.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>720 rows × 8 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                   time         code  open  close  high   low     volume  \\\n",
+       "0   2021-01-05 09:31:00  600409.XSHG  9.23   9.16  9.23  9.08  1417593.0   \n",
+       "1   2021-01-05 09:32:00  600409.XSHG  9.17   9.17  9.17  9.15   394595.0   \n",
+       "2   2021-01-05 09:33:00  600409.XSHG  9.16   9.15  9.16  9.13   492224.0   \n",
+       "3   2021-01-05 09:34:00  600409.XSHG  9.14   9.21  9.22  9.14   451578.0   \n",
+       "4   2021-01-05 09:35:00  600409.XSHG  9.20   9.20  9.22  9.20   503040.0   \n",
+       "..                  ...          ...   ...    ...   ...   ...        ...   \n",
+       "715 2021-01-05 11:26:00  600415.XSHG  5.87   5.87  5.87  5.87   279118.0   \n",
+       "716 2021-01-05 11:27:00  600415.XSHG  5.87   5.87  5.87  5.87   253667.0   \n",
+       "717 2021-01-05 11:28:00  600415.XSHG  5.87   5.87  5.87  5.87   137293.0   \n",
+       "718 2021-01-05 11:29:00  600415.XSHG  5.87   5.87  5.87  5.87   218351.0   \n",
+       "719 2021-01-05 11:30:00  600415.XSHG  5.87   5.87  5.87  5.87   248941.0   \n",
+       "\n",
+       "          money  \n",
+       "0    13000397.0  \n",
+       "1     3613283.0  \n",
+       "2     4501667.0  \n",
+       "3     4152355.0  \n",
+       "4     4631781.0  \n",
+       "..          ...  \n",
+       "715   1639550.0  \n",
+       "716   1490048.0  \n",
+       "717    806465.0  \n",
+       "718   1282600.0  \n",
+       "719   1462285.0  \n",
+       "\n",
+       "[720 rows x 8 columns]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# range of multiple stocks price in different resolution\n",
+    "mul_stocks_df = jq.get_..0n63\n",
+    "\n",
+    "\n",
+    "price(['002709.XSHE',\n",
+    " '002920.XSHE',\n",
+    " '300274.XSHE',\n",
+    " '600409.XSHG',\n",
+    " '600415.XSHG',\n",
+    " '603882.XSHG'],start_date='2021-01-05 9:00:00', end_date='2021-01-05 12:00:00', frequency='1m')\n",
+    "mul_stocks_df\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>display_name</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>000008.XSHE</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.088</td>\n",
+       "      <td>神州高铁</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>000009.XSHE</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.344</td>\n",
+       "      <td>中国宝安</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>000012.XSHE</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.180</td>\n",
+       "      <td>南玻A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>000021.XSHE</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.297</td>\n",
+       "      <td>深科技</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>001872.XSHE</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.030</td>\n",
+       "      <td>招商港口</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688002.XSHG</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.438</td>\n",
+       "      <td>睿创微纳</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688099.XSHG</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.287</td>\n",
+       "      <td>晶晨股份</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688088.XSHG</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.252</td>\n",
+       "      <td>虹软科技</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688321.XSHG</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.134</td>\n",
+       "      <td>微芯生物</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688029.XSHG</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.130</td>\n",
+       "      <td>南微医学</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>500 rows × 3 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                   date  weight display_name\n",
+       "000008.XSHE  2020-12-31   0.088         神州高铁\n",
+       "000009.XSHE  2020-12-31   0.344         中国宝安\n",
+       "000012.XSHE  2020-12-31   0.180          南玻A\n",
+       "000021.XSHE  2020-12-31   0.297          深科技\n",
+       "001872.XSHE  2020-12-31   0.030         招商港口\n",
+       "...                 ...     ...          ...\n",
+       "688002.XSHG  2020-12-31   0.438         睿创微纳\n",
+       "688099.XSHG  2020-12-31   0.287         晶晨股份\n",
+       "688088.XSHG  2020-12-31   0.252         虹软科技\n",
+       "688321.XSHG  2020-12-31   0.134         微芯生物\n",
+       "688029.XSHG  2020-12-31   0.130         南微医学\n",
+       "\n",
+       "[500 rows x 3 columns]"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# composition of benchmark\n",
+    "jq.get_index_weights('000905.XSHG',date='2021-01-20 9:00:00')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'600409.XSHG': {'sw_l1': {'industry_code': '801030', 'industry_name': '化工I'},\n",
+       "  'sw_l2': {'industry_code': '801033', 'industry_name': '化学原料II'},\n",
+       "  'sw_l3': {'industry_code': '850321', 'industry_name': '纯碱III'},\n",
+       "  'zjw': {'industry_code': 'C26', 'industry_name': '化学原料和化学制品制造业'},\n",
+       "  'jq_l2': {'industry_code': 'HY02107', 'industry_name': '粘胶'},\n",
+       "  'jq_l1': {'industry_code': 'HY002', 'industry_name': '原材料'}},\n",
+       " '603882.XSHG': {'sw_l1': {'industry_code': '801150',\n",
+       "   'industry_name': '医药生物I'},\n",
+       "  'sw_l2': {'industry_code': '801156', 'industry_name': '医疗服务II'},\n",
+       "  'sw_l3': {'industry_code': '851562', 'industry_name': '诊断服务III'},\n",
+       "  'zjw': {'industry_code': 'Q83', 'industry_name': '卫生'},\n",
+       "  'jq_l2': {'industry_code': 'HY06103', 'industry_name': '体外诊断'},\n",
+       "  'jq_l1': {'industry_code': 'HY006', 'industry_name': '医药卫生'}},\n",
+       " '300274.XSHE': {'sw_l1': {'industry_code': '801730',\n",
+       "   'industry_name': '电气设备I'},\n",
+       "  'sw_l2': {'industry_code': '801735', 'industry_name': '光伏设备II'},\n",
+       "  'sw_l3': {'industry_code': '857353', 'industry_name': '逆变器III'},\n",
+       "  'zjw': {'industry_code': 'C38', 'industry_name': '电气机械和器材制造业'},\n",
+       "  'jq_l2': {'industry_code': 'HY03111', 'industry_name': '光伏设备'},\n",
+       "  'jq_l1': {'industry_code': 'HY003', 'industry_name': '工业'}},\n",
+       " '002709.XSHE': {'sw_l1': {'industry_code': '801730',\n",
+       "   'industry_name': '电气设备I'},\n",
+       "  'sw_l2': {'industry_code': '801737', 'industry_name': '电池II'},\n",
+       "  'sw_l3': {'industry_code': '857372', 'industry_name': '电池化学品III'},\n",
+       "  'zjw': {'industry_code': 'C26', 'industry_name': '化学原料和化学制品制造业'},\n",
+       "  'jq_l2': {'industry_code': 'HY03119', 'industry_name': '电池部件及材料'},\n",
+       "  'jq_l1': {'industry_code': 'HY003', 'industry_name': '工业'}},\n",
+       " '002920.XSHE': {'sw_l1': {'industry_code': '801750', 'industry_name': '计算机I'},\n",
+       "  'sw_l2': {'industry_code': '801104', 'industry_name': '软件开发II'},\n",
+       "  'sw_l3': {'industry_code': '851041', 'industry_name': '垂直应用软件III'},\n",
+       "  'zjw': {'industry_code': 'C36', 'industry_name': '汽车制造业'},\n",
+       "  'jq_l2': {'industry_code': 'HY04103', 'industry_name': '汽车电子'},\n",
+       "  'jq_l1': {'industry_code': 'HY004', 'industry_name': '可选消费'}},\n",
+       " '600415.XSHG': {'sw_l1': {'industry_code': '801200',\n",
+       "   'industry_name': '商业贸易I'},\n",
+       "  'sw_l2': {'industry_code': '801203', 'industry_name': '一般零售II'},\n",
+       "  'sw_l3': {'industry_code': '852034', 'industry_name': '商业物业经营III'},\n",
+       "  'zjw': {'industry_code': 'L72', 'industry_name': '商务服务业'},\n",
+       "  'jq_l2': {'industry_code': 'HY03147', 'industry_name': '市场服务'},\n",
+       "  'jq_l1': {'industry_code': 'HY003', 'industry_name': '工业'}}}"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# sector information of each stock\n",
+    "jq.get_industry(['002709.XSHE',\n",
+    " '002920.XSHE',\n",
+    " '300274.XSHE',\n",
+    " '600409.XSHG',\n",
+    " '600415.XSHG',\n",
+    " '603882.XSHG'])\n",
+    "# display name of each stock"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([datetime.date(2005, 1, 4), datetime.date(2005, 1, 5),\n",
+       "       datetime.date(2005, 1, 6), ..., datetime.date(2025, 8, 11),\n",
+       "       datetime.date(2025, 8, 12), datetime.date(2025, 8, 13)],\n",
+       "      dtype=object)"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get all trading days\n",
+    "jq.get_all_trade_days()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>display_name</th>\n",
+       "      <th>name</th>\n",
+       "      <th>start_date</th>\n",
+       "      <th>end_date</th>\n",
+       "      <th>type</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>000001.XSHE</th>\n",
+       "      <td>平安银行</td>\n",
+       "      <td>PAYH</td>\n",
+       "      <td>1991-04-03</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>000002.XSHE</th>\n",
+       "      <td>万科A</td>\n",
+       "      <td>WKA</td>\n",
+       "      <td>1991-01-29</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>000004.XSHE</th>\n",
+       "      <td>国华网安</td>\n",
+       "      <td>GHWA</td>\n",
+       "      <td>1990-12-01</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>000005.XSHE</th>\n",
+       "      <td>ST星源</td>\n",
+       "      <td>STXY</td>\n",
+       "      <td>1990-12-10</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>000006.XSHE</th>\n",
+       "      <td>深振业A</td>\n",
+       "      <td>SZYA</td>\n",
+       "      <td>1992-04-27</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688799.XSHG</th>\n",
+       "      <td>华纳药厂</td>\n",
+       "      <td>HNYC</td>\n",
+       "      <td>2021-07-13</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688800.XSHG</th>\n",
+       "      <td>瑞可达</td>\n",
+       "      <td>RKD</td>\n",
+       "      <td>2021-07-22</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688819.XSHG</th>\n",
+       "      <td>天能股份</td>\n",
+       "      <td>TNGF</td>\n",
+       "      <td>2021-01-18</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>688981.XSHG</th>\n",
+       "      <td>中芯国际</td>\n",
+       "      <td>ZXGJ</td>\n",
+       "      <td>2020-07-16</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>689009.XSHG</th>\n",
+       "      <td>九号公司</td>\n",
+       "      <td>JHGS</td>\n",
+       "      <td>2020-10-29</td>\n",
+       "      <td>2200-01-01</td>\n",
+       "      <td>stock</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5248 rows × 5 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            display_name  name start_date   end_date   type\n",
+       "000001.XSHE         平安银行  PAYH 1991-04-03 2200-01-01  stock\n",
+       "000002.XSHE          万科A   WKA 1991-01-29 2200-01-01  stock\n",
+       "000004.XSHE         国华网安  GHWA 1990-12-01 2200-01-01  stock\n",
+       "000005.XSHE         ST星源  STXY 1990-12-10 2200-01-01  stock\n",
+       "000006.XSHE         深振业A  SZYA 1992-04-27 2200-01-01  stock\n",
+       "...                  ...   ...        ...        ...    ...\n",
+       "688799.XSHG         华纳药厂  HNYC 2021-07-13 2200-01-01  stock\n",
+       "688800.XSHG          瑞可达   RKD 2021-07-22 2200-01-01  stock\n",
+       "688819.XSHG         天能股份  TNGF 2021-01-18 2200-01-01  stock\n",
+       "688981.XSHG         中芯国际  ZXGJ 2020-07-16 2200-01-01  stock\n",
+       "689009.XSHG         九号公司  JHGS 2020-10-29 2200-01-01  stock\n",
+       "\n",
+       "[5248 rows x 5 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "jq.get_all_securities()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'engine' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[4], line 9\u001b[0m\n\u001b[1;32m      7\u001b[0m     df[\u001b[39m'\u001b[39m\u001b[39mdate\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mto_datetime(df[\u001b[39m'\u001b[39m\u001b[39mdate\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[1;32m      8\u001b[0m     \u001b[39mreturn\u001b[39;00m df\n\u001b[0;32m----> 9\u001b[0m get_most_recent_profile(\u001b[39m'\u001b[39;49m\u001b[39mportfolio\u001b[39;49m\u001b[39m'\u001b[39;49m)\n",
+      "Cell \u001b[0;32mIn[4], line 5\u001b[0m, in \u001b[0;36mget_most_recent_profile\u001b[0;34m(type)\u001b[0m\n\u001b[1;32m      3\u001b[0m table_name \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mbenchmark_profile\u001b[39m\u001b[39m'\u001b[39m \u001b[39mif\u001b[39;00m \u001b[39mtype\u001b[39m \u001b[39m==\u001b[39m \u001b[39m'\u001b[39m\u001b[39mbenchmark\u001b[39m\u001b[39m'\u001b[39m \u001b[39melse\u001b[39;00m \u001b[39m'\u001b[39m\u001b[39mportfolio_profile\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m      4\u001b[0m query \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mSELECT * FROM \u001b[39m\u001b[39m{\u001b[39;00mtable_name\u001b[39m}\u001b[39;00m\u001b[39m WHERE date = (SELECT MAX(date) FROM \u001b[39m\u001b[39m{\u001b[39;00mtable_name\u001b[39m}\u001b[39;00m\u001b[39m)\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m----> 5\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mread_sql(query, con\u001b[39m=\u001b[39mengine)\n\u001b[1;32m      6\u001b[0m \u001b[39m# convert date to datetime object\u001b[39;00m\n\u001b[1;32m      7\u001b[0m df[\u001b[39m'\u001b[39m\u001b[39mdate\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mto_datetime(df[\u001b[39m'\u001b[39m\u001b[39mdate\u001b[39m\u001b[39m'\u001b[39m])\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'engine' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "# when update stock price just need the latest portfolio frame\n",
+    "def get_most_recent_profile(type):\n",
+    "    table_name = 'benchmark_profile' if type == 'benchmark' else 'portfolio_profile'\n",
+    "    query = f\"SELECT * FROM {table_name} WHERE date = (SELECT MAX(date) FROM {table_name})\"\n",
+    "    with create_engine(db_url).connect() as conn:\n",
+    "        df = pd.read_sql(query, con=engine)\n",
+    "        # convert date to datetime object\n",
+    "        df['date'] = pd.to_datetime(df['date'])\n",
+    "        return df\n",
+    "get_most_recent_profile('portfolio')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

appComponents.py

@@ -1,3 +1,817 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:60c6ae82e349660ba410e78e371fb97ced027c60bd53401bb8524dca04818d3e
-size 35435
+from script import processing
+from datetime import datetime, timedelta
+import panel as pn
+import pandas as pd
+import hvplot.pandas  # noqa
+import plotly.express as px
+import numpy as np
+import hvplot.pandas  # noqa
+from panel.viewable import Viewer
+import param
+from script import styling
+from script import description
+import plotly.graph_objs as go
+# import warnings
+pn.extension('mathjax')
+pn.extension('plotly')
+pn.extension('plotly')
+# warnings.filterwarnings("ignore", category=pd.core.common.SettingWithCopyWarning)
+# overal performance default to 30 days
+
+
+def create_portfolio_overview(df_list):
+    calculated_b_stock, calculated_p_stock, p_eval_df, sector_eval_df = df_list
+
+    range_slider = pn.widgets.DateRangeSlider(name='date range',
+                                              start=sector_eval_df.date.min(),
+                                              end=sector_eval_df.date.max(),
+                                              value=(sector_eval_df.date.max() - timedelta(days=30),
+                                                     sector_eval_df.date.max()),
+                                              align='center',
+                                              sizing_mode='stretch_width',
+                                              )
+    size = dict(width=780)
+    option = dict(legend_position="left")
+    active_tools = dict(tools=['hover'], active_tools=[], axiswise=True)
+
+    # def create_overview_panel()
+    ip_eval_df = p_eval_df.interactive()
+    isector_eval_df = sector_eval_df.interactive()
+
+    # TODO convert below to a class
+    ranged_ip_eval_df = ip_eval_df[ip_eval_df.date.between(
+        range_slider.param.value_start, range_slider.param.value_end)]
+    ranged_isector_eval_df = isector_eval_df[isector_eval_df.date.between(
+        range_slider.param.value_start, range_slider.param.value_end)]
+    # return
+    return_plot = ranged_ip_eval_df.hvplot.line(x='date', y=['portfolio_return_p', 'portfolio_return_b'])\
+        .opts(title='投资组合总回报 v.s benchmark总回报', **size, **option)
+    # active return
+    active_return_plot = ranged_ip_eval_df.hvplot.line(x='date', y=['active_return'])\
+        .opts(title='每日主动回报',  **size, axiswise=True)
+
+    # total risk and tracking error
+    risk_tracking_plot = ranged_ip_eval_df.hvplot.line(x='date', y=['risk', 'tracking_error'])\
+        .opts(title='风险和追踪误差', **size, **option)
+
+    # sector return
+    sector_return_plot = ranged_isector_eval_df.hvplot.line(x='date', y=['portfolio_return_p'], by='aggregate_sector')\
+        .opts(title='投资组合各行业总回报',  **size, **option)
+
+    # bsector_return_plot = ranged_isector_eval_df.hvplot.line(x='date', y=['portfolio_return_b'], by='aggregate_sector')\
+    #     .opts(title='benchmark sector return',  **size, **option)
+
+    # sector active return
+    s_active_return_plot = ranged_isector_eval_df.hvplot.line(x='date', y=['active_return'], by='aggregate_sector')\
+        .opts(title='投资组合各行业每日主动回报',  **size, **option)
+
+    # sector risk and tracking error
+    s_risk_plot = ranged_isector_eval_df.hvplot.line(x='date', y=['tracking_error'], by='aggregate_sector')\
+        .opts(title='投资组合各行业追踪误差',  **size, **option)
+    s_tracking_plot = ranged_isector_eval_df.hvplot.line(x='date', y=['risk'], by='aggregate_sector')\
+        .opts(title='投资组合各行业风险', **size, **option)
+
+    # attribute
+    def create_attribute_plot(start, end, calculated_b_stock, calculated_p_stock):
+        result = processing.calculate_attributes_between_dates(
+            start, end, calculated_b_stock, calculated_p_stock)
+        portfolio_attribute = result.aggregate({
+            'interaction': 'sum',
+            'allocation': 'sum',
+            'selection': 'sum',
+        })
+        layout = pn.Column(
+            pn.pane.DataFrame(portfolio_attribute.transpose()),
+            result.hvplot.bar(x='display_name_p',
+                              y=['interaction', 'allocation', 'selection'],
+                              shared_axes=False,
+                              stacked=True,
+                              rot=90).opts(**size, **option, title='投资组合总主动回报归因')
+        )
+        return layout
+
+    attribute_plot = pn.bind(create_attribute_plot,
+                             start=range_slider.param.value_start,
+                             end=range_slider.param.value_end,
+                             calculated_b_stock=calculated_b_stock,
+                             calculated_p_stock=calculated_p_stock)
+
+    # stock performance
+    # selected_p_stock = calculated_p_stock[calculated_p_stock.date ==
+    #                                       calculated_p_stock.date.max()]
+    # stock_radar_plot = go.Figure()
+    # category = ['return', 'risk', 'portfolio_return', 'prev_w_in_p']
+    # for display_name, group in selected_p_stock.groupby('display_name'):
+    #     stock_radar_plot.add_trace(go.Scatterpolar(
+    #         r=group[category].values[0],
+    #         theta=category,
+    #         fill='toself',
+    #         name=display_name
+    #     ))
+    total_view_plots = pn.Column(return_plot.opts(**active_tools).output(),
+                                 risk_tracking_plot.opts(
+                                     **active_tools).output(),
+                                 active_return_plot.opts(
+                                     **active_tools).output(),
+                                 attribute_plot,
+                                 height=1000,
+                                 scroll=True)
+    sector_view_plots = pn.Column(sector_return_plot.opts(**active_tools).output(),
+                                  s_risk_plot.opts(**active_tools).output(),
+                                  s_tracking_plot.opts(
+                                      **active_tools).output(),
+                                  s_active_return_plot.opts(
+                                      **active_tools).output(),
+                                  height=1000,
+                                  scroll=True)
+
+    return pn.Column(
+        # pn.Row(align='center'),
+        range_slider,
+        pn.Row(total_view_plots, sector_view_plots, align='center'))
+
+
+def attribution_view(daily_bnb_result, daily_sector_bnb_result, p_eval_df):
+    p_eval_df.date = pd.to_datetime(p_eval_df.date)
+    daily_bnb_result.date = pd.to_datetime(daily_bnb_result.date)
+    daily_sector_bnb_result.date = pd.to_datetime(daily_sector_bnb_result.date)
+
+    # interactive widget
+    dt_range = pn.widgets.DateRangeSlider(start=p_eval_df.date.min(
+    ), end=p_eval_df.date.max(), value=(p_eval_df.date.min(), p_eval_df.date.max()))
+    # total attribution and return
+    p_eval_df_i = p_eval_df.interactive()
+    daily_bnb_result_i = daily_bnb_result.interactive()
+    daily_return_plot = p_eval_df_i[(p_eval_df_i.date >= dt_range.param.value_start) & (
+        p_eval_df_i.date <= dt_range.param.value_end)].hvplot(x='date', y=['portfolio_return_p', 'portfolio_return_b'], title='投资组合总回报 v.s benchmark总回报').output()
+    daily_bnb_plot = daily_bnb_result_i[daily_bnb_result_i.date.between(dt_range.param.value_start, dt_range.param.value_end)]\
+        .hvplot.bar(x='date', y=['allocation', 'selection', 'interaction', "active_return"], stacked=True, title='每日主动收益归因', yformatter='%.2f', xlabel='日期', shared_axes=False).output()
+
+    # return
+    daily_sector_bnb_df_i = daily_sector_bnb_result.interactive()
+    selected_range_df = daily_sector_bnb_df_i[daily_sector_bnb_df_i.date.between(
+        dt_range.param.value_start, dt_range.param.value_end)]
+    sector_active_return_plot = selected_range_df.hvplot.line(
+        x='date', y='active_return', by='aggregate_sector', width=1000, height=400, title='投资组合行业每日主动回报').output()
+
+    # attribution
+    def plot_attribute_by_sector(sector):
+        selected_sector_df = selected_range_df[selected_range_df.aggregate_sector == sector]
+        return selected_sector_df.hvplot.bar(x='date', y=['active_return', 'allocation', 'selection', 'interaction'], title='投资组合行业每日主动收入归因', stacked=True, shared_axes=False).output()
+    sector_attr_plot_tabs = pn.Tabs(*[(sector, plot_attribute_by_sector(sector))
+                                    for sector in daily_sector_bnb_result.aggregate_sector.unique()], dymacic=True)
+
+    # layout
+    sector_view = pn.Column(sector_attr_plot_tabs, sector_active_return_plot)
+    total_view = pn.Column(daily_return_plot, daily_bnb_plot)
+    return pn.Column(
+        pn.Row(dt_range),
+        pn.Row(total_view, sector_view)
+    )
+
+# plot explore
+
+
+def create_hvplot_explore(calculated_b_stock, calculated_p_stock, p_eval_df, sector_eval_df, attribution_result_df, s_attribution_result_df):
+
+    options = ['calculated_b_stock', 'calculated_p_stock', 'p_eval_df',
+               'sector_eval_df', 'attribution_result_df', 's_attribution_result_df']
+    name_to_df = {
+        'calculated_b_stock': calculated_b_stock,
+        'calculated_p_stock': calculated_p_stock,
+        'p_eval_df': p_eval_df,
+        'sector_eval_df': sector_eval_df,
+        'attribution_result_df': attribution_result_df,
+        's_attribution_result_df': s_attribution_result_df
+
+    }
+
+    selector = pn.widgets.Select(
+        name='Select', options=options, value=options[0])
+
+    def create_exploer(name):
+        df = name_to_df[name]
+        explorer = hvplot.explorer(df)
+
+        def plot_code(**kwargs):
+            code = f'```python\n{explorer.plot_code()}\n```'
+            return pn.pane.Markdown(code, sizing_mode='stretch_width')
+        pn.Column(
+            explorer,
+            '**Code**:',
+            pn.bind(plot_code, **explorer.param.objects())
+        )
+        return explorer
+
+    def create_perspective(name):
+        df = name_to_df[name]
+        return pn.pane.Perspective(df, columns=list(df.columns), width=1500, height=800)
+    perspective = pn.bind(create_perspective, name=selector)
+    exploer = pn.bind(create_exploer, name=selector)
+    exploer_component = pn.Column(selector, exploer, perspective)
+    return exploer_component
+
+
+class TotalReturnCard(Viewer):
+
+    value = param.Range(doc="A numeric range.")
+    width = param.Integer(default=300)
+    start_date = param.Parameter()
+    end_date = param.Parameter()
+    eval_df = param.Parameter()
+    b_stock_df = param.Parameter()
+    p_stock_df = param.Parameter()
+    selected_df = param.Parameter()
+    plot_pane = param.Parameter()
+    report = param.Parameter()
+
+    def format_number(self, num):
+        return f'{round(num * 100, 2)}%'
+
+    def get_color(self, num):
+        return 'green' if num >= 0 else 'red'
+
+    def create_report(self):
+        # Calculate the total return and risk
+        result = processing.calculate_return(
+            self.eval_df, self.start_date, self.end_date)
+        most_recent_row = result.tail(1)
+        active_return = most_recent_row.active_return.values[0]
+        tracking_error = result.active_return.std() * np.sqrt(252)
+        total_return = most_recent_row.return_p.values[0]
+        mkt_cap = most_recent_row.mkt_cap.values[0]
+        risk = result['return_b'].std() * np.sqrt(252)
+
+        # Calculate the total attribution
+        attributes = processing.calculate_attributes_between_dates(
+            self.start_date, self.end_date, self.p_stock_df, self.b_stock_df)
+        total_attributes = attributes.aggregate({
+            'interaction': 'sum',
+            'allocation': 'sum',
+            'selection': 'sum',
+            'active_return': 'sum',
+            'notional_return': 'sum'
+        })
+        active_return_from_stock = total_attributes.active_return
+        notional_return = total_attributes.notional_return
+        interaction = total_attributes.interaction
+        allocation = total_attributes.allocation
+        selection = total_attributes.selection
+
+        # Create a function for text report 
+        report = f"""
+<style>
+    .compact-container {{
+        display: flex;
+        flex-direction: column;
+        gap: 5px;
+    }}
+    
+    .compact-container > div {{
+        display: flex;
+        justify-content: space-between;
+        margin-bottom: 2px;
+    }}
+    
+    .compact-container > div > h2,
+    .compact-container > div > h3,
+    .compact-container > div > p,
+    .compact-container > div > ul > li {{
+        margin: 0;
+    }}
+    
+    .compact-container > ul {{
+        padding: 0;
+        margin: 0;
+        list-style-type: none;
+    }}
+    
+    .compact-container > ul > li {{
+        display: flex;
+        margin-bottom: 2px;
+    }}
+</style>
+
+<div class="compact-container">
+    <u><b>总市值</b></u>
+    <div>
+        <h2 style="margin: 0;">¥{round(mkt_cap,2)}</h2>
+        <h2 style='color: {self.get_color(total_return)}; margin: 0;'>{self.format_number(total_return)}</h2>
+    </div>
+    <div>
+        <p style="margin: 0;">追踪误差</p>
+        <p style='color: {self.get_color(tracking_error)}; margin: 0;'>{self.format_number(tracking_error)}</p>
+    </div>
+    <div>
+        <p style="margin: 0;">风险</p>
+        <p style='color: {self.get_color(risk)}; margin: 0;'>{self.format_number(risk)}</p>
+    </div>
+    <div>
+        <p style="margin: 0;">归因</p>
+        <ul style="padding: 0; margin: 0; list-style-type: none;">
+            <li style="margin-bottom: 2px;">
+                <div style="display: flex;">
+                    <p style="margin: 0;">主动回报:</p>
+                    <p style="color: {self.get_color(active_return)}; margin: 0;">{self.format_number(active_return)}</p>
+                </div>
+            </li>
+            <li style="margin-bottom: 2px;">
+                <div style="display: flex;">
+                    <p style="margin: 0;">交互:</p>
+                    <p style="color: {self.get_color(interaction)}; margin: 0;">{self.format_number(interaction)}</p>
+                </div>
+            </li>
+            <li style="margin-bottom: 2px;">
+                <div style="display: flex;">
+                    <p style="margin: 0;">名义主动回报:</p>
+                    <p style="color: {self.get_color(notional_return)}; margin: 0;">{self.format_number(notional_return)}</p>
+                </div>
+            </li>
+            <li style="margin-bottom: 2px;">
+                <div style="display: flex;">
+                    <p style="margin: 0;">选择:</p>
+                    <p style="color: {self.get_color(selection)}; margin: 0;">{self.format_number(selection)}</p>
+                </div>
+            </li>
+            <li style="margin-bottom: 2px;">
+                <div style="display: flex;">
+                    <p style="margin: 0;">分配:</p>
+                    <p style="color: {self.get_color(allocation)}; margin: 0;">{self.format_number(allocation)}</p>
+                </div>
+            </li>
+        </ul>
+    </div>
+</div>
+"""
+
+        return report
+
+    def create_plot(self):
+        result = processing.calculate_return(
+            self.eval_df, self.start_date, self.end_date)
+        fig = px.line(result, x="date", y=['return_p', 'return_b'])
+        fig.update_traces(mode="lines+markers",
+                          marker=dict(size=5), line=dict(width=2))
+        fig.update_layout(styling.plot_layout)
+        colname_to_name = {
+            'return_p': 'Portfolio回报',
+            'return_b': 'benchmark回报'
+        }
+        fig.for_each_trace(lambda t: t.update(name=colname_to_name.get(t.name, t.name),
+                                              legendgroup=colname_to_name.get(
+            t.name, t.name),
+            hovertemplate=t.hovertemplate.replace(
+            t.name, colname_to_name.get(t.name, t.name))
+        ))
+        # fig.layout.autosize = True
+        return fig.to_dict()
+
+    @param.depends('start_date', 'end_date', 'eval_df', watch=True)
+    def update(self):
+        fig = self.create_plot()
+        report = self.create_report()
+        self.report.object = report
+        self.plot_pane.object = fig
+
+    def __init__(self, eval_df, b_stock_df, p_stock_df, **params):
+        self.eval_df = eval_df
+        self.b_stock_df = b_stock_df
+        self.p_stock_df = p_stock_df
+        self._date_range = pn.widgets.DateRangeSlider(
+            start=eval_df.date.min(),
+            end=eval_df.date.max(),
+            value=(eval_df.date.max() - timedelta(days=7), eval_df.date.max())
+        )
+        self.start_date = self._date_range.value_start
+        self.end_date = self._date_range.value_end
+        self.plot_pane = pn.pane.Plotly(
+            self.create_plot(), sizing_mode='stretch_width')
+
+        self.report = pn.pane.HTML(
+            self.create_report(), sizing_mode='stretch_width')
+        super().__init__(**params)
+        self._sync_widgets()
+
+    def __panel__(self):
+        self._layout = pn.Card(self._date_range, self.report, self.plot_pane,
+                               width=500, header=pn.Row(pn.pane.Str('投资组合总结'),
+                                                        pn.widgets.TooltipIcon(value=description.summary_card)))
+        return self._layout
+
+    @param.depends('start_date', 'end_date', 'eval_df', watch=True)
+    def update_selected_df(self):
+        self.selected_df = self.eval_df[self.eval_df.date.between(
+            self.start_date, self.end_date
+        )]
+
+    @param.depends('value', 'width', watch=True)
+    def _sync_widgets(self):
+        pass
+
+    @param.depends('_date_range.value', watch=True)
+    def _sync_params(self):
+        self.start_date = self._date_range.value[0]
+        self.end_date = self._date_range.value[1]
+
+
+class DrawDownCard(Viewer):
+    def __init__(self, eval_df, calculated_p_stock, calculated_b_stock, **params):
+        self.eval_df = eval_df
+        self.calculated_p_stock = calculated_p_stock
+        self.calculated_b_stock = calculated_b_stock
+        self.drawdown_plot = pn.pane.Plotly(self.plot_drawdown())
+        super().__init__(**params)
+
+    def calculate_drawdown(self):
+        df = self.eval_df.copy()
+        # rolling max return
+        # TODO: consider adding this to the processing code
+        df['rolling_max_return_p'] = df['portfolio_return_p'].rolling(
+            window=len(df), min_periods=1).max()
+        # calculate drawdown
+        df['drawn_down'] = abs(
+            (1 + df.portfolio_return_p) / (1 + df.rolling_max_return_p) - 1)
+        return df
+
+    def plot_drawdown(self):
+        df = self.calculate_drawdown()
+        fig = px.line(df, x="date", y=['drawn_down'])
+        # add scatter to represetn new high
+        new_height_pnl = df[df.portfolio_return_p == df.rolling_max_return_p]
+        fig.add_trace(go.Scatter(
+            x=new_height_pnl['date'], y=new_height_pnl['drawn_down'], mode='markers', name='新的最高总回报'))
+        colname_to_name = {
+            'drawn_down': '回撤'
+        }
+        fig.update_layout(styling.plot_layout)
+        fig.for_each_trace(lambda t: t.update(name=colname_to_name.get(t.name, t.name),
+                                              legendgroup=colname_to_name.get(
+            t.name, t.name),
+            # hovertemplate=t.hovertemplate.replace(
+            # t.name, colname_to_name.get(t.name, t.name))
+        ))
+        return fig
+
+    def update(self):
+        pass
+
+    def __panel__(self):
+        self._layout = pn.Card(self.drawdown_plot,
+                               header=pn.Row(pn.pane.Str('回撤分析')),
+                               width=500
+                               )
+        return self._layout
+
+
+class HistReturnCard(Viewer):
+    eval_df = param.Parameter()
+    return_barplot = param.Parameterized()
+    select_resolution = param.ObjectSelector(
+        default='每月回报', objects=['每日回报', '每周回报', '每月回报', '每年回报'])
+
+    def update_aggregate_df(self):
+        freq = None
+        if self.select_resolution == "每日回报":
+            return self.eval_df
+        elif self.select_resolution == "每月回报":
+            freq = 'M'
+        elif self.select_resolution == "每年回报":
+            freq = 'Y'
+        elif self.select_resolution == "每周回报":
+            freq = 'W'
+        # I don't think this formula is correct, check this later
+        agg_df = self.eval_df.groupby([pd.Grouper(key='date', freq=freq)])\
+            .aggregate({'portfolio_pct_p': 'sum', 'portfolio_pct_b': 'sum'})
+        agg_df['portfolio_return_p'] = np.exp(agg_df.portfolio_pct_p) - 1
+        agg_df['portfolio_return_b'] = np.exp(agg_df.portfolio_pct_b) - 1
+        return agg_df.reset_index()
+
+    def create_attributes_barplot(self):
+        self.attribute_df = self.update_attributes_df()
+        fig = px.bar(self.attribute_df, x='date', y=[
+                     'allocation', 'selection', 'interaction', 'notional_return', 'active_return'])
+        colname_to_name = {
+            'allocation': '分配',
+            'selection': '选择',
+            'interaction': '交互',
+            'notional_return': '名义主动回报',
+            'active_return': '实际主动回报'
+        }
+        fig.for_each_trace(lambda t: t.update(name=colname_to_name.get(t.name, t.name),
+                                              legendgroup=colname_to_name.get(
+            t.name, t.name),
+            hovertemplate=t.hovertemplate.replace(
+            t.name, colname_to_name.get(t.name, t.name))
+        ))
+        fig.update_layout(barmode='group', title='主动回报归因',
+                          bargap=0.0, bargroupgap=0.0)
+        fig.update_layout(**styling.plot_layout)
+        fig.update_traces(**styling.barplot_trace)
+        return fig.to_dict()
+
+    def create_return_barplot(self):
+        self.agg_df = self.update_aggregate_df()
+        fig = px.bar(self.agg_df, x='date', y=[
+                     'portfolio_return_p', 'portfolio_return_b'],
+                     barmode='overlay',
+                     title='周期回报',
+                     )
+        # update legend
+        colname_to_name = {
+            'portfolio_return_p': 'portfolio回报率',
+            'portfolio_return_b': 'benchmark回报率'
+        }
+        fig.for_each_trace(lambda t: t.update(name=colname_to_name.get(t.name, t.name),
+                                              legendgroup=colname_to_name.get(
+                                                  t.name, t.name),
+                                              hovertemplate=t.hovertemplate.replace(
+                                                  t.name, colname_to_name.get(t.name, t.name))
+                                              ))
+
+        fig.update_layout(**styling.plot_layout)
+
+        fig.update_traces(**styling.barplot_trace)
+
+        return fig.to_dict()
+
+    @param.depends('eval_df', 'select_resolution', watch=True)
+    def update(self):
+        return_barplot = self.create_return_barplot()
+        self.return_barplot.object = return_barplot
+        attributes_barplot = self.create_attributes_barplot()
+        self.attribute_barplot.object = attributes_barplot
+
+    def update_attributes_df(self):
+        freq = None
+        if self.select_resolution == "每日回报":
+            freq = 'D'
+        elif self.select_resolution == "每月回报":
+            freq = 'M'
+        elif self.select_resolution == "每年回报":
+            freq = 'Y'
+        elif self.select_resolution == "每周回报":
+            freq = 'W'
+        p_stock = processing.change_resolution(self.calculated_p_stock, freq)
+        b_stock = processing.change_resolution(self.calculated_b_stock, freq)
+        return processing.calculate_total_attribution(p_stock, b_stock)
+
+    def __init__(self, eval_df, calculated_p_stock, calculated_b_stock, **params):
+        self.eval_df = eval_df
+        self.calculated_p_stock = calculated_p_stock
+        self.calculated_b_stock = calculated_b_stock
+        self._range_slider = pn.widgets.DateRangeSlider(
+            name='Date Range Slider',
+            start=self.eval_df.date.min(), end=self.eval_df.date.max(),
+            value=(self.eval_df.date.min(), self.eval_df.date.max()),
+
+        )
+        self.return_barplot = pn.pane.Plotly(self.create_return_barplot())
+        self.attribute_barplot = pn.pane.Plotly(
+            self.create_attributes_barplot())
+        super().__init__(**params)
+
+    def __panel__(self):
+        self._layout = pn.Card(pn.Param(self.param.select_resolution, name='选择周期'),
+                               self.return_barplot, self.attribute_barplot, width=500, header=pn.Row(pn.pane.Str('周期回报'),
+                                                                                                     pn.widgets.TooltipIcon(value=description.periodic_return_report)))
+        return self._layout
+
+
+class PortfolioComposationCard(Viewer):
+    p_stock_df = param.Parameterized()
+
+    def create_cash_position_df(self):
+        aggregate_df = self.p_stock_df.groupby('date', as_index=False).agg({
+            'current_weight': 'sum'
+        })
+        aggregate_df['type'] = 'portfolio'
+        not_in_portfolio_df = aggregate_df.copy()
+        not_in_portfolio_df['type'] = 'not_in_portfolio'
+        not_in_portfolio_df['current_weight'] = 1000
+        # append df
+        aggregate_df = pd.concat([aggregate_df, not_in_portfolio_df])
+        # sort
+        aggregate_df.sort_values(by=['date'], inplace=True)
+        return aggregate_df[aggregate_df.date.between(self.date_range.value[0], self.date_range.value[1])]
+
+    @param.depends('p_stock_df', 'date_range.value', watch=True)
+    def update_trend_plot(self):
+        self.trend_plot.object = self.create_trend_plot()
+
+    def create_trend_plot(self):
+        aggregate_df = self.create_cash_position_df()
+        fig = px.bar(aggregate_df, x='date', y='current_weight', color='type')
+        fig.update_layout(legend=dict(
+            orientation="h",
+            yanchor="bottom",
+            y=1.02,
+            xanchor="right",
+            x=1
+        ))
+        fig.update_traces(
+            marker_line_width=0,
+            selector=dict(type="bar"))
+        fig.update_layout(bargap=0,
+                          bargroupgap=0,
+                          )
+        fig.update_layout(uniformtext_minsize=8, uniformtext_mode='hide',
+                          yaxis_title=None, xaxis_title=None,
+                          margin=dict(l=0, r=0, t=0, b=0))
+        return fig.to_dict()
+
+    def create_treemap(self):
+        self.selected_df['position'] = 'portfolio'
+        not_in_portfolio_row = pd.DataFrame({
+            'display_name': ['不在portfolio中'],
+            'position': ['not_in_portfolio'],
+            'aggregate_sector': ['不在portfolio中'],
+            'current_weight': [1000],
+            'portfolio_return': [0],
+            'portfolio_pct': [0]
+        })
+        df = pd.concat([self.selected_df, not_in_portfolio_row],
+                       ignore_index=True)
+        fig = px.treemap(df, path=[px.Constant('cash_position'), 'position', 'aggregate_sector', 'display_name'], values='current_weight',
+                         color='portfolio_return', hover_data=['portfolio_return', 'portfolio_pct'],
+                         color_continuous_scale='RdBu',
+                         color_continuous_midpoint=np.average(
+                             df['portfolio_return'])
+                         )
+        fig.update_layout(styling.plot_layout)
+        fig.update_layout(coloraxis_colorbar=dict(
+            title="weighted return"))
+        colname_to_name = {
+            'cash_position': '现金分布',
+            'portfolio_return': '加权回报',
+            'not_in_portfolio': '不在portfolio中',
+            'current_weight': '现金',
+
+        }
+        fig.for_each_trace(lambda t: t.update(name=colname_to_name.get(t.name, t.name),
+                                              hovertemplate=t.hovertemplate.replace(
+            t.name, colname_to_name.get(t.name, t.name))
+        ))
+        return fig.to_dict()
+
+    def __init__(self, p_stock_df, **params):
+        self.p_stock_df = p_stock_df
+        self.date_picker = pn.widgets.DatetimePicker(name='选择某日资金分布',
+                                                     start=self.p_stock_df.date.min(),
+                                                     end=self.p_stock_df.date.max(),
+                                                     value=self.p_stock_df.date.max(),
+                                                     enabled_dates=[datetime_object.date(
+                                                     ) for datetime_object in self.p_stock_df.date.unique()],
+                                                     enable_time=False,
+                                                     )
+        self.date_range = pn.widgets.DateRangeSlider(name='选择资金分布走势区间',
+                                                     start=self.p_stock_df.date.min(),
+                                                     end=self.p_stock_df.date.max(),
+                                                     value=(self.p_stock_df.date.min(
+                                                     ), self.p_stock_df.date.max()),
+                                                     )
+        self.selected_df = self.p_stock_df[self.p_stock_df.date ==
+                                           self.date_picker.value]
+        self.tree_plot = pn.pane.Plotly(self.create_treemap())
+        self.trend_plot = pn.pane.Plotly(self.create_trend_plot())
+
+        # calculate money position
+        super().__init__(**params)
+
+    def __panel__(self):
+        self._layout = pn.Card(self.date_picker, self.tree_plot, self.date_range, self.trend_plot,
+                               width=500, header=pn.pane.Str('资金分布'))
+        return self._layout
+
+    @param.depends('date_picker.value', 'p_stock_df', watch=True)
+    def update(self):
+        self.selected_df = self.p_stock_df[self.p_stock_df.date ==
+                                           self.date_picker.value]
+        tree_plot = self.create_treemap()
+        self.tree_plot.object = tree_plot
+
+
+class BestAndWorstStocks(Viewer):
+    p_stock_df = param.Parameter()
+    b_stock_df = param.Parameter()
+    start_date = param.Parameter()
+    end_date = param.Parameter()
+
+    def calculate_attributes(self):
+        result_df = processing.calculate_attributes_between_dates(self.start_date,
+                                                                  self.end_date,
+                                                                  self.p_stock_df,
+                                                                  self.b_stock_df)
+
+        return result_df
+
+    def create_tabulator(self, df):
+        col_title_map = {
+            'display_name_p': '股票名称',
+            'ticker': '股票代码',
+            'pct_p': '加权回报率',
+            'prev_w_in_p_b': '在benchmark中的权重',
+            'prev_w_in_p_p': '在portfolio中的权重',
+            'allocation': '分配分数',
+            'selection': '选择分数',
+            'interaction': '交互分数',
+            'return': '未加权回报率',
+            'active_return': '加权主动回报率',
+        }
+        return pn.widgets.Tabulator(df, sizing_mode='stretch_width',
+                                    hidden_columns=['index', 'display_name_b',
+                                                    'pct_b', 'in_portfolio',
+                                                    ],
+                                    frozen_columns=['display_name_p'],
+                                    titles=col_title_map)
+
+    @param.depends('start_date', 'end_date', watch=True)
+    def update(self):
+        result_df = self.get_processed_df()
+        self.best_5_tabulator.value = result_df.tail(5)
+        self.worst_5_tabulator.value = result_df.head(5)
+
+    def get_processed_df(self):
+        '''
+        calculate attributes and return a sorted dataframe on weighted return
+        '''
+
+        result_df = self.calculate_attributes()
+        result_df = result_df[result_df.in_portfolio]
+        result_df.sort_values(by='return', inplace=True)
+        return result_df
+
+    def __init__(self, p_stock_df, b_stock_df, **params):
+        self.p_stock_df = p_stock_df
+        self.b_stock_df = b_stock_df
+        self._date_range = pn.widgets.DateRangeSlider(
+            name='选择计算回报的时间区间',
+            start=p_stock_df.date.min(),
+            end=p_stock_df.date.max(),
+            value=(p_stock_df.date.max() -
+                   timedelta(days=7), p_stock_df.date.max())
+        )
+        self.start_date = self._date_range.value_start
+        self.end_date = self._date_range.value_end
+        result_df = self.get_processed_df()
+        self.best_5_tabulator = self.create_tabulator(result_df.tail(5))
+        self.worst_5_tabulator = self.create_tabulator(result_df.head(5))
+        super().__init__(**params)
+
+    @param.depends('_date_range.value', watch=True)
+    def _sync_params(self):
+        self.start_date = self._date_range.value[0]
+        self.end_date = self._date_range.value[1]
+        # print('update range...')
+
+    def __panel__(self):
+        self._layout = pn.Card(self._date_range,
+                               pn.pane.Str('加权回报率最高回报5只股票'),
+                               self.best_5_tabulator,
+                               pn.pane.Str('加权回报率最低回报5只股票'),
+                               self.worst_5_tabulator,
+                               max_width=500, header=pn.pane.Str('Portfolio中最高回报和最低加权回报率股票'))
+        return self._layout
+
+
+class TopHeader(Viewer):
+    '''
+    display up to todays' PnL, total return and max drawdown
+    '''
+    eval_df = param.Parameter()
+
+    @param.depends('eval_df', watch=True)
+    def update(self):
+        '''
+        update Pnl, total return and max drawdown when df is updated
+        '''
+        return
+
+    def calculation(self):
+        '''calculate PnL, total return and max drawdown'''
+        pnl = self.eval_df[self.eval_df.date ==
+                           self.eval_df.date.max()].cum_pnl.values[0]
+        total_return = self.eval_df[self.eval_df.date ==
+                                    self.eval_df.date.max()].portfolio_return_p.values[0]
+        # max draw down
+        self.eval_df['rolling_max_return'] = self.eval_df.portfolio_return_p.rolling(
+            window=len(self.eval_df), min_periods=1).max()
+        self.eval_df.draw_down = abs(
+            (1 + self.eval_df.portfolio_return_p) /
+            (1 + self.eval_df.rolling_max_return) - 1
+        )
+        max_drawdown = self.eval_df.draw_down.max()
+        return pnl, total_return, max_drawdown
+
+    def create_report(self, pnl, total_return, max_drawdown):
+        return pn.FlexBox(
+            f"PnL:{round(pnl,2)}¥", f"回报：{round(total_return * 100,2)}%", f'最大回撤:{round(max_drawdown * 100,2)}%', justify_content='space-evenly')
+
+    def __init__(self, eval_df, **params):
+        self.eval_df = eval_df
+        pnl, total_return, max_drawdown = self.calculation()
+        self.report = self.create_report(pnl, total_return, max_drawdown)
+        super().__init__(**params)
+
+    def __panel__(self):
+        self._layout = pn.Card(self.report, sizing_mode='stretch_width')
+        return self._layout

app_ini.ipynb

@@ -1,3 +1,723 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1178e33b9c1d97aac01ebe954a92585270ea63da5fca3f3ae388a3ae70b317ad
-size 56961
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.1.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n  var reloading = false;\n  var Bokeh = root.Bokeh;\n  var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n      require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n      })\n      require([\"jspanel-modal\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-tooltip\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-hint\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-layout\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-dock\"], function() {\n\ton_load()\n      })\n      require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n      })\n      require([\"notyf\"], function() {\n\ton_load()\n      })\n      root._bokeh_is_loading = css_urls.length + 9;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.holoviz.org/panel/1.1.1/dist/panel.min.js\"];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      Bokeh = root.Bokeh;\n      bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n  window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n    function JupyterCommManager() {\n    }\n\n    JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n      if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        comm_manager.register_target(comm_id, function(comm) {\n          comm.on_msg(msg_handler);\n        });\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n          comm.onMsg = msg_handler;\n        });\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n          var messages = comm.messages[Symbol.asyncIterator]();\n          function processIteratorResult(result) {\n            var message = result.value;\n            console.log(message)\n            var content = {data: message.data, comm_id};\n            var buffers = []\n            for (var buffer of message.buffers || []) {\n              buffers.push(new DataView(buffer))\n            }\n            var metadata = message.metadata || {};\n            var msg = {content, buffers, metadata}\n            msg_handler(msg);\n            return messages.next().then(processIteratorResult);\n          }\n          return messages.next().then(processIteratorResult);\n        })\n      }\n    }\n\n    JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n      if (comm_id in window.PyViz.comms) {\n        return window.PyViz.comms[comm_id];\n      } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n        if (msg_handler) {\n          comm.on_msg(msg_handler);\n        }\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n        comm.open();\n        if (msg_handler) {\n          comm.onMsg = msg_handler;\n        }\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        var comm_promise = google.colab.kernel.comms.open(comm_id)\n        comm_promise.then((comm) => {\n          window.PyViz.comms[comm_id] = comm;\n          if (msg_handler) {\n            var messages = comm.messages[Symbol.asyncIterator]();\n            function processIteratorResult(result) {\n              var message = result.value;\n              var content = {data: message.data};\n              var metadata = message.metadata || {comm_id};\n              var msg = {content, metadata}\n              msg_handler(msg);\n              return messages.next().then(processIteratorResult);\n            }\n            return messages.next().then(processIteratorResult);\n          }\n        }) \n        var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n          return comm_promise.then((comm) => {\n            comm.send(data, metadata, buffers, disposeOnDone);\n          });\n        };\n        var comm = {\n          send: sendClosure\n        };\n      }\n      window.PyViz.comms[comm_id] = comm;\n      return comm;\n    }\n    window.PyViz.comm_manager = new JupyterCommManager();\n    \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n  var div = document.createElement(\"div\");\n  var script = document.createElement(\"script\");\n  node.appendChild(div);\n  node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n  var output_area = handle.output_area;\n  var output = handle.output;\n  if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n    return\n  }\n  var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n  var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n  if (id !== undefined) {\n    var nchildren = toinsert.length;\n    var html_node = toinsert[nchildren-1].children[0];\n    html_node.innerHTML = output.data[HTML_MIME_TYPE];\n    var scripts = [];\n    var nodelist = html_node.querySelectorAll(\"script\");\n    for (var i in nodelist) {\n      if (nodelist.hasOwnProperty(i)) {\n        scripts.push(nodelist[i])\n      }\n    }\n\n    scripts.forEach( function (oldScript) {\n      var newScript = document.createElement(\"script\");\n      var attrs = [];\n      var nodemap = oldScript.attributes;\n      for (var j in nodemap) {\n        if (nodemap.hasOwnProperty(j)) {\n          attrs.push(nodemap[j])\n        }\n      }\n      attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n      newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n      oldScript.parentNode.replaceChild(newScript, oldScript);\n    });\n    if (JS_MIME_TYPE in output.data) {\n      toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n    }\n    output_area._hv_plot_id = id;\n    if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n      window.PyViz.plot_index[id] = Bokeh.index[id];\n    } else {\n      window.PyViz.plot_index[id] = null;\n    }\n  } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n    var bk_div = document.createElement(\"div\");\n    bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n    var script_attrs = bk_div.children[0].attributes;\n    for (var i = 0; i < script_attrs.length; i++) {\n      toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n    }\n    // store reference to server id on output_area\n    output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n  }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n  var id = handle.cell.output_area._hv_plot_id;\n  var server_id = handle.cell.output_area._bokeh_server_id;\n  if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n  var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n  if (server_id !== null) {\n    comm.send({event_type: 'server_delete', 'id': server_id});\n    return;\n  } else if (comm !== null) {\n    comm.send({event_type: 'delete', 'id': id});\n  }\n  delete PyViz.plot_index[id];\n  if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n    var doc = window.Bokeh.index[id].model.document\n    doc.clear();\n    const i = window.Bokeh.documents.indexOf(doc);\n    if (i > -1) {\n      window.Bokeh.documents.splice(i, 1);\n    }\n  }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n  delete PyViz.comms[\"hv-extension-comm\"];\n  window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n  handle_clear_output(event, {cell: {output_area: handle.output_area}})\n  handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n  function append_mime(data, metadata, element) {\n    // create a DOM node to render to\n    var toinsert = this.create_output_subarea(\n    metadata,\n    CLASS_NAME,\n    EXEC_MIME_TYPE\n    );\n    this.keyboard_manager.register_events(toinsert);\n    // Render to node\n    var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n    render(props, toinsert[0]);\n    element.append(toinsert);\n    return toinsert\n  }\n\n  events.on('output_added.OutputArea', handle_add_output);\n  events.on('output_updated.OutputArea', handle_update_output);\n  events.on('clear_output.CodeCell', handle_clear_output);\n  events.on('delete.Cell', handle_clear_output);\n  events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n  OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n    safe: true,\n    index: 0\n  });\n}\n\nif (window.Jupyter !== undefined) {\n  try {\n    var events = require('base/js/events');\n    var OutputArea = require('notebook/js/outputarea').OutputArea;\n    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n      register_renderer(events, OutputArea);\n    }\n  } catch(err) {\n  }\n}\n",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style>*[data-root-id],\n",
+       "*[data-root-id] > * {\n",
+       "  box-sizing: border-box;\n",
+       "  font-family: var(--jp-ui-font-family);\n",
+       "  font-size: var(--jp-ui-font-size1);\n",
+       "  color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
+       "}\n",
+       "\n",
+       "/* Override VSCode background color */\n",
+       ".cell-output-ipywidget-background:has(> .cell-output-ipywidget-background\n",
+       "    > .lm-Widget\n",
+       "    > *[data-root-id]),\n",
+       ".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
+       "  background-color: transparent !important;\n",
+       "}\n",
+       "</style>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from datetime import datetime, timedelta\n",
+    "from script import processing\n",
+    "from script import api\n",
+    "from sqlalchemy import create_engine\n",
+    "import pytz\n",
+    "import numpy as np\n",
+    "import hvplot.pandas\n",
+    "db_url = 'sqlite:///local.db'\n",
+    "engine = create_engine(db_url)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "[Errno 2] No such file or directory: './data/p_profile.pkl'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[2], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[39m## initialize by batchprocess to have initial result \u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m p_profile \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mread_pickle(\u001b[39m'\u001b[39;49m\u001b[39m./data/p_profile.pkl\u001b[39;49m\u001b[39m'\u001b[39;49m)\n\u001b[1;32m      3\u001b[0m start_date \u001b[39m=\u001b[39m p_profile\u001b[39m.\u001b[39mdate\u001b[39m.\u001b[39mmin()\n\u001b[1;32m      4\u001b[0m end_date \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mto_datetime(datetime\u001b[39m.\u001b[39mnow()\u001b[39m-\u001b[39mtimedelta(days\u001b[39m=\u001b[39m\u001b[39m7\u001b[39m))\n",
+      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/portfolio_risk_assesment/lib/python3.11/site-packages/pandas/io/pickle.py:179\u001b[0m, in \u001b[0;36mread_pickle\u001b[0;34m(filepath_or_buffer, compression, storage_options)\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    116\u001b[0m \u001b[39mLoad pickled pandas object (or any object) from file.\u001b[39;00m\n\u001b[1;32m    117\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    176\u001b[0m \u001b[39m4    4    9\u001b[39;00m\n\u001b[1;32m    177\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    178\u001b[0m excs_to_catch \u001b[39m=\u001b[39m (\u001b[39mAttributeError\u001b[39;00m, \u001b[39mImportError\u001b[39;00m, \u001b[39mModuleNotFoundError\u001b[39;00m, \u001b[39mTypeError\u001b[39;00m)\n\u001b[0;32m--> 179\u001b[0m \u001b[39mwith\u001b[39;00m get_handle(\n\u001b[1;32m    180\u001b[0m     filepath_or_buffer,\n\u001b[1;32m    181\u001b[0m     \u001b[39m\"\u001b[39;49m\u001b[39mrb\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[1;32m    182\u001b[0m     compression\u001b[39m=\u001b[39;49mcompression,\n\u001b[1;32m    183\u001b[0m     is_text\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    184\u001b[0m     storage_options\u001b[39m=\u001b[39;49mstorage_options,\n\u001b[1;32m    185\u001b[0m ) \u001b[39mas\u001b[39;00m handles:\n\u001b[1;32m    186\u001b[0m     \u001b[39m# 1) try standard library Pickle\u001b[39;00m\n\u001b[1;32m    187\u001b[0m     \u001b[39m# 2) try pickle_compat (older pandas version) to handle subclass changes\u001b[39;00m\n\u001b[1;32m    188\u001b[0m     \u001b[39m# 3) try pickle_compat with latin-1 encoding upon a UnicodeDecodeError\u001b[39;00m\n\u001b[1;32m    190\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    191\u001b[0m         \u001b[39m# TypeError for Cython complaints about object.__new__ vs Tick.__new__\u001b[39;00m\n\u001b[1;32m    192\u001b[0m         \u001b[39mtry\u001b[39;00m:\n",
+      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/portfolio_risk_assesment/lib/python3.11/site-packages/pandas/io/common.py:868\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    859\u001b[0m         handle \u001b[39m=\u001b[39m \u001b[39mopen\u001b[39m(\n\u001b[1;32m    860\u001b[0m             handle,\n\u001b[1;32m    861\u001b[0m             ioargs\u001b[39m.\u001b[39mmode,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    864\u001b[0m             newline\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m    865\u001b[0m         )\n\u001b[1;32m    866\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    867\u001b[0m         \u001b[39m# Binary mode\u001b[39;00m\n\u001b[0;32m--> 868\u001b[0m         handle \u001b[39m=\u001b[39m \u001b[39mopen\u001b[39;49m(handle, ioargs\u001b[39m.\u001b[39;49mmode)\n\u001b[1;32m    869\u001b[0m     handles\u001b[39m.\u001b[39mappend(handle)\n\u001b[1;32m    871\u001b[0m \u001b[39m# Convert BytesIO or file objects passed with an encoding\u001b[39;00m\n",
+      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: './data/p_profile.pkl'"
+     ]
+    }
+   ],
+   "source": [
+    "## initialize by batchprocess to have initial result \n",
+    "p_profile = pd.read_pickle('./data/p_profile.pkl')\n",
+    "start_date = p_profile.date.min()\n",
+    "end_date = pd.to_datetime(datetime.now()-timedelta(days=7))\n",
+    "# collect data upto 7 days ago  \n",
+    "b_profile, error  = api.update_benchmark_profile(start_date, end_date)\n",
+    "p_stocks, error = api.get_stocks_price(p_profile, start_date, end_date)\n",
+    "b_stocks, error = api.get_stocks_price(b_profile, start_date, end_date)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save result \n",
+    "# p_profile.to_pickle('./data/p_profile.pkl')\n",
+    "# b_profile.to_pickle('./data/b_profile.pkl')\n",
+    "p_stocks.to_pickle('./data/p_stocks.pkl')\n",
+    "b_stocks.to_pickle('./data/b_stocks.pkl')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/lamonkey/Desktop/risk monitor/script/processing.py:262: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  df.fillna(0, inplace=True)\n",
+      "/Users/lamonkey/Desktop/risk monitor/script/processing.py:263: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  df['active_return'] = df.pct_p * \\\n",
+      "/Users/lamonkey/Desktop/risk monitor/script/processing.py:266: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  df['allocation'] = (df.prev_w_in_p_p - df.prev_w_in_p_b) * df.pct_b\n",
+      "/Users/lamonkey/Desktop/risk monitor/script/processing.py:267: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  df['selection'] = (df.pct_p - df.pct_b) * df.prev_w_in_p_b\n",
+      "/Users/lamonkey/Desktop/risk monitor/script/processing.py:268: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  df['interaction'] = (df.pct_p - df.pct_b) * \\\n",
+      "/Users/lamonkey/Desktop/risk monitor/script/processing.py:270: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  df['notinal_return'] = df.allocation + df.selection + df.interaction\n"
+     ]
+    }
+   ],
+   "source": [
+    "## batch processing \n",
+    "calculated_p_stock = processing.get_processing_result_of_stocks_df(p_stocks, p_profile)\n",
+    "calculated_b_stock = processing.get_processing_result_of_stocks_df(b_stocks, b_profile)\n",
+    "p_eval_df = processing.get_portfolio_evaluation(calculated_p_stock, calculated_b_stock, p_profile)\n",
+    "sector_eval_df = processing.get_portfolio_sector_evaluation(calculated_p_stock, calculated_b_stock)\n",
+    "attribution_result_df = processing.calculate_total_attribution(calculated_p_stock, calculated_b_stock)\n",
+    "s_attribution_result_df = processing.calculate_total_attribution_by_sector(calculated_p_stock, calculated_b_stock)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## save result to db\n",
+    "with engine.connect() as connection:\n",
+    "    # all_stock_info.to_sql('all_stock_info', con=connection, if_exists='replace', index=False)\n",
+    "    calculated_b_stock.to_sql('calculated_b_stock', con=connection, if_exists='replace', index=False)\n",
+    "    calculated_p_stock.to_sql('calculated_p_stock', con=connection, if_exists='replace', index=False)\n",
+    "    p_eval_df.to_sql('p_eval_result', con=connection, if_exists='replace', index=False)\n",
+    "    sector_eval_df.to_sql('sector_eval_result', con=connection, if_exists='replace', index=False)\n",
+    "    attribution_result_df.to_sql('attribution_result', con=connection, if_exists='replace', index=False)\n",
+    "    s_attribution_result_df.to_sql('s_attribution_result', con=connection, if_exists='replace', index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load from sql\n",
+    "name_df_map = dict()\n",
+    "with engine.connect() as connection:\n",
+    "    for table in ['calculated_b_stock','calculated_p_stock','p_eval_result','sector_eval_result']:\n",
+    "        try:\n",
+    "            df = pd.read_sql_table(table, con=connection)\n",
+    "            name_df_map[table] = df\n",
+    "        except:\n",
+    "            pass\n",
+    "            # TODO load data from api and calculate result \n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load data upto now\n",
+    "# Get the current time in UTC\n",
+    "current_time = datetime.datetime.utcnow()\n",
+    "# Set the timezone to Beijing\n",
+    "beijing_timezone = pytz.timezone('Asia/Shanghai')\n",
+    "# Convert the current time to Beijing time\n",
+    "end_time = pd.to_datetime(current_time.astimezone(beijing_timezone).date())\n",
+    "start_time = name_df_map['p_eval_result'].date.max() + timedelta(days=1)\n",
+    "\n",
+    "# get data up to today\n",
+    "b_profile, error  = api.update_benchmark_profile(start_time, end_time)\n",
+    "p_stocks, error = api.get_stocks_price(p_profile, start_time, end_time)\n",
+    "b_stocks, error = api.get_stocks_price(b_profile, start_time, end_time)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 185,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ValueError",
+     "evalue": "The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "\u001b[0;32m/var/folders/v5/2108rh5964q9j741wg_s8r1w0000gn/T/ipykernel_35506/2587190678.py\u001b[0m in \u001b[0;36m?\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmost_recent_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname_df_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'calculated_p_stock'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ticker'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mconcat_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmost_recent_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_stocks\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'outer'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_processing_result_of_stocks_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcat_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_profile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/Desktop/risk monitor/script/processing.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(stock_df, profile_df)\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m             \u001b[0mcur_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'nan'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m             \u001b[0;31m# if has initial weight, the following row all use this initial weight\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'initial_weight'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m                 \u001b[0mini_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'initial_weight'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m                 \u001b[0mcur_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mini_w\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/portfolio_risk_assesment/lib/python3.11/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1464\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1465\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__nonzero__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mNoReturn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1466\u001b[0;31m         raise ValueError(\n\u001b[0m\u001b[1;32m   1467\u001b[0m             \u001b[0;34mf\"The truth value of a {type(self).__name__} is ambiguous. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1468\u001b[0m             \u001b[0;34m\"Use a.empty, a.bool(), a.item(), a.any() or a.all().\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1469\u001b[0m         )\n",
+      "\u001b[0;31mValueError\u001b[0m: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all()."
+     ]
+    }
+   ],
+   "source": [
+    "most_recent_df = name_df_map['calculated_p_stock'].groupby('ticker').last().reset_index()\n",
+    "concat_df = pd.concat([most_recent_df, p_stocks], axis=0, join='outer')\n",
+    "processing.get_processing_result_of_stocks_df(concat_df, p_profile)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 182,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "      <th>display_name</th>\n",
+       "      <th>name</th>\n",
+       "      <th>...</th>\n",
+       "      <th>portfolio_pct</th>\n",
+       "      <th>prev_w_in_sectore</th>\n",
+       "      <th>ini_w_in_sector</th>\n",
+       "      <th>sector_pct</th>\n",
+       "      <th>portfolio_return</th>\n",
+       "      <th>cum_pct</th>\n",
+       "      <th>return</th>\n",
+       "      <th>sector_return</th>\n",
+       "      <th>cur_w_in_p</th>\n",
+       "      <th>pre_w_in_sector</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>9.23</td>\n",
+       "      <td>9.57</td>\n",
+       "      <td>9.66</td>\n",
+       "      <td>9.08</td>\n",
+       "      <td>82669289.0</td>\n",
+       "      <td>7.803391e+08</td>\n",
+       "      <td>三友化工</td>\n",
+       "      <td>SYHG</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>300274.XSHE</td>\n",
+       "      <td>76.03</td>\n",
+       "      <td>76.45</td>\n",
+       "      <td>80.20</td>\n",
+       "      <td>75.27</td>\n",
+       "      <td>51384827.0</td>\n",
+       "      <td>3.961995e+09</td>\n",
+       "      <td>阳光电源</td>\n",
+       "      <td>YGDY</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>002920.XSHE</td>\n",
+       "      <td>85.44</td>\n",
+       "      <td>87.25</td>\n",
+       "      <td>87.95</td>\n",
+       "      <td>84.07</td>\n",
+       "      <td>3852674.0</td>\n",
+       "      <td>3.322598e+08</td>\n",
+       "      <td>德赛西威</td>\n",
+       "      <td>DSXW</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>002709.XSHE</td>\n",
+       "      <td>32.54</td>\n",
+       "      <td>33.89</td>\n",
+       "      <td>34.22</td>\n",
+       "      <td>31.39</td>\n",
+       "      <td>59152352.0</td>\n",
+       "      <td>1.942406e+09</td>\n",
+       "      <td>天赐材料</td>\n",
+       "      <td>TCCL</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>603882.XSHG</td>\n",
+       "      <td>125.25</td>\n",
+       "      <td>124.64</td>\n",
+       "      <td>128.31</td>\n",
+       "      <td>121.68</td>\n",
+       "      <td>6803710.0</td>\n",
+       "      <td>8.458543e+08</td>\n",
+       "      <td>金域医学</td>\n",
+       "      <td>JYYX</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3609</th>\n",
+       "      <td>2023-06-27</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>8.52</td>\n",
+       "      <td>8.69</td>\n",
+       "      <td>8.78</td>\n",
+       "      <td>8.40</td>\n",
+       "      <td>151396630.0</td>\n",
+       "      <td>1.305075e+09</td>\n",
+       "      <td>小商品城</td>\n",
+       "      <td>XSPC</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.027187</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.301143</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3610</th>\n",
+       "      <td>2023-06-28</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>8.60</td>\n",
+       "      <td>8.63</td>\n",
+       "      <td>8.68</td>\n",
+       "      <td>8.37</td>\n",
+       "      <td>103167271.0</td>\n",
+       "      <td>8.798186e+08</td>\n",
+       "      <td>小商品城</td>\n",
+       "      <td>XSPC</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>-0.006904</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.299958</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3611</th>\n",
+       "      <td>2023-06-29</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>8.60</td>\n",
+       "      <td>8.74</td>\n",
+       "      <td>8.88</td>\n",
+       "      <td>8.58</td>\n",
+       "      <td>128969467.0</td>\n",
+       "      <td>1.125704e+09</td>\n",
+       "      <td>小商品城</td>\n",
+       "      <td>XSPC</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.012746</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.301804</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3612</th>\n",
+       "      <td>2023-06-30</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>8.74</td>\n",
+       "      <td>8.53</td>\n",
+       "      <td>8.77</td>\n",
+       "      <td>8.48</td>\n",
+       "      <td>103029932.0</td>\n",
+       "      <td>8.844883e+08</td>\n",
+       "      <td>小商品城</td>\n",
+       "      <td>XSPC</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>-0.024027</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.293612</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3613</th>\n",
+       "      <td>2023-07-03</td>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>8.45</td>\n",
+       "      <td>8.37</td>\n",
+       "      <td>8.46</td>\n",
+       "      <td>8.05</td>\n",
+       "      <td>133732493.0</td>\n",
+       "      <td>1.108033e+09</td>\n",
+       "      <td>小商品城</td>\n",
+       "      <td>XSPC</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>-0.018757</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.286010</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>3614 rows × 27 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           date       ticker    open   close    high     low       volume  \\\n",
+       "0    2021-01-05  600409.XSHG    9.23    9.57    9.66    9.08   82669289.0   \n",
+       "1    2021-01-05  300274.XSHE   76.03   76.45   80.20   75.27   51384827.0   \n",
+       "2    2021-01-05  002920.XSHE   85.44   87.25   87.95   84.07    3852674.0   \n",
+       "3    2021-01-05  002709.XSHE   32.54   33.89   34.22   31.39   59152352.0   \n",
+       "4    2021-01-05  603882.XSHG  125.25  124.64  128.31  121.68    6803710.0   \n",
+       "...         ...          ...     ...     ...     ...     ...          ...   \n",
+       "3609 2023-06-27  600415.XSHG    8.52    8.69    8.78    8.40  151396630.0   \n",
+       "3610 2023-06-28  600415.XSHG    8.60    8.63    8.68    8.37  103167271.0   \n",
+       "3611 2023-06-29  600415.XSHG    8.60    8.74    8.88    8.58  128969467.0   \n",
+       "3612 2023-06-30  600415.XSHG    8.74    8.53    8.77    8.48  103029932.0   \n",
+       "3613 2023-07-03  600415.XSHG    8.45    8.37    8.46    8.05  133732493.0   \n",
+       "\n",
+       "             money display_name  name  ... portfolio_pct  prev_w_in_sectore  \\\n",
+       "0     7.803391e+08         三友化工  SYHG  ...           NaN                NaN   \n",
+       "1     3.961995e+09         阳光电源  YGDY  ...           NaN                NaN   \n",
+       "2     3.322598e+08         德赛西威  DSXW  ...           NaN                NaN   \n",
+       "3     1.942406e+09         天赐材料  TCCL  ...           NaN                NaN   \n",
+       "4     8.458543e+08         金域医学  JYYX  ...           NaN                NaN   \n",
+       "...            ...          ...   ...  ...           ...                ...   \n",
+       "3609  1.305075e+09         小商品城  XSPC  ...           NaN                NaN   \n",
+       "3610  8.798186e+08         小商品城  XSPC  ...           NaN                NaN   \n",
+       "3611  1.125704e+09         小商品城  XSPC  ...           NaN                NaN   \n",
+       "3612  8.844883e+08         小商品城  XSPC  ...           NaN                NaN   \n",
+       "3613  1.108033e+09         小商品城  XSPC  ...           NaN                NaN   \n",
+       "\n",
+       "      ini_w_in_sector  sector_pct  portfolio_return  cum_pct  return  \\\n",
+       "0                 1.0         NaN               NaN      NaN     NaN   \n",
+       "1                 0.5         NaN               NaN      NaN     NaN   \n",
+       "2                 1.0         NaN               NaN      NaN     NaN   \n",
+       "3                 0.5         NaN               NaN      NaN     NaN   \n",
+       "4                 1.0         NaN               NaN      NaN     NaN   \n",
+       "...               ...         ...               ...      ...     ...   \n",
+       "3609              NaN    0.027187               NaN      NaN     NaN   \n",
+       "3610              NaN   -0.006904               NaN      NaN     NaN   \n",
+       "3611              NaN    0.012746               NaN      NaN     NaN   \n",
+       "3612              NaN   -0.024027               NaN      NaN     NaN   \n",
+       "3613              NaN   -0.018757               NaN      NaN     NaN   \n",
+       "\n",
+       "      sector_return  cur_w_in_p  pre_w_in_sector  \n",
+       "0               NaN         NaN              NaN  \n",
+       "1               NaN         NaN              NaN  \n",
+       "2               NaN         NaN              NaN  \n",
+       "3               NaN         NaN              NaN  \n",
+       "4               NaN         NaN              NaN  \n",
+       "...             ...         ...              ...  \n",
+       "3609            NaN    0.301143              1.0  \n",
+       "3610            NaN    0.299958              1.0  \n",
+       "3611            NaN    0.301804              1.0  \n",
+       "3612            NaN    0.293612              1.0  \n",
+       "3613            NaN    0.286010              1.0  \n",
+       "\n",
+       "[3614 rows x 27 columns]"
+      ]
+     },
+     "execution_count": 182,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "most_recent_df = name_df_map['calculated_p_stock'].groupby('ticker').last().reset_index()\n",
+    "\n",
+    "def get_last_values(row):\n",
+    "    ticker = row['ticker']\n",
+    "    if ticker in p_profile['ticker'].values:\n",
+    "        return p_profile.loc[p_profile['ticker'] == ticker, ['display_name', 'name', 'aggregate_sector']].iloc[-1]\n",
+    "    else:\n",
+    "        return pd.Series([np.nan, np.nan, np.nan], index=['display_name', 'name', 'aggregate_sector'])\n",
+    "# dispaly_name, name and aggregate_sector\n",
+    "p_stocks[['display_name', 'name', 'aggregate_sector']] = p_stocks.apply(get_last_values, axis=1)\n",
+    "\n",
+    "# use the most recent result to resume calculation\n",
+    "concat_df = pd.concat([most_recent_df, p_stocks], axis=0, join='outer')\n",
+    "\n",
+    "# pct\n",
+    "concat_df['pct'] = concat_df.groupby('ticker')['close'].pct_change()\n",
+    "\n",
+    "# calculate not normalized previous weight and current weight\n",
+    "groups = concat_df.groupby('ticker')\n",
+    "for _, group in groups:\n",
+    "    cur_weight = np.nan\n",
+    "    for index, row in group.iterrows():\n",
+    "        if pd.notna(row['current_weight']):\n",
+    "            cur_weight = row['current_weight']\n",
+    "        else:\n",
+    "            concat_df.loc[index, 'previous_weight'] = cur_weight\n",
+    "            cur_weight = cur_weight * (1 + row['pct'])\n",
+    "            concat_df.loc[index, 'current_weight'] = cur_weight\n",
+    "\n",
+    "# calculate normalized previous and current weight\n",
+    "concat_df['prev_w_in_p'] = concat_df['previous_weight'] / \\\n",
+    "    concat_df.groupby('date')['previous_weight'].transform('sum')\n",
+    "concat_df['cur_w_in_p'] = concat_df['current_weight'] / \\\n",
+    "    concat_df.groupby('date')['current_weight'].transform('sum')\n",
+    "\n",
+    "# calculate previous weight in sector\n",
+    "concat_df['pre_w_in_sector'] = concat_df['prev_w_in_p'] / \\\n",
+    "    concat_df.groupby(['date', 'aggregate_sector'])['prev_w_in_p'].transform('sum')\n",
+    "\n",
+    "# calculate pct in sector\n",
+    "concat_df['sector_pct'] = concat_df['pct'] * concat_df['pre_w_in_sector']\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "# remove group with first date\n",
+    "min_date_group = concat_df.groupby('date')['date'].idxmin()\n",
+    "concat_df = concat_df.drop(min_date_group)\n",
+    "\n",
+    "# merge back to calculated_stock\n",
+    "pd.concat([name_df_map['calculated_p_stock'],concat_df]).reset_index(drop=True)\n",
+    "\n",
+    "# concat_df[concat_df.ticker == '002709.XSHE'][['date','pct','current_weight','previous_weight','prev_w_in_p','cur_w_in_p']]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

find_outlier.ipynb

@@ -1,3 +1,120 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d032b15baf8608f845eb33d643f02a1deb7543fc883732c7bae925bb06a5b281
-size 2947
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import panel as pn\n",
+    "import pandas as pd\n",
+    "import scipy.stats as stats\n",
+    "from sqlalchemy import create_engine\n",
+    "import hvplot.pandas\n",
+    "pn.extension()\n",
+    "db_url = 'sqlite:///local.db'\n",
+    "engine = create_engine(db_url)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Expirement to use z-score to find outlier of daily pct, not sure if this is helpful but here is it"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "b_stocks_df = None\n",
+    "# load benchmark stock\n",
+    "with engine.connect() as connection:\n",
+    "    b_stocks_df = pd.read_sql('calculated_b_stock', con=connection)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### z = (x - μ) / σ"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "b_stocks_df['pct_mean'] = b_stocks_df.groupby('ticker')['pct']\\\n",
+    "    .transform(lambda x: x.rolling(7, min_periods=1).mean())\n",
+    "b_stocks_df['pct_std'] = b_stocks_df.groupby('ticker')['pct']\\\n",
+    "    .transform(lambda x: x.rolling(7, min_periods=1).std())\n",
+    "b_stocks_df['pct_z_score'] = (b_stocks_df['pct'] - b_stocks_df['pct_mean']) / b_stocks_df['pct_std']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "select = pn.widgets.Select(name='Select', options=b_stocks_df.display_name.unique().tolist())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ib_stocks_df = b_stocks_df.interactive()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "selected_stock_df = ib_stocks_df[ib_stocks_df.display_name == select]\n",
+    "outliers = selected_stock_df[((ib_stocks_df['pct_z_score'] > 2) | (ib_stocks_df['pct_z_score'] < -2))]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "outlier_plot = outliers.hvplot.scatter(x='date',y='pct',color='red')\n",
+    "pct_plot = selected_stock_df.hvplot(x='date',y='pct')\n",
+    "outlier_plot * pct_plot\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

index_page.py

@@ -1,3 +1,73 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ff4adeb8072294a101900b8d10e3d544db694872c9cdb31353d5bf59ac3351d5
-size 2544
+import panel as pn
+import pandas as pd
+from datetime import datetime, timedelta
+import plotly.express as px
+import holoviews as hv
+import numpy as np
+import random
+import scipy.stats as stats
+import hvplot.pandas  # noqa
+from sqlalchemy import create_engine
+from . import api
+# from backgroundTask import stocks_stream
+from functools import partial
+import plotly.graph_objects as go
+from panel.viewable import Viewer
+from script import processing
+import appComponents
+import param
+# import warnings
+pn.extension('mathjax')
+# warnings.filterwarnings("ignore", category=pd.core.common.SettingWithCopyWarning)
+pn.extension('plotly')
+pn.extension('tabulator')
+db_url = 'sqlite:///local.db'
+engine = create_engine(db_url)
+
+
+p_eval_df = None
+calculated_b_stock = None
+calculated_p_stock = None
+# load benchmark stock
+with engine.connect() as connection:
+    calculated_b_stock = pd.read_sql('calculated_b_stock', con=connection)
+    calculated_p_stock = pd.read_sql('calculated_p_stock', con=connection)
+    p_eval_df = pd.read_sql('p_eval_result', con=connection)
+
+stock_overview = appComponents.BestAndWorstStocks(
+    p_stock_df=calculated_p_stock, b_stock_df=calculated_b_stock)
+composation_card = appComponents.PortfolioComposationCard(
+    p_stock_df=calculated_p_stock)
+monthly_return_card = appComponents.HistReturnCard(
+    eval_df=p_eval_df, calculated_p_stock=calculated_p_stock, calculated_b_stock=calculated_b_stock)
+total_return_card = appComponents.TotalReturnCard(name='Range', eval_df=p_eval_df,
+                                                  b_stock_df=calculated_b_stock,
+                                                  p_stock_df=calculated_p_stock,
+                                                  value=(0, 20))
+drawdown_card = appComponents.DrawDownCard(
+    eval_df=p_eval_df, calculated_p_stock=calculated_p_stock, calculated_b_stock=calculated_b_stock)
+
+top_header = appComponents.TopHeader(
+    eval_df=p_eval_df
+)
+
+template = pn.template.FastListTemplate(
+    title="Portfolio一览",
+    # sidebar=[freq, phase],
+)
+template.main.extend(
+    [pn.Row(top_header),
+     pn.Row(
+        pn.Column(monthly_return_card, stock_overview,
+                  width=500, margin=(10, 10, 10, 10)),
+        pn.Column(total_return_card, drawdown_card, margin=(10, 10, 10, 10)),
+        pn.Column(composation_card, margin=(10, 10, 10, 10)),
+    )]
+)
+template.servable()
+# pn.Row(
+
+#     pn.Column(monthly_return_card, stock_overview, width=500),
+#     pn.Column(total_return_card),
+#     pn.Column(composation_card)
+# ).servable()

initialize_db.py

@@ -1,3 +1,28 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8f0779a757c8a2a2ab42df0d79a4a932642fca6184fcaf95884af0b3f98bd66c
-size 998
+from sqlalchemy import create_engine
+import pandas as pd
+import table_schema as ts
+db_url = "sqlite:///local.db"
+
+
+def _create_table_with_schema(table_name: str, table_schema: dict):
+    with create_engine(db_url).connect() as conn:
+        df = pd.DataFrame(
+            columns=table_schema.keys()).astype(table_schema)
+        df.to_sql(
+            table_name, conn, if_exists='replace', index=False)
+        return True
+
+
+def initialize_db():
+    # initialize portfolio profile table
+    if not _create_table_with_schema(ts.PORTFOLIO_TABLE, ts.PORTFOLIO_TABLE_SCHEMA):
+        raise Exception(
+            f'INITIALIZATION ERROR: cannot create table {ts.PORTFOLIO_TABLE} ')
+    # initialize stocks details table
+    if not _create_table_with_schema(ts.STOCKS_DETAILS_TABLE, ts.STOCKS_DETAILS_TABLE_SCHEMA):
+        raise Exception(
+            f'INITIALIZATION ERROR: cannot create table {ts.STOCKS_DETAILS_TABLE} ')
+
+# allow to be run as script
+if __name__ == '__main__':
+    initialize_db()

newBackgroundTask.py

@@ -1,3 +1,12 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:87f3213d968c3f343949a31758a09973c8e189b2c13d4ec710130f0e5c96bde2
-size 285
+import sys
+sys.path.append('/Users/lamonkey/Desktop/risk-monitor-dashboard')
+from pipeline import update
+import panel as pn
+from datetime import timedelta
+
+# pn.state.schedule_task(
+#     'task', run, period=timedelta(seconds=3)
+# )
+
+# update stock price and benchmark profile
+update()

pipeline.py

@@ -1,3 +1,337 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ddf43d57700db70f9b7e3b5a4de30913ee461a7d8d0360a7b654155ed2db2105
-size 10888
+import sys
+# sys.path.append('/Users/lamonkey/Desktop/risk-monitor-dashboard')
+import panel as pn
+import datetime as dt
+import asyncio
+import random
+from sqlalchemy import create_engine, text
+import pandas as pd
+from streamz import Stream
+from datetime import timedelta
+import settings
+import os
+import utils
+import api
+import numpy as np
+import pytz
+import table_schema as table_s
+# fetch new stock price
+stock_price_stream = Stream()
+
+# save stock price to db
+# stock_price_stream.sink(save_stock_price)
+# from dask.distributed import Client
+# client = Client()
+# import nest_asyncio
+# nest_asyncio.apply()
+# import settings
+
+# run using  --setup
+db_url = "sqlite:///local.db"
+
+
+def create_portfolio_profile_df(stocks: list[dict]):
+    profile_df = pd.DataFrame(stocks)
+    profile_df = add_details_to_stock_df(profile_df)
+
+    # check if there is duplicate ticker
+    if profile_df.ticker.duplicated().any():
+        raise Exception(
+            'VALIDATION ERROR: cannot have duplicate ticker with the same date')
+
+    return profile_df
+
+
+def need_to_update(table_name: str, freq: dt.datetime):
+    '''check table with table_name need to update
+    Return
+    ------
+    None if no need to update
+    (start_date, end_date, freq) if need to update
+    '''
+    with create_engine(db_url).connect() as conn:
+        max_date = conn.execute(
+            text(f"SELECT MAX(date) FROM {table_name}")).fetchone()[0]
+        max_date = utils.convert_string_to_datetime(max_date)
+        current_time = utils.time_in_beijing()
+        if current_time - max_date > freq:
+            return (max_date + freq, current_time, freq)
+        else:
+            return None
+
+
+def need_to_fetch_new_stock_price():
+    '''
+    check if need to pull new stock price from jq
+
+    RETURN
+    ------
+    (min_date, max_date) : if update is needed
+       the start and end date need to fetch new stock price
+    None if no need to fetch new stock price
+
+    '''
+    # get min date from portfolio_profile
+    with create_engine(db_url).connect() as conn:
+        table_name = 'portfolio_profile'
+        query = f"SELECT DISTINCT date FROM {table_name} ORDER BY date ASC LIMIT 1"
+        df = pd.read_sql(query, con=conn)
+        df.date = pd.to_datetime(df.date)
+        min_date = df.date[0]
+
+    # compare to min date from stocks_price
+    with create_engine(db_url).connect() as conn:
+        table_name = 'stocks_price'
+        query = f"SELECT DISTINCT time FROM {table_name} ORDER BY time ASC LIMIT 1"
+        df = pd.read_sql(query, con=conn)
+        df.time = pd.to_datetime(df.time)
+
+    # return
+    if min_date <= df.time[0]:
+        return (min_date, df.time[0] - dt.timedelta(days=1))
+    else:
+        return None
+
+
+def get_most_recent_profile(type):
+    table_name = 'benchmark_profile' if type == 'benchmark' else 'portfolio_profile'
+    query = f"SELECT * FROM {table_name} WHERE date = (SELECT MAX(date) FROM {table_name})"
+    with create_engine(db_url).connect() as conn:
+        df = pd.read_sql(query, con=conn)
+        # convert date to datetime object
+        df['date'] = pd.to_datetime(df['date'])
+        return df
+
+
+def update_stocks_details_to_db():
+    '''create table contain all stocks detail in db
+    will override existing table if exists
+    Table Schema
+    ------------
+    'display_name', 'name', 'start_date', 'end_date', 'type', 'ticker',
+       'sector', 'aggregate_sector'
+    '''
+    df = api.get_all_stocks_detail()
+    # validation
+    if not _validate_schema(df, table_s.STOCKS_DETAILS_TABLE_SCHEMA):
+        raise ValueError(
+            'df has different schema than STOCKS_DETAILS_TABLE_SCHEMA')
+    with create_engine(db_url).connect() as conn:
+        df.to_sql(table_s.STOCKS_DETAILS_TABLE, con=conn,
+                  if_exists='replace', index=False)
+
+
+def fetch_new_stocks_price():
+    '''
+    get a df contain updated stock prices for both benchmark and portfolio,
+    also indicate if the stock is in portfolio and benchmark
+    '''
+    # most recent profiles
+    p_portfolio = get_most_recent_profile('portfolio')
+    p_benchmark = get_most_recent_profile('benchmark')
+    # combine ticker
+    unique_tickers = pd.concat([p_portfolio, p_benchmark])[
+        'ticker'].unique().tolist()
+    # fetch list of stock
+    # TODO: hard code delta time to 1 day
+    start_date = p_portfolio.date[0] + dt.timedelta(days=1)
+    end_date = utils.time_in_beijing()
+    freq = 'daily'
+    stock_df = api.fetch_stocks_price(
+        unique_tickers, start_date, end_date, freq)
+    stock_df['in_portfolio'] = stock_df['ticker'].isin(
+        p_portfolio['ticker'].unique().tolist())
+    stock_df['in_benchmark'] = stock_df['ticker'].isin(
+        p_benchmark['ticker'].unique().tolist())
+    return stock_df
+
+
+def need_to_update_stocks_price(delta_time):
+    # convert p_portfolio.date[0] to timezone-aware datetime object
+    tz = pytz.timezone('Asia/Shanghai')
+    # get stock price df
+    with create_engine(db_url).connect() as conn:
+        # check if a table exist
+        if not conn.dialect.has_table(conn, 'stocks_price'):
+            return True
+        else:
+            query = "SELECT * FROM stocks_price WHERE time = (SELECT MAX(time) FROM stocks_price)"
+            most_recent_price = pd.read_sql(query, con=conn)
+            most_recent_price.time = pd.to_datetime(most_recent_price.time)
+            date_time = tz.localize(most_recent_price.time[0].to_pydatetime())
+            if utils.time_in_beijing() - date_time > delta_time:
+                return True
+            else:
+                return False
+
+
+def processing():
+    '''
+    run the whole processing pipeline here
+    '''
+    pass
+
+
+def add_details_to_stock_df(stock_df):
+    with create_engine(db_url).connect() as conn:
+        detail_df = pd.read_sql('stocks_details', con=conn)
+        merged_df = pd.merge(stock_df, detail_df[
+            ['sector', 'name',
+             'aggregate_sector',
+             'display_name',
+             'ticker']
+        ], on='ticker', how='left')
+        merged_df['aggregate_sector'].fillna('其他', inplace=True)
+        return merged_df
+
+
+def _validate_schema(df, schema):
+    '''
+    validate df has the same columns and data types as schema
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    schema: dict
+        {column_name: data_type}
+
+    Returns
+    -------
+    bool
+        True if df has the same columns and data types as schema
+        False otherwise
+    '''
+
+    # check if the DataFrame has the same columns as the schema
+    if set(df.columns) != set(schema.keys()):
+        return False
+    # check if the data types of the columns match the schema
+    # TODO: ignoring type check for now
+    # for col, dtype in schema.items():
+    #     if df[col].dtype != dtype:
+    #         return False
+    return True
+
+
+def save_stock_price_to_db(df: pd.DataFrame):
+    print('saving to stock to db')
+    with create_engine(db_url).connect() as conn:
+        df.to_sql('stocks_price', con=conn, if_exists='append', index=False)
+
+
+def update_portfolio_profile_to_db(portfolio_df):
+    '''overwrite the portfolio profile table in db'''
+
+    if (_validate_schema(portfolio_df, table_s.PORTFOLIO_TABLE_SCHEMA)):
+        raise ValueError(
+            'portfoliijuo_df has different schema than PORTFOLIO_DB_SCHEMA')
+
+    with create_engine(db_url).connect() as conn:
+        print("updating profile to db")
+        try:
+            portfolio_df[table_s.PORTFOLIO_TABLE_SCHEMA.keys()].to_sql(
+                table_s.PORTFOLIO_TABLE, con=conn, if_exists='append', index=False)
+            return True
+        except:
+            return False
+    # TODO trigger recomputation of analysis
+
+
+def update_stock_price():
+    '''get daily stocks price until today'''
+    # most recent profiles
+    p_portfolio = get_most_recent_profile('portfolio')
+    p_benchmark = get_most_recent_profile('benchmark')
+    # combine ticker
+    unique_tickers = pd.concat([p_portfolio, p_benchmark])[
+        'ticker'].unique().tolist()
+    # fetch list of stock
+    # TODO: hard code delta time to 1 day
+    start_date = p_portfolio.date[0] + dt.timedelta(days=1)
+    end_date = utils.time_in_beijing()
+    freq = 'daily'
+    stock_df = api.fetch_stocks_price(
+        unique_tickers, start_date, end_date, freq)
+    stock_df['in_portfolio'] = stock_df['ticker'].isin(
+        p_portfolio['ticker'].unique().tolist())
+    stock_df['in_benchmark'] = stock_df['ticker'].isin(
+        p_benchmark['ticker'].unique().tolist())
+    return stock_df
+
+
+def patch_stock_prices_db(window):
+    '''
+    patch stock price db with all daily stock price within window
+
+    Parameters
+    ----------
+    window: tuple
+        (start, end) date of the window
+
+    Returns
+    -------
+    None
+    '''
+    start, end = window
+    # all trading stock between start day and end date
+    with create_engine(db_url).connect() as conn:
+        all_stocks = pd.read_sql('stocks_details', con=conn)
+
+    selected_stocks = all_stocks[(all_stocks.start_date <= end) & (
+        all_stocks.end_date >= start)]
+    tickers = selected_stocks.ticker.to_list()
+
+    # fetch stock price and append to db
+    stock_price = api.fetch_stocks_price(tickers, start, end, 'daily')
+    detailed_stock_df = add_details_to_stock_df(stock_price)
+    # drop where closing price is null
+    detailed_stock_df.dropna(subset=['close'], inplace=True)
+    with create_engine(db_url).connect() as conn:
+        detailed_stock_df.to_sql(
+            'stocks_price', con=conn, if_exists='append', index=False)
+    return detailed_stock_df
+
+
+def update():
+    '''
+    run only once, update stock price and benchmark profile
+    '''
+    print("Checking stock_price table")
+    # collect daily stock price until today in beijing time
+    if need_to_update_stocks_price(dt.timedelta(days=1)):
+        print("Updating stock_price table")
+        stock_df = update_stock_price()
+        stock_df = add_details_to_stock_df(stock_df)
+        save_stock_price_to_db(stock_df)
+        stock_price_stream.emit(stock_df)
+
+
+async def run():
+    '''
+    start the pipeline here to check update and fetch new data
+    '''
+    print("background_task running!")
+    # TODO: update benchmark_profile
+    # if (need_to_update_stocks_price()):
+    if True:
+        print("running update")
+        # TODO testing code get stock price df
+        with create_engine(db_url).connect() as conn:
+            stock_df = pd.read_sql('stocks_price', con=conn)
+            print('sending data!')
+            # print(stock_df)
+            stock_price_stream.emit(stock_df)
+
+        # # latest stock price
+        # stock_df = update_stocks_price()
+        # # add display name and sector to stock_df
+        # stock_df = add_details_to_stock_df(stock_df)
+        # save_stock_price_to_db(stock_df)
+        # stock_price_stream.emit(stock_df)
+    # update sotck_price
+
+    # send fetched data
+
+    # run processing
+
+    # send fetched data

pipeline/create_dumpy_data.ipynb

@@ -1,3 +1,36 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b4e751763ecdbde68a22e49c407308cafb8b6d48bffa145c2ae8d2027f8ab9f8
-size 683
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from test_db_peration import db_operator\n",
+    "from datetime import datetime\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

pipeline/db_operation.py

@@ -1,3 +1,35 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38192fe7d6dba1eb0c98f2f17c3acf4d662777defe5031fd6157601b35726466
-size 1003
+from model import Stock
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+
+class db_operator():
+    def __init__(self, db_url):
+        self.engine = create_engine(db_url)
+
+    def get_stocks_between(self, d1, d2):
+        with Session(self.engine) as session:
+            return session.query(Stock).filter(
+                Stock.date.between(d1, d2)
+                ).all()
+    
+    def add_stock(self, stock_data: dict):
+        with Session(self.engine) as session:
+            new_stock = Stock(**stock_data)
+            session.add(new_stock)
+            session.commit()
+    
+    def delete_stocks_between(self, d1, d2):
+        with Session(self.engine) as session:
+            session.query(Stock).filter(
+                Stock.date.between(d1, d2)
+                ).delete()
+            session.commit()
+    
+    def delete_all_stocks(self):
+        with Session(self.engine) as session:
+            session.query(Stock).delete()
+            session.commit()
+    
+
+    

pipeline/model.py

@@ -1,3 +1,41 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:138123323449aabb36ea46e03512564a88c13ebebd1a82c2a7730ba5f9196daa
-size 1335
+from sqlalchemy import Column, Integer, String, Float, create_engine, DateTime, PickleType
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+Base = declarative_base()
+
+class Stock(Base):
+    __tablename__ = 'stock'
+
+    id = Column(Integer, primary_key=True)
+    ticker = Column(String(50))
+    weight = Column(Float, nullable=True)
+    display_name = Column(String(50), nullable=False)
+    date = Column(DateTime, nullable=False)
+    order = Column(Integer, autoincrement=True, nullable=True)
+
+    def __repr__(self):
+        return f"<{self.ticker}\
+                    {self.date}\
+                        {self.display_name}\
+                            {round(self.weight * 100)}%>"
+
+
+class Portfolio(Base):
+    __tablename__ = 'portfolio'
+    
+    id = Column(Integer, primary_key=True)
+    stocks = Column(PickleType, nullable=False)
+    cached_result = Column(PickleType, nullable=True)
+    # data = Column(PickleType, nullable=False)
+    date = Column(DateTime, nullable=False)
+    order = Column(Integer, autoincrement=True, nullable=True)
+    
+
+# db_url = 'sqlite:///local_db.db'  # Replace 'stocks.db' with the desired database name and location
+
+# engine = create_engine(db_url)
+# Base.metadata.create_all(engine)
+
+# Session = sessionmaker(bind=engine)
+# session = Session()

pipeline/test_db_peration.py

@@ -1,3 +1,107 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0de9a67f5db93566eb0eb9003dc54a51c253d1c2b33cbeb3a8122b7929701b3f
-size 4275
+from unittest import TestCase, main
+from db_operation import db_operator
+from datetime import datetime
+class test_db_operation(TestCase):
+    def setUp(self) -> None:
+        self.db_operator = db_operator('sqlite:///local_db.db')
+        self.db_operator.delete_all_stocks()
+        self.stock1 = {
+            "ticker": 'AAPL',
+            "weight": 1.0,
+            "display_name": 'Apple Inc.',
+            "date": datetime(2021, 1, 1)
+        }
+        self.stock2 = {
+            "ticker": 'MSFT',
+            "weight": 1.0,
+            "display_name": 'Microsoft Corporation',
+            "date": datetime(2021, 1, 1)
+        }
+        
+    def test_insert(self):
+        stock1 = {
+            "ticker": 'AAPL',
+            "weight": 1.0,
+            "display_name": 'Apple Inc.',
+            "date": datetime(2021, 1, 1)
+        }
+        self.db_operator.add_stock(stock1)
+        retrieved_stock = self.db_operator.get_stocks_between(datetime(2021, 1, 1), datetime(2021, 1, 1))[0]
+        self.assertEqual(retrieved_stock.ticker, 'AAPL')
+        self.assertEqual(retrieved_stock.weight, 1.0)
+        self.assertEqual(retrieved_stock.display_name, 'Apple Inc.')
+        self.assertEqual(retrieved_stock.date, datetime(2021, 1, 1))
+
+    def test_delete(self):
+        self.db_operator.add_stock(self.stock1)
+        self.db_operator.add_stock(self.stock2)
+        self.db_operator.delete_stocks_between(
+            datetime(2021, 1, 1),
+            datetime(2021, 1, 1))
+        retrieved_stocks = self.db_operator.get_stocks_between(
+            datetime(2021, 1, 1),
+            datetime(2021, 1, 1))
+        self.assertEqual(len(retrieved_stocks), 0)
+
+    def test_query_window_1d(self):
+        # insert 2 stocks between 2021-01-01 and 2021-01-01 every hour
+        for i in range(24):
+            self.stock1['date'] = datetime(2021, 1, 1, i)
+            self.stock2['date'] = datetime(2021, 1, 1, i)
+            self.db_operator.add_stock(self.stock1)
+            self.db_operator.add_stock(self.stock2)
+        # insert two on 2021-01-02
+        self.stock1['date'] = datetime(2021, 1, 2)
+        self.stock2['date'] = datetime(2021, 1, 2)
+        self.db_operator.add_stock(self.stock1)
+        self.db_operator.add_stock(self.stock2)
+        # query 1d
+        retrieved_stocks = self.db_operator.get_stocks_between(
+            datetime(2021, 1, 1),
+            datetime(2021, 1, 2))
+        self.assertEqual(len(retrieved_stocks), 50)
+    
+    def test_query_window_12h(self):
+        # insert 2 stocks every hour between 2021-01-01 and 2021-01-01
+        for i in range(24):
+            self.stock1['date'] = datetime(2021, 1, 1, i)
+            self.stock2['date'] = datetime(2021, 1, 1, i)
+            self.db_operator.add_stock(self.stock1)
+            self.db_operator.add_stock(self.stock2)
+        # query 12h
+        retrieved_stocks = self.db_operator.get_stocks_between(
+            datetime(2021, 1, 1, 0),
+            datetime(2021, 1, 1, 12))
+        self.assertEqual(len(retrieved_stocks), 26)
+        # self.assertTrue(False)
+    
+    def test_query_window_1h(self):
+       # insert 2 stocks every mins between 2021-01-01 and 2021-01-01
+        for i in range(60):
+            self.stock1['date'] = datetime(2021, 1, 1, 0, i)
+            self.stock2['date'] = datetime(2021, 1, 1, 0, i)
+            self.db_operator.add_stock(self.stock1)
+            self.db_operator.add_stock(self.stock2)
+        # query 1h
+        retrieved_stocks = self.db_operator.get_stocks_between(
+            datetime(2021, 1, 1, 0),
+            datetime(2021, 1, 1, 1))
+        self.assertEqual(len(retrieved_stocks), 120)
+    
+    def test_query_window_30m(self):
+        # insert 2 stocks every 1 between 2021-01-01-00:00 and 2021-01-01-00:20
+        for i in range(20):
+            self.stock1['date'] = datetime(2021, 1, 1, 0, i)
+            self.stock2['date'] = datetime(2021, 1, 1, 0, i)
+            self.db_operator.add_stock(self.stock1)
+            self.db_operator.add_stock(self.stock2)
+        # query 30m
+        retrieved_stocks = self.db_operator.get_stocks_between(
+            datetime(2021, 1, 1, 0),
+            datetime(2021, 1, 1, 0, 30))
+        self.assertEqual(len(retrieved_stocks), 40)
+    
+    
+
+if __name__ == '__main__':
+    main()

portfolioEditingPage.py

@@ -1,3 +1,377 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d46685058cab392b20622cd6f2e346aa3c7fde90e17d82a8b585d787f9c8c03c
-size 15763
+# %%
+# load portfolio
+import panel as pn
+from utils import create_stocks_entry_from_excel, style_number, create_share_changes_report
+import datetime as dt
+import pytz
+import io
+from bokeh.models.widgets.tables import CheckboxEditor, NumberEditor, SelectEditor
+from utils import time_in_beijing
+import api
+import pandas as pd
+from sqlalchemy import create_engine
+from pipeline import update_portfolio_profile_to_db
+import table_schema
+import pipeline
+db_url = 'sqlite:///local.db'
+pn.extension()
+pn.extension('tabulator')
+pn.extension('plotly')
+pn.extension('floatpanel')
+
+
+# %%
+# the width of iphone se
+MIN_COMPONENT_WIDTH = 375
+MAX_COMPONENT_WIDTH = 600
+
+# %%
+
+
+def app():
+
+    # load portfolio df
+    with create_engine(db_url).connect() as conn:
+        p_profile = pd.read_sql_table(table_schema.PORTFOLIO_TABLE, con=conn)
+        p_profile.date = pd.to_datetime(p_profile.date)
+        p_profile.sort_values(by=['date'], inplace=True)
+        # change in shares for same ticker
+        p_profile['share_changes'] = p_profile.groupby(['ticker'])[
+            'shares'].diff()
+        p_profile['share_changes'] = p_profile['share_changes'].fillna(
+            p_profile['shares'])
+        # indicate if change is saved
+        p_profile['change_saved'] = True
+        p_profile['sync_to_db'] = True
+
+        # get all stocks ticker for auto fill
+        stock_details = pd.read_sql_table(table_schema.STOCKS_DETAILS_TABLE, con=conn)
+        all_tickers = stock_details.ticker.to_list()
+
+    # get most recent portfolio for auto generate entry
+    most_recent_portfolio = None
+    if len(p_profile) == 0:
+        most_recent_portfolio = p_profile
+    else:
+        most_recent_portfolio = p_profile[p_profile.date == max(
+            p_profile.date)]
+
+    # create portfolio table tabulator
+    hidden_column = ['index', 'sector', 'name']
+    col_to_titles = {'ticker': '证劵代码', 'weight': '权重',
+                     'date': '时间', 'aggregate_sector': '分类',
+                     'display_name': '名称',
+                     'shares': '持仓', 'change_saved': '已同步',
+                     'sync_to_db': '存入', 'share_changes': '持仓变化',
+                     'cash': '现金', 'ave_price': '平均成本',
+                     }
+    # styling
+    tabulator_formatters = {
+        # 'float': {'type': 'progress', 'max': 10},
+        'sync_to_db': {'type': 'tickCross'},
+        'change_saved': {'type': 'tickCross'},
+    }
+    bokeh_editors = {
+        'ticker': SelectEditor(options=all_tickers),
+        'shares': NumberEditor(),
+    }
+    # frozen_columns = ['date','ticker','display_name','shares','sync_to_db','change_saved']
+
+    portfolio_tabulator = pn.widgets.Tabulator(p_profile,
+                                               layout='fit_columns',
+                                               height_policy='max',
+                                               width=1000,
+                                               groupby=['date'],
+                                               hidden_columns=hidden_column, titles=col_to_titles,
+                                               formatters=tabulator_formatters,
+                                               editors=bokeh_editors,
+                                               pagination='local',
+                                               # page_size=25,
+                                               #    frozen_columns=frozen_columns
+                                               )
+
+    portfolio_tabulator.style.apply(style_number, subset=['share_changes'])
+
+    # history tabulator
+    history_dt = p_profile[['date', 'sync_to_db', 'change_saved']].copy()
+    history_dt = history_dt.groupby('date').agg({
+        "sync_to_db": lambda x: all(x),
+        'change_saved': lambda x: all(x),
+    })
+    history_dt['date'] = history_dt.index
+    history_dt.reset_index(drop=True, inplace=True)
+    history_tabulator = pn.widgets.Tabulator(history_dt,
+                                             formatters=tabulator_formatters,
+                                             buttons={'detail': "<i>📋</i>"},
+                                             hidden_columns=hidden_column,
+                                             height_policy='max',
+                                             titles=col_to_titles)
+
+    # create component
+    new_stock_btn = pn.widgets.Button(
+        name='增加新股票', button_type='primary', sizing_mode='stretch_width')
+    preview_btn = pn.widgets.Button(
+        name='预览', button_type='primary', sizing_mode='stretch_width')
+    file_input = pn.widgets.FileInput(
+        accept='.xlsx', sizing_mode='stretch_width')
+    # strip timezone info
+    datetime_picker = pn.widgets.DatetimePicker(name='Datetime Picker',
+                                                value=time_in_beijing().replace(tzinfo=None),
+                                                sizing_mode='stretch_width')
+    upload_to_db_btn = pn.widgets.Button(
+        name='保存到数据库', button_type='warning', sizing_mode='stretch_width')
+    # emtpy stock_column to display new entires
+    stock_column = pn.Column(
+        width_policy='max', height_policy='max', scroll=True)
+    # floating window row
+    floating_windows = pn.Row()
+
+    def _update_history_tabulator(action, df=None):
+        '''handle update history tabulator'''
+        # handle add new entires to view
+        if action == 'append' and df is not None:
+            index = history_tabulator.value[history_tabulator.value.date ==
+                                            df.date[0]].index.to_list()
+            if len(index) == 0:
+                # drop duplicate date in df
+                df = df.drop_duplicates(subset='date', keep='first')
+                # if not in history tabulator add new entry
+                selected_df = df[['date', 'sync_to_db', 'change_saved']]
+                # if stream to empty tabulator, index will be mismatched
+                if (len(history_tabulator.value) == 0):
+                    history_tabulator.value = selected_df
+                else:
+                    history_tabulator.stream(
+                        df[['date', 'sync_to_db', 'change_saved']], follow=True)
+            else:
+                # if in history tabulator patch change_saved to false
+                history_tabulator.patch({
+                    'change_saved': [(index[0], False)]
+                }, as_index=True)
+        # hanlde editing portoflio tabulator
+        elif action == 'edit':
+            # mark synced_to_db to false when entry is edited
+            date = df
+            index = history_tabulator.value[history_tabulator.value.date == date].index.to_list(
+            )
+            history_tabulator.patch({
+                'change_saved': [(index[0], False)]
+            }, as_index=True)
+        # handle sync to db
+        elif action == 'sync':
+            # patch all synced_to_db to true
+            indices = history_tabulator.value[
+                ~history_tabulator.value['change_saved']].index.to_list()
+            
+            # add an offset to address the issue when df is empty index start from 1
+            
+            history_tabulator.patch({
+                'change_saved': [(index, True) for index in indices]
+            }, as_index=True)
+            # mark synced_to_db to false when editing or select not synced_to_db
+        # if dt is not None and df.date[0] in history_tabulator.value.date.values:
+        #     history_tabulator.stream(df[['date','sync_to_db','change_saved']], follow=True)
+
+        # update mark all synced_to_db to true when update
+
+    def delete_stock(row):
+        '''delete a stock entry'''
+        stock_column.remove(row)
+
+    def create_new_stock_entry(ticker=None, shares=0, ave_price=0.0, disable_ticker=True):
+        '''create a new new stock entry'''
+        delete_btn = pn.widgets.Button(
+            name='❌', width=50, height=60, sizing_mode='fixed')
+        ticker_selector = pn.widgets.AutocompleteInput(
+            value=ticker,
+            name='证劵代码',
+            sizing_mode='stretch_width',
+            options=all_tickers,
+            placeholder='input ticker',
+
+        )
+        share_input = pn.widgets.IntInput(
+            name='持仓',
+            value=shares,
+            step=1,
+            start=0,
+            sizing_mode='stretch_width')
+
+        mean_price_input = pn.widgets.FloatInput(
+            name='平均成本',
+            value=ave_price, step=0.01, start=0, sizing_mode='stretch_width')
+
+        row = pn.Row(
+            delete_btn,
+            ticker_selector,
+            share_input,
+            mean_price_input,
+            width_policy='max',
+        )
+        delete_btn.on_click(lambda _, row=row: delete_stock(row))
+        return row
+
+    def update_stock_column(xlsx_file=None):
+        stock_entries = []
+        if xlsx_file is None:
+            for ticker, shares in most_recent_portfolio[['ticker', 'shares']].values:
+                stock_entries.append(create_new_stock_entry(
+                    ticker=ticker, shares=shares))
+        # create from xlsx_file
+        else:
+            stocks_list = create_stocks_entry_from_excel(xlsx_file)
+            for entry in stocks_list:
+                stock_entries.append(create_new_stock_entry(
+                    ave_price=entry['mean_price'],
+                    ticker=entry['ticker'],
+                    shares=entry['shares']))
+            # modify time
+            datetime_picker.value = stocks_list[0]['date']
+            file_input.value = None
+
+        # update
+        stock_column.clear()
+        stock_column.extend(stock_entries)
+
+    def _get_stocks_price(df):
+        '''return a df with latest stock price added the new portfolio entry'''
+        stock_price = api.fetch_stocks_price(
+            security=df.ticker.to_list(),
+            end_date=df.date[0],
+            count=1,
+            frequency='minute',
+        )
+        stock_price.rename(columns={'time': 'stock_price_ts'}, inplace=True)
+        merged_df = df.merge(
+            stock_price[['ticker', 'stock_price_ts', 'close']], on='ticker', how='left')
+        return merged_df
+
+    def _calculate_weigth(df):
+        '''
+        calculate weight on new portfolio entry
+        '''
+        df['total_value'] = df.shares * df.close
+        df['weight'] = df.total_value / df.total_value.sum()
+
+    def update_profile_tabulator(e):
+        '''add all stocks entry to ui'''
+        new_entry = [dict(ticker=row[1].value,
+                          shares=row[2].value,
+                          ave_price=row[3].value,
+                          date=datetime_picker.value) for row in stock_column]
+        
+        if len(new_entry) == 0:
+            print("no entry added")
+            return 
+        
+        new_profile = pipeline.create_portfolio_profile_df(new_entry)
+        # calculate share changes
+        tmp_profile = pd.concat([p_profile, new_profile], ignore_index=True)
+        tmp_profile.sort_values(by='date', inplace=True)
+        tmp_profile['share_changes'] = tmp_profile.groupby('ticker')[
+            'shares'].diff()
+        tmp_profile['share_changes'] = tmp_profile['share_changes'].fillna(
+            tmp_profile['shares'])
+        new_profile = new_profile.merge(tmp_profile[[
+                                        'ticker', 'date', 'share_changes', 'change_saved']], on=['ticker', 'date'], how='left')
+        # fill emtpy change_saved to False
+        new_profile['change_saved'] = new_profile['change_saved'].fillna(False)
+        new_profile['sync_to_db'] = True
+        # calculate cash and weight
+        new_profile['cash'] = new_profile.shares * new_profile.ave_price
+        new_profile['weight'] = new_profile.cash / new_profile.cash.sum()
+
+        # update history tabulator
+        _update_history_tabulator('append', new_profile)
+        _stream_to_portfolio_tabulator(new_profile)
+
+    def add_new_stock(e):
+        row = create_new_stock_entry()
+        stock_column.append(row)
+
+    def _stream_to_portfolio_tabulator(entry):
+        if len(portfolio_tabulator.value) == 0:
+            portfolio_tabulator.value = entry
+        else:
+            portfolio_tabulator.stream(entry, follow=True)
+
+    def handle_click_on_history_tabulator(e):
+        '''handle click click on history tabulator'''
+        if e.column == 'detail':
+            row_index = e.row
+            date = history_tabulator.value.iloc[row_index]['date']
+            date_str = date.strftime("%Y-%m-%d : %H:%M:%S")
+            record_df = portfolio_tabulator.value[portfolio_tabulator.value.date == date]
+            floatpanel = pn.layout.FloatPanel(create_share_changes_report(
+                record_df), name=date_str, margin=20, position='right-top')
+            floating_windows.append(floatpanel)
+
+    def handle_sync_to_db(e):
+        # TODO: change to use profile df instead, because tabulator might not contain all entry
+        '''sync selected entry to db'''
+        new_portfolio = portfolio_tabulator.value
+        # TODO when initially df is empty, there is a 0 row in df as place holder
+        # only update selected row to db
+        selected_portfolio = new_portfolio[new_portfolio['sync_to_db']]
+        successed = update_portfolio_profile_to_db(selected_portfolio)
+        # update history tabulator and portfolio tabulator
+        if successed:
+            # mark changes as saved
+            indices = selected_portfolio[~selected_portfolio['change_saved']].index.to_list()
+            portfolio_tabulator.patch({
+                'change_saved': [(index, True) for index in indices]
+            }, as_index=True)
+
+        
+            _update_history_tabulator('sync')
+
+    def handle_edit_portfolio_tabulator(e):
+        date = portfolio_tabulator.value.iloc[e.row]['date']
+        _update_history_tabulator(df=date, action='edit')
+        print(date)
+
+    # %%
+    # register event handler
+    upload_to_db_btn.on_click(handle_sync_to_db)
+    preview_btn.on_click(update_profile_tabulator)
+    new_stock_btn.on_click(add_new_stock)
+    history_tabulator.on_click(
+        handle_click_on_history_tabulator
+    )
+    portfolio_tabulator.on_edit(handle_edit_portfolio_tabulator)
+
+    # %%
+    # create handler component to add to panel so can be listened to
+    upload_xlsx_handler = pn.bind(update_stock_column, file_input)
+
+    # %%
+    # layout
+
+    editor_widget = pn.Column(floating_windows, datetime_picker, upload_to_db_btn, new_stock_btn,
+                              preview_btn, file_input, pn.widgets.TooltipIcon(
+                                  value="用于更新修改持仓信息,默认股票为最近持仓，默认时间为目前北京时间，点击增加新股票按钮,输入股票代码和持仓选择日期(北京时间),点击预览,确认无误后点击保存到数据库。或者直接���拽excel文件到下方上传按钮"),
+                              stock_column, width=MIN_COMPONENT_WIDTH, height_policy='max')
+    # tooltip
+    toolTip2 = pn.widgets.TooltipIcon(
+        value="持仓总结,每一行的已同步到数据库代表所做更改是否已同步到数据库,点击保存到数据库将上传所有更改。点击右侧📋按钮查看详细持仓变化报告")
+
+    return pn.Row(
+        pn.layout.HSpacer(),
+        editor_widget,
+        pn.Spacer(width=10),
+        history_tabulator,
+        pn.Spacer(width=10),
+        portfolio_tabulator,
+        pn.Spacer(width=10),
+        upload_xlsx_handler,
+        pn.layout.HSpacer(),
+        height=1500,
+        # width_policy='max', height_policy='max')
+        # sizing_mode='stretch_both',
+    )
+
+
+# app
+template = pn.template.FastListTemplate(title='portfolio编辑')
+template.main.append(app())
+template.servable()

portfolio_page.py

@@ -1,3 +1,4 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0b8e6557dc04a241847cf244f74af06c7830b18eef037126cdc88db2c252d6d6
-size 66
+'''
+A page to view history of portfolio and update portfolio
+'''
+

requirements.txt

@@ -1,3 +1,136 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2a41e4b76e79fa147f1a49339e56ec5ff8e22f21fc60f5e51437dd159bd99600
-size 2430
+anyio==3.7.0
+appnope==0.1.3
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+arrow==1.2.3
+asttokens==2.2.1
+async-lru==2.0.2
+attrs==23.1.0
+Babel==2.12.1
+backcall==0.2.0
+backports.functools-lru-cache==1.6.4
+beautifulsoup4==4.12.2
+bleach==6.0.0
+bokeh==3.1.1
+certifi==2023.5.7
+cffi==1.15.1
+charset-normalizer==3.1.0
+colorcet==3.0.1
+comm==0.1.3
+contourpy==1.1.0
+cycler==0.11.0
+debugpy==1.6.7
+decorator==5.1.1
+defusedxml==0.7.1
+et-xmlfile==1.1.0
+executing==1.2.0
+fastjsonschema==2.17.1
+fonttools==4.40.0
+fqdn==1.5.1
+holoviews==1.16.2
+hvplot==0.8.4
+idna==3.4
+importlib-metadata==6.7.0
+ipykernel==6.23.2
+ipython==8.14.0
+ipywidgets==8.0.6
+isoduration==20.11.0
+jedi==0.18.2
+Jinja2==3.1.2
+jqdatasdk==1.8.11
+json5==0.9.14
+jsonpointer==2.4
+jsonschema==4.17.3
+jupyter-bokeh==3.0.7
+jupyter_client==8.2.0
+jupyter_core==5.3.1
+jupyter-events==0.6.3
+jupyter-lsp==2.2.0
+jupyter_server==2.6.0
+jupyter_server_terminals==0.4.4
+jupyterlab==4.0.2
+jupyterlab-pygments==0.2.2
+jupyterlab_server==2.23.0
+jupyterlab-widgets==3.0.7
+kiwisolver==1.4.4
+linkify-it-py==2.0.2
+Markdown==3.4.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.3
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mistune==3.0.1
+msgpack==1.0.5
+nbclient==0.8.0
+nbconvert==7.6.0
+nbformat==5.9.0
+nest-asyncio==1.5.6
+notebook_shim==0.2.3
+numpy==1.25.0
+openpyxl==3.1.2
+overrides==7.3.1
+packaging==23.1
+pandas==2.0.2
+pandocfilters==1.5.0
+panel==1.1.1
+param==1.13.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.5.0
+pip==23.1.2
+platformdirs==3.6.0
+plotly==5.15.0
+ply==3.11
+prometheus-client==0.17.0
+prompt-toolkit==3.0.38
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pycparser==2.21
+pyct==0.5.0
+Pygments==2.15.1
+PyMySQL==1.0.3
+pyparsing==3.1.0
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+pytz==2023.3
+pyviz-comms==2.3.2
+PyYAML==6.0
+pyzmq==25.1.0
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+scipy==1.10.1
+Send2Trash==1.8.2
+setuptools==67.7.2
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.4.1
+SQLAlchemy==2.0.16
+stack-data==0.6.2
+tenacity==8.2.2
+terminado==0.17.1
+thriftpy2==0.4.16
+tinycss2==1.2.1
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+typing_extensions==4.6.3
+tzdata==2023.3
+uc-micro-py==1.0.2
+uri-template==1.3.0
+urllib3==2.0.3
+wcwidth==0.2.6
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.6.0
+wheel==0.40.0
+widgetsnbextension==4.0.7
+xyzservices==2023.5.0
+zipp==3.15.0
+streamz==0.6.4
+python-dotenv==1.0.0

script/description.py

@@ -1,3 +1,14 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a9ebc0011567cb67b90e946044ba3e92107255fb7c963044358438586fd1193f
-size 411
+'''
+This file store the tooltip to show in the GUI
+
+'''
+#周期回报
+periodic_return_report = '''
+选择周期查看每个周期的回报率,以及每个周期主动回报的归因。
+周期回报的数据点代表周期结束。
+主动回报归因的数据点代表周期开始。
+'''
+
+summary_card = '''
+选择周期查看每个周期的总市值，总回报率，主动回报率，主动回报归因。
+'''

script/downloadData.ipynb

@@ -1,3 +1,751 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e9498800f6ef5fa8a6901907b8902f0c5a1f783ae8e2d453d6fd1194d9a92f59
-size 27769
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from api import create_portfolio, get_portfile_data, get_benchmark_portfolio\n",
+    "from datetime import datetime"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df = pd.read_pickle('../dummy_portfolio.pkl')\n",
+    "# df.head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ## create a portfolio\n",
+    "\n",
+    "# # weight can use mony\n",
+    "# portfolio_profile_data = [\n",
+    "# {'ticker': '002709.XSHE', 'date': datetime(2021, 1, 5), 'weight': 100},\n",
+    "# {'ticker': '002920.XSHE', 'date': datetime(2021, 1, 5), 'weight': 100},\n",
+    "# {'ticker': '300274.XSHE', 'date': datetime(2021, 1, 5), 'weight': 100},\n",
+    "# {'ticker': '600409.XSHG', 'date': datetime(2021, 1, 5), 'weight': 100},\n",
+    "# {'ticker': '600415.XSHG', 'date': datetime(2021, 1, 5), 'weight': 100},\n",
+    "# {'ticker': '603882.XSHG', 'date': datetime(2021, 1, 5), 'weight': 100},\n",
+    "# ]\n",
+    "# # ten thousand\n",
+    "# mkt_cap = 100000\n",
+    "# portfolio_profile, error = create_portfolio(portfolio_profile_data, mkt_cap)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# print(error)\n",
+    "# # save \n",
+    "# portfolio_profile.to_pickle('../data/portfolio_portfile.pkl')\n",
+    "# portfolio_profile.head(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>date</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>display_name</th>\n",
+       "      <th>name</th>\n",
+       "      <th>sector</th>\n",
+       "      <th>aggregate_sector</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>002709.XSHE</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>100</td>\n",
+       "      <td>天赐材料</td>\n",
+       "      <td>TCCL</td>\n",
+       "      <td>电气设备I 电池II 电池化学品III 化学原料和化学制品制造业 电池部件及材料 工业</td>\n",
+       "      <td>工业</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>002920.XSHE</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>100</td>\n",
+       "      <td>德赛西威</td>\n",
+       "      <td>DSXW</td>\n",
+       "      <td>计算机I 软件开发II 垂直应用软件III 汽车制造业 汽车电子 可选消费</td>\n",
+       "      <td>信息与通信</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>300274.XSHE</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>100</td>\n",
+       "      <td>阳光电源</td>\n",
+       "      <td>YGDY</td>\n",
+       "      <td>电气设备I 光伏设备II 逆变器III 电气机械和器材制造业 光伏设备 工业</td>\n",
+       "      <td>工业</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>100</td>\n",
+       "      <td>三友化工</td>\n",
+       "      <td>SYHG</td>\n",
+       "      <td>化工I 化学原料II 纯碱III 化学原料和化学制品制造业 粘胶 原材料</td>\n",
+       "      <td>原料与能源</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>100</td>\n",
+       "      <td>小商品城</td>\n",
+       "      <td>XSPC</td>\n",
+       "      <td>商业贸易I 一般零售II 商业物业经营III 商务服务业 市场服务 工业</td>\n",
+       "      <td>消费</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>603882.XSHG</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>100</td>\n",
+       "      <td>金域医学</td>\n",
+       "      <td>JYYX</td>\n",
+       "      <td>医药生物I 医疗服务II 诊断服务III 卫生 体外诊断 医药卫生</td>\n",
+       "      <td>医药卫生</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        ticker       date  weight display_name  name  \\\n",
+       "0  002709.XSHE 2021-01-05     100         天赐材料  TCCL   \n",
+       "1  002920.XSHE 2021-01-05     100         德赛西威  DSXW   \n",
+       "2  300274.XSHE 2021-01-05     100         阳光电源  YGDY   \n",
+       "3  600409.XSHG 2021-01-05     100         三友化工  SYHG   \n",
+       "4  600415.XSHG 2021-01-05     100         小商品城  XSPC   \n",
+       "5  603882.XSHG 2021-01-05     100         金域医学  JYYX   \n",
+       "\n",
+       "                                        sector aggregate_sector  \n",
+       "0  电气设备I 电池II 电池化学品III 化学原料和化学制品制造业 电池部件及材料 工业               工业  \n",
+       "1        计算机I 软件开发II 垂直应用软件III 汽车制造业 汽车电子 可选消费            信息与通信  \n",
+       "2       电气设备I 光伏设备II 逆变器III 电气机械和器材制造业 光伏设备 工业               工业  \n",
+       "3         化工I 化学原料II 纯碱III 化学原料和化学制品制造业 粘胶 原材料            原料与能源  \n",
+       "4         商业贸易I 一般零售II 商业物业经营III 商务服务业 市场服务 工业               消费  \n",
+       "5            医药生物I 医疗服务II 诊断服务III 卫生 体外诊断 医药卫生             医药卫生  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## load portfolio_profile\n",
+    "portfolio_profile = pd.read_pickle('../data/portfolio_portfile.pkl')\n",
+    "portfolio_profile.head(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start_date = datetime(2021, 1, 5)\n",
+    "end_date = datetime(2022, 1, 10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "auth success \n"
+     ]
+    }
+   ],
+   "source": [
+    "# get portfolio data\n",
+    "portfile_data, error = get_portfile_data(portfolio_profile, start_date, end_date)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[]\n",
+      "(1482, 8)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>date</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2022-01-07</th>\n",
+       "      <td>603882.XSHG</td>\n",
+       "      <td>2022-01-07</td>\n",
+       "      <td>91.13</td>\n",
+       "      <td>87.99</td>\n",
+       "      <td>91.17</td>\n",
+       "      <td>87.72</td>\n",
+       "      <td>6971998.0</td>\n",
+       "      <td>6.176535e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-07</th>\n",
+       "      <td>002709.XSHE</td>\n",
+       "      <td>2022-01-07</td>\n",
+       "      <td>51.28</td>\n",
+       "      <td>51.72</td>\n",
+       "      <td>52.62</td>\n",
+       "      <td>50.47</td>\n",
+       "      <td>32210458.0</td>\n",
+       "      <td>1.661823e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-07</th>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>2022-01-07</td>\n",
+       "      <td>8.23</td>\n",
+       "      <td>8.22</td>\n",
+       "      <td>8.29</td>\n",
+       "      <td>8.19</td>\n",
+       "      <td>35003739.0</td>\n",
+       "      <td>2.884990e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-07</th>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>2022-01-07</td>\n",
+       "      <td>4.74</td>\n",
+       "      <td>4.70</td>\n",
+       "      <td>4.79</td>\n",
+       "      <td>4.68</td>\n",
+       "      <td>24902567.0</td>\n",
+       "      <td>1.178837e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-10</th>\n",
+       "      <td>300274.XSHE</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>127.49</td>\n",
+       "      <td>124.09</td>\n",
+       "      <td>127.49</td>\n",
+       "      <td>123.29</td>\n",
+       "      <td>17238708.0</td>\n",
+       "      <td>2.148032e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-10</th>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>8.24</td>\n",
+       "      <td>8.35</td>\n",
+       "      <td>8.39</td>\n",
+       "      <td>8.21</td>\n",
+       "      <td>32516017.0</td>\n",
+       "      <td>2.699300e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-10</th>\n",
+       "      <td>002920.XSHE</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>130.36</td>\n",
+       "      <td>138.43</td>\n",
+       "      <td>141.96</td>\n",
+       "      <td>130.11</td>\n",
+       "      <td>5005400.0</td>\n",
+       "      <td>6.901614e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-10</th>\n",
+       "      <td>002709.XSHE</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>51.63</td>\n",
+       "      <td>50.73</td>\n",
+       "      <td>51.93</td>\n",
+       "      <td>50.03</td>\n",
+       "      <td>29821246.0</td>\n",
+       "      <td>1.518902e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-10</th>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>4.70</td>\n",
+       "      <td>4.75</td>\n",
+       "      <td>4.85</td>\n",
+       "      <td>4.67</td>\n",
+       "      <td>39278041.0</td>\n",
+       "      <td>1.859827e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2022-01-10</th>\n",
+       "      <td>603882.XSHG</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>88.45</td>\n",
+       "      <td>95.53</td>\n",
+       "      <td>95.59</td>\n",
+       "      <td>88.39</td>\n",
+       "      <td>6991445.0</td>\n",
+       "      <td>6.468392e+08</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                 ticker       date    open   close    high     low  \\\n",
+       "2022-01-07  603882.XSHG 2022-01-07   91.13   87.99   91.17   87.72   \n",
+       "2022-01-07  002709.XSHE 2022-01-07   51.28   51.72   52.62   50.47   \n",
+       "2022-01-07  600409.XSHG 2022-01-07    8.23    8.22    8.29    8.19   \n",
+       "2022-01-07  600415.XSHG 2022-01-07    4.74    4.70    4.79    4.68   \n",
+       "2022-01-10  300274.XSHE 2022-01-10  127.49  124.09  127.49  123.29   \n",
+       "2022-01-10  600409.XSHG 2022-01-10    8.24    8.35    8.39    8.21   \n",
+       "2022-01-10  002920.XSHE 2022-01-10  130.36  138.43  141.96  130.11   \n",
+       "2022-01-10  002709.XSHE 2022-01-10   51.63   50.73   51.93   50.03   \n",
+       "2022-01-10  600415.XSHG 2022-01-10    4.70    4.75    4.85    4.67   \n",
+       "2022-01-10  603882.XSHG 2022-01-10   88.45   95.53   95.59   88.39   \n",
+       "\n",
+       "                volume         money  \n",
+       "2022-01-07   6971998.0  6.176535e+08  \n",
+       "2022-01-07  32210458.0  1.661823e+09  \n",
+       "2022-01-07  35003739.0  2.884990e+08  \n",
+       "2022-01-07  24902567.0  1.178837e+08  \n",
+       "2022-01-10  17238708.0  2.148032e+09  \n",
+       "2022-01-10  32516017.0  2.699300e+08  \n",
+       "2022-01-10   5005400.0  6.901614e+08  \n",
+       "2022-01-10  29821246.0  1.518902e+09  \n",
+       "2022-01-10  39278041.0  1.859827e+08  \n",
+       "2022-01-10   6991445.0  6.468392e+08  "
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(error)\n",
+    "print(portfile_data.shape)\n",
+    "portfile_data.sort_values(by=['date'], inplace=True)\n",
+    "portfile_data.tail(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save\n",
+    "portfile_data.to_pickle('../data/portfolio_data.pkl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load benchmark portfolio\n",
+    "benchmark_portfolio, error = get_benchmark_portfolio(start_date, end_date)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[]\n",
+      "(185500, 15)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>display_name_x</th>\n",
+       "      <th>actual_data</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "      <th>display_name_y</th>\n",
+       "      <th>name</th>\n",
+       "      <th>sector</th>\n",
+       "      <th>aggregate_sector</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>185163</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.274</td>\n",
+       "      <td>厦门钨业</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600549.XSHG</td>\n",
+       "      <td>21.50</td>\n",
+       "      <td>21.59</td>\n",
+       "      <td>21.75</td>\n",
+       "      <td>21.43</td>\n",
+       "      <td>10451128.0</td>\n",
+       "      <td>2.254199e+08</td>\n",
+       "      <td>厦门钨业</td>\n",
+       "      <td>XMWY</td>\n",
+       "      <td>有色金属I 稀有金属II 钨III 有色金属冶炼和压延加工业 钨钼 原材料</td>\n",
+       "      <td>原料与能源</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185162</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.116</td>\n",
+       "      <td>山煤国际</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600546.XSHG</td>\n",
+       "      <td>6.74</td>\n",
+       "      <td>6.86</td>\n",
+       "      <td>6.86</td>\n",
+       "      <td>6.71</td>\n",
+       "      <td>66929559.0</td>\n",
+       "      <td>4.539690e+08</td>\n",
+       "      <td>山煤国际</td>\n",
+       "      <td>SMGJ</td>\n",
+       "      <td>煤炭I 煤炭开采II 动力煤III 批发业 煤炭 能源</td>\n",
+       "      <td>原料与能源</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185161</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.211</td>\n",
+       "      <td>中国软件</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600536.XSHG</td>\n",
+       "      <td>29.01</td>\n",
+       "      <td>30.90</td>\n",
+       "      <td>31.45</td>\n",
+       "      <td>28.98</td>\n",
+       "      <td>35489167.0</td>\n",
+       "      <td>1.087404e+09</td>\n",
+       "      <td>中国软件</td>\n",
+       "      <td>ZGRJ</td>\n",
+       "      <td>计算机I IT服务II IT服务III 软件和信息技术服务业 行业应用软件 信息技术</td>\n",
+       "      <td>信息与通信</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185160</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.205</td>\n",
+       "      <td>天士力</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600535.XSHG</td>\n",
+       "      <td>15.04</td>\n",
+       "      <td>15.52</td>\n",
+       "      <td>15.75</td>\n",
+       "      <td>14.95</td>\n",
+       "      <td>46823950.0</td>\n",
+       "      <td>7.247465e+08</td>\n",
+       "      <td>天士力</td>\n",
+       "      <td>TSL</td>\n",
+       "      <td>医药生物I 中药II 中药III 医药制造业 中成药 医药卫生</td>\n",
+       "      <td>医药卫生</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185159</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.297</td>\n",
+       "      <td>山东药玻</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600529.XSHG</td>\n",
+       "      <td>42.39</td>\n",
+       "      <td>40.14</td>\n",
+       "      <td>42.39</td>\n",
+       "      <td>39.47</td>\n",
+       "      <td>12543641.0</td>\n",
+       "      <td>5.049473e+08</td>\n",
+       "      <td>山东药玻</td>\n",
+       "      <td>SDYB</td>\n",
+       "      <td>医药生物I 医疗器械II 医疗耗材III 非金属矿物制品业 医疗耗材 医药卫生</td>\n",
+       "      <td>医药卫生</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185158</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.156</td>\n",
+       "      <td>中铁工业</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600528.XSHG</td>\n",
+       "      <td>8.69</td>\n",
+       "      <td>9.08</td>\n",
+       "      <td>9.26</td>\n",
+       "      <td>8.68</td>\n",
+       "      <td>98213530.0</td>\n",
+       "      <td>8.961637e+08</td>\n",
+       "      <td>中铁工业</td>\n",
+       "      <td>ZTGY</td>\n",
+       "      <td>机械设备I 运输设备II 铁路设备III 专用设备制造业 城轨铁路 工业</td>\n",
+       "      <td>工业</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185157</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.642</td>\n",
+       "      <td>中天科技</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600522.XSHG</td>\n",
+       "      <td>15.21</td>\n",
+       "      <td>14.76</td>\n",
+       "      <td>15.26</td>\n",
+       "      <td>14.53</td>\n",
+       "      <td>112744173.0</td>\n",
+       "      <td>1.670952e+09</td>\n",
+       "      <td>中天科技</td>\n",
+       "      <td>ZTKJ</td>\n",
+       "      <td>通信I 通信设备II 通信线缆及配套III 电气机械和器材制造业 通信系统设备及组件 通信服务</td>\n",
+       "      <td>信息与通信</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185156</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.276</td>\n",
+       "      <td>华海药业</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600521.XSHG</td>\n",
+       "      <td>22.32</td>\n",
+       "      <td>23.32</td>\n",
+       "      <td>23.43</td>\n",
+       "      <td>21.90</td>\n",
+       "      <td>29810065.0</td>\n",
+       "      <td>6.827298e+08</td>\n",
+       "      <td>华海药业</td>\n",
+       "      <td>HHYY</td>\n",
+       "      <td>医药生物I 化学制药II 化学制剂III 医药制造业 药品制剂 医药卫生</td>\n",
+       "      <td>医药卫生</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185169</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.240</td>\n",
+       "      <td>卧龙电驱</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>600580.XSHG</td>\n",
+       "      <td>16.66</td>\n",
+       "      <td>16.51</td>\n",
+       "      <td>16.68</td>\n",
+       "      <td>16.23</td>\n",
+       "      <td>19294606.0</td>\n",
+       "      <td>3.175398e+08</td>\n",
+       "      <td>卧龙电驱</td>\n",
+       "      <td>WLDQ</td>\n",
+       "      <td>电气设备I 电机II 电机III 电气机械和器材制造业 电动机与工控自动化 工业</td>\n",
+       "      <td>工业</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>185499</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>思瑞浦</td>\n",
+       "      <td>2021-12-31</td>\n",
+       "      <td>688536.XSHG</td>\n",
+       "      <td>476.70</td>\n",
+       "      <td>463.10</td>\n",
+       "      <td>476.70</td>\n",
+       "      <td>446.03</td>\n",
+       "      <td>924992.0</td>\n",
+       "      <td>4.254053e+08</td>\n",
+       "      <td>思瑞浦</td>\n",
+       "      <td>SRP</td>\n",
+       "      <td>电子I 半导体II 模拟芯片设计III 软件和信息技术服务业 集成电路设计 信息技术</td>\n",
+       "      <td>信息与通信</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             date  weight display_name_x actual_data       ticker    open  \\\n",
+       "185163 2022-01-10   0.274           厦门钨业  2021-12-31  600549.XSHG   21.50   \n",
+       "185162 2022-01-10   0.116           山煤国际  2021-12-31  600546.XSHG    6.74   \n",
+       "185161 2022-01-10   0.211           中国软件  2021-12-31  600536.XSHG   29.01   \n",
+       "185160 2022-01-10   0.205            天士力  2021-12-31  600535.XSHG   15.04   \n",
+       "185159 2022-01-10   0.297           山东药玻  2021-12-31  600529.XSHG   42.39   \n",
+       "185158 2022-01-10   0.156           中铁工业  2021-12-31  600528.XSHG    8.69   \n",
+       "185157 2022-01-10   0.642           中天科技  2021-12-31  600522.XSHG   15.21   \n",
+       "185156 2022-01-10   0.276           华海药业  2021-12-31  600521.XSHG   22.32   \n",
+       "185169 2022-01-10   0.240           卧龙电驱  2021-12-31  600580.XSHG   16.66   \n",
+       "185499 2022-01-10   0.350            思瑞浦  2021-12-31  688536.XSHG  476.70   \n",
+       "\n",
+       "         close    high     low       volume         money display_name_y  \\\n",
+       "185163   21.59   21.75   21.43   10451128.0  2.254199e+08           厦门钨业   \n",
+       "185162    6.86    6.86    6.71   66929559.0  4.539690e+08           山煤国际   \n",
+       "185161   30.90   31.45   28.98   35489167.0  1.087404e+09           中国软件   \n",
+       "185160   15.52   15.75   14.95   46823950.0  7.247465e+08            天士力   \n",
+       "185159   40.14   42.39   39.47   12543641.0  5.049473e+08           山东药玻   \n",
+       "185158    9.08    9.26    8.68   98213530.0  8.961637e+08           中铁工业   \n",
+       "185157   14.76   15.26   14.53  112744173.0  1.670952e+09           中天科技   \n",
+       "185156   23.32   23.43   21.90   29810065.0  6.827298e+08           华海药业   \n",
+       "185169   16.51   16.68   16.23   19294606.0  3.175398e+08           卧龙电驱   \n",
+       "185499  463.10  476.70  446.03     924992.0  4.254053e+08            思瑞浦   \n",
+       "\n",
+       "        name                                           sector aggregate_sector  \n",
+       "185163  XMWY            有色金属I 稀有金属II 钨III 有色金属冶炼和压延加工业 钨钼 原材料            原料与能源  \n",
+       "185162  SMGJ                      煤炭I 煤炭开采II 动力煤III 批发业 煤炭 能源            原料与能源  \n",
+       "185161  ZGRJ       计算机I IT服务II IT服务III 软件和信息技术服务业 行业应用软件 信息技术            信息与通信  \n",
+       "185160   TSL                  医药生物I 中药II 中药III 医药制造业 中成药 医药卫生             医药卫生  \n",
+       "185159  SDYB          医药生物I 医疗器械II 医疗耗材III 非金属矿物制品业 医疗耗材 医药卫生             医药卫生  \n",
+       "185158  ZTGY             机械设备I 运输设备II 铁路设备III 专用设备制造业 城轨铁路 工业               工业  \n",
+       "185157  ZTKJ  通信I 通信设备II 通信线缆及配套III 电气机械和器材制造业 通信系统设备及组件 通信服务            信息与通信  \n",
+       "185156  HHYY             医药生物I 化学制药II 化学制剂III 医药制造业 药品制剂 医药卫生             医药卫生  \n",
+       "185169  WLDQ         电气设备I 电机II 电机III 电气机械和器材制造业 电动机与工控自动化 工业               工业  \n",
+       "185499   SRP       电子I 半导体II 模拟芯片设计III 软件和信息技术服务业 集成电路设计 信息技术            信息与通信  "
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(error)\n",
+    "print(benchmark_portfolio.shape)\n",
+    "benchmark_portfolio.sort_values(by=['date'], inplace=True)\n",
+    "benchmark_portfolio.tail(10)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save\n",
+    "benchmark_portfolio.to_pickle('../data/benchmark_portfolio.pkl')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

script/downloadData.py

@@ -1,3 +1,10 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c09badb2fa652cafccfc5ac41fe1955279bcf476e066e5d7f6eeaf29e2421f8f
-size 177
+import pandas as pd
+from api import update_portfolio_profile, get_stocks_price, get_benchmark_portfolio
+
+portfolio_profile_data = [
+    
+]
+
+
+def main():
+    # create a portfolio

script/pipeline.ipynb

@@ -1,3 +1,521 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:20e6c5c9a495ce9720e90e775ced8bf83bfe9ce072faa30cb5984dfa7758c8cb
-size 18969
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import math\n",
+    "from datetime import datetime\n",
+    "import hvplot.pandas\n",
+    "import math\n",
+    "import numpy as np\n",
+    "# load data\n",
+    "profile_df = pd.read_pickle('../data/portfolio_portfile.pkl')\n",
+    "benchmark_df = pd.read_pickle('../data/benchmark_portfolio.pkl')\n",
+    "portfolio_df = pd.read_pickle('../data/portfolio_data.pkl')"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## This section transfrom benchmark_df and creat an benchmark profile to accomadate current pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# drop weight\n",
+    "# benchmark_df = benchmark_df.drop(columns=['weight'])\n",
+    "\n",
+    "## simulate update potfolio weigth at 2021-01-10\n",
+    "update_profile_df = profile_df.copy()\n",
+    "update_profile_df['date'] = datetime(2021,1,10)\n",
+    "update_profile_df['weight'] = [50,100,200,300,400,500]\n",
+    "profile_df = pd.concat([profile_df, update_profile_df])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## create a profile for benchmark\n",
+    "b_profile = benchmark_df.drop_duplicates(subset=['ticker', 'actual_data'])\n",
+    "# df_unique[df_unique.ticker == \"000008.XSHE\"]\n",
+    "# only keep ticker\tdate\tweight\tdisplay_name\tname\tsector\taggregate_sector column\n",
+    "b_profile = b_profile[['ticker','date','weight','name','sector','aggregate_sector','display_name_x']]\n",
+    "b_profile.rename(columns={'display_name_x': 'display_name'}, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>000008.XSHE</td>\n",
+       "      <td>2.52</td>\n",
+       "      <td>2.57</td>\n",
+       "      <td>2.67</td>\n",
+       "      <td>2.49</td>\n",
+       "      <td>33215803.0</td>\n",
+       "      <td>85358605.99</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        date       ticker  open  close  high   low      volume        money\n",
+       "0 2021-01-05  000008.XSHE  2.52   2.57  2.67  2.49  33215803.0  85358605.99"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# drop weight in benchmark\n",
+    "benchmark_df = benchmark_df.drop(columns=['weight'])\n",
+    "benchmark_df = benchmark_df.drop(columns=['display_name_x'])\n",
+    "# drop display_name_y\n",
+    "benchmark_df = benchmark_df.drop(columns=['display_name_y'])\n",
+    "# drop actual_data\n",
+    "benchmark_df = benchmark_df.drop(columns=['actual_data'])\n",
+    "# drop name\n",
+    "benchmark_df = benchmark_df.drop(columns=['name'])\n",
+    "# drop aggregate_sector\n",
+    "benchmark_df = benchmark_df.drop(columns=['aggregate_sector'])\n",
+    "# drop sector\n",
+    "benchmark_df = benchmark_df.drop(columns=['sector'])\n",
+    "benchmark_df.head(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for col_name in benchmark_df.columns:\n",
+    "    if (col_name not in portfolio_df.columns):\n",
+    "        print(f'portfolio does not have {col_name}')\n",
+    "    \n",
+    "\n",
+    "for col_name in portfolio_df.columns:\n",
+    "    if (col_name not in benchmark_df.columns):\n",
+    "        print(f'benchmark does not have {col_name}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n",
+      "True\n"
+     ]
+    }
+   ],
+   "source": [
+    "for col_name in b_profile.columns:\n",
+    "    print(col_name in profile_df.columns)\n",
+    "\n",
+    "for col_name in profile_df.columns:\n",
+    "    print(col_name in b_profile.columns)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## calculate result for each individual stock this part should return a table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_processing_result_of_stocks_df(stock_df, profile_df):\n",
+    "    ## add sector_name display_name name\n",
+    "    ticker_sector_map = dict(zip(profile_df['ticker'], profile_df['aggregate_sector']))\n",
+    "    ticker_display_name_map = dict(zip(profile_df['ticker'], profile_df['display_name']))\n",
+    "    ticker_name_map = dict(zip(profile_df['ticker'], profile_df['name']))\n",
+    "\n",
+    "    stock_df['display_name'] = stock_df['ticker'].map(ticker_display_name_map)\n",
+    "    stock_df['name'] = stock_df['ticker'].map(ticker_name_map)\n",
+    "    stock_df['aggregate_sector'] = stock_df['ticker'].map(ticker_sector_map)\n",
+    "\n",
+    "    ## calculate pct using closing price\n",
+    "    stock_df.sort_values(by=['date'], inplace=True)\n",
+    "    stock_df['pct'] = stock_df.groupby('ticker')['close'].pct_change()\n",
+    "\n",
+    "    ## calculate weight TODO: think about how to optimize this\n",
+    "    stock_df = stock_df.merge(profile_df[['weight', 'date', 'ticker']], on=['ticker', 'date'], how='outer')\n",
+    "    stock_df.rename(columns={'weight': 'initial_weight'}, inplace=True)\n",
+    "    stock_df['current_weight'] = float('nan')\n",
+    "    stock_df['previous_weight'] = float('nan')\n",
+    "    df_grouped = stock_df.groupby('ticker')\n",
+    "    for _, group in df_grouped:\n",
+    "        pre_w = float('nan')\n",
+    "        ini_w = float('nan')\n",
+    "        for index, row in group.iterrows():\n",
+    "            cur_w = float('nan')\n",
+    "\n",
+    "            # if has initial weight, the following row all use this initial weight\n",
+    "            if not pd.isna(row['initial_weight']):\n",
+    "                ini_w = row['initial_weight']\n",
+    "                cur_w = ini_w\n",
+    "\n",
+    "            # just calculate current weight based on previous weight\n",
+    "            else:\n",
+    "                cur_w = pre_w * (1 + row['pct'])\n",
+    "\n",
+    "            stock_df.loc[index, 'current_weight'] = cur_w    \n",
+    "            stock_df.loc[index, 'previous_weight'] = pre_w\n",
+    "            stock_df.loc[index, 'initial_weight'] = ini_w\n",
+    "            pre_w = cur_w\n",
+    "\n",
+    "    stock_df.rename(columns={'weight': 'initial_weight'}, inplace=True)\n",
+    "    stock_df.dropna(subset=['close'], inplace=True)\n",
+    "\n",
+    "    ## normalize weight\n",
+    "    stock_df['prev_w_in_p'] = stock_df['previous_weight'] / \\\n",
+    "                                    stock_df.groupby('date')['previous_weight'].transform('sum')\n",
+    "\n",
+    "    stock_df['ini_w_in_p'] = stock_df['initial_weight'] / \\\n",
+    "                                    stock_df.groupby('date')['initial_weight'].transform('sum')\n",
+    "\n",
+    "    ## calculate weighted pct in portfolio\n",
+    "    stock_df['portfolio_pct'] = stock_df['pct'] * stock_df['prev_w_in_p']\n",
+    "\n",
+    "    ## calculate weight in sector TODO: remove\n",
+    "    stock_df['prev_w_in_sectore'] = stock_df['previous_weight'] / \\\n",
+    "                                                stock_df.groupby(['date', 'aggregate_sector'])['previous_weight'].transform('sum')\n",
+    "    stock_df['ini_w_in_sector'] = stock_df['initial_weight'] / \\\n",
+    "                                                stock_df.groupby(['date', 'aggregate_sector'])['initial_weight'].transform('sum')\n",
+    "    ## weighted pct in sector TODO: remove\n",
+    "    stock_df['sector_pct'] = stock_df['pct'] * stock_df['prev_w_in_sectore']\n",
+    "\n",
+    "    ## portfolio return\n",
+    "    stock_df['cum_p_pct'] = stock_df.groupby('ticker')['portfolio_pct'].cumsum()\n",
+    "    stock_df['portfolio_return'] = np.exp(stock_df['cum_p_pct']) -1 \n",
+    "    # drop intermediate columns\n",
+    "    stock_df = stock_df.drop(columns=['cum_p_pct'])\n",
+    "\n",
+    "\n",
+    "    ## sector return TODO:remove \n",
+    "    # stock_df['sector_return'] = stock_df['ini_w_in_sector'] * stock_df['return']\n",
+    "\n",
+    "    return stock_df\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# portfolio_stock = get_processing_result_of_stocks_df(portfolio_df, profile_df)\n",
+    "# portfolio_stock[portfolio_stock.ticker == '002709.XSHE'][['date','portfolio_pct','prev_w_in_p','portfolio_return']]\n",
+    "# # benchmark_stock = get_processing_result_of_stocks_df(benchmark_df, b_profile)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# profile_df.groupby('date')['weight'].sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## total return by date\n",
+    "def get_portfolio_evaluation(portfolio_stock, benchmark_stock, profile_df):\n",
+    "    # add pct of benchmark \n",
+    "    merged_df = portfolio_stock.merge(benchmark_stock[['ticker','date','portfolio_pct','portfolio_return']], \n",
+    "                                    on=['ticker','date'],how='left',suffixes=('_p','_b'))\n",
+    "\n",
+    "    # sum up pct and return from portfolio and benchmark\n",
+    "    merged_df = merged_df.groupby('date',as_index=False).agg({'portfolio_return_p':'sum',\n",
+    "                                                            'portfolio_return_b':'sum',\n",
+    "                                                            'portfolio_pct_p':'sum',\n",
+    "                                                            'portfolio_pct_b':'sum'})\n",
+    "\n",
+    "    # portfolio mkt cap\n",
+    "    mkt_adjustment = pd.DataFrame(profile_df.groupby('date')['weight'].sum())\n",
+    "    mkt_adjustment.rename(columns={'weight':'mkt_cap'}, inplace=True)\n",
+    "    merged_df = merged_df.merge(mkt_adjustment, on=['date'], how='outer')\n",
+    "\n",
+    "    for i in range(len(merged_df)):\n",
+    "        if pd.isna(merged_df.loc[i, 'mkt_cap']):\n",
+    "            merged_df.loc[i, 'mkt_cap'] = merged_df.loc[i-1, 'mkt_cap'] * (1 + merged_df.loc[i, 'portfolio_pct_p'])\n",
+    "    # drop where portfolio_return_p is nan\n",
+    "    merged_df.dropna(subset=['portfolio_return_p'], inplace=True)\n",
+    "    # portfolio pnl TODO seem I can just use current wegith to do this\n",
+    "    merged_df['prev_mkt_cap'] = merged_df['mkt_cap'].shift(1)\n",
+    "    merged_df['pnl'] = merged_df['prev_mkt_cap'] * merged_df['portfolio_pct_p']\n",
+    "\n",
+    "    # risk std(pct)\n",
+    "    merged_df['risk'] = merged_df['portfolio_pct_p'].rolling(len(merged_df), min_periods=1).std() * math.sqrt(252)\n",
+    "\n",
+    "    # active return\n",
+    "    merged_df['active_return'] = merged_df['portfolio_pct_p'] - merged_df['portfolio_pct_b']\n",
+    "\n",
+    "    # tracking errro std(active return)\n",
+    "    merged_df['tracking_error'] = merged_df['active_return'].rolling(len(merged_df), min_periods=1).std() * math.sqrt(252)\n",
+    "\n",
+    "    # cum pnl\n",
+    "    merged_df['cum_pnl'] = merged_df['pnl'].cumsum()\n",
+    "\n",
+    "    return merged_df\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# portfolio_eval_df = get_portfolio_evaluation(portfolio_stock, benchmark_stock, profile_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# portfolio_stock.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## TODO convert below to funciton\n",
+    "\n",
+    "def get_portfolio_sector_evaluation(portfolio_stock,benchmark_df):\n",
+    "# aggregate on sector and day\n",
+    "    p_sector_df = portfolio_stock.groupby(['date','aggregate_sector'], as_index=False)\\\n",
+    "        .agg({'prev_w_in_p': 'sum','ini_w_in_p':\"sum\",\"current_weight\":'sum',\\\n",
+    "            \"portfolio_pct\":\"sum\", \"portfolio_return\":\"sum\"})\n",
+    "    # TODO shrink it down before aggregate\n",
+    "    b_sector_df = benchmark_df.groupby(['date','aggregate_sector'], as_index=False)\\\n",
+    "        .agg({'prev_w_in_p': 'sum','ini_w_in_p':\"sum\",\"current_weight\":'sum',\\\n",
+    "            \"portfolio_pct\":\"sum\", \"portfolio_return\":\"sum\"})\n",
+    "    \n",
+    "    # merge portfolio and benchmark\n",
+    "    merge_df = p_sector_df.merge(b_sector_df, on=['date','aggregate_sector'], how='left', suffixes=('_p','_b'))\n",
+    "\n",
+    "    return merge_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# sector_eval_df = get_portfolio_sector_evaluation(portfolio_stock, benchmark_stock)\n",
+    "# sector_eval_df[sector_eval_df.date == datetime(2021, 10,13)].hvplot.bar(x='aggregate_sector', y=['portfolio_pct_p','portfolio_pct_b'], stacked=True, rot=90, title='sector pct')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def merge_on_date(portfolio_stock, benchmark_df):\n",
+    "    p_selected = portfolio_stock.reset_index()[['ini_w_in_p', 'portfolio_return', 'date', 'ticker', 'display_name']]\n",
+    "    b_selected = benchmark_df.reset_index()[['ini_w_in_p', 'portfolio_return', 'date', 'ticker']]\n",
+    "    merged_stock_df = pd.merge(p_selected, b_selected, on=['date', 'ticker'], how='outer', suffixes=('_p', '_b'))\n",
+    "    return merged_stock_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# merged_df = merge_on_date(portfolio_stock, benchmark_stock)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_bhb_result(merged_stock_df):\n",
+    "    # merged_stock_df['ini_w_in_p_p'].fillna(0, inplace=True)\n",
+    "    # merged_stock_df['ini_w_in_p_b'].fillna(0, inplace=True)\n",
+    "    # merged_stock_df['portfolio_return_b'].fillna(0, inplace=True)\n",
+    "    # merged_stock_df['portfolio_return_p'].fillna(0, inplace=True)\n",
+    "    # allocation\n",
+    "    merged_stock_df['allocation'] = (merged_stock_df['ini_w_in_p_p'] - merged_stock_df['ini_w_in_p_b']) \\\n",
+    "        * merged_stock_df['portfolio_return_b']\n",
+    "\n",
+    "    # selection\n",
+    "    merged_stock_df['selection'] = merged_stock_df['ini_w_in_p_b'] * \\\n",
+    "        (merged_stock_df['portfolio_return_p'] - merged_stock_df['portfolio_return_b'])\n",
+    "\n",
+    "    # interaction\n",
+    "    merged_stock_df['interaction'] = (merged_stock_df['ini_w_in_p_p'] - merged_stock_df['ini_w_in_p_b']) * \\\n",
+    "        (merged_stock_df['portfolio_return_p'] - merged_stock_df['portfolio_return_b'])\n",
+    "\n",
+    "    # excess\n",
+    "    merged_stock_df['excess'] = merged_stock_df['portfolio_return_p'] - merged_stock_df['portfolio_return_b']\n",
+    "\n",
+    "    # replace inf with nan\n",
+    "    merged_stock_df.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
+    "    return merged_stock_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test ing pipeline here "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "portfolio_stock = get_processing_result_of_stocks_df(portfolio_df, profile_df)\n",
+    "benchmark_stock = get_processing_result_of_stocks_df(benchmark_df, b_profile)\n",
+    "\n",
+    "portfolio_eval_df = get_portfolio_evaluation(portfolio_stock, benchmark_stock, profile_df)\n",
+    "sector_eval_df = get_portfolio_sector_evaluation(portfolio_stock, benchmark_stock)\n",
+    "merged_df = merge_on_date(portfolio_stock, benchmark_stock)\n",
+    "bnb_sector_result = get_bhb_result(sector_eval_df)\n",
+    "bnb_stock_result = get_bhb_result(merged_df)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save result \n",
+    "portfolio_eval_df.to_pickle('../data/portfolio_eval_df.pkl')\n",
+    "sector_eval_df.to_pickle('../data/sector_eval_df.pkl')\n",
+    "# merged_df.to_csv('merged_df.csv')\n",
+    "bnb_sector_result.to_pickle('../data/bnb_sector_result.pkl')\n",
+    "bnb_stock_result.to_pickle('../data/bnb_stock_result.pkl')\n",
+    "profile_df.to_pickle('../data/protfolio_profile.pkl')\n",
+    "b_profile.to_pickle('../data/benchmark_profile.pkl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# bnb_sector_result[bnb_sector_result.date == datetime(2021, 10,13)].hvplot.bar(x='aggregate_sector', y=['allocation','selection','interaction'], stacked=True, rot=90, title='sector allocation')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

script/processing.ipynb

@@ -1,3 +1,501 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8a5fbb45900fe3b82e3ebf11e97ec3808f865f816a46782063de9d0c08eaf038
-size 38406
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.1.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n  var reloading = false;\n  var Bokeh = root.Bokeh;\n  var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n      require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n      })\n      require([\"jspanel-modal\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-tooltip\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-hint\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-layout\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-dock\"], function() {\n\ton_load()\n      })\n      require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n      })\n      require([\"notyf\"], function() {\n\ton_load()\n      })\n      root._bokeh_is_loading = css_urls.length + 9;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.holoviz.org/panel/1.1.1/dist/panel.min.js\"];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      Bokeh = root.Bokeh;\n      bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n  window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n    function JupyterCommManager() {\n    }\n\n    JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n      if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        comm_manager.register_target(comm_id, function(comm) {\n          comm.on_msg(msg_handler);\n        });\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n          comm.onMsg = msg_handler;\n        });\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n          var messages = comm.messages[Symbol.asyncIterator]();\n          function processIteratorResult(result) {\n            var message = result.value;\n            console.log(message)\n            var content = {data: message.data, comm_id};\n            var buffers = []\n            for (var buffer of message.buffers || []) {\n              buffers.push(new DataView(buffer))\n            }\n            var metadata = message.metadata || {};\n            var msg = {content, buffers, metadata}\n            msg_handler(msg);\n            return messages.next().then(processIteratorResult);\n          }\n          return messages.next().then(processIteratorResult);\n        })\n      }\n    }\n\n    JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n      if (comm_id in window.PyViz.comms) {\n        return window.PyViz.comms[comm_id];\n      } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n        if (msg_handler) {\n          comm.on_msg(msg_handler);\n        }\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n        comm.open();\n        if (msg_handler) {\n          comm.onMsg = msg_handler;\n        }\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        var comm_promise = google.colab.kernel.comms.open(comm_id)\n        comm_promise.then((comm) => {\n          window.PyViz.comms[comm_id] = comm;\n          if (msg_handler) {\n            var messages = comm.messages[Symbol.asyncIterator]();\n            function processIteratorResult(result) {\n              var message = result.value;\n              var content = {data: message.data};\n              var metadata = message.metadata || {comm_id};\n              var msg = {content, metadata}\n              msg_handler(msg);\n              return messages.next().then(processIteratorResult);\n            }\n            return messages.next().then(processIteratorResult);\n          }\n        }) \n        var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n          return comm_promise.then((comm) => {\n            comm.send(data, metadata, buffers, disposeOnDone);\n          });\n        };\n        var comm = {\n          send: sendClosure\n        };\n      }\n      window.PyViz.comms[comm_id] = comm;\n      return comm;\n    }\n    window.PyViz.comm_manager = new JupyterCommManager();\n    \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n  var div = document.createElement(\"div\");\n  var script = document.createElement(\"script\");\n  node.appendChild(div);\n  node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n  var output_area = handle.output_area;\n  var output = handle.output;\n  if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n    return\n  }\n  var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n  var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n  if (id !== undefined) {\n    var nchildren = toinsert.length;\n    var html_node = toinsert[nchildren-1].children[0];\n    html_node.innerHTML = output.data[HTML_MIME_TYPE];\n    var scripts = [];\n    var nodelist = html_node.querySelectorAll(\"script\");\n    for (var i in nodelist) {\n      if (nodelist.hasOwnProperty(i)) {\n        scripts.push(nodelist[i])\n      }\n    }\n\n    scripts.forEach( function (oldScript) {\n      var newScript = document.createElement(\"script\");\n      var attrs = [];\n      var nodemap = oldScript.attributes;\n      for (var j in nodemap) {\n        if (nodemap.hasOwnProperty(j)) {\n          attrs.push(nodemap[j])\n        }\n      }\n      attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n      newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n      oldScript.parentNode.replaceChild(newScript, oldScript);\n    });\n    if (JS_MIME_TYPE in output.data) {\n      toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n    }\n    output_area._hv_plot_id = id;\n    if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n      window.PyViz.plot_index[id] = Bokeh.index[id];\n    } else {\n      window.PyViz.plot_index[id] = null;\n    }\n  } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n    var bk_div = document.createElement(\"div\");\n    bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n    var script_attrs = bk_div.children[0].attributes;\n    for (var i = 0; i < script_attrs.length; i++) {\n      toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n    }\n    // store reference to server id on output_area\n    output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n  }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n  var id = handle.cell.output_area._hv_plot_id;\n  var server_id = handle.cell.output_area._bokeh_server_id;\n  if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n  var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n  if (server_id !== null) {\n    comm.send({event_type: 'server_delete', 'id': server_id});\n    return;\n  } else if (comm !== null) {\n    comm.send({event_type: 'delete', 'id': id});\n  }\n  delete PyViz.plot_index[id];\n  if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n    var doc = window.Bokeh.index[id].model.document\n    doc.clear();\n    const i = window.Bokeh.documents.indexOf(doc);\n    if (i > -1) {\n      window.Bokeh.documents.splice(i, 1);\n    }\n  }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n  delete PyViz.comms[\"hv-extension-comm\"];\n  window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n  handle_clear_output(event, {cell: {output_area: handle.output_area}})\n  handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n  function append_mime(data, metadata, element) {\n    // create a DOM node to render to\n    var toinsert = this.create_output_subarea(\n    metadata,\n    CLASS_NAME,\n    EXEC_MIME_TYPE\n    );\n    this.keyboard_manager.register_events(toinsert);\n    // Render to node\n    var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n    render(props, toinsert[0]);\n    element.append(toinsert);\n    return toinsert\n  }\n\n  events.on('output_added.OutputArea', handle_add_output);\n  events.on('output_updated.OutputArea', handle_update_output);\n  events.on('clear_output.CodeCell', handle_clear_output);\n  events.on('delete.Cell', handle_clear_output);\n  events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n  OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n    safe: true,\n    index: 0\n  });\n}\n\nif (window.Jupyter !== undefined) {\n  try {\n    var events = require('base/js/events');\n    var OutputArea = require('notebook/js/outputarea').OutputArea;\n    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n      register_renderer(events, OutputArea);\n    }\n  } catch(err) {\n  }\n}\n",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style>*[data-root-id],\n",
+       "*[data-root-id] > * {\n",
+       "  box-sizing: border-box;\n",
+       "  font-family: var(--jp-ui-font-family);\n",
+       "  font-size: var(--jp-ui-font-size1);\n",
+       "  color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
+       "}\n",
+       "\n",
+       "/* Override VSCode background color */\n",
+       ".cell-output-ipywidget-background:has(> .cell-output-ipywidget-background\n",
+       "    > .lm-Widget\n",
+       "    > *[data-root-id]),\n",
+       ".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
+       "  background-color: transparent !important;\n",
+       "}\n",
+       "</style>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import math\n",
+    "from datetime import datetime\n",
+    "import hvplot.pandas\n",
+    "# load data\n",
+    "profile_df = pd.read_pickle('../data/portfolio_portfile.pkl')\n",
+    "benchmark_df = pd.read_pickle('../data/benchmark_portfolio.pkl')\n",
+    "portfolio_df = pd.read_pickle('../data/portfolio_data.pkl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## calculate pct using closing price for each date\n",
+    "portfolio_df.sort_values(by=['date'], inplace=True)\n",
+    "portfolio_df['pct'] = portfolio_df.groupby(['ticker'])['close'].pct_change()\n",
+    "\n",
+    "benchmark_df.sort_values(by=['date'], inplace=True)\n",
+    "benchmark_df['pct'] = benchmark_df.groupby(['ticker'])['close'].pct_change()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## return and weight\n",
+    "merge_df = portfolio_df.merge(profile_df[['ticker','weight','date']], on=['ticker','date'], how='left')\n",
+    "merge_df.sort_values(by=['date'], inplace=True)\n",
+    "benchmark_df.sort_values(by=['date'], inplace=True)\n",
+    "\n",
+    "# return for each stock\n",
+    "# portfolio_df['return'] = portfolio_df['close'] / portfolio_df.groupby(['ticker'])['close'].transform('first') - 1\n",
+    "# benchmark_df['return'] = benchmark_df['close'] / benchmark_df.groupby(['ticker'])['close'].transform('first') - 1\n",
+    "\n",
+    "# stock return in portfolio\n",
+    "first_close_per_ticker = merge_df.groupby('ticker')['close'].transform('first')\n",
+    "merge_df['return'] = merge_df['close'] / first_close_per_ticker - 1\n",
+    "\n",
+    "# stock return in benchmark\n",
+    "first_close_per_ticker = benchmark_df.groupby('ticker')['close'].transform('first')\n",
+    "benchmark_df['return'] = benchmark_df['close'] / first_close_per_ticker - 1\n",
+    "\n",
+    "# weight\n",
+    "initial_weight = merge_df.groupby('ticker')['weight'].transform('first') \n",
+    "merge_df['weight'] =  initial_weight + (merge_df['return'] * initial_weight)\n",
+    "\n",
+    "# initial_weight\n",
+    "# merge_df[merge_df.ticker==\"002709.XSHE\"]\n",
+    "# merge_df\n",
+    "# benchmark_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "portfolio_df = merge_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## normalize weight\n",
+    "benchmark_df['norm_weight'] = benchmark_df['weight'] / benchmark_df.groupby(['date'])['weight'].transform('sum')\n",
+    "portfolio_df['norm_weight'] = portfolio_df['weight'] / portfolio_df.groupby(['date'])['weight'].transform('sum')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## calculate weighted return\n",
+    "portfolio_df['weighted_pct'] = portfolio_df['pct'] * portfolio_df['norm_weight']\n",
+    "benchmark_df['weighted_pct'] = benchmark_df['pct'] * benchmark_df['norm_weight']\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Calculate portfolio pct of return by summing the 'weighted_return' for each date\n",
+    "\n",
+    "# reset index becaue its a new table\n",
+    "portfolio_pct = portfolio_df.groupby('date')['weighted_pct'].sum().reset_index()\n",
+    "portfolio_pct.columns = ['date', 'pct']\n",
+    "\n",
+    "# benchmark pct summing the 'weighted_return' for each date\n",
+    "#TODO chagne to use new weight in order to perform simulation\n",
+    "benchmark_pct = benchmark_df.groupby('date')['weighted_pct'].sum().reset_index()\n",
+    "benchmark_pct.columns = ['date', 'pct']\n",
+    "\n",
+    "# 'date', 'portfolio_return', and 'benchmark_return' columns\n",
+    "return_result = pd.merge(portfolio_pct, benchmark_pct, on='date', suffixes=('_portfolio', '_benchmark'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## add total return\n",
+    "portfolio_df['weighted_return'] = portfolio_df['return'] * portfolio_df['norm_weight']\n",
+    "portfolio_return = portfolio_df.groupby('date')['weighted_return'].sum().reset_index()\n",
+    "portfolio_return.columns = ['date', 'return']\n",
+    "return_result = return_result.merge(portfolio_return, on='date')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#  daily_active_return\n",
+    "return_result['daily_active_return'] = return_result['pct_portfolio'] - return_result['pct_benchmark']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# daily pnl\n",
+    "return_result['mkt_cap'] = None\n",
+    "\n",
+    "# assign initial market cap\n",
+    "# sum weight because it is the money invested in the stock\n",
+    "# TODO chagne here to add simulation functionality\n",
+    "return_result.loc[return_result['date'] == profile_df['date'].values[0],'mkt_cap'] = profile_df['weight'].sum()\n",
+    "return_result.sort_values(by=['date'], inplace=True)\n",
+    "\n",
+    "# calculate daily mkt_cap\n",
+    "for i in range(1, len(return_result)):\n",
+    "    return_result.loc[i, 'mkt_cap'] = return_result.loc[i-1, 'mkt_cap'] * (1 + return_result.loc[i, 'pct_portfolio'])\n",
+    "\n",
+    "# calculate daily pnl\n",
+    "return_result['daily_pnl'] = return_result['mkt_cap'].diff()\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## return volatilty mean return tracking erro cumulative pnl\n",
+    "# volatitly\n",
+    "return_result['pct_volitity_p'] = return_result['pct_portfolio'].expanding().std() * math.sqrt(252)\n",
+    "return_result['pct_volitity_b'] = return_result['pct_benchmark'].expanding().std() * math.sqrt(252)\n",
+    "# mean return\n",
+    "return_result['mean_return'] = return_result['return'].mean()\n",
+    "# tracking error \n",
+    "return_result['tracking_error'] = return_result['daily_active_return'].expanding().std() * math.sqrt(252)\n",
+    "# cumulative pnl\n",
+    "return_result['cum_pnl'] = return_result['daily_pnl'].cumsum()\n",
+    "# drawdown \n",
+    "return_result['drawdown'] = (return_result['cum_pnl'] - return_result['cum_pnl'].cummax()) / return_result['cum_pnl'].cummax()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## save\n",
+    "return_result.to_pickle('../data/risk_return_result.pkl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# add sector info to portfolio_df\n",
+    "portfolio_df = pd.merge(portfolio_df, profile_df[['ticker','aggregate_sector']], on='ticker', how='left')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/v5/2108rh5964q9j741wg_s8r1w0000gn/T/ipykernel_1263/1040563745.py:2: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  benchmark_df[benchmark_df.aggregate_sector.isna()].aggregate_sector = '其他'\n"
+     ]
+    }
+   ],
+   "source": [
+    "# hanlde sector is nan\n",
+    "benchmark_df[benchmark_df.aggregate_sector.isna()].aggregate_sector = '其他'\n",
+    "portfolio_df[portfolio_df.aggregate_sector.isna()].aggregate_sector = '其他'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>display_name_x</th>\n",
+       "      <th>actual_data</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "      <th>display_name_y</th>\n",
+       "      <th>name</th>\n",
+       "      <th>sector</th>\n",
+       "      <th>aggregate_sector</th>\n",
+       "      <th>pct</th>\n",
+       "      <th>return</th>\n",
+       "      <th>norm_weight</th>\n",
+       "      <th>weighted_pct</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>0.088</td>\n",
+       "      <td>神州高铁</td>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>000008.XSHE</td>\n",
+       "      <td>2.52</td>\n",
+       "      <td>2.57</td>\n",
+       "      <td>2.67</td>\n",
+       "      <td>2.49</td>\n",
+       "      <td>33215803.0</td>\n",
+       "      <td>85358605.99</td>\n",
+       "      <td>神州高铁</td>\n",
+       "      <td>SZGT</td>\n",
+       "      <td>机械设备I 运输设备II 铁路设备III 铁路、船舶、航空航天和其他运输设备制造业 城轨铁路 工业</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.00088</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        date  weight display_name_x actual_data       ticker  open  close  \\\n",
+       "0 2021-01-05   0.088           神州高铁  2020-12-31  000008.XSHE  2.52   2.57   \n",
+       "\n",
+       "   high   low      volume        money display_name_y  name  \\\n",
+       "0  2.67  2.49  33215803.0  85358605.99           神州高铁  SZGT   \n",
+       "\n",
+       "                                              sector aggregate_sector  pct  \\\n",
+       "0  机械设备I 运输设备II 铁路设备III 铁路、船舶、航空航天和其他运输设备制造业 城轨铁路 工业               工业  NaN   \n",
+       "\n",
+       "   return  norm_weight  weighted_pct  \n",
+       "0     0.0      0.00088           NaN  "
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "benchmark_df.head(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## aggregate as sector \n",
+    "\n",
+    "# weighted return in a sector, weight in a sector\n",
+    "portfolio_df['weight_in_sector'] = portfolio_df['norm_weight']/ portfolio_df.groupby(['aggregate_sector','date'])['weight'].transform('sum')\n",
+    "portfolio_df['weighted_s_return'] = portfolio_df['return'] * portfolio_df['weight_in_sector']\n",
+    "\n",
+    "# return and weight by sector\n",
+    "return_by_sector = portfolio_df.groupby(['aggregate_sector','date'])['weighted_s_return'].sum().reset_index()\n",
+    "normed_sector_weight = portfolio_df.groupby(['aggregate_sector','date'])['norm_weight'].sum().reset_index()\n",
+    "p_sector_result = pd.merge(return_by_sector, normed_sector_weight, on=['aggregate_sector','date'], how='left')\n",
+    "p_sector_result.rename(columns={'weighted_s_return':'return'}, inplace=True)\n",
+    "\n",
+    "# same thing for benchmark\n",
+    "benchmark_df['weight_in_sector'] = benchmark_df['norm_weight']/ benchmark_df.groupby(['aggregate_sector','date'])['norm_weight'].transform('sum')\n",
+    "benchmark_df['weighted_s_return'] = benchmark_df['return'] * benchmark_df['weight_in_sector']\n",
+    "return_by_sector = benchmark_df.groupby(['aggregate_sector','date'])['weighted_s_return'].sum().reset_index()\n",
+    "normed_sector_weight = benchmark_df.groupby(['aggregate_sector','date'])['norm_weight'].sum().reset_index()\n",
+    "b_sector_result = pd.merge(return_by_sector, normed_sector_weight, on=['aggregate_sector','date'], how='left')\n",
+    "b_sector_result.rename(columns={'weighted_s_return':'return'}, inplace=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## selection, allocation, interaction for sector\n",
+    "\n",
+    "\n",
+    "bnb_result_sector = pd.merge(\n",
+    "    p_sector_result, b_sector_result,\n",
+    "    on=['date','aggregate_sector',], how='outer', suffixes=('_p', '_b'))\n",
+    "\n",
+    "# replace nan norm_weight_b with 0\n",
+    "bnb_result_sector['norm_weight_b'] = bnb_result_sector['norm_weight_b'].fillna(0)\n",
+    "bnb_result_sector['norm_weight_p'] = bnb_result_sector['norm_weight_p'].fillna(0)\n",
+    "\n",
+    "# active weight\n",
+    "bnb_result_sector['active_weight'] = bnb_result_sector['norm_weight_p'] - bnb_result_sector['norm_weight_b']\n",
+    "\n",
+    "# allocation \n",
+    "bnb_result_sector['allocation'] = bnb_result_sector['active_weight'] * bnb_result_sector['return_b']\n",
+    "\n",
+    "#selection\n",
+    "bnb_result_sector['selection'] = bnb_result_sector['norm_weight_p'] * (bnb_result_sector['return_p'] - bnb_result_sector['return_b'])\n",
+    "\n",
+    "#inetration\n",
+    "bnb_result_sector['interaction'] = (bnb_result_sector['norm_weight_p'] - bnb_result_sector['norm_weight_b']) *\\\n",
+    "                            (bnb_result_sector['return_p'] - bnb_result_sector['return_b'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## save result \n",
+    "bnb_result_sector.to_pickle('../data/bnb_result_sector.pkl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## selection, allocation, interaction for stocks\n",
+    "\n",
+    "# I keep weight to exame the stock not in benchmark but in portfolio and vice versa\n",
+    "bnb_result = pd.merge(\n",
+    "    portfolio_df[['date','ticker','norm_weight','weight','return','aggregate_sector']],\n",
+    "    benchmark_df[['date','ticker','norm_weight','weight','return']], on=['date','ticker',], how='outer', suffixes=('_p', '_b'))\n",
+    "\n",
+    "# replace nan norm_weight_b with 0\n",
+    "bnb_result['norm_weight_b'] = bnb_result['norm_weight_b'].fillna(0)\n",
+    "bnb_result['norm_weight_p'] = bnb_result['norm_weight_p'].fillna(0)\n",
+    "\n",
+    "# active weight\n",
+    "bnb_result['active_weight'] = bnb_result['norm_weight_p'] - bnb_result['norm_weight_b']\n",
+    "\n",
+    "# allocation \n",
+    "bnb_result['allocation'] = bnb_result['active_weight'] * bnb_result['return_b']\n",
+    "\n",
+    "#selection\n",
+    "bnb_result['selection'] = bnb_result['norm_weight_p'] * (bnb_result['return_p'] - bnb_result['return_b'])\n",
+    "\n",
+    "#inetration\n",
+    "bnb_result['interaction'] = (bnb_result['norm_weight_p'] - bnb_result['norm_weight_b']) *\\\n",
+    "                            (bnb_result['return_p'] - bnb_result['return_b'])\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## svae\n",
+    "bnb_result.to_pickle('../data/bnb_result.pkl')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

script/processing.py

@@ -1,3 +1,369 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c1e5a2142c83b558c112d0682f11907b6365d4873690c2e39d8ebc6a56a88c35
-size 16107
+import pandas as pd
+import math
+from datetime import datetime
+import hvplot.pandas
+import math
+import numpy as np
+# load data
+
+
+def get_processing_result_of_stocks_df(stock_df, profile_df):
+
+    # add sector_name display_name name
+    ticker_sector_map = dict(
+        zip(profile_df['ticker'], profile_df['aggregate_sector']))
+    ticker_display_name_map = dict(
+        zip(profile_df['ticker'], profile_df['display_name']))
+    ticker_name_map = dict(zip(profile_df['ticker'], profile_df['name']))
+    stock_df['display_name'] = stock_df['ticker'].map(ticker_display_name_map)
+    stock_df['name'] = stock_df['ticker'].map(ticker_name_map)
+    stock_df['aggregate_sector'] = stock_df['ticker'].map(ticker_sector_map)
+
+    # calculate pct using closing price
+    stock_df.sort_values(by=['date'], inplace=True)
+    stock_df['pct'] = stock_df.groupby('ticker')['close'].pct_change()
+
+    # calculate weight TODO: think about how to optimize this
+    stock_df = stock_df.merge(profile_df[['weight', 'date', 'ticker']], on=[
+                              'ticker', 'date'], how='outer')
+    stock_df.rename(columns={'weight': 'initial_weight'}, inplace=True)
+    # create if not in stock_df
+
+    stock_df['current_weight'] = float('nan')
+    stock_df['previous_weight'] = float('nan')
+    df_grouped = stock_df.groupby('ticker')
+    for _, group in df_grouped:
+        pre_w = float('nan')
+        ini_w = float('nan')
+        for index, row in group.iterrows():
+            cur_w = float('nan')
+
+            # if has initial weight, the following row all use this initial weight
+            if not pd.isna(row['initial_weight']):
+                ini_w = row['initial_weight']
+                pre_w = ini_w
+                cur_w = ini_w
+
+            # just calculate current weight based on previous weight
+            else:
+                cur_w = pre_w * (1 + row['pct'])
+
+            stock_df.loc[index, 'current_weight'] = cur_w
+            stock_df.loc[index, 'previous_weight'] = pre_w
+            stock_df.loc[index, 'initial_weight'] = ini_w
+            pre_w = cur_w
+
+    stock_df.rename(columns={'weight': 'initial_weight'}, inplace=True)
+    stock_df.dropna(subset=['close'], inplace=True)
+
+    # normalize weight
+    stock_df['prev_w_in_p'] = stock_df['previous_weight'] / \
+        stock_df.groupby('date')['previous_weight'].transform('sum')
+
+    stock_df['ini_w_in_p'] = stock_df['initial_weight'] / \
+        stock_df.groupby('date')['initial_weight'].transform('sum')
+
+    # calculate weighted pct in portfolio
+    stock_df['portfolio_pct'] = stock_df['pct'] * stock_df['prev_w_in_p']
+
+    # calculate weight in sector TODO: remove
+    stock_df['prev_w_in_sectore'] = stock_df['previous_weight'] / \
+        stock_df.groupby(['date', 'aggregate_sector'])[
+        'previous_weight'].transform('sum')
+    stock_df['ini_w_in_sector'] = stock_df['initial_weight'] / \
+        stock_df.groupby(['date', 'aggregate_sector'])[
+        'initial_weight'].transform('sum')
+    # weighted pct in sector TODO: remove
+    stock_df['sector_pct'] = stock_df['pct'] * stock_df['prev_w_in_sectore']
+
+    # portfolio return
+    stock_df['portfolio_return'] = (stock_df.groupby('ticker')['portfolio_pct'].cumprod() + 1) - 1
+    # stock_df['cum_p_pct'] = stock_df.groupby(
+    #     'ticker')['portfolio_pct'].cumsum()
+    # stock_df['portfolio_return'] = np.exp(stock_df['cum_p_pct']) - 1
+
+    # stock return
+    stock_df['return'] = (stock_df.groupby('ticker')['pct'].cumprod() + 1) - 1
+    # stock_df['cum_pct'] = stock_df.groupby(
+    #     'ticker')['pct'].cumsum()
+    # stock_df['return'] = np.exp(stock_df['cum_pct']) - 1
+
+    # drop intermediate columns
+    stock_df = stock_df.drop(columns=['cum_p_pct'])
+
+    # risk
+    stock_df['risk'] = stock_df.groupby('ticker')['pct']\
+        .transform(lambda x: x.rolling(len(x), min_periods=1).std() * math.sqrt(252))
+
+    # fill na aggregate_sector
+    stock_df['aggregate_sector'].fillna('其他', inplace=True)
+    # sector return
+    stock_df['sector_return'] = stock_df['ini_w_in_sector'] * \
+        stock_df['return']
+
+    return stock_df
+
+
+# total return by date
+def get_portfolio_evaluation(portfolio_stock, benchmark_stock, profile_df):
+    # agg by date
+    agg_p_stock = portfolio_stock\
+        .groupby('date', as_index=False)\
+        .agg({'portfolio_return': 'sum', 'portfolio_pct': 'sum'})
+    agg_b_stock = benchmark_stock\
+        .groupby('date', as_index=False)\
+        .agg({'portfolio_return': 'sum', 'portfolio_pct': 'sum'})
+
+    # add pct of benchmark
+    merged_df = pd.merge(agg_p_stock, agg_b_stock, on=[
+                         'date'], how='left', suffixes=('_p', '_b'))
+
+    # portfolio mkt cap
+    mkt_adjustment = pd.DataFrame(profile_df.groupby('date')['weight'].sum())
+    mkt_adjustment.rename(columns={'weight': 'mkt_cap'}, inplace=True)
+    merged_df = merged_df.merge(mkt_adjustment, on=['date'], how='outer')
+
+    for i in range(len(merged_df)):
+        if pd.isna(merged_df.loc[i, 'mkt_cap']) and i > 0:
+            merged_df.loc[i, 'mkt_cap'] = merged_df.loc[i-1,
+                                                        'mkt_cap'] * (1 + merged_df.loc[i, 'portfolio_pct_p'])
+    # drop where portfolio_return_p is nan
+    merged_df.dropna(subset=['portfolio_return_p'], inplace=True)
+    # portfolio pnl TODO seem I can just use current wegith to do this
+    merged_df['prev_mkt_cap'] = merged_df['mkt_cap'].shift(1)
+    merged_df['pnl'] = merged_df['prev_mkt_cap'] * merged_df['portfolio_pct_p']
+
+    # risk std(pct)
+    merged_df['risk'] = merged_df['portfolio_pct_p'].rolling(
+        len(merged_df), min_periods=1).std() * math.sqrt(252)
+
+    # active return
+    merged_df['active_return'] = merged_df['portfolio_pct_p'] - \
+        merged_df['portfolio_pct_b']
+
+    # tracking errro std(active return)
+    merged_df['tracking_error'] = merged_df['active_return'].rolling(
+        len(merged_df), min_periods=1).std() * math.sqrt(252)
+
+    # cum pnl
+    merged_df['cum_pnl'] = merged_df['pnl'].cumsum()
+
+    return merged_df
+
+
+# TODO convert below to funciton
+
+def get_portfolio_sector_evaluation(portfolio_stock, benchmark_df):
+    # aggregate on sector and day
+    p_sector_df = portfolio_stock.groupby(['date', 'aggregate_sector'], as_index=False)\
+        .agg({'prev_w_in_p': 'sum', 'ini_w_in_p': "sum", "current_weight": 'sum',
+              "portfolio_pct": "sum", 'sector_return': "sum", 'ini_w_in_sector': 'sum', "portfolio_return": "sum"})
+    b_sector_df = benchmark_df.groupby(['date', 'aggregate_sector'], as_index=False)\
+        .agg({'prev_w_in_p': 'sum', 'ini_w_in_p': "sum", "current_weight": 'sum',
+              "portfolio_pct": "sum", "portfolio_return": "sum", 'sector_return': "sum", 'ini_w_in_sector': 'sum'})
+
+    # merge portfolio and benchmark
+    merge_df = p_sector_df.merge(
+        b_sector_df, on=['date', 'aggregate_sector'], how='outer', suffixes=('_p', '_b'))
+
+    # to acomendate bhb result
+    merge_df.rename(columns={'sector_return_p': 'return_p',
+                    'sector_return_b': 'return_b'}, inplace=True)
+    # active return
+    merge_df['active_return'] = merge_df['portfolio_return_p'] - \
+        merge_df['portfolio_return_b']
+
+    # risk
+    merge_df['risk'] = merge_df.groupby('aggregate_sector')['portfolio_pct_p']\
+        .transform(lambda x: x.rolling(len(x), min_periods=1).std() * math.sqrt(252))
+
+    # tracking error
+    merge_df['tracking_error'] = merge_df.groupby('aggregate_sector')['active_return']\
+        .transform(lambda x: x.rolling(len(x), min_periods=1).std() * math.sqrt(252))
+    return merge_df
+
+
+# sector_eval_df = get_portfolio_sector_evaluation(portfolio_stock, benchmark_stock)
+# sector_eval_df[sector_eval_df.date == datetime(2021, 10,13)].hvplot.bar(x='aggregate_sector', y=['portfolio_pct_p','portfolio_pct_b'], stacked=True, rot=90, title='sector pct')
+
+
+def merge_on_date(calculated_ps, calculated_bs):
+    p_selected = calculated_ps.reset_index(
+    )[['ini_w_in_p', 'portfolio_return', 'date', 'ticker', 'display_name', 'return']]
+    b_selected = calculated_bs.reset_index(
+    )[['ini_w_in_p', 'portfolio_return', 'date', 'ticker', 'return']]
+    merged_stock_df = pd.merge(p_selected, b_selected, on=[
+                               'date', 'ticker'], how='outer', suffixes=('_p', '_b'))
+    return merged_stock_df
+
+
+# merged_df = merge_on_date(portfolio_stock, benchmark_stock)
+
+
+def get_bhb_result(merged_stock_df):
+    # merged_stock_df['ini_w_in_p_p'].fillna(0, inplace=True)
+    # merged_stock_df['ini_w_in_p_b'].fillna(0, inplace=True)
+    # merged_stock_df['portfolio_return_b'].fillna(0, inplace=True)
+    # merged_stock_df['portfolio_return_p'].fillna(0, inplace=True)
+    # allocation
+    merged_stock_df['allocation'] = (merged_stock_df['ini_w_in_p_p'] - merged_stock_df['ini_w_in_p_b']) \
+        * merged_stock_df['return_b']
+
+    # selection
+    merged_stock_df['selection'] = merged_stock_df['ini_w_in_p_b'] * \
+        (merged_stock_df['return_p'] -
+         merged_stock_df['return_b'])
+
+    # interaction
+    merged_stock_df['interaction'] = (merged_stock_df['ini_w_in_p_p'] - merged_stock_df['ini_w_in_p_b']) * \
+        (merged_stock_df['return_p'] -
+         merged_stock_df['return_b'])
+
+    # excess
+    merged_stock_df['excess'] = merged_stock_df['portfolio_return_p'] - \
+        merged_stock_df['portfolio_return_b']
+
+    # replace inf with nan
+    # merged_stock_df.replace([np.inf, -np.inf], np.nan, inplace=True)
+    return merged_stock_df
+
+
+def calculate_total_attribution_by_sector(calculated_p_stock, calculated_b_stock):
+    sector_view_p = calculated_p_stock.groupby(['date', 'aggregate_sector']).aggregate({
+        'prev_w_in_p': 'sum', 'sector_pct': 'sum'})
+    sector_view_b = calculated_b_stock.groupby(['date', 'aggregate_sector']).aggregate({
+        'prev_w_in_p': 'sum', 'sector_pct': 'sum'})
+
+    sector_view_p['weighted_return'] = sector_view_p.prev_w_in_p * \
+        sector_view_p.sector_pct
+    sector_view_b['weighted_return'] = sector_view_b.prev_w_in_p * \
+        sector_view_b.sector_pct
+
+    merged_df = pd.merge(sector_view_p, sector_view_b, left_index=True,
+                         right_index=True, how='outer', suffixes=['_b', '_p'])
+    merged_df.fillna(0, inplace=True)
+    merged_df['active_return'] = merged_df['weighted_return_p'] - \
+        merged_df['weighted_return_b']
+    merged_df['allocation'] = (
+        merged_df.prev_w_in_p_p - merged_df.prev_w_in_p_b) * merged_df.sector_pct_b
+    merged_df['selection'] = (
+        merged_df.sector_pct_p - merged_df.sector_pct_b) * merged_df.prev_w_in_p_b
+    merged_df['interaction'] = (merged_df.sector_pct_p - merged_df.sector_pct_b) * (
+        merged_df.prev_w_in_p_p - merged_df.prev_w_in_p_b)
+    merged_df['notinal_return'] = merged_df.allocation + \
+        merged_df.selection + merged_df.interaction
+    return merged_df.reset_index()
+
+
+def calculate_total_attribution(calculated_p_stock, calculated_b_stock):
+    '''
+    using pct between two row's data of ticker to calculate the attribute,
+    use this method if need to calculate weekly attribut, yearly attribut, etc.
+    '''
+    merged_df = pd.merge(calculated_b_stock, calculated_p_stock, on=[
+                         'date', 'ticker'], how='outer', suffixes=['_b', '_p'])
+    df = merged_df[['pct_p', 'pct_b', 'prev_w_in_p_p',
+                    'prev_w_in_p_b', 'ticker', 'date']]
+    df.fillna(0, inplace=True)
+    df['active_return'] = df.pct_p * \
+        df.prev_w_in_p_p - df.pct_b * df.prev_w_in_p_b
+    # allocation
+    df['allocation'] = (df.prev_w_in_p_p - df.prev_w_in_p_b) * df.pct_b
+    df['selection'] = (df.pct_p - df.pct_b) * df.prev_w_in_p_b
+    df['interaction'] = (df.pct_p - df.pct_b) * \
+        (df.prev_w_in_p_p - df.prev_w_in_p_b)
+    df['notional_return'] = df.allocation + df.selection + df.interaction
+
+    daily_bnb_result = df.groupby(['date']).aggregate(
+        {'allocation': 'sum', 'selection': 'sum', 'interaction': 'sum', 'notional_return': 'sum', 'active_return': 'sum'})
+    daily_bnb_result['date'] = daily_bnb_result.index
+
+    return daily_bnb_result.reset_index(drop=True)
+    # return df
+
+
+def calculate_return(df, start, end):
+    '''
+    return a df consist of total return for each day,
+    the return at start date would be 0
+    '''
+    selected_df = df[df.date.between(start, end)].copy()
+    # set the pct of first row to null
+    selected_df.iloc[0, selected_df.columns.get_indexer(
+        ['portfolio_pct_p', 'portfolio_pct_b'])] = 0
+    selected_df['return_p'] = (1 + selected_df['portfolio_pct_p']).cumprod() - 1
+    selected_df['return_b'] = (1 + selected_df['portfolio_pct_b']).cumprod() - 1
+    selected_df['active_return'] = selected_df.return_p - selected_df.return_b
+    return selected_df
+
+
+def calculate_attributes_between_dates(start, end, calculated_p_stock, calculated_b_stock):
+    '''
+    calculate the attributes to explain the active return between two time series entries, the time series entry
+    right after or at start and another time serie right before or at end
+    return a df with attributes to explain the active return between start and end time series
+    '''
+    p_ranged_df = calculated_p_stock[(calculated_p_stock.date >= start) & (
+        calculated_p_stock.date <= end)]
+    b_ranged_df = calculated_b_stock[(calculated_b_stock.date >= start) & (
+        calculated_b_stock.date <= end)]
+
+    # return and weight of portfolio
+    p_start_df = p_ranged_df[p_ranged_df.date == p_ranged_df.date.min()]
+    p_end_df = p_ranged_df[p_ranged_df.date == p_ranged_df.date.max()]
+    p_concat = pd.concat([p_start_df, p_end_df])
+    # pct is unweighted return
+    p_concat['pct'] = p_concat.groupby('ticker')['close'].pct_change()
+    p_concat = p_concat.dropna(subset=['pct'])
+    p_concat['prev_w_in_p'] = p_concat['ticker'].map(
+        lambda x: p_start_df[p_start_df.ticker == x]['prev_w_in_p'].values[0])
+    # p_concatp_concat[['date', 'display_name', 'pct',
+    #           'close', 'prev_w_in_p', 'ini_w_in_p']]
+    # return and weight of benchmark
+    b_start_df = b_ranged_df[b_ranged_df.date == b_ranged_df.date.min()]
+    b_end_df = b_ranged_df[b_ranged_df.date == b_ranged_df.date.max()]
+    b_concat = pd.concat([b_start_df, b_end_df])
+    b_concat['pct'] = b_concat.groupby('ticker')['close'].pct_change()
+    b_concat = b_concat.dropna(subset=['pct'])
+    b_concat['prev_w_in_p'] = b_concat['ticker'].map(
+        lambda x: b_concat[b_concat.ticker == x]['prev_w_in_p'].values[0])
+    # b_concat = b_concat[['date', 'display_name', 'pct',
+    #           'close', 'prev_w_in_p', 'ini_w_in_p']]
+    merged_df = pd.merge(b_concat, p_concat, on=[
+                         'ticker', 'date'], suffixes=('_b', '_p'), how='outer')
+    df = merged_df[['display_name_p', 'display_name_b', 'ticker',
+                    'pct_b', 'pct_p', 'prev_w_in_p_b', 'prev_w_in_p_p']].copy()
+
+    # indicate weather stock is in portfolio
+    df['in_portfolio'] = False
+    df.loc[df.display_name_p.notnull(), 'in_portfolio'] = True
+
+    # fill display_name
+    df['display_name_p'] = df['display_name_p'].fillna(df['display_name_b'])
+    df['display_name_b'] = df['display_name_b'].fillna(df['display_name_p'])
+
+    # treat nan weight and pct as 0
+    df.fillna(0, inplace=True)
+
+    # allocation, selection, interaction, notional return, active return
+    df['allocation'] = (df.prev_w_in_p_p - df.prev_w_in_p_b) * df.pct_b
+    df['selection'] = (df.pct_p - df.pct_b) * df.prev_w_in_p_b
+    df['interaction'] = (df.pct_p - df.pct_b) * \
+        (df.prev_w_in_p_p - df.prev_w_in_p_b)
+    df['notional_return'] = df.allocation + df.selection + df.interaction
+    # weighted return
+    df['return'] = df.prev_w_in_p_p * df.pct_p
+    # weight * prev_w is the weighted return
+    df['active_return'] = df.prev_w_in_p_p * \
+        df.pct_p - df.prev_w_in_p_b * df.pct_b
+
+    return df
+
+
+def change_resolution(df, freq='W'):
+    '''
+    aggregate by keeping the first entry of the freq period,
+    the resolution of the df, default to weekly
+    '''
+    df['freq'] = pd.to_datetime(df['date']).dt.to_period(freq)
+    return df.groupby('freq').first().reset_index()

script/processing2.ipynb

@@ -1,3 +1,1836 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f2b581ac16ff53983fcf43922dd3896d186279b8868b1080cd274938babaf627
-size 59598
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 142,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import math\n",
+    "from datetime import datetime\n",
+    "import hvplot.pandas\n",
+    "import math\n",
+    "# load data\n",
+    "profile_df = pd.read_pickle('../data/portfolio_portfile.pkl')\n",
+    "benchmark_df = pd.read_pickle('../data/benchmark_portfolio.pkl')\n",
+    "portfolio_df = pd.read_pickle('../data/portfolio_data.pkl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 143,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# to acoomodate the current pipe line\n",
+    "min_dates = benchmark_df.groupby('ticker')['date'].min()\n",
+    "\n",
+    "for ticker, min_date in min_dates.items():\n",
+    "   benchmark_df.loc[(benchmark_df['ticker'] == ticker) & (benchmark_df['date'] != min_date), 'weight'] = float('nan')\n",
+    "\n",
+    "benchmark_df['initial_weight'] = benchmark_df['weight']\n",
+    "# drop weight\n",
+    "benchmark_df = benchmark_df.drop(columns=['weight'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 144,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Series([], Name: initial_weight, dtype: int64)"
+      ]
+     },
+     "execution_count": 144,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# check if all unique ticker has an weight\n",
+    "count_list = benchmark_df.groupby('ticker')['initial_weight'].count().sort_values(ascending=False)\n",
+    "count_list[count_list != 1]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 145,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "update_profile_df = profile_df.copy()\n",
+    "update_profile_df['date'] = datetime(2021,1,10)\n",
+    "update_profile_df['weight'] = [50,100,200,300,400,500]\n",
+    "profile_df = pd.concat([profile_df, update_profile_df])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 146,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def calculate_pct(stock_df):\n",
+    "    stock_df['pct'] = stock_df.groupby(['ticker'])['close'].pct_change()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 147,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# step 1 pct\n",
+    "calculate_pct(portfolio_df)\n",
+    "calculate_pct(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 148,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def return_weighted_stock_df(stock_price_df, profile_df=None):\n",
+    "    # TODO change later this a temporary solution\n",
+    "    # initialize weight if profile_df is not none\n",
+    "    merged_df = pd.DataFrame()\n",
+    "    if profile_df is not None:\n",
+    "        merged_df = stock_price_df.merge(profile_df[['weight', 'date', 'ticker']], on=['ticker', 'date'], how='outer')\n",
+    "        merged_df.sort_values(by=['date'], inplace=True)\n",
+    "        merged_df.rename(columns={'weight': 'initial_weight'}, inplace=True)\n",
+    "    else:\n",
+    "        merged_df = stock_price_df.copy()\n",
+    "    merged_df['current_weight'] = float('nan')\n",
+    "    merged_df['previous_weight'] = float('nan')\n",
+    "    df_grouped = merged_df.groupby('ticker')\n",
+    "    for _, group in df_grouped:\n",
+    "        pre_w = float('nan')\n",
+    "        ini_w = float('nan')\n",
+    "        for index, row in group.iterrows():\n",
+    "            cur_w = float('nan')\n",
+    "            # if has initial weight, the following row all use this initial weight\n",
+    "            if not pd.isna(row['initial_weight']):\n",
+    "                ini_w = row['initial_weight']\n",
+    "                cur_w = ini_w\n",
+    "            # just calculate current weight based on previous weight\n",
+    "            else:\n",
+    "                cur_w = pre_w * (1 + row['pct'])\n",
+    "\n",
+    "            merged_df.loc[index, 'current_weight'] = cur_w    \n",
+    "            merged_df.loc[index, 'previous_weight'] = pre_w\n",
+    "            merged_df.loc[index, 'initial_weight'] = ini_w\n",
+    "            pre_w = cur_w\n",
+    "            \n",
+    "    # drop row where closing price is none\n",
+    "    merged_df = merged_df[~pd.isna(merged_df['close'])]\n",
+    "    # drop index\n",
+    "    return merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 149,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO consider save the weight calculation\n",
+    "portfolio_df = return_weighted_stock_df(portfolio_df, profile_df)\n",
+    "benchmark_df = return_weighted_stock_df(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 150,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# benchmark_df[benchmark_df.ticker =='000008.XSHE']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 151,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## normalize all weight\n",
+    "def normalize_weight(stock_df):\n",
+    "    stock_df['current_weight'] = stock_df['current_weight'] / \\\n",
+    "                                    stock_df.groupby('date')['current_weight'].transform('sum')\n",
+    "\n",
+    "    stock_df['previous_weight'] = stock_df['previous_weight'] / \\\n",
+    "                                    stock_df.groupby('date')['previous_weight'].transform('sum')\n",
+    "\n",
+    "    stock_df['initial_weight'] = stock_df['initial_weight'] / \\\n",
+    "                                    stock_df.groupby('date')['initial_weight'].transform('sum')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 152,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "normalize_weight(portfolio_df)\n",
+    "normalize_weight(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 153,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "260    0.032258\n",
+       "258    0.064516\n",
+       "262    0.129032\n",
+       "263    0.258065\n",
+       "259    0.322581\n",
+       "261    0.193548\n",
+       "Name: initial_weight, dtype: float64"
+      ]
+     },
+     "execution_count": 153,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "portfolio_df[portfolio_df.date == datetime(2021, 3, 12)]['initial_weight']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 154,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1.0\n",
+      "1.0\n",
+      "1.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(benchmark_df[benchmark_df.date == datetime(2021, 3, 12)]['initial_weight'].sum())\n",
+    "print(benchmark_df[benchmark_df.date == datetime(2021, 3, 12)]['current_weight'].sum())\n",
+    "print(benchmark_df[benchmark_df.date == datetime(2021, 3, 12)]['previous_weight'].sum())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 155,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# step 3 sector wegiht\n",
+    "\n",
+    "# add sector information first\n",
+    "def create_sector_weight(stock_df, profile_df=None):\n",
+    "    # if profile_df is none assume the aggregate_sector stock info already in stock_df\n",
+    "    merged_df = None\n",
+    "    if profile_df is not None:\n",
+    "        merged_df = stock_df.merge(profile_df[['ticker', 'aggregate_sector']], on='ticker', how='left')\n",
+    "    else:\n",
+    "        merged_df = stock_df.copy()\n",
+    "    # set null to others\n",
+    "    merged_df['aggregate_sector'] = merged_df['aggregate_sector'].fillna('其他')\n",
+    "    # calculate previous_sector_weight\n",
+    "    merged_df['previous_sector_weight'] = merged_df['previous_weight'] / \\\n",
+    "                                            merged_df.groupby(['date', 'aggregate_sector'])['previous_weight'].transform('sum')\n",
+    "    # calculate initial sectore weight\n",
+    "    merged_df['initial_sector_weight'] = merged_df['initial_weight'] / \\\n",
+    "                                            merged_df.groupby(['date', 'aggregate_sector'])['initial_weight'].transform('sum')\n",
+    "    \n",
+    "    return merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 156,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "portfolio_df = create_sector_weight(stock_df = portfolio_df, profile_df = profile_df)\n",
+    "benchmark_df = create_sector_weight(benchmark_df)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 157,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "aggregate_sector\n",
+      "信息与通信    1.0\n",
+      "公用事业     1.0\n",
+      "其他       1.0\n",
+      "医药卫生     1.0\n",
+      "原料与能源    1.0\n",
+      "工业       1.0\n",
+      "消费       1.0\n",
+      "金融与地产    1.0\n",
+      "Name: previous_sector_weight, dtype: float64\n",
+      "aggregate_sector\n",
+      "信息与通信    1.0\n",
+      "公用事业     1.0\n",
+      "其他       1.0\n",
+      "医药卫生     1.0\n",
+      "原料与能源    1.0\n",
+      "工业       1.0\n",
+      "消费       1.0\n",
+      "金融与地产    1.0\n",
+      "Name: initial_sector_weight, dtype: float64\n"
+     ]
+    }
+   ],
+   "source": [
+    "# check result \n",
+    "print(benchmark_df[benchmark_df.date == datetime(2021, 3, 12)].groupby('aggregate_sector')['previous_sector_weight'].sum())\n",
+    "print(benchmark_df[benchmark_df.date == datetime(2021, 3, 12)].groupby('aggregate_sector')['initial_sector_weight'].sum())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 158,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "aggregate_sector\n",
+      "信息与通信    1.0\n",
+      "医药卫生     1.0\n",
+      "原料与能源    1.0\n",
+      "工业       1.0\n",
+      "消费       1.0\n",
+      "Name: previous_sector_weight, dtype: float64\n",
+      "aggregate_sector\n",
+      "信息与通信    1.0\n",
+      "医药卫生     1.0\n",
+      "原料与能源    1.0\n",
+      "工业       1.0\n",
+      "消费       1.0\n",
+      "Name: initial_sector_weight, dtype: float64\n"
+     ]
+    }
+   ],
+   "source": [
+    "# check result \n",
+    "print(portfolio_df[portfolio_df.date == datetime(2021, 3, 12)].groupby('aggregate_sector')['previous_sector_weight'].sum())\n",
+    "print(portfolio_df[portfolio_df.date == datetime(2021, 3, 12)].groupby('aggregate_sector')['initial_sector_weight'].sum())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 159,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## return define as the total return since the portfolio created\n",
+    "def calcualte_return(stock_df):\n",
+    "    stock_df['return'] = stock_df['close'] / stock_df.groupby(['ticker'])['close'].transform('first') - 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 160,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "calcualte_return(portfolio_df)\n",
+    "calcualte_return(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 161,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def calculate_weighted_sector_return(stock_df):\n",
+    "    stock_df['weighted_sectore_return'] = stock_df['return'] * stock_df['initial_sector_weight']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 162,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "calculate_weighted_sector_return(portfolio_df)\n",
+    "calculate_weighted_sector_return(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 163,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## weighted return and sector weighred return \n",
+    "def calculate_weighted_return(stock_df):\n",
+    "    stock_df['weighted_return'] = stock_df['return'] * stock_df['initial_weight']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 164,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# step\n",
+    "calculate_weighted_return(portfolio_df)\n",
+    "calculate_weighted_return(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 165,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def calculate_weighted_sector_return(stock_df):\n",
+    "    stock_df['weighted_sector_return'] = stock_df['return'] * stock_df['initial_sector_weight']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 166,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "calculate_weighted_sector_return(portfolio_df)\n",
+    "calculate_weighted_sector_return(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 167,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## calcualte weighted pc\n",
+    "def calculate_weighted_pct(stock_df):\n",
+    "    stock_df['weighted_pct'] = stock_df['pct'] * stock_df['previous_weight']\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 168,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def calculate_weighted_sector_pct(stock_df):\n",
+    "    stock_df['weighted_sector_pct'] = stock_df['pct'] * stock_df['previous_sector_weight']\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 169,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "calculate_weighted_sector_pct(portfolio_df)\n",
+    "calculate_weighted_sector_pct(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 170,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "calculate_weighted_pct(portfolio_df)\n",
+    "calculate_weighted_pct(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 171,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "calculate_weighted_sector_return(portfolio_df)\n",
+    "calculate_weighted_sector_return(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 172,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## aggregate by date\n",
+    "\n",
+    "# pct and weighted_return\n",
+    "# def agg_by_date(stock_df)\n",
+    "def agg_by_date(stock_df):\n",
+    "    agg_on_date_df = pd.DataFrame(stock_df.groupby('date')[['weighted_return','weighted_pct']].sum())\n",
+    "    agg_on_date_df.rename(columns={'weighted_return': 'return', 'weighted_pct': 'pct'}, inplace=True)\n",
+    "    return agg_on_date_df\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 173,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "p_total_view = agg_by_date(portfolio_df)\n",
+    "b_total_view = agg_by_date(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 174,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## aggregate by sector\n",
+    "def agg_by_sector(stock_df):\n",
+    "    agg_on_sector_df = pd.DataFrame(stock_df.groupby(['aggregate_sector','date'])[['weighted_sector_return','weighted_sector_pct']].sum())\n",
+    "    agg_on_sector_df.rename(columns={'weighted_sector_return': 'return', 'weighted_sector_pct': 'pct'}, inplace=True)\n",
+    "    return agg_on_sector_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 175,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "p_sector_view = agg_by_sector(portfolio_df)\n",
+    "b_sector_view = agg_by_sector(benchmark_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 200,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_risk_table(portfolio_summary, benchmark_summary):\n",
+    "    # total risk tracking error  \n",
+    "    merged_df = pd.merge(portfolio_summary, benchmark_summary, on='date', how='outer', suffixes=('_p', '_b'))\n",
+    "    merged_df['risk_p'] = merged_df['return_p'].expanding().std() * math.sqrt(252)\n",
+    "    merged_df['risk_b'] = merged_df['return_b'].expanding().std() * math.sqrt(252)\n",
+    "    merged_df['active_return'] = merged_df['return_p'] - merged_df['return_b']\n",
+    "    merged_df['tracking_error'] = merged_df['active_return'].expanding().std() * math.sqrt(252)\n",
+    "    merged_df['date'] = merged_df.index\n",
+    "    # drop index\n",
+    "    merged_df.reset_index(drop=True, inplace=True)\n",
+    "    return merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 201,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "portfolio_risk_by_date_df = create_risk_table(p_total_view, b_total_view)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 202,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>return_p</th>\n",
+       "      <th>pct_p</th>\n",
+       "      <th>return_b</th>\n",
+       "      <th>pct_b</th>\n",
+       "      <th>risk_p</th>\n",
+       "      <th>risk_b</th>\n",
+       "      <th>active_return</th>\n",
+       "      <th>tracking_error</th>\n",
+       "      <th>date</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.012146</td>\n",
+       "      <td>0.012146</td>\n",
+       "      <td>-0.001934</td>\n",
+       "      <td>-0.001934</td>\n",
+       "      <td>0.136341</td>\n",
+       "      <td>0.021705</td>\n",
+       "      <td>0.014080</td>\n",
+       "      <td>0.158046</td>\n",
+       "      <td>2021-01-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0.086830</td>\n",
+       "      <td>0.074233</td>\n",
+       "      <td>-0.000811</td>\n",
+       "      <td>0.001125</td>\n",
+       "      <td>0.746402</td>\n",
+       "      <td>0.015414</td>\n",
+       "      <td>0.087641</td>\n",
+       "      <td>0.747127</td>\n",
+       "      <td>2021-01-07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0.089435</td>\n",
+       "      <td>0.002496</td>\n",
+       "      <td>0.002535</td>\n",
+       "      <td>0.003349</td>\n",
+       "      <td>0.756382</td>\n",
+       "      <td>0.030137</td>\n",
+       "      <td>0.086900</td>\n",
+       "      <td>0.740979</td>\n",
+       "      <td>2021-01-08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0.148063</td>\n",
+       "      <td>0.029363</td>\n",
+       "      <td>-0.013015</td>\n",
+       "      <td>-0.015511</td>\n",
+       "      <td>0.970984</td>\n",
+       "      <td>0.095654</td>\n",
+       "      <td>0.161078</td>\n",
+       "      <td>1.032423</td>\n",
+       "      <td>2021-01-11</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>242</th>\n",
+       "      <td>0.028005</td>\n",
+       "      <td>-0.071081</td>\n",
+       "      <td>0.086827</td>\n",
+       "      <td>0.000156</td>\n",
+       "      <td>2.097631</td>\n",
+       "      <td>0.886298</td>\n",
+       "      <td>-0.058822</td>\n",
+       "      <td>1.856213</td>\n",
+       "      <td>2022-01-04</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>243</th>\n",
+       "      <td>-0.033053</td>\n",
+       "      <td>-0.059582</td>\n",
+       "      <td>0.067931</td>\n",
+       "      <td>-0.017386</td>\n",
+       "      <td>2.099052</td>\n",
+       "      <td>0.884891</td>\n",
+       "      <td>-0.100984</td>\n",
+       "      <td>1.861347</td>\n",
+       "      <td>2022-01-05</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>244</th>\n",
+       "      <td>-0.042238</td>\n",
+       "      <td>-0.008112</td>\n",
+       "      <td>0.069522</td>\n",
+       "      <td>0.001490</td>\n",
+       "      <td>2.101118</td>\n",
+       "      <td>0.883542</td>\n",
+       "      <td>-0.111761</td>\n",
+       "      <td>1.867445</td>\n",
+       "      <td>2022-01-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>245</th>\n",
+       "      <td>-0.073118</td>\n",
+       "      <td>-0.031015</td>\n",
+       "      <td>0.062056</td>\n",
+       "      <td>-0.006981</td>\n",
+       "      <td>2.105760</td>\n",
+       "      <td>0.881986</td>\n",
+       "      <td>-0.135174</td>\n",
+       "      <td>1.875959</td>\n",
+       "      <td>2022-01-07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>246</th>\n",
+       "      <td>-0.029944</td>\n",
+       "      <td>0.044300</td>\n",
+       "      <td>0.064588</td>\n",
+       "      <td>0.002384</td>\n",
+       "      <td>2.106749</td>\n",
+       "      <td>0.880502</td>\n",
+       "      <td>-0.094532</td>\n",
+       "      <td>1.880060</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>247 rows × 9 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     return_p     pct_p  return_b     pct_b    risk_p    risk_b  \\\n",
+       "0    0.000000  0.000000  0.000000  0.000000       NaN       NaN   \n",
+       "1    0.012146  0.012146 -0.001934 -0.001934  0.136341  0.021705   \n",
+       "2    0.086830  0.074233 -0.000811  0.001125  0.746402  0.015414   \n",
+       "3    0.089435  0.002496  0.002535  0.003349  0.756382  0.030137   \n",
+       "4    0.148063  0.029363 -0.013015 -0.015511  0.970984  0.095654   \n",
+       "..        ...       ...       ...       ...       ...       ...   \n",
+       "242  0.028005 -0.071081  0.086827  0.000156  2.097631  0.886298   \n",
+       "243 -0.033053 -0.059582  0.067931 -0.017386  2.099052  0.884891   \n",
+       "244 -0.042238 -0.008112  0.069522  0.001490  2.101118  0.883542   \n",
+       "245 -0.073118 -0.031015  0.062056 -0.006981  2.105760  0.881986   \n",
+       "246 -0.029944  0.044300  0.064588  0.002384  2.106749  0.880502   \n",
+       "\n",
+       "     active_return  tracking_error       date  \n",
+       "0         0.000000             NaN 2021-01-05  \n",
+       "1         0.014080        0.158046 2021-01-06  \n",
+       "2         0.087641        0.747127 2021-01-07  \n",
+       "3         0.086900        0.740979 2021-01-08  \n",
+       "4         0.161078        1.032423 2021-01-11  \n",
+       "..             ...             ...        ...  \n",
+       "242      -0.058822        1.856213 2022-01-04  \n",
+       "243      -0.100984        1.861347 2022-01-05  \n",
+       "244      -0.111761        1.867445 2022-01-06  \n",
+       "245      -0.135174        1.875959 2022-01-07  \n",
+       "246      -0.094532        1.880060 2022-01-10  \n",
+       "\n",
+       "[247 rows x 9 columns]"
+      ]
+     },
+     "execution_count": 202,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# add mkt cap\n",
+    "portfolio_risk_by_date_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 217,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "date\n",
+       "2021-01-05     600\n",
+       "2021-01-10    1550\n",
+       "Name: weight, dtype: int64"
+      ]
+     },
+     "execution_count": 217,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "profile_df.groupby('date')['weight'].sum()\n",
+    "\n",
+    "# for i in range(1, len(portfolio_risk_by_date_df)):\n",
+    "#     cur_mkt = portfolio_risk_by_date_df.loc[i, 'mkt_cap']\n",
+    "#     if pd.isna(cur_mkt):\n",
+    "#         portfolio_risk_by_date_df.loc[i, 'mkt_cap'] = portfolio_risk_by_date_df.loc[i-1, 'mkt_cap'] * (1 + portfolio_risk_by_date_df.loc[i, 'pct_p'])\n",
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 216,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>return_p</th>\n",
+       "      <th>pct_p</th>\n",
+       "      <th>return_b</th>\n",
+       "      <th>pct_b</th>\n",
+       "      <th>risk_p</th>\n",
+       "      <th>risk_b</th>\n",
+       "      <th>active_return</th>\n",
+       "      <th>tracking_error</th>\n",
+       "      <th>date</th>\n",
+       "      <th>mkt_cap</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>600.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   return_p  pct_p  return_b  pct_b  risk_p  risk_b  active_return  \\\n",
+       "0       0.0    0.0       0.0    0.0     NaN     NaN            0.0   \n",
+       "\n",
+       "   tracking_error       date  mkt_cap  \n",
+       "0             NaN 2021-01-05    600.0  "
+      ]
+     },
+     "execution_count": 216,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# display row where mkt_cap is not nana\n",
+    "portfolio_risk_by_date_df[portfolio_risk_by_date_df['mkt_cap'].notna()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/v5/2108rh5964q9j741wg_s8r1w0000gn/T/ipykernel_23255/2871737262.py:10: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  pct['weighted_pct'] = pct['pct'] * pct['norm_weight']\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>return</th>\n",
+       "      <th>pct</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2021-01-06</td>\n",
+       "      <td>0.007011</td>\n",
+       "      <td>0.036439</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2021-01-07</td>\n",
+       "      <td>0.047531</td>\n",
+       "      <td>0.218707</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2021-01-08</td>\n",
+       "      <td>0.047111</td>\n",
+       "      <td>0.013639</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2021-01-11</td>\n",
+       "      <td>0.052768</td>\n",
+       "      <td>0.014559</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>242</th>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>0.363845</td>\n",
+       "      <td>-0.199827</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>243</th>\n",
+       "      <td>2022-01-05</td>\n",
+       "      <td>0.306697</td>\n",
+       "      <td>-0.193598</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>244</th>\n",
+       "      <td>2022-01-06</td>\n",
+       "      <td>0.331291</td>\n",
+       "      <td>0.023418</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>245</th>\n",
+       "      <td>2022-01-07</td>\n",
+       "      <td>0.313726</td>\n",
+       "      <td>-0.080728</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>246</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.313262</td>\n",
+       "      <td>0.110254</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>247 rows × 3 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          date    return       pct\n",
+       "0   2021-01-05  0.000000  0.000000\n",
+       "1   2021-01-06  0.007011  0.036439\n",
+       "2   2021-01-07  0.047531  0.218707\n",
+       "3   2021-01-08  0.047111  0.013639\n",
+       "4   2021-01-11  0.052768  0.014559\n",
+       "..         ...       ...       ...\n",
+       "242 2022-01-04  0.363845 -0.199827\n",
+       "243 2022-01-05  0.306697 -0.193598\n",
+       "244 2022-01-06  0.331291  0.023418\n",
+       "245 2022-01-07  0.313726 -0.080728\n",
+       "246 2022-01-10  0.313262  0.110254\n",
+       "\n",
+       "[247 rows x 3 columns]"
+      ]
+     },
+     "execution_count": 191,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## aggregate by date\n",
+    "# step 7 aggregate (get portfolio return and pct(change of daily return))by date\n",
+    "def create_agg_by_date(stock_df):\n",
+    "    # sum up weighted return to get return \n",
+    "    agg_return = stock_df.groupby(['date'])['weighted_return'].sum().reset_index()\n",
+    "    agg_return.rename(columns={'weighted_return':'return'}, inplace=True)\n",
+    "\n",
+    "    # sum up weighted pct to get pct\n",
+    "    pct = stock_df[['date','pct','norm_weight','ticker']]\n",
+    "    pct['weighted_pct'] = pct['pct'] * pct['norm_weight']\n",
+    "    agg_pct = pct.groupby(['date'])['pct'].sum().reset_index()\n",
+    "\n",
+    "    agg_df = pd.merge(agg_return, agg_pct, on='date', how='outer')\n",
+    "    return agg_df\n",
+    "\n",
+    "\n",
+    "\n",
+    "p_perform_result = create_agg_by_date(p_stock_df)\n",
+    "p_perform_result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>date</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "      <th>pct</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>return</th>\n",
+       "      <th>norm_weight</th>\n",
+       "      <th>weighted_return</th>\n",
+       "      <th>aggregate_sector</th>\n",
+       "      <th>display_name</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1452</th>\n",
+       "      <td>603882.XSHG</td>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>106.89</td>\n",
+       "      <td>98.84</td>\n",
+       "      <td>106.89</td>\n",
+       "      <td>98.67</td>\n",
+       "      <td>5140406.0</td>\n",
+       "      <td>5.181929e+08</td>\n",
+       "      <td>-0.076262</td>\n",
+       "      <td>79.300385</td>\n",
+       "      <td>-0.206996</td>\n",
+       "      <td>0.107586</td>\n",
+       "      <td>-0.022270</td>\n",
+       "      <td>医药卫生</td>\n",
+       "      <td>金域医学</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1453</th>\n",
+       "      <td>002709.XSHE</td>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>57.64</td>\n",
+       "      <td>54.64</td>\n",
+       "      <td>57.87</td>\n",
+       "      <td>54.29</td>\n",
+       "      <td>42150916.0</td>\n",
+       "      <td>2.333429e+09</td>\n",
+       "      <td>-0.028277</td>\n",
+       "      <td>161.227501</td>\n",
+       "      <td>0.612275</td>\n",
+       "      <td>0.218735</td>\n",
+       "      <td>0.133926</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>天赐材料</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1454</th>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>8.16</td>\n",
+       "      <td>8.21</td>\n",
+       "      <td>8.25</td>\n",
+       "      <td>8.15</td>\n",
+       "      <td>27288613.0</td>\n",
+       "      <td>2.237925e+08</td>\n",
+       "      <td>0.007362</td>\n",
+       "      <td>85.788924</td>\n",
+       "      <td>-0.142111</td>\n",
+       "      <td>0.116389</td>\n",
+       "      <td>-0.016540</td>\n",
+       "      <td>原料与能源</td>\n",
+       "      <td>三友化工</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1455</th>\n",
+       "      <td>002920.XSHE</td>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>139.71</td>\n",
+       "      <td>131.69</td>\n",
+       "      <td>140.91</td>\n",
+       "      <td>131.45</td>\n",
+       "      <td>5410083.0</td>\n",
+       "      <td>7.233361e+08</td>\n",
+       "      <td>-0.060833</td>\n",
+       "      <td>150.934097</td>\n",
+       "      <td>0.509341</td>\n",
+       "      <td>0.204770</td>\n",
+       "      <td>0.104298</td>\n",
+       "      <td>信息与通信</td>\n",
+       "      <td>德赛西威</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1456</th>\n",
+       "      <td>300274.XSHE</td>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>146.52</td>\n",
+       "      <td>134.96</td>\n",
+       "      <td>148.46</td>\n",
+       "      <td>134.61</td>\n",
+       "      <td>24205007.0</td>\n",
+       "      <td>3.333125e+09</td>\n",
+       "      <td>-0.071291</td>\n",
+       "      <td>176.533682</td>\n",
+       "      <td>0.765337</td>\n",
+       "      <td>0.239501</td>\n",
+       "      <td>0.183299</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>阳光电源</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1457</th>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>4.80</td>\n",
+       "      <td>4.89</td>\n",
+       "      <td>4.90</td>\n",
+       "      <td>4.78</td>\n",
+       "      <td>58291943.0</td>\n",
+       "      <td>2.832956e+08</td>\n",
+       "      <td>0.029474</td>\n",
+       "      <td>83.304940</td>\n",
+       "      <td>-0.166951</td>\n",
+       "      <td>0.113019</td>\n",
+       "      <td>-0.018869</td>\n",
+       "      <td>消费</td>\n",
+       "      <td>小商品城</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           ticker       date    open   close    high     low      volume  \\\n",
+       "1452  603882.XSHG 2022-01-04  106.89   98.84  106.89   98.67   5140406.0   \n",
+       "1453  002709.XSHE 2022-01-04   57.64   54.64   57.87   54.29  42150916.0   \n",
+       "1454  600409.XSHG 2022-01-04    8.16    8.21    8.25    8.15  27288613.0   \n",
+       "1455  002920.XSHE 2022-01-04  139.71  131.69  140.91  131.45   5410083.0   \n",
+       "1456  300274.XSHE 2022-01-04  146.52  134.96  148.46  134.61  24205007.0   \n",
+       "1457  600415.XSHG 2022-01-04    4.80    4.89    4.90    4.78  58291943.0   \n",
+       "\n",
+       "             money       pct      weight    return  norm_weight  \\\n",
+       "1452  5.181929e+08 -0.076262   79.300385 -0.206996     0.107586   \n",
+       "1453  2.333429e+09 -0.028277  161.227501  0.612275     0.218735   \n",
+       "1454  2.237925e+08  0.007362   85.788924 -0.142111     0.116389   \n",
+       "1455  7.233361e+08 -0.060833  150.934097  0.509341     0.204770   \n",
+       "1456  3.333125e+09 -0.071291  176.533682  0.765337     0.239501   \n",
+       "1457  2.832956e+08  0.029474   83.304940 -0.166951     0.113019   \n",
+       "\n",
+       "      weighted_return aggregate_sector display_name  \n",
+       "1452        -0.022270             医药卫生         金域医学  \n",
+       "1453         0.133926               工业         天赐材料  \n",
+       "1454        -0.016540            原料与能源         三友化工  \n",
+       "1455         0.104298            信息与通信         德赛西威  \n",
+       "1456         0.183299               工业         阳光电源  \n",
+       "1457        -0.018869               消费         小商品城  "
+      ]
+     },
+     "execution_count": 194,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p_stock_df[p_stock_df.date==datetime(2022,1,4)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>mkt_cap</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2021-01-10</td>\n",
+       "      <td>1550</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        date  mkt_cap\n",
+       "0 2021-01-05      600\n",
+       "1 2021-01-10     1550"
+      ]
+     },
+     "execution_count": 102,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mkt_cap_df =  pd.DataFrame(profile_df.groupby(['date'])['weight'].sum()).reset_index()\n",
+    "mkt_cap_df.rename(columns={'weight':'mkt_cap'}, inplace=True)\n",
+    "mkt_cap_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>return</th>\n",
+       "      <th>pct</th>\n",
+       "      <th>mkt_cap</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>600.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2021-01-06</td>\n",
+       "      <td>0.007011</td>\n",
+       "      <td>0.036439</td>\n",
+       "      <td>621.863161</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2021-01-07</td>\n",
+       "      <td>0.047531</td>\n",
+       "      <td>0.218707</td>\n",
+       "      <td>757.869005</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2021-01-08</td>\n",
+       "      <td>0.047111</td>\n",
+       "      <td>0.013639</td>\n",
+       "      <td>768.205269</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        date    return       pct     mkt_cap\n",
+       "0 2021-01-05  0.000000  0.000000  600.000000\n",
+       "1 2021-01-06  0.007011  0.036439  621.863161\n",
+       "2 2021-01-07  0.047531  0.218707  757.869005\n",
+       "3 2021-01-08  0.047111  0.013639  768.205269"
+      ]
+     },
+     "execution_count": 103,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get mkt adjustment (weight is the fund in a stock)\n",
+    "mkt_adjustment =  pd.DataFrame(profile_df.groupby(['date'])['weight'].sum()).reset_index()\n",
+    "mkt_adjustment.rename(columns={'weight':'mkt_cap'}, inplace=True)\n",
+    "merge_df = p_perform_result.merge(mkt_adjustment, on='date', how='outer')\n",
+    "\n",
+    "\n",
+    "for i in range(1, len(merge_df)):\n",
+    "    merge_df.loc[i, 'mkt_cap'] = merge_df.loc[i-1, 'mkt_cap'] * (1 + merge_df.loc[i, 'pct'])\n",
+    "\n",
+    "# # calculate daily mkt_cap\n",
+    "# # initial_mkt_cap = merge_df.loc[0, 'mkt_cap']\n",
+    "# for i in range(1, len(merge_df)):\n",
+    "#     row = merge_df.loc[i]\n",
+    "#     if pd.isna(row['mkt_cap']):\n",
+    "#         merge_df.loc[i, 'mkt_cap'] = merge_df.loc[i-1, 'mkt_cap'] * (1 + merge_df.loc[i, 'pct_portfolio'])\n",
+    "    \n",
+    "# # step 8 calculate daily mkt cap\n",
+    "\n",
+    "merge_df[merge_df.date < datetime(2021,1,10)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>date</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "      <th>pct</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>return</th>\n",
+       "      <th>norm_weight</th>\n",
+       "      <th>weighted_return</th>\n",
+       "      <th>aggregate_sector</th>\n",
+       "      <th>display_name</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>002709.XSHE</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>32.54</td>\n",
+       "      <td>33.89</td>\n",
+       "      <td>34.22</td>\n",
+       "      <td>31.39</td>\n",
+       "      <td>59152352.0</td>\n",
+       "      <td>1.942406e+09</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>100.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.166667</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>天赐材料</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>5.33</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.87</td>\n",
+       "      <td>5.22</td>\n",
+       "      <td>180936477.0</td>\n",
+       "      <td>1.010225e+09</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>100.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.166667</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>消费</td>\n",
+       "      <td>小商品城</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>9.23</td>\n",
+       "      <td>9.57</td>\n",
+       "      <td>9.66</td>\n",
+       "      <td>9.08</td>\n",
+       "      <td>82669289.0</td>\n",
+       "      <td>7.803391e+08</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>100.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.166667</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>原料与能源</td>\n",
+       "      <td>三友化工</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>300274.XSHE</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>76.03</td>\n",
+       "      <td>76.45</td>\n",
+       "      <td>80.20</td>\n",
+       "      <td>75.27</td>\n",
+       "      <td>51384827.0</td>\n",
+       "      <td>3.961995e+09</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>100.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.166667</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>阳光电源</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>002920.XSHE</td>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>85.44</td>\n",
+       "      <td>87.25</td>\n",
+       "      <td>87.95</td>\n",
+       "      <td>84.07</td>\n",
+       "      <td>3852674.0</td>\n",
+       "      <td>3.322598e+08</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>100.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.166667</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>信息与通信</td>\n",
+       "      <td>德赛西威</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1477</th>\n",
+       "      <td>600409.XSHG</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>8.24</td>\n",
+       "      <td>8.35</td>\n",
+       "      <td>8.39</td>\n",
+       "      <td>8.21</td>\n",
+       "      <td>32516017.0</td>\n",
+       "      <td>2.699300e+08</td>\n",
+       "      <td>0.015815</td>\n",
+       "      <td>87.251829</td>\n",
+       "      <td>-0.127482</td>\n",
+       "      <td>0.121949</td>\n",
+       "      <td>-0.015546</td>\n",
+       "      <td>原料与能源</td>\n",
+       "      <td>三友化工</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1478</th>\n",
+       "      <td>002920.XSHE</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>130.36</td>\n",
+       "      <td>138.43</td>\n",
+       "      <td>141.96</td>\n",
+       "      <td>130.11</td>\n",
+       "      <td>5005400.0</td>\n",
+       "      <td>6.901614e+08</td>\n",
+       "      <td>0.046888</td>\n",
+       "      <td>158.659026</td>\n",
+       "      <td>0.586590</td>\n",
+       "      <td>0.221752</td>\n",
+       "      <td>0.130077</td>\n",
+       "      <td>信息与通信</td>\n",
+       "      <td>德赛西威</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1479</th>\n",
+       "      <td>002709.XSHE</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>51.63</td>\n",
+       "      <td>50.73</td>\n",
+       "      <td>51.93</td>\n",
+       "      <td>50.03</td>\n",
+       "      <td>29821246.0</td>\n",
+       "      <td>1.518902e+09</td>\n",
+       "      <td>-0.019142</td>\n",
+       "      <td>149.690174</td>\n",
+       "      <td>0.496902</td>\n",
+       "      <td>0.209216</td>\n",
+       "      <td>0.103960</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>天赐材料</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1480</th>\n",
+       "      <td>600415.XSHG</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>4.70</td>\n",
+       "      <td>4.75</td>\n",
+       "      <td>4.85</td>\n",
+       "      <td>4.67</td>\n",
+       "      <td>39278041.0</td>\n",
+       "      <td>1.859827e+08</td>\n",
+       "      <td>0.010638</td>\n",
+       "      <td>80.919932</td>\n",
+       "      <td>-0.190801</td>\n",
+       "      <td>0.113099</td>\n",
+       "      <td>-0.021579</td>\n",
+       "      <td>消费</td>\n",
+       "      <td>小商品城</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1481</th>\n",
+       "      <td>603882.XSHG</td>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>88.45</td>\n",
+       "      <td>95.53</td>\n",
+       "      <td>95.59</td>\n",
+       "      <td>88.39</td>\n",
+       "      <td>6991445.0</td>\n",
+       "      <td>6.468392e+08</td>\n",
+       "      <td>0.085692</td>\n",
+       "      <td>76.644737</td>\n",
+       "      <td>-0.233553</td>\n",
+       "      <td>0.107123</td>\n",
+       "      <td>-0.025019</td>\n",
+       "      <td>医药卫生</td>\n",
+       "      <td>金域医学</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1482 rows × 15 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           ticker       date    open   close    high     low       volume  \\\n",
+       "0     002709.XSHE 2021-01-05   32.54   33.89   34.22   31.39   59152352.0   \n",
+       "1     600415.XSHG 2021-01-05    5.33    5.87    5.87    5.22  180936477.0   \n",
+       "2     600409.XSHG 2021-01-05    9.23    9.57    9.66    9.08   82669289.0   \n",
+       "3     300274.XSHE 2021-01-05   76.03   76.45   80.20   75.27   51384827.0   \n",
+       "4     002920.XSHE 2021-01-05   85.44   87.25   87.95   84.07    3852674.0   \n",
+       "...           ...        ...     ...     ...     ...     ...          ...   \n",
+       "1477  600409.XSHG 2022-01-10    8.24    8.35    8.39    8.21   32516017.0   \n",
+       "1478  002920.XSHE 2022-01-10  130.36  138.43  141.96  130.11    5005400.0   \n",
+       "1479  002709.XSHE 2022-01-10   51.63   50.73   51.93   50.03   29821246.0   \n",
+       "1480  600415.XSHG 2022-01-10    4.70    4.75    4.85    4.67   39278041.0   \n",
+       "1481  603882.XSHG 2022-01-10   88.45   95.53   95.59   88.39    6991445.0   \n",
+       "\n",
+       "             money       pct      weight    return  norm_weight  \\\n",
+       "0     1.942406e+09       NaN  100.000000  0.000000     0.166667   \n",
+       "1     1.010225e+09       NaN  100.000000  0.000000     0.166667   \n",
+       "2     7.803391e+08       NaN  100.000000  0.000000     0.166667   \n",
+       "3     3.961995e+09       NaN  100.000000  0.000000     0.166667   \n",
+       "4     3.322598e+08       NaN  100.000000  0.000000     0.166667   \n",
+       "...            ...       ...         ...       ...          ...   \n",
+       "1477  2.699300e+08  0.015815   87.251829 -0.127482     0.121949   \n",
+       "1478  6.901614e+08  0.046888  158.659026  0.586590     0.221752   \n",
+       "1479  1.518902e+09 -0.019142  149.690174  0.496902     0.209216   \n",
+       "1480  1.859827e+08  0.010638   80.919932 -0.190801     0.113099   \n",
+       "1481  6.468392e+08  0.085692   76.644737 -0.233553     0.107123   \n",
+       "\n",
+       "      weighted_return aggregate_sector display_name  \n",
+       "0            0.000000               工业         天赐材料  \n",
+       "1            0.000000               消费         小商品城  \n",
+       "2            0.000000            原料与能源         三友化工  \n",
+       "3            0.000000               ��业         阳光电源  \n",
+       "4            0.000000            信息与通信         德赛西威  \n",
+       "...               ...              ...          ...  \n",
+       "1477        -0.015546            原料与能源         三友化工  \n",
+       "1478         0.130077            信息与通信         德赛西威  \n",
+       "1479         0.103960               工业         天赐材料  \n",
+       "1480        -0.021579               消费         小商品城  \n",
+       "1481        -0.025019             医药卫生         金域医学  \n",
+       "\n",
+       "[1482 rows x 15 columns]"
+      ]
+     },
+     "execution_count": 127,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## agg by sector and day\n",
+    "p_stock_df['weight_in_sector'] = p_stock_df.groupby"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def creaet_portfolio_return(stock_df):\n",
+    "    portfolio_df = stock_df.groupby(['date'])['weighted_return'].sum().reset_index()\n",
+    "    portfolio_df.rename(columns={'weighted_return':'portfolio_return'}, inplace=True)\n",
+    "    return portfolio_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>portfolio_return</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2021-01-05</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2021-01-06</td>\n",
+       "      <td>0.007011</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2021-01-07</td>\n",
+       "      <td>0.047531</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2021-01-08</td>\n",
+       "      <td>0.047111</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2021-01-11</td>\n",
+       "      <td>0.052768</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>242</th>\n",
+       "      <td>2022-01-04</td>\n",
+       "      <td>0.363845</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>243</th>\n",
+       "      <td>2022-01-05</td>\n",
+       "      <td>0.306697</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>244</th>\n",
+       "      <td>2022-01-06</td>\n",
+       "      <td>0.331291</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>245</th>\n",
+       "      <td>2022-01-07</td>\n",
+       "      <td>0.313726</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>246</th>\n",
+       "      <td>2022-01-10</td>\n",
+       "      <td>0.313262</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>247 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          date  portfolio_return\n",
+       "0   2021-01-05          0.000000\n",
+       "1   2021-01-06          0.007011\n",
+       "2   2021-01-07          0.047531\n",
+       "3   2021-01-08          0.047111\n",
+       "4   2021-01-11          0.052768\n",
+       "..         ...               ...\n",
+       "242 2022-01-04          0.363845\n",
+       "243 2022-01-05          0.306697\n",
+       "244 2022-01-06          0.331291\n",
+       "245 2022-01-07          0.313726\n",
+       "246 2022-01-10          0.313262\n",
+       "\n",
+       "[247 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 58,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "portfolio_df = creaet_portfolio_return(p_stock_df)\n",
+    "portfolio_df"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

script/stream_pricessing.ipynb

@@ -1,3 +1,617 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a654077c9f8cfd02c5894f6905f2156c814e6421b612c94f170363b8ee793e85
-size 41951
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.1.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n  var reloading = false;\n  var Bokeh = root.Bokeh;\n  var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n      require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n      })\n      require([\"jspanel-modal\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-tooltip\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-hint\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-layout\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-dock\"], function() {\n\ton_load()\n      })\n      require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n      })\n      require([\"notyf\"], function() {\n\ton_load()\n      })\n      root._bokeh_is_loading = css_urls.length + 9;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.holoviz.org/panel/1.1.1/dist/panel.min.js\"];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      Bokeh = root.Bokeh;\n      bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n  window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n    function JupyterCommManager() {\n    }\n\n    JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n      if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        comm_manager.register_target(comm_id, function(comm) {\n          comm.on_msg(msg_handler);\n        });\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n          comm.onMsg = msg_handler;\n        });\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n          var messages = comm.messages[Symbol.asyncIterator]();\n          function processIteratorResult(result) {\n            var message = result.value;\n            console.log(message)\n            var content = {data: message.data, comm_id};\n            var buffers = []\n            for (var buffer of message.buffers || []) {\n              buffers.push(new DataView(buffer))\n            }\n            var metadata = message.metadata || {};\n            var msg = {content, buffers, metadata}\n            msg_handler(msg);\n            return messages.next().then(processIteratorResult);\n          }\n          return messages.next().then(processIteratorResult);\n        })\n      }\n    }\n\n    JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n      if (comm_id in window.PyViz.comms) {\n        return window.PyViz.comms[comm_id];\n      } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n        if (msg_handler) {\n          comm.on_msg(msg_handler);\n        }\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n        comm.open();\n        if (msg_handler) {\n          comm.onMsg = msg_handler;\n        }\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        var comm_promise = google.colab.kernel.comms.open(comm_id)\n        comm_promise.then((comm) => {\n          window.PyViz.comms[comm_id] = comm;\n          if (msg_handler) {\n            var messages = comm.messages[Symbol.asyncIterator]();\n            function processIteratorResult(result) {\n              var message = result.value;\n              var content = {data: message.data};\n              var metadata = message.metadata || {comm_id};\n              var msg = {content, metadata}\n              msg_handler(msg);\n              return messages.next().then(processIteratorResult);\n            }\n            return messages.next().then(processIteratorResult);\n          }\n        }) \n        var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n          return comm_promise.then((comm) => {\n            comm.send(data, metadata, buffers, disposeOnDone);\n          });\n        };\n        var comm = {\n          send: sendClosure\n        };\n      }\n      window.PyViz.comms[comm_id] = comm;\n      return comm;\n    }\n    window.PyViz.comm_manager = new JupyterCommManager();\n    \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n  var div = document.createElement(\"div\");\n  var script = document.createElement(\"script\");\n  node.appendChild(div);\n  node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n  var output_area = handle.output_area;\n  var output = handle.output;\n  if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n    return\n  }\n  var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n  var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n  if (id !== undefined) {\n    var nchildren = toinsert.length;\n    var html_node = toinsert[nchildren-1].children[0];\n    html_node.innerHTML = output.data[HTML_MIME_TYPE];\n    var scripts = [];\n    var nodelist = html_node.querySelectorAll(\"script\");\n    for (var i in nodelist) {\n      if (nodelist.hasOwnProperty(i)) {\n        scripts.push(nodelist[i])\n      }\n    }\n\n    scripts.forEach( function (oldScript) {\n      var newScript = document.createElement(\"script\");\n      var attrs = [];\n      var nodemap = oldScript.attributes;\n      for (var j in nodemap) {\n        if (nodemap.hasOwnProperty(j)) {\n          attrs.push(nodemap[j])\n        }\n      }\n      attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n      newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n      oldScript.parentNode.replaceChild(newScript, oldScript);\n    });\n    if (JS_MIME_TYPE in output.data) {\n      toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n    }\n    output_area._hv_plot_id = id;\n    if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n      window.PyViz.plot_index[id] = Bokeh.index[id];\n    } else {\n      window.PyViz.plot_index[id] = null;\n    }\n  } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n    var bk_div = document.createElement(\"div\");\n    bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n    var script_attrs = bk_div.children[0].attributes;\n    for (var i = 0; i < script_attrs.length; i++) {\n      toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n    }\n    // store reference to server id on output_area\n    output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n  }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n  var id = handle.cell.output_area._hv_plot_id;\n  var server_id = handle.cell.output_area._bokeh_server_id;\n  if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n  var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n  if (server_id !== null) {\n    comm.send({event_type: 'server_delete', 'id': server_id});\n    return;\n  } else if (comm !== null) {\n    comm.send({event_type: 'delete', 'id': id});\n  }\n  delete PyViz.plot_index[id];\n  if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n    var doc = window.Bokeh.index[id].model.document\n    doc.clear();\n    const i = window.Bokeh.documents.indexOf(doc);\n    if (i > -1) {\n      window.Bokeh.documents.splice(i, 1);\n    }\n  }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n  delete PyViz.comms[\"hv-extension-comm\"];\n  window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n  handle_clear_output(event, {cell: {output_area: handle.output_area}})\n  handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n  function append_mime(data, metadata, element) {\n    // create a DOM node to render to\n    var toinsert = this.create_output_subarea(\n    metadata,\n    CLASS_NAME,\n    EXEC_MIME_TYPE\n    );\n    this.keyboard_manager.register_events(toinsert);\n    // Render to node\n    var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n    render(props, toinsert[0]);\n    element.append(toinsert);\n    return toinsert\n  }\n\n  events.on('output_added.OutputArea', handle_add_output);\n  events.on('output_updated.OutputArea', handle_update_output);\n  events.on('clear_output.CodeCell', handle_clear_output);\n  events.on('delete.Cell', handle_clear_output);\n  events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n  OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n    safe: true,\n    index: 0\n  });\n}\n\nif (window.Jupyter !== undefined) {\n  try {\n    var events = require('base/js/events');\n    var OutputArea = require('notebook/js/outputarea').OutputArea;\n    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n      register_renderer(events, OutputArea);\n    }\n  } catch(err) {\n  }\n}\n",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style>*[data-root-id],\n",
+       "*[data-root-id] > * {\n",
+       "  box-sizing: border-box;\n",
+       "  font-family: var(--jp-ui-font-family);\n",
+       "  font-size: var(--jp-ui-font-size1);\n",
+       "  color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
+       "}\n",
+       "\n",
+       "/* Override VSCode background color */\n",
+       ".cell-output-ipywidget-background:has(> .cell-output-ipywidget-background\n",
+       "    > .lm-Widget\n",
+       "    > *[data-root-id]),\n",
+       ".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
+       "  background-color: transparent !important;\n",
+       "}\n",
+       "</style>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import math\n",
+    "from datetime import datetime\n",
+    "import hvplot.pandas\n",
+    "import math\n",
+    "import numpy as np\n",
+    "from streamz import Stream"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "b_stocks = pd.read_pickle('../data/b_stocks.pkl')\n",
+    "p_stocks = pd.read_pickle('../data/p_stocks.pkl')\n",
+    "p_profile = pd.read_pickle('../data/p_profile.pkl')\n",
+    "b_profile = pd.read_pickle('../data/b_profile.pkl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>002233.XSHE</td>\n",
+       "      <td>11.27</td>\n",
+       "      <td>11.28</td>\n",
+       "      <td>11.34</td>\n",
+       "      <td>11.17</td>\n",
+       "      <td>16262377.0</td>\n",
+       "      <td>1.829668e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>247</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>601778.XSHG</td>\n",
+       "      <td>7.27</td>\n",
+       "      <td>7.65</td>\n",
+       "      <td>7.77</td>\n",
+       "      <td>7.25</td>\n",
+       "      <td>59723781.0</td>\n",
+       "      <td>4.523540e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>493</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>002368.XSHE</td>\n",
+       "      <td>25.41</td>\n",
+       "      <td>28.18</td>\n",
+       "      <td>28.18</td>\n",
+       "      <td>25.27</td>\n",
+       "      <td>17448308.0</td>\n",
+       "      <td>4.729692e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>739</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>001914.XSHE</td>\n",
+       "      <td>21.21</td>\n",
+       "      <td>20.39</td>\n",
+       "      <td>21.33</td>\n",
+       "      <td>20.26</td>\n",
+       "      <td>6619778.0</td>\n",
+       "      <td>1.366024e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>985</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>002384.XSHE</td>\n",
+       "      <td>25.66</td>\n",
+       "      <td>25.98</td>\n",
+       "      <td>26.00</td>\n",
+       "      <td>25.17</td>\n",
+       "      <td>50695885.0</td>\n",
+       "      <td>1.296706e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>147355</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>600511.XSHG</td>\n",
+       "      <td>46.39</td>\n",
+       "      <td>45.10</td>\n",
+       "      <td>46.55</td>\n",
+       "      <td>44.32</td>\n",
+       "      <td>45375375.0</td>\n",
+       "      <td>2.043297e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>147601</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>600236.XSHG</td>\n",
+       "      <td>4.05</td>\n",
+       "      <td>4.05</td>\n",
+       "      <td>4.05</td>\n",
+       "      <td>4.02</td>\n",
+       "      <td>5788783.0</td>\n",
+       "      <td>2.339606e+07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>147847</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>000807.XSHE</td>\n",
+       "      <td>7.32</td>\n",
+       "      <td>7.58</td>\n",
+       "      <td>7.71</td>\n",
+       "      <td>7.13</td>\n",
+       "      <td>136647514.0</td>\n",
+       "      <td>1.027073e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>148093</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>002815.XSHE</td>\n",
+       "      <td>13.41</td>\n",
+       "      <td>13.55</td>\n",
+       "      <td>13.67</td>\n",
+       "      <td>13.29</td>\n",
+       "      <td>5410989.0</td>\n",
+       "      <td>7.290842e+07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>148339</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>002690.XSHE</td>\n",
+       "      <td>31.68</td>\n",
+       "      <td>31.68</td>\n",
+       "      <td>32.12</td>\n",
+       "      <td>31.26</td>\n",
+       "      <td>3641409.0</td>\n",
+       "      <td>1.153387e+08</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>604 rows × 8 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             date       ticker   open  close   high    low       volume  \\\n",
+       "1      2021-01-04  002233.XSHE  11.27  11.28  11.34  11.17   16262377.0   \n",
+       "247    2021-01-04  601778.XSHG   7.27   7.65   7.77   7.25   59723781.0   \n",
+       "493    2021-01-04  002368.XSHE  25.41  28.18  28.18  25.27   17448308.0   \n",
+       "739    2021-01-04  001914.XSHE  21.21  20.39  21.33  20.26    6619778.0   \n",
+       "985    2021-01-04  002384.XSHE  25.66  25.98  26.00  25.17   50695885.0   \n",
+       "...           ...          ...    ...    ...    ...    ...          ...   \n",
+       "147355 2021-01-04  600511.XSHG  46.39  45.10  46.55  44.32   45375375.0   \n",
+       "147601 2021-01-04  600236.XSHG   4.05   4.05   4.05   4.02    5788783.0   \n",
+       "147847 2021-01-04  000807.XSHE   7.32   7.58   7.71   7.13  136647514.0   \n",
+       "148093 2021-01-04  002815.XSHE  13.41  13.55  13.67  13.29    5410989.0   \n",
+       "148339 2021-01-04  002690.XSHE  31.68  31.68  32.12  31.26    3641409.0   \n",
+       "\n",
+       "               money  \n",
+       "1       1.829668e+08  \n",
+       "247     4.523540e+08  \n",
+       "493     4.729692e+08  \n",
+       "739     1.366024e+08  \n",
+       "985     1.296706e+09  \n",
+       "...              ...  \n",
+       "147355  2.043297e+09  \n",
+       "147601  2.339606e+07  \n",
+       "147847  1.027073e+09  \n",
+       "148093  7.290842e+07  \n",
+       "148339  1.153387e+08  \n",
+       "\n",
+       "[604 rows x 8 columns]"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# start stream here\n",
+    "dates = b_stocks.date.unique()\n",
+    "b_stocks[b_stocks.date == dates[1]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data1 = b_stocks[b_stocks.date == dates[0]]\n",
+    "data2 = b_stocks[b_stocks.date == dates[1]]\n",
+    "data3 = b_stocks[b_stocks.date == dates[2]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>002233.XSHE</td>\n",
+       "      <td>11.23</td>\n",
+       "      <td>11.30</td>\n",
+       "      <td>11.38</td>\n",
+       "      <td>11.19</td>\n",
+       "      <td>9712496.0</td>\n",
+       "      <td>1.096390e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>246</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>601778.XSHG</td>\n",
+       "      <td>7.28</td>\n",
+       "      <td>7.23</td>\n",
+       "      <td>7.39</td>\n",
+       "      <td>7.20</td>\n",
+       "      <td>29971398.0</td>\n",
+       "      <td>2.181267e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>492</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>002368.XSHE</td>\n",
+       "      <td>24.96</td>\n",
+       "      <td>25.62</td>\n",
+       "      <td>25.92</td>\n",
+       "      <td>24.96</td>\n",
+       "      <td>7090839.0</td>\n",
+       "      <td>1.811902e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>738</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>001914.XSHE</td>\n",
+       "      <td>20.52</td>\n",
+       "      <td>21.26</td>\n",
+       "      <td>21.36</td>\n",
+       "      <td>20.52</td>\n",
+       "      <td>5598757.0</td>\n",
+       "      <td>1.171876e+08</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>984</th>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>002384.XSHE</td>\n",
+       "      <td>25.67</td>\n",
+       "      <td>25.65</td>\n",
+       "      <td>25.90</td>\n",
+       "      <td>25.15</td>\n",
+       "      <td>49307624.0</td>\n",
+       "      <td>1.256593e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>147355</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>600511.XSHG</td>\n",
+       "      <td>46.39</td>\n",
+       "      <td>45.10</td>\n",
+       "      <td>46.55</td>\n",
+       "      <td>44.32</td>\n",
+       "      <td>45375375.0</td>\n",
+       "      <td>2.043297e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>147601</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>600236.XSHG</td>\n",
+       "      <td>4.05</td>\n",
+       "      <td>4.05</td>\n",
+       "      <td>4.05</td>\n",
+       "      <td>4.02</td>\n",
+       "      <td>5788783.0</td>\n",
+       "      <td>2.339606e+07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>147847</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>000807.XSHE</td>\n",
+       "      <td>7.32</td>\n",
+       "      <td>7.58</td>\n",
+       "      <td>7.71</td>\n",
+       "      <td>7.13</td>\n",
+       "      <td>136647514.0</td>\n",
+       "      <td>1.027073e+09</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>148093</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>002815.XSHE</td>\n",
+       "      <td>13.41</td>\n",
+       "      <td>13.55</td>\n",
+       "      <td>13.67</td>\n",
+       "      <td>13.29</td>\n",
+       "      <td>5410989.0</td>\n",
+       "      <td>7.290842e+07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>148339</th>\n",
+       "      <td>2021-01-04</td>\n",
+       "      <td>002690.XSHE</td>\n",
+       "      <td>31.68</td>\n",
+       "      <td>31.68</td>\n",
+       "      <td>32.12</td>\n",
+       "      <td>31.26</td>\n",
+       "      <td>3641409.0</td>\n",
+       "      <td>1.153387e+08</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1208 rows × 8 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             date       ticker   open  close   high    low       volume  \\\n",
+       "0      2020-12-31  002233.XSHE  11.23  11.30  11.38  11.19    9712496.0   \n",
+       "246    2020-12-31  601778.XSHG   7.28   7.23   7.39   7.20   29971398.0   \n",
+       "492    2020-12-31  002368.XSHE  24.96  25.62  25.92  24.96    7090839.0   \n",
+       "738    2020-12-31  001914.XSHE  20.52  21.26  21.36  20.52    5598757.0   \n",
+       "984    2020-12-31  002384.XSHE  25.67  25.65  25.90  25.15   49307624.0   \n",
+       "...           ...          ...    ...    ...    ...    ...          ...   \n",
+       "147355 2021-01-04  600511.XSHG  46.39  45.10  46.55  44.32   45375375.0   \n",
+       "147601 2021-01-04  600236.XSHG   4.05   4.05   4.05   4.02    5788783.0   \n",
+       "147847 2021-01-04  000807.XSHE   7.32   7.58   7.71   7.13  136647514.0   \n",
+       "148093 2021-01-04  002815.XSHE  13.41  13.55  13.67  13.29    5410989.0   \n",
+       "148339 2021-01-04  002690.XSHE  31.68  31.68  32.12  31.26    3641409.0   \n",
+       "\n",
+       "               money  \n",
+       "0       1.096390e+08  \n",
+       "246     2.181267e+08  \n",
+       "492     1.811902e+08  \n",
+       "738     1.171876e+08  \n",
+       "984     1.256593e+09  \n",
+       "...              ...  \n",
+       "147355  2.043297e+09  \n",
+       "147601  2.339606e+07  \n",
+       "147847  1.027073e+09  \n",
+       "148093  7.290842e+07  \n",
+       "148339  1.153387e+08  \n",
+       "\n",
+       "[1208 rows x 8 columns]"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "merged_df = pd.concat([data1, data2])\n",
+    "merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "             date       ticker   open  close   high    low      volume  \\\n",
+      "0      2020-12-31  002233.XSHE  11.23  11.30  11.38  11.19   9712496.0   \n",
+      "246    2020-12-31  601778.XSHG   7.28   7.23   7.39   7.20  29971398.0   \n",
+      "492    2020-12-31  002368.XSHE  24.96  25.62  25.92  24.96   7090839.0   \n",
+      "738    2020-12-31  001914.XSHE  20.52  21.26  21.36  20.52   5598757.0   \n",
+      "984    2020-12-31  002384.XSHE  25.67  25.65  25.90  25.15  49307624.0   \n",
+      "...           ...          ...    ...    ...    ...    ...         ...   \n",
+      "147354 2020-12-31  600511.XSHG  44.43  46.33  48.45  44.16  69779041.0   \n",
+      "147600 2020-12-31  600236.XSHG   4.07   4.05   4.10   4.02   6542015.0   \n",
+      "147846 2020-12-31  000807.XSHE   7.34   7.35   7.48   7.20  72260375.0   \n",
+      "148092 2020-12-31  002815.XSHE  13.22  13.41  13.51  13.19   4198958.0   \n",
+      "148338 2020-12-31  002690.XSHE  31.14  31.48  31.60  30.79   4010199.0   \n",
+      "\n",
+      "               money  \n",
+      "0       1.096390e+08  \n",
+      "246     2.181267e+08  \n",
+      "492     1.811902e+08  \n",
+      "738     1.171876e+08  \n",
+      "984     1.256593e+09  \n",
+      "...              ...  \n",
+      "147354  3.258608e+09  \n",
+      "147600  2.649244e+07  \n",
+      "147846  5.305392e+08  \n",
+      "148092  5.621386e+07  \n",
+      "148338  1.257796e+08  \n",
+      "\n",
+      "[604 rows x 8 columns]\n",
+      "             date       ticker   open  close   high    low       volume  \\\n",
+      "0      2020-12-31  002233.XSHE  11.23  11.30  11.38  11.19    9712496.0   \n",
+      "246    2020-12-31  601778.XSHG   7.28   7.23   7.39   7.20   29971398.0   \n",
+      "492    2020-12-31  002368.XSHE  24.96  25.62  25.92  24.96    7090839.0   \n",
+      "738    2020-12-31  001914.XSHE  20.52  21.26  21.36  20.52    5598757.0   \n",
+      "984    2020-12-31  002384.XSHE  25.67  25.65  25.90  25.15   49307624.0   \n",
+      "...           ...          ...    ...    ...    ...    ...          ...   \n",
+      "147355 2021-01-04  600511.XSHG  46.39  45.10  46.55  44.32   45375375.0   \n",
+      "147601 2021-01-04  600236.XSHG   4.05   4.05   4.05   4.02    5788783.0   \n",
+      "147847 2021-01-04  000807.XSHE   7.32   7.58   7.71   7.13  136647514.0   \n",
+      "148093 2021-01-04  002815.XSHE  13.41  13.55  13.67  13.29    5410989.0   \n",
+      "148339 2021-01-04  002690.XSHE  31.68  31.68  32.12  31.26    3641409.0   \n",
+      "\n",
+      "               money  \n",
+      "0       1.096390e+08  \n",
+      "246     2.181267e+08  \n",
+      "492     1.811902e+08  \n",
+      "738     1.171876e+08  \n",
+      "984     1.256593e+09  \n",
+      "...              ...  \n",
+      "147355  2.043297e+09  \n",
+      "147601  2.339606e+07  \n",
+      "147847  1.027073e+09  \n",
+      "148093  7.290842e+07  \n",
+      "148339  1.153387e+08  \n",
+      "\n",
+      "[1208 rows x 8 columns]\n",
+      "             date       ticker   open  close   high    low       volume  \\\n",
+      "0      2020-12-31  002233.XSHE  11.23  11.30  11.38  11.19    9712496.0   \n",
+      "246    2020-12-31  601778.XSHG   7.28   7.23   7.39   7.20   29971398.0   \n",
+      "492    2020-12-31  002368.XSHE  24.96  25.62  25.92  24.96    7090839.0   \n",
+      "738    2020-12-31  001914.XSHE  20.52  21.26  21.36  20.52    5598757.0   \n",
+      "984    2020-12-31  002384.XSHE  25.67  25.65  25.90  25.15   49307624.0   \n",
+      "...           ...          ...    ...    ...    ...    ...          ...   \n",
+      "147356 2021-01-05  600511.XSHG  45.00  45.58  46.79  43.58   45296155.0   \n",
+      "147602 2021-01-05  600236.XSHG   4.04   4.01   4.05   4.00    4302703.0   \n",
+      "147848 2021-01-05  000807.XSHE   7.61   7.99   8.21   7.48  177268867.0   \n",
+      "148094 2021-01-05  002815.XSHE  13.52  13.63  13.65  13.44    5800866.0   \n",
+      "148340 2021-01-05  002690.XSHE  31.68  31.85  32.14  31.42    5082815.0   \n",
+      "\n",
+      "               money  \n",
+      "0       1.096390e+08  \n",
+      "246     2.181267e+08  \n",
+      "492     1.811902e+08  \n",
+      "738     1.171876e+08  \n",
+      "984     1.256593e+09  \n",
+      "...              ...  \n",
+      "147356  2.035299e+09  \n",
+      "147602  1.726972e+07  \n",
+      "147848  1.383377e+09  \n",
+      "148094  7.856429e+07  \n",
+      "148340  1.612986e+08  \n",
+      "\n",
+      "[1812 rows x 8 columns]\n"
+     ]
+    }
+   ],
+   "source": [
+    "def add(prev_df, new_df):\n",
+    "    merged_df = pd.concat([prev_df, new_df])\n",
+    "    return merged_df\n",
+    "\n",
+    "source = Stream()\n",
+    "source.accumulate(add).sink(print)\n",
+    "source.emit(b_stocks[b_stocks.date == dates[0]])\n",
+    "source.emit(b_stocks[b_stocks.date == dates[1]])\n",
+    "source.emit(b_stocks[b_stocks.date == dates[2]])\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

script/stream_processing.py

@@ -1,3 +1,32 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e193f46fa91cc5f42ea6117dd54ae0aebe6ff173ed452f505915d10089ebabf8
-size 948
+import pandas as pd
+import math
+from datetime import datetime
+import hvplot.pandas
+import math
+import numpy as np
+import time
+from streamz import Stream
+
+b_stocks = pd.read_pickle('../data/b_stocks.pkl')
+p_stocks = pd.read_pickle('../data/p_stocks.pkl')
+p_profile = pd.read_pickle('../data/p_profile.pkl')
+b_profile = pd.read_pickle('../data/b_profile.pkl')
+
+# start stream here
+dates = b_stocks.date.unique()
+b_stocks[b_stocks.date == dates[1]]
+
+def add(prev_df, new_df):
+    merged_df = pd.concat([prev_df, new_df])
+    merged_df.sort_values(by=['date'], inplace=True)
+    merged_df['pct'] = merged_df.groupby('ticker')['close'].pct_change()
+
+    # remove prev_df
+    merged_df = merged_df[~merged_df.isin(prev_df)].dropna()
+    return merged_df
+
+source = Stream()
+source.accumulate(add).sink(print)
+source.emit(b_stocks[b_stocks.date == dates[0]])
+source.emit(b_stocks[b_stocks.date == dates[1]])
+source.emit(b_stocks[b_stocks.date == dates[2]])

script/styling.py

@@ -1,3 +1,15 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:455ecf338b18610c747777df147049b6b7c4e01f63fd0ca35c5fb20d79c00a41
-size 268
+plot_layout = dict(
+    legend=dict(
+        orientation="h",
+    ),
+    yaxis_title=None,
+    xaxis_title=None,
+    margin=dict(l=0, r=0, t=30, b=0),
+    uniformtext_mode='hide',
+    
+)
+
+barplot_trace = dict(
+    marker_line_width=0,
+    selector=dict(type="bar"),
+)

settings.py

@@ -1,3 +1,9 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0fd4fd43c9eee777f8959e01aded629a34384181af34f1532a27409682460b81
-size 212
+from datetime import timedelta
+stream_frequency = timedelta(seconds=60)
+
+TABLE_NAME_AND_FREQ = [
+    ('benchmark_profile', timedelta(days=1)),
+    ('portfolio_profile', timedelta(days=1))
+]
+
+COMPONENT_WIDTH = 375

styling.py

@@ -1,3 +1,15 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:455ecf338b18610c747777df147049b6b7c4e01f63fd0ca35c5fb20d79c00a41
-size 268
+plot_layout = dict(
+    legend=dict(
+        orientation="h",
+    ),
+    yaxis_title=None,
+    xaxis_title=None,
+    margin=dict(l=0, r=0, t=30, b=0),
+    uniformtext_mode='hide',
+    
+)
+
+barplot_trace = dict(
+    marker_line_width=0,
+    selector=dict(type="bar"),
+)

table_schema.py

@@ -1,3 +1,28 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d7f88f15456c20d7ebef37e45f78ecb96276acdc26f696572c4d867a3b7b5674
-size 590
+
+'''
+create df schema for db
+'''
+PORTFOLIO_TABLE = 'portfolio_profile'
+PORTFOLIO_TABLE_SCHEMA = {
+    'ticker': str,
+    'shares': int,
+    'date': 'datetime64[ns]',
+    'sector': str,
+    'aggregate_sector': str,
+    'display_name': str,
+    'name': str,
+    'cash': float,
+    'weight': float,
+    'ave_price': float
+}
+STOCKS_DETAILS_TABLE = 'all_stock_info'
+STOCKS_DETAILS_TABLE_SCHEMA = {
+    'display_name': str,
+    'name': str,
+    'start_date': 'datetime64[ns]',
+    'end_date': 'datetime64[ns]',
+    'type': str,
+    'ticker': str,
+    'sector': str,
+    'aggregate_sector': str
+}

test.ipynb

@@ -1,3 +1,47 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a7aa0baf72e8e836a10321925eccc7cad01f6f6f0062a811f571486ef0176060
-size 1171
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import panel as pn\n",
+    "\n",
+    "pn.extension('perspective', template='fast', sizing_mode='stretch_width')\n",
+    "df = pd.DataFrame(np.random.randn(10, 4), columns=list('ABCD')).cumsum()\n",
+    "\n",
+    "rollover = pn.widgets.IntInput(name='Rollover', value=15)\n",
+    "\n",
+    "perspective = pn.pane.Perspective(df, height=400)\n",
+    "\n",
+    "def stream():\n",
+    "    data = df.iloc[-1] + np.random.randn(4)\n",
+    "    perspective.stream(data, rollover=rollover.value)\n",
+    "\\\n",
+    "cb = pn.state.add_periodic_callback(stream, 50)\n",
+    "\n",
+    "pn.Column(\n",
+    "    pn.Row(cb.param.period, rollover, perspective.param.theme),\n",
+    "    perspective\n",
+    ").servable()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

test_background_task.py

@@ -1,3 +1,26 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:045734bbc99c4c78abfb10c850ed49a55e4cf7434595137022446074f3e44802
-size 617
+import pipeline
+import settings
+
+def test_need_to_update():
+    table_list = settings.TABLE_NAME_AND_FREQ
+    for table, freq in table_list:
+        result = pipeline.need_to_update(table, freq)
+        print(result)
+
+# test_need_to_update()
+
+def test_fetch_stock_price():
+    df = pipeline.fetch_stock_price()
+    return df
+
+def test_need_to_update_stocks_price():
+    print(pipeline.need_to_update_stocks_price())
+
+
+def test_add_details_to_stock_df():
+    stock_df = pipeline.update_stocks_price()
+    stock_df = pipeline.add_details_to_stock_df(stock_df)
+
+    return stock_df
+
+print(test_add_details_to_stock_df())

test_responsive.py

@@ -1,3 +1,14 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a935ade97cc93f866e1d21a344bd4986d8ac6a8656ae4d356a40a64c0fb27d98
-size 398
+import random
+import panel.widgets as pnw
+import panel as pn
+pn.extension()
+
+
+def rcolor(): return "#%06x" % random.randint(0, 0xFFFFFF)
+
+
+box = pn.FlexBox(*[pn.pane.HTML(str(i), styles=dict(background=rcolor()),
+                 width=370, height=100, sizing_mode='fixed') for i in range(24)],
+                 align_conten='start', styles={'background': 'black'}
+                 )
+box.servable()

testing_pipeline.ipynb

@@ -1,3 +1,300 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3333498618a0ef539b89cb37d51d165ec901c3e058225ba511b1ed4a6f258d90
-size 8920
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pipeline\n",
+    "import settings\n",
+    "from sqlalchemy import create_engine\n",
+    "import pandas as pd\n",
+    "import datetime as dt\n",
+    "import api\n",
+    "db_url = 'sqlite:///local.db'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "auth success \n"
+     ]
+    }
+   ],
+   "source": [
+    "pipeline.update_stocks_details_to_db()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 147,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## calculate weight for benchmark and portfolio \n",
+    "with create_engine(db_url).connect() as conn:\n",
+    "    p_profile = pd.read_sql('portfolio_profile', con=conn)\n",
+    "    b_profile = pd.read_sql('benchmark_profile', con=conn)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 167,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "date = dt.datetime(2023, 8, 2, 15, 0,0)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 168,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>time</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>open</th>\n",
+       "      <th>close</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>money</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2023-08-02 15:00:00</td>\n",
+       "      <td>603288.XSHG</td>\n",
+       "      <td>45.85</td>\n",
+       "      <td>45.85</td>\n",
+       "      <td>45.85</td>\n",
+       "      <td>45.85</td>\n",
+       "      <td>156700.0</td>\n",
+       "      <td>7184695.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                 time       ticker   open  close   high    low    volume  \\\n",
+       "0 2023-08-02 15:00:00  603288.XSHG  45.85  45.85  45.85  45.85  156700.0   \n",
+       "\n",
+       "       money  \n",
+       "0  7184695.0  "
+      ]
+     },
+     "execution_count": 168,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "api.fetch_stocks_price(security=['603288.XSHG'],end_date=date,count=1, frequency='minute')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 151,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Timestamp('2023-08-02 21:20:02')"
+      ]
+     },
+     "execution_count": 151,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "date = dt.datetime(2023, 8, 2)\n",
+    "p_profile.iloc[0].date"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 149,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>index</th>\n",
+       "      <th>date</th>\n",
+       "      <th>weight</th>\n",
+       "      <th>display_name</th>\n",
+       "      <th>ticker</th>\n",
+       "      <th>sector</th>\n",
+       "      <th>aggregate_sector</th>\n",
+       "      <th>name</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1500</td>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.088</td>\n",
+       "      <td>神州高铁</td>\n",
+       "      <td>000008.XSHE</td>\n",
+       "      <td>机械设备I 运输设备II 铁路设备III 铁路、船舶、航空航天和其他运输设备制造业 城轨铁路 工业</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>SZGT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1501</td>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.344</td>\n",
+       "      <td>中国宝安</td>\n",
+       "      <td>000009.XSHE</td>\n",
+       "      <td>电气设备I 电池II 电池化学品III 综合 工业集团企业 工业</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>ZGBA</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>1502</td>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.180</td>\n",
+       "      <td>南玻A</td>\n",
+       "      <td>000012.XSHE</td>\n",
+       "      <td>建筑材料I 玻璃制造II 玻璃制造III 非金属矿物制品业 玻璃 原材料</td>\n",
+       "      <td>原料与能源</td>\n",
+       "      <td>NBA</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1503</td>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.297</td>\n",
+       "      <td>深科技</td>\n",
+       "      <td>000021.XSHE</td>\n",
+       "      <td>电子I 电子制造II 消费电子零部件及组装III 计算机、通信和其他电子设备制造业 安防设备...</td>\n",
+       "      <td>信息与通信</td>\n",
+       "      <td>SKJ</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>1504</td>\n",
+       "      <td>2020-12-31</td>\n",
+       "      <td>0.030</td>\n",
+       "      <td>招商港口</td>\n",
+       "      <td>001872.XSHE</td>\n",
+       "      <td>交通运输I 航运港口II 港口III 水上运输业 港口 工业</td>\n",
+       "      <td>工业</td>\n",
+       "      <td>ZSGK</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   index       date  weight display_name       ticker  \\\n",
+       "0   1500 2020-12-31   0.088         神州高铁  000008.XSHE   \n",
+       "1   1501 2020-12-31   0.344         中国宝安  000009.XSHE   \n",
+       "2   1502 2020-12-31   0.180          南玻A  000012.XSHE   \n",
+       "3   1503 2020-12-31   0.297          深科技  000021.XSHE   \n",
+       "4   1504 2020-12-31   0.030         招商港口  001872.XSHE   \n",
+       "\n",
+       "                                              sector aggregate_sector  name  \n",
+       "0  机械设备I 运输设备II 铁路设备III 铁路、船舶、航空航天和其他运输设备制造业 城轨铁路 工业               工业  SZGT  \n",
+       "1                   电气设备I 电池II 电池化学品III 综合 工业集团企业 工业               工业  ZGBA  \n",
+       "2               建筑材料I 玻璃制造II 玻璃制造III 非金属矿物制品业 玻璃 原材料            原料与能源   NBA  \n",
+       "3  电子I 电子制造II 消费电子零部件及组装III 计算机、通信和其他电子设备制造业 安防设备...            信息与通信   SKJ  \n",
+       "4                     交通运输I 航运港口II 港口III 水上运输业 港口 工业               工业  ZSGK  "
+      ]
+     },
+     "execution_count": 149,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "b_profile.head()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "risk-dashboard",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

total_return.ipynb

@@ -1,3 +1,713 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:652c33dc1d1fad4473c1bfd23f19bc11b2e718e2dbaf5057a2716a958de2f7f3
-size 84782
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.1.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n  var reloading = false;\n  var Bokeh = root.Bokeh;\n  var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n      require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n      })\n      require([\"jspanel-modal\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-tooltip\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-hint\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-layout\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-dock\"], function() {\n\ton_load()\n      })\n      require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n      })\n      require([\"notyf\"], function() {\n\ton_load()\n      })\n      root._bokeh_is_loading = css_urls.length + 9;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.holoviz.org/panel/1.1.1/dist/panel.min.js\"];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      Bokeh = root.Bokeh;\n      bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n  window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n    function JupyterCommManager() {\n    }\n\n    JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n      if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        comm_manager.register_target(comm_id, function(comm) {\n          comm.on_msg(msg_handler);\n        });\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n          comm.onMsg = msg_handler;\n        });\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n          var messages = comm.messages[Symbol.asyncIterator]();\n          function processIteratorResult(result) {\n            var message = result.value;\n            console.log(message)\n            var content = {data: message.data, comm_id};\n            var buffers = []\n            for (var buffer of message.buffers || []) {\n              buffers.push(new DataView(buffer))\n            }\n            var metadata = message.metadata || {};\n            var msg = {content, buffers, metadata}\n            msg_handler(msg);\n            return messages.next().then(processIteratorResult);\n          }\n          return messages.next().then(processIteratorResult);\n        })\n      }\n    }\n\n    JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n      if (comm_id in window.PyViz.comms) {\n        return window.PyViz.comms[comm_id];\n      } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n        if (msg_handler) {\n          comm.on_msg(msg_handler);\n        }\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n        comm.open();\n        if (msg_handler) {\n          comm.onMsg = msg_handler;\n        }\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        var comm_promise = google.colab.kernel.comms.open(comm_id)\n        comm_promise.then((comm) => {\n          window.PyViz.comms[comm_id] = comm;\n          if (msg_handler) {\n            var messages = comm.messages[Symbol.asyncIterator]();\n            function processIteratorResult(result) {\n              var message = result.value;\n              var content = {data: message.data};\n              var metadata = message.metadata || {comm_id};\n              var msg = {content, metadata}\n              msg_handler(msg);\n              return messages.next().then(processIteratorResult);\n            }\n            return messages.next().then(processIteratorResult);\n          }\n        }) \n        var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n          return comm_promise.then((comm) => {\n            comm.send(data, metadata, buffers, disposeOnDone);\n          });\n        };\n        var comm = {\n          send: sendClosure\n        };\n      }\n      window.PyViz.comms[comm_id] = comm;\n      return comm;\n    }\n    window.PyViz.comm_manager = new JupyterCommManager();\n    \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n  var div = document.createElement(\"div\");\n  var script = document.createElement(\"script\");\n  node.appendChild(div);\n  node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n  var output_area = handle.output_area;\n  var output = handle.output;\n  if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n    return\n  }\n  var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n  var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n  if (id !== undefined) {\n    var nchildren = toinsert.length;\n    var html_node = toinsert[nchildren-1].children[0];\n    html_node.innerHTML = output.data[HTML_MIME_TYPE];\n    var scripts = [];\n    var nodelist = html_node.querySelectorAll(\"script\");\n    for (var i in nodelist) {\n      if (nodelist.hasOwnProperty(i)) {\n        scripts.push(nodelist[i])\n      }\n    }\n\n    scripts.forEach( function (oldScript) {\n      var newScript = document.createElement(\"script\");\n      var attrs = [];\n      var nodemap = oldScript.attributes;\n      for (var j in nodemap) {\n        if (nodemap.hasOwnProperty(j)) {\n          attrs.push(nodemap[j])\n        }\n      }\n      attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n      newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n      oldScript.parentNode.replaceChild(newScript, oldScript);\n    });\n    if (JS_MIME_TYPE in output.data) {\n      toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n    }\n    output_area._hv_plot_id = id;\n    if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n      window.PyViz.plot_index[id] = Bokeh.index[id];\n    } else {\n      window.PyViz.plot_index[id] = null;\n    }\n  } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n    var bk_div = document.createElement(\"div\");\n    bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n    var script_attrs = bk_div.children[0].attributes;\n    for (var i = 0; i < script_attrs.length; i++) {\n      toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n    }\n    // store reference to server id on output_area\n    output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n  }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n  var id = handle.cell.output_area._hv_plot_id;\n  var server_id = handle.cell.output_area._bokeh_server_id;\n  if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n  var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n  if (server_id !== null) {\n    comm.send({event_type: 'server_delete', 'id': server_id});\n    return;\n  } else if (comm !== null) {\n    comm.send({event_type: 'delete', 'id': id});\n  }\n  delete PyViz.plot_index[id];\n  if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n    var doc = window.Bokeh.index[id].model.document\n    doc.clear();\n    const i = window.Bokeh.documents.indexOf(doc);\n    if (i > -1) {\n      window.Bokeh.documents.splice(i, 1);\n    }\n  }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n  delete PyViz.comms[\"hv-extension-comm\"];\n  window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n  handle_clear_output(event, {cell: {output_area: handle.output_area}})\n  handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n  function append_mime(data, metadata, element) {\n    // create a DOM node to render to\n    var toinsert = this.create_output_subarea(\n    metadata,\n    CLASS_NAME,\n    EXEC_MIME_TYPE\n    );\n    this.keyboard_manager.register_events(toinsert);\n    // Render to node\n    var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n    render(props, toinsert[0]);\n    element.append(toinsert);\n    return toinsert\n  }\n\n  events.on('output_added.OutputArea', handle_add_output);\n  events.on('output_updated.OutputArea', handle_update_output);\n  events.on('clear_output.CodeCell', handle_clear_output);\n  events.on('delete.Cell', handle_clear_output);\n  events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n  OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n    safe: true,\n    index: 0\n  });\n}\n\nif (window.Jupyter !== undefined) {\n  try {\n    var events = require('base/js/events');\n    var OutputArea = require('notebook/js/outputarea').OutputArea;\n    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n      register_renderer(events, OutputArea);\n    }\n  } catch(err) {\n  }\n}\n",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style>*[data-root-id],\n",
+       "*[data-root-id] > * {\n",
+       "  box-sizing: border-box;\n",
+       "  font-family: var(--jp-ui-font-family);\n",
+       "  font-size: var(--jp-ui-font-size1);\n",
+       "  color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
+       "}\n",
+       "\n",
+       "/* Override VSCode background color */\n",
+       ".cell-output-ipywidget-background:has(> .cell-output-ipywidget-background\n",
+       "    > .lm-Widget\n",
+       "    > *[data-root-id]),\n",
+       ".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
+       "  background-color: transparent !important;\n",
+       "}\n",
+       "</style>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.1.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n  var reloading = true;\n  var Bokeh = root.Bokeh;\n  var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n      require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n      })\n      require([\"jspanel-modal\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-tooltip\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-hint\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-layout\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-dock\"], function() {\n\ton_load()\n      })\n      require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n      })\n      require([\"notyf\"], function() {\n\ton_load()\n      })\n      root._bokeh_is_loading = css_urls.length + 9;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      Bokeh = root.Bokeh;\n      bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n  window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n    function JupyterCommManager() {\n    }\n\n    JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n      if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        comm_manager.register_target(comm_id, function(comm) {\n          comm.on_msg(msg_handler);\n        });\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n          comm.onMsg = msg_handler;\n        });\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n          var messages = comm.messages[Symbol.asyncIterator]();\n          function processIteratorResult(result) {\n            var message = result.value;\n            console.log(message)\n            var content = {data: message.data, comm_id};\n            var buffers = []\n            for (var buffer of message.buffers || []) {\n              buffers.push(new DataView(buffer))\n            }\n            var metadata = message.metadata || {};\n            var msg = {content, buffers, metadata}\n            msg_handler(msg);\n            return messages.next().then(processIteratorResult);\n          }\n          return messages.next().then(processIteratorResult);\n        })\n      }\n    }\n\n    JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n      if (comm_id in window.PyViz.comms) {\n        return window.PyViz.comms[comm_id];\n      } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n        if (msg_handler) {\n          comm.on_msg(msg_handler);\n        }\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n        comm.open();\n        if (msg_handler) {\n          comm.onMsg = msg_handler;\n        }\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        var comm_promise = google.colab.kernel.comms.open(comm_id)\n        comm_promise.then((comm) => {\n          window.PyViz.comms[comm_id] = comm;\n          if (msg_handler) {\n            var messages = comm.messages[Symbol.asyncIterator]();\n            function processIteratorResult(result) {\n              var message = result.value;\n              var content = {data: message.data};\n              var metadata = message.metadata || {comm_id};\n              var msg = {content, metadata}\n              msg_handler(msg);\n              return messages.next().then(processIteratorResult);\n            }\n            return messages.next().then(processIteratorResult);\n          }\n        }) \n        var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n          return comm_promise.then((comm) => {\n            comm.send(data, metadata, buffers, disposeOnDone);\n          });\n        };\n        var comm = {\n          send: sendClosure\n        };\n      }\n      window.PyViz.comms[comm_id] = comm;\n      return comm;\n    }\n    window.PyViz.comm_manager = new JupyterCommManager();\n    \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n  var div = document.createElement(\"div\");\n  var script = document.createElement(\"script\");\n  node.appendChild(div);\n  node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n  var output_area = handle.output_area;\n  var output = handle.output;\n  if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n    return\n  }\n  var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n  var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n  if (id !== undefined) {\n    var nchildren = toinsert.length;\n    var html_node = toinsert[nchildren-1].children[0];\n    html_node.innerHTML = output.data[HTML_MIME_TYPE];\n    var scripts = [];\n    var nodelist = html_node.querySelectorAll(\"script\");\n    for (var i in nodelist) {\n      if (nodelist.hasOwnProperty(i)) {\n        scripts.push(nodelist[i])\n      }\n    }\n\n    scripts.forEach( function (oldScript) {\n      var newScript = document.createElement(\"script\");\n      var attrs = [];\n      var nodemap = oldScript.attributes;\n      for (var j in nodemap) {\n        if (nodemap.hasOwnProperty(j)) {\n          attrs.push(nodemap[j])\n        }\n      }\n      attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n      newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n      oldScript.parentNode.replaceChild(newScript, oldScript);\n    });\n    if (JS_MIME_TYPE in output.data) {\n      toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n    }\n    output_area._hv_plot_id = id;\n    if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n      window.PyViz.plot_index[id] = Bokeh.index[id];\n    } else {\n      window.PyViz.plot_index[id] = null;\n    }\n  } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n    var bk_div = document.createElement(\"div\");\n    bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n    var script_attrs = bk_div.children[0].attributes;\n    for (var i = 0; i < script_attrs.length; i++) {\n      toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n    }\n    // store reference to server id on output_area\n    output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n  }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n  var id = handle.cell.output_area._hv_plot_id;\n  var server_id = handle.cell.output_area._bokeh_server_id;\n  if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n  var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n  if (server_id !== null) {\n    comm.send({event_type: 'server_delete', 'id': server_id});\n    return;\n  } else if (comm !== null) {\n    comm.send({event_type: 'delete', 'id': id});\n  }\n  delete PyViz.plot_index[id];\n  if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n    var doc = window.Bokeh.index[id].model.document\n    doc.clear();\n    const i = window.Bokeh.documents.indexOf(doc);\n    if (i > -1) {\n      window.Bokeh.documents.splice(i, 1);\n    }\n  }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n  delete PyViz.comms[\"hv-extension-comm\"];\n  window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n  handle_clear_output(event, {cell: {output_area: handle.output_area}})\n  handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n  function append_mime(data, metadata, element) {\n    // create a DOM node to render to\n    var toinsert = this.create_output_subarea(\n    metadata,\n    CLASS_NAME,\n    EXEC_MIME_TYPE\n    );\n    this.keyboard_manager.register_events(toinsert);\n    // Render to node\n    var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n    render(props, toinsert[0]);\n    element.append(toinsert);\n    return toinsert\n  }\n\n  events.on('output_added.OutputArea', handle_add_output);\n  events.on('output_updated.OutputArea', handle_update_output);\n  events.on('clear_output.CodeCell', handle_clear_output);\n  events.on('delete.Cell', handle_clear_output);\n  events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n  OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n    safe: true,\n    index: 0\n  });\n}\n\nif (window.Jupyter !== undefined) {\n  try {\n    var events = require('base/js/events');\n    var OutputArea = require('notebook/js/outputarea').OutputArea;\n    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n      register_renderer(events, OutputArea);\n    }\n  } catch(err) {\n  }\n}\n",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style>*[data-root-id],\n",
+       "*[data-root-id] > * {\n",
+       "  box-sizing: border-box;\n",
+       "  font-family: var(--jp-ui-font-family);\n",
+       "  font-size: var(--jp-ui-font-size1);\n",
+       "  color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
+       "}\n",
+       "\n",
+       "/* Override VSCode background color */\n",
+       ".cell-output-ipywidget-background:has(> .cell-output-ipywidget-background\n",
+       "    > .lm-Widget\n",
+       "    > *[data-root-id]),\n",
+       ".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
+       "  background-color: transparent !important;\n",
+       "}\n",
+       "</style>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.1.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n  var reloading = true;\n  var Bokeh = root.Bokeh;\n  var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {'mathjax': '//cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS_HTML', 'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'mathjax': {'exports': 'MathJax'}, 'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n      require([\"mathjax\"], function() {\n\ton_load()\n      })\n      require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n      })\n      require([\"jspanel-modal\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-tooltip\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-hint\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-layout\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n      })\n      require([\"jspanel-dock\"], function() {\n\ton_load()\n      })\n      require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n      })\n      require([\"notyf\"], function() {\n\ton_load()\n      })\n      root._bokeh_is_loading = css_urls.length + 10;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    if (((window['MathJax'] !== undefined) && (!(window['MathJax'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_CHTML'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.1.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n      var urls = ['https://cdn.holoviz.org/panel/1.1.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n      for (var i = 0; i < urls.length; i++) {\n        skip.push(urls[i])\n      }\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_CHTML\"];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      Bokeh = root.Bokeh;\n      bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n  window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n    function JupyterCommManager() {\n    }\n\n    JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n      if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        comm_manager.register_target(comm_id, function(comm) {\n          comm.on_msg(msg_handler);\n        });\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n          comm.onMsg = msg_handler;\n        });\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n          var messages = comm.messages[Symbol.asyncIterator]();\n          function processIteratorResult(result) {\n            var message = result.value;\n            console.log(message)\n            var content = {data: message.data, comm_id};\n            var buffers = []\n            for (var buffer of message.buffers || []) {\n              buffers.push(new DataView(buffer))\n            }\n            var metadata = message.metadata || {};\n            var msg = {content, buffers, metadata}\n            msg_handler(msg);\n            return messages.next().then(processIteratorResult);\n          }\n          return messages.next().then(processIteratorResult);\n        })\n      }\n    }\n\n    JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n      if (comm_id in window.PyViz.comms) {\n        return window.PyViz.comms[comm_id];\n      } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n        var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n        var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n        if (msg_handler) {\n          comm.on_msg(msg_handler);\n        }\n      } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n        var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n        comm.open();\n        if (msg_handler) {\n          comm.onMsg = msg_handler;\n        }\n      } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n        var comm_promise = google.colab.kernel.comms.open(comm_id)\n        comm_promise.then((comm) => {\n          window.PyViz.comms[comm_id] = comm;\n          if (msg_handler) {\n            var messages = comm.messages[Symbol.asyncIterator]();\n            function processIteratorResult(result) {\n              var message = result.value;\n              var content = {data: message.data};\n              var metadata = message.metadata || {comm_id};\n              var msg = {content, metadata}\n              msg_handler(msg);\n              return messages.next().then(processIteratorResult);\n            }\n            return messages.next().then(processIteratorResult);\n          }\n        }) \n        var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n          return comm_promise.then((comm) => {\n            comm.send(data, metadata, buffers, disposeOnDone);\n          });\n        };\n        var comm = {\n          send: sendClosure\n        };\n      }\n      window.PyViz.comms[comm_id] = comm;\n      return comm;\n    }\n    window.PyViz.comm_manager = new JupyterCommManager();\n    \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n  var div = document.createElement(\"div\");\n  var script = document.createElement(\"script\");\n  node.appendChild(div);\n  node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n  var output_area = handle.output_area;\n  var output = handle.output;\n  if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n    return\n  }\n  var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n  var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n  if (id !== undefined) {\n    var nchildren = toinsert.length;\n    var html_node = toinsert[nchildren-1].children[0];\n    html_node.innerHTML = output.data[HTML_MIME_TYPE];\n    var scripts = [];\n    var nodelist = html_node.querySelectorAll(\"script\");\n    for (var i in nodelist) {\n      if (nodelist.hasOwnProperty(i)) {\n        scripts.push(nodelist[i])\n      }\n    }\n\n    scripts.forEach( function (oldScript) {\n      var newScript = document.createElement(\"script\");\n      var attrs = [];\n      var nodemap = oldScript.attributes;\n      for (var j in nodemap) {\n        if (nodemap.hasOwnProperty(j)) {\n          attrs.push(nodemap[j])\n        }\n      }\n      attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n      newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n      oldScript.parentNode.replaceChild(newScript, oldScript);\n    });\n    if (JS_MIME_TYPE in output.data) {\n      toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n    }\n    output_area._hv_plot_id = id;\n    if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n      window.PyViz.plot_index[id] = Bokeh.index[id];\n    } else {\n      window.PyViz.plot_index[id] = null;\n    }\n  } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n    var bk_div = document.createElement(\"div\");\n    bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n    var script_attrs = bk_div.children[0].attributes;\n    for (var i = 0; i < script_attrs.length; i++) {\n      toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n    }\n    // store reference to server id on output_area\n    output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n  }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n  var id = handle.cell.output_area._hv_plot_id;\n  var server_id = handle.cell.output_area._bokeh_server_id;\n  if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n  var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n  if (server_id !== null) {\n    comm.send({event_type: 'server_delete', 'id': server_id});\n    return;\n  } else if (comm !== null) {\n    comm.send({event_type: 'delete', 'id': id});\n  }\n  delete PyViz.plot_index[id];\n  if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n    var doc = window.Bokeh.index[id].model.document\n    doc.clear();\n    const i = window.Bokeh.documents.indexOf(doc);\n    if (i > -1) {\n      window.Bokeh.documents.splice(i, 1);\n    }\n  }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n  delete PyViz.comms[\"hv-extension-comm\"];\n  window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n  handle_clear_output(event, {cell: {output_area: handle.output_area}})\n  handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n  function append_mime(data, metadata, element) {\n    // create a DOM node to render to\n    var toinsert = this.create_output_subarea(\n    metadata,\n    CLASS_NAME,\n    EXEC_MIME_TYPE\n    );\n    this.keyboard_manager.register_events(toinsert);\n    // Render to node\n    var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n    render(props, toinsert[0]);\n    element.append(toinsert);\n    return toinsert\n  }\n\n  events.on('output_added.OutputArea', handle_add_output);\n  events.on('output_updated.OutputArea', handle_update_output);\n  events.on('clear_output.CodeCell', handle_clear_output);\n  events.on('delete.Cell', handle_clear_output);\n  events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n  OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n    safe: true,\n    index: 0\n  });\n}\n\nif (window.Jupyter !== undefined) {\n  try {\n    var events = require('base/js/events');\n    var OutputArea = require('notebook/js/outputarea').OutputArea;\n    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n      register_renderer(events, OutputArea);\n    }\n  } catch(err) {\n  }\n}\n",
+      "application/vnd.holoviews_load.v0+json": ""
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style>*[data-root-id],\n",
+       "*[data-root-id] > * {\n",
+       "  box-sizing: border-box;\n",
+       "  font-family: var(--jp-ui-font-family);\n",
+       "  font-size: var(--jp-ui-font-size1);\n",
+       "  color: var(--vscode-editor-foreground, var(--jp-ui-font-color1));\n",
+       "}\n",
+       "\n",
+       "/* Override VSCode background color */\n",
+       ".cell-output-ipywidget-background:has(> .cell-output-ipywidget-background\n",
+       "    > .lm-Widget\n",
+       "    > *[data-root-id]),\n",
+       ".cell-output-ipywidget-background:has(> .lm-Widget > *[data-root-id]) {\n",
+       "  background-color: transparent !important;\n",
+       "}\n",
+       "</style>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import panel as pn\n",
+    "import pandas as pd\n",
+    "import scipy.stats as stats\n",
+    "from sqlalchemy import create_engine\n",
+    "import hvplot.pandas\n",
+    "from datetime import datetime, timedelta\n",
+    "from script import processing\n",
+    "import numpy as np\n",
+    "pn.extension()\n",
+    "db_url = 'sqlite:///local.db'\n",
+    "engine = create_engine(db_url)\n",
+    "import plotly.express as px\n",
+    "pn.extension('mathjax')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "p_eval_df = None\n",
+    "calculated_b_stock = None\n",
+    "calculated_p_stock = None\n",
+    "# load benchmark stock\n",
+    "with engine.connect() as connection:\n",
+    "    calculated_b_stock = pd.read_sql('calculated_b_stock', con=connection)\n",
+    "    calculated_p_stock = pd.read_sql('calculated_p_stock', con=connection)\n",
+    "    p_eval_df = pd.read_sql('p_eval_result', con=connection)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 139,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2023-06-29 00:00:00\n",
+      "interaction        0.012116\n",
+      "allocation         0.002211\n",
+      "selection         -0.012116\n",
+      "active_return      0.002211\n",
+      "notional_return    0.002211\n",
+      "dtype: float64\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>portfolio_return_p</th>\n",
+       "      <th>portfolio_pct_p</th>\n",
+       "      <th>portfolio_return_b</th>\n",
+       "      <th>portfolio_pct_b</th>\n",
+       "      <th>mkt_cap</th>\n",
+       "      <th>prev_mkt_cap</th>\n",
+       "      <th>pnl</th>\n",
+       "      <th>risk</th>\n",
+       "      <th>active_return</th>\n",
+       "      <th>tracking_error</th>\n",
+       "      <th>cum_pnl</th>\n",
+       "      <th>return_p</th>\n",
+       "      <th>return_b</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>600</th>\n",
+       "      <td>2023-06-29</td>\n",
+       "      <td>0.343213</td>\n",
+       "      <td>0.006552</td>\n",
+       "      <td>-0.017097</td>\n",
+       "      <td>-0.000125</td>\n",
+       "      <td>1219.334595</td>\n",
+       "      <td>1211.396987</td>\n",
+       "      <td>7.937608</td>\n",
+       "      <td>0.309501</td>\n",
+       "      <td>0.002785</td>\n",
+       "      <td>0.218361</td>\n",
+       "      <td>140.338486</td>\n",
+       "      <td>0.014219</td>\n",
+       "      <td>0.011434</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          date  portfolio_return_p  portfolio_pct_p  portfolio_return_b  \\\n",
+       "600 2023-06-29            0.343213         0.006552           -0.017097   \n",
+       "\n",
+       "     portfolio_pct_b      mkt_cap  prev_mkt_cap       pnl      risk  \\\n",
+       "600        -0.000125  1219.334595   1211.396987  7.937608  0.309501   \n",
+       "\n",
+       "     active_return  tracking_error     cum_pnl  return_p  return_b  \n",
+       "600       0.002785        0.218361  140.338486  0.014219  0.011434  "
+      ]
+     },
+     "execution_count": 139,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## check why activate return and nominal return is not same \n",
+    "# start_date = p_eval_df['date'].min()\n",
+    "# start_date = datetime(2022, 12, 14)\n",
+    "start_date = datetime(2023,6,25)\n",
+    "end_date = p_eval_df['date'].max()\n",
+    "print(end_date)\n",
+    "ticker = \"601117.XSHG\"\n",
+    "attributes_df = processing.calculate_attributes_between_dates(\n",
+    "    start_date, end_date, calculated_b_stock=calculated_b_stock, calculated_p_stock=calculated_p_stock)\n",
+    "total_attributes = attributes_df.aggregate({\n",
+    "        'interaction': 'sum',\n",
+    "        'allocation': 'sum',\n",
+    "        'selection': 'sum',\n",
+    "        'active_return': 'sum',\n",
+    "        'notional_return': 'sum'\n",
+    "    }) \n",
+    "# print(attributes_df.columns)\n",
+    "print(total_attributes)\n",
+    "return_df = processing.calculate_return(p_eval_df, start_date, end_date)\n",
+    "return_df\n",
+    "most_recent_row = return_df.tail(1)\n",
+    "# print(return_df.head(1))\n",
+    "most_recent_row\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 124,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1219.334714348576"
+      ]
+     },
+     "execution_count": 124,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "1038.573137 * (1 + 0.174048)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 137,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.01448137201285984"
+      ]
+     },
+     "execution_count": 137,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# portfolio return\n",
+    "attributes_df['w_return_p'] = attributes_df.pct_p * attributes_df.prev_w_in_p_p\n",
+    "attributes_df.w_return_p.sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 129,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.09551009084622147"
+      ]
+     },
+     "execution_count": 129,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "0.17190200433908703 - 0.07639191349286556"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 138,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.011250737770502067"
+      ]
+     },
+     "execution_count": 138,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# benchmark return\n",
+    "attributes_df['w_return_b'] = attributes_df.pct_b * attributes_df.prev_w_in_p_b\n",
+    "attributes_df.w_return_b.sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 120,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/v5/2108rh5964q9j741wg_s8r1w0000gn/T/ipykernel_87460/2230555833.py:4: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  selected_df['return_p'] = (1 + selected_df.portfolio_pct_p).cumprod()\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>portfolio_return_p</th>\n",
+       "      <th>portfolio_pct_p</th>\n",
+       "      <th>portfolio_return_b</th>\n",
+       "      <th>portfolio_pct_b</th>\n",
+       "      <th>mkt_cap</th>\n",
+       "      <th>prev_mkt_cap</th>\n",
+       "      <th>pnl</th>\n",
+       "      <th>risk</th>\n",
+       "      <th>active_return</th>\n",
+       "      <th>tracking_error</th>\n",
+       "      <th>cum_pnl</th>\n",
+       "      <th>return_p</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>471</th>\n",
+       "      <td>2022-12-14</td>\n",
+       "      <td>0.151775</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.002796</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>1038.573137</td>\n",
+       "      <td>1043.453255</td>\n",
+       "      <td>-4.880117</td>\n",
+       "      <td>0.327324</td>\n",
+       "      <td>-0.000508</td>\n",
+       "      <td>0.225463</td>\n",
+       "      <td>-40.422972</td>\n",
+       "      <td>1.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>472</th>\n",
+       "      <td>2022-12-15</td>\n",
+       "      <td>0.166816</td>\n",
+       "      <td>0.013638</td>\n",
+       "      <td>0.004505</td>\n",
+       "      <td>0.001707</td>\n",
+       "      <td>1052.737107</td>\n",
+       "      <td>1038.573137</td>\n",
+       "      <td>14.163970</td>\n",
+       "      <td>0.327122</td>\n",
+       "      <td>0.011931</td>\n",
+       "      <td>0.225384</td>\n",
+       "      <td>-26.259002</td>\n",
+       "      <td>1.013638</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>473</th>\n",
+       "      <td>2022-12-16</td>\n",
+       "      <td>0.156618</td>\n",
+       "      <td>-0.009792</td>\n",
+       "      <td>-0.000671</td>\n",
+       "      <td>-0.005172</td>\n",
+       "      <td>1042.429190</td>\n",
+       "      <td>1052.737107</td>\n",
+       "      <td>-10.307917</td>\n",
+       "      <td>0.326859</td>\n",
+       "      <td>-0.004619</td>\n",
+       "      <td>0.225175</td>\n",
+       "      <td>-36.566919</td>\n",
+       "      <td>1.003713</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>474</th>\n",
+       "      <td>2022-12-19</td>\n",
+       "      <td>0.144766</td>\n",
+       "      <td>-0.012217</td>\n",
+       "      <td>-0.022720</td>\n",
+       "      <td>-0.022046</td>\n",
+       "      <td>1029.693832</td>\n",
+       "      <td>1042.429190</td>\n",
+       "      <td>-12.735359</td>\n",
+       "      <td>0.326642</td>\n",
+       "      <td>0.009829</td>\n",
+       "      <td>0.225043</td>\n",
+       "      <td>-49.302277</td>\n",
+       "      <td>0.991450</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>475</th>\n",
+       "      <td>2022-12-20</td>\n",
+       "      <td>0.140936</td>\n",
+       "      <td>-0.003925</td>\n",
+       "      <td>-0.032668</td>\n",
+       "      <td>-0.009947</td>\n",
+       "      <td>1025.652205</td>\n",
+       "      <td>1029.693832</td>\n",
+       "      <td>-4.041626</td>\n",
+       "      <td>0.326312</td>\n",
+       "      <td>0.006022</td>\n",
+       "      <td>0.224844</td>\n",
+       "      <td>-53.343904</td>\n",
+       "      <td>0.987559</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>596</th>\n",
+       "      <td>2023-06-21</td>\n",
+       "      <td>0.338226</td>\n",
+       "      <td>-0.012860</td>\n",
+       "      <td>-0.012265</td>\n",
+       "      <td>-0.020746</td>\n",
+       "      <td>1213.343064</td>\n",
+       "      <td>1229.149762</td>\n",
+       "      <td>-15.806698</td>\n",
+       "      <td>0.310372</td>\n",
+       "      <td>0.007886</td>\n",
+       "      <td>0.219003</td>\n",
+       "      <td>134.346955</td>\n",
+       "      <td>1.168279</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>597</th>\n",
+       "      <td>2023-06-26</td>\n",
+       "      <td>0.327466</td>\n",
+       "      <td>-0.009151</td>\n",
+       "      <td>-0.028546</td>\n",
+       "      <td>-0.016269</td>\n",
+       "      <td>1202.239760</td>\n",
+       "      <td>1213.343064</td>\n",
+       "      <td>-11.103304</td>\n",
+       "      <td>0.310175</td>\n",
+       "      <td>0.007118</td>\n",
+       "      <td>0.218860</td>\n",
+       "      <td>123.243651</td>\n",
+       "      <td>1.157588</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>598</th>\n",
+       "      <td>2023-06-27</td>\n",
+       "      <td>0.339179</td>\n",
+       "      <td>0.010630</td>\n",
+       "      <td>-0.016308</td>\n",
+       "      <td>0.012234</td>\n",
+       "      <td>1215.019735</td>\n",
+       "      <td>1202.239760</td>\n",
+       "      <td>12.779975</td>\n",
+       "      <td>0.309985</td>\n",
+       "      <td>-0.001604</td>\n",
+       "      <td>0.218682</td>\n",
+       "      <td>136.023626</td>\n",
+       "      <td>1.169893</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>599</th>\n",
+       "      <td>2023-06-28</td>\n",
+       "      <td>0.335389</td>\n",
+       "      <td>-0.002982</td>\n",
+       "      <td>-0.016974</td>\n",
+       "      <td>-0.000665</td>\n",
+       "      <td>1211.396987</td>\n",
+       "      <td>1215.019735</td>\n",
+       "      <td>-3.622748</td>\n",
+       "      <td>0.309735</td>\n",
+       "      <td>-0.002316</td>\n",
+       "      <td>0.218507</td>\n",
+       "      <td>132.400878</td>\n",
+       "      <td>1.166405</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>600</th>\n",
+       "      <td>2023-06-29</td>\n",
+       "      <td>0.343213</td>\n",
+       "      <td>0.006552</td>\n",
+       "      <td>-0.017097</td>\n",
+       "      <td>-0.000125</td>\n",
+       "      <td>1219.334595</td>\n",
+       "      <td>1211.396987</td>\n",
+       "      <td>7.937608</td>\n",
+       "      <td>0.309501</td>\n",
+       "      <td>0.006678</td>\n",
+       "      <td>0.218361</td>\n",
+       "      <td>140.338486</td>\n",
+       "      <td>1.174048</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>130 rows × 13 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          date  portfolio_return_p  portfolio_pct_p  portfolio_return_b  \\\n",
+       "471 2022-12-14            0.151775         0.000000            0.002796   \n",
+       "472 2022-12-15            0.166816         0.013638            0.004505   \n",
+       "473 2022-12-16            0.156618        -0.009792           -0.000671   \n",
+       "474 2022-12-19            0.144766        -0.012217           -0.022720   \n",
+       "475 2022-12-20            0.140936        -0.003925           -0.032668   \n",
+       "..         ...                 ...              ...                 ...   \n",
+       "596 2023-06-21            0.338226        -0.012860           -0.012265   \n",
+       "597 2023-06-26            0.327466        -0.009151           -0.028546   \n",
+       "598 2023-06-27            0.339179         0.010630           -0.016308   \n",
+       "599 2023-06-28            0.335389        -0.002982           -0.016974   \n",
+       "600 2023-06-29            0.343213         0.006552           -0.017097   \n",
+       "\n",
+       "     portfolio_pct_b      mkt_cap  prev_mkt_cap        pnl      risk  \\\n",
+       "471         0.000000  1038.573137   1043.453255  -4.880117  0.327324   \n",
+       "472         0.001707  1052.737107   1038.573137  14.163970  0.327122   \n",
+       "473        -0.005172  1042.429190   1052.737107 -10.307917  0.326859   \n",
+       "474        -0.022046  1029.693832   1042.429190 -12.735359  0.326642   \n",
+       "475        -0.009947  1025.652205   1029.693832  -4.041626  0.326312   \n",
+       "..               ...          ...           ...        ...       ...   \n",
+       "596        -0.020746  1213.343064   1229.149762 -15.806698  0.310372   \n",
+       "597        -0.016269  1202.239760   1213.343064 -11.103304  0.310175   \n",
+       "598         0.012234  1215.019735   1202.239760  12.779975  0.309985   \n",
+       "599        -0.000665  1211.396987   1215.019735  -3.622748  0.309735   \n",
+       "600        -0.000125  1219.334595   1211.396987   7.937608  0.309501   \n",
+       "\n",
+       "     active_return  tracking_error     cum_pnl  return_p  \n",
+       "471      -0.000508        0.225463  -40.422972  1.000000  \n",
+       "472       0.011931        0.225384  -26.259002  1.013638  \n",
+       "473      -0.004619        0.225175  -36.566919  1.003713  \n",
+       "474       0.009829        0.225043  -49.302277  0.991450  \n",
+       "475       0.006022        0.224844  -53.343904  0.987559  \n",
+       "..             ...             ...         ...       ...  \n",
+       "596       0.007886        0.219003  134.346955  1.168279  \n",
+       "597       0.007118        0.218860  123.243651  1.157588  \n",
+       "598      -0.001604        0.218682  136.023626  1.169893  \n",
+       "599      -0.002316        0.218507  132.400878  1.166405  \n",
+       "600       0.006678        0.218361  140.338486  1.174048  \n",
+       "\n",
+       "[130 rows x 13 columns]"
+      ]
+     },
+     "execution_count": 120,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "selected_df = p_eval_df[p_eval_df.date.between(start_date, end_date)]\n",
+    "selected_df.iloc[0, selected_df.columns.get_indexer(\n",
+    "    ['portfolio_pct_p', 'portfolio_pct_b'])] = 0\n",
+    "selected_df['return_p'] = (1 + selected_df.portfolio_pct_p).cumprod()\n",
+    "selected_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 140,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['date', 'portfolio_return_p', 'portfolio_pct_p', 'portfolio_return_b',\n",
+       "       'portfolio_pct_b', 'mkt_cap', 'prev_mkt_cap', 'pnl', 'risk',\n",
+       "       'active_return', 'tracking_error', 'cum_pnl'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 140,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p_eval_df.columns"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "portfolio_risk_assesment",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

utils.py

@@ -1,3 +1,110 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:873819e905c0ff108acd3da231a5b6c00c8830d4ae8cbe99b280899c7893e026
-size 3697
+import pytz
+import datetime
+import io
+import pandas as pd
+
+
+def time_in_beijing():
+    '''
+    return current time in Beijing as datetime object
+    '''
+    tz = pytz.timezone('Asia/Shanghai')
+    dt = datetime.datetime.now(tz)
+    return dt
+
+
+def convert_string_to_datetime(date_string, time_zone="Asia/Shanghai"):
+    '''
+    Convert a string to a datetime object with the timezone by default,
+    Shanghai
+    '''
+    dt = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f')
+    tz = pytz.timezone(time_zone)
+    dt = tz.localize(dt)
+    return dt
+
+
+def create_stocks_entry_from_excel(byte_string):
+    '''create stock entry from excel file
+    Parameters
+    ----------
+    byte_string: bytes
+        the byte string of the excel file
+    Returns
+    -------
+    new_stock_entry: list
+        [{ticker:str, shares:int, mean_price: float, date:datetime.datetime}]
+        the list of stock entry
+    '''
+    uploaded_df = None
+    with io.BytesIO(byte_string) as f:
+        uploaded_df = pd.read_excel(f)
+
+    # throw exception if doesn't have required columns
+    if not set(['证券代码', '持仓数量', '平均建仓成本', 'time_stamp']).issubset(uploaded_df.columns):
+        raise Exception('Missing required columns')
+    # print(uploaded_df)
+    # uploaded_df = pd.read_excel()
+    uploaded_df.drop(columns='Unnamed: 0', inplace=True)
+    # Define the regular expression pattern to match the string endings
+    pattern = r'\.(sz|sh)$'
+    # Define the replacement strings for each match group
+    replacements = {'.sz': '.XSHE', '.sh': '.XSHG'}
+    # Use the str.replace method with the pattern and replacements
+    uploaded_df['证券代码'] = uploaded_df['证券代码'].str.lower()
+    uploaded_df['证券代码'] = uploaded_df['证券代码'].str.replace(
+        pattern, lambda m: replacements[m.group()], regex=True)
+    new_stock_entry = [
+        dict(ticker=ticker, shares=shares, date=time, mean_price=mean_price)
+        for ticker, shares, mean_price, time in zip(
+            uploaded_df['证券代码'],
+            uploaded_df['持仓数量'],
+            uploaded_df['平均建仓成本'],
+            pd.to_datetime(uploaded_df['time_stamp']))]
+    # new_profile, error = api.update_portfolio_profile(new_stock_entry)
+    print(new_stock_entry)
+    return new_stock_entry
+
+
+def style_number(vals):
+    '''color negative number as red, positive as green
+    Parameters
+    ----------
+    vals: df columns
+        the columns to be styled
+    Returns
+    -------
+    list
+        the list of style
+
+    '''
+    return ['color: red' if v < 0 else 'color: green' for v in vals]
+
+
+def create_share_changes_report(df):
+    '''Create a markdown report of the share changes for certain date
+    Parameters
+    ----------
+    df: pd.DataFrame
+        the dataframe of profile for a specific date
+    Returns
+    -------
+    markdown: str
+    '''
+
+    date_str = df.date.to_list()[0].strftime('%Y-%m-%d %H:%M:%S')
+    markdown = f"### {date_str}\n\n"
+    markdown += 'Ticker | Display Name | Share Changes\n'
+    markdown += '--- | --- | ---\n'
+    for _, row in df.iterrows():
+        share_changes = row['share_changes']
+        # Apply green color to positive numbers and red color to negative numbers
+        if share_changes > 0:
+            share_changes_str = f'<span style="color:green">{share_changes}</span>'
+        elif share_changes < 0:
+            share_changes_str = f'<span style="color:red">{share_changes}</span>'
+        else:
+            share_changes_str = str(share_changes)
+        markdown += '{} | {} | {}\n'.format(row['ticker'],
+                                            row['display_name'], share_changes_str)
+    return markdown