Kaggle competition: Two Sigma: Using News to Predict Stock Movements
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<load_pretrained>
import numpy as np
import lightgbm as lgb
import pandas as pd
from kaggle.competitions import twosigmanews
import matplotlib.pyplot as plt
import random
from datetime import datetime, date
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time
import math
import gc
from statsmodels.nonparametric.smoothers_lowess import lowess
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
import inspect, sys
from multiprocessing import Pool
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
dic_assetName={}
dic_assetNameSNo={}
dic_assetCodes={}
dic_assetCode={}
dic_assetCodeSNo={}
dic_rng = {}
dic_min = {}
set_sector = {"TECH","ENER","BMAT","INDS","CYCS","NCYC","FINS","HECA","TECH","TCOM","UTIL"}
dic_sector_code_name = {"ENER":"Energy", "BMAT":"Basic Materials", "INDS":"Industrials", "CYCS":"Cyclical Consumer Goods & Services", \
"NCYC":"Non Cyclical Consumer Goods & Services", "FINS": "Financials", "HECA":"Healthcare","TECH":"Technology", \
"TCOM":"Telecommunication Services", "UTIL":"Utilities"}
sector_col = list(set_sector)
 sector_col.append("assetNameSNo")
 dic_sectorSNo = {v: k for v,k in enumerate(list(set_sector)) }
dic_sector = {k: v for v,k in enumerate(list(set_sector)) }
dic_sectorSNo["-1"] = "Unknown"
dic_sector["Unknown"] = -1
dic_asset_sector = {}
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
total_news_obs_df = []
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10',"SenSR"]
sector_return_features = ["weighted_SenSR"]
n_lag = [3,7,14]
debug_func = set()
prv_mkt_data = []
prv_news_data = []
data = pd.read_pickle('data.pkl' )<drop_column>
env = twosigmanews.make_env()
days = env.get_prediction_days()
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
( market_train_df, news_train_df)= env.get_training_data()
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
if 1==2:
prv_time_list = market_train_df["time"].dt.date.unique() [-14:]
prv_time_list.sort()
prv_time_list
for prv_time in prv_time_list:
prv_mkt_data.append(market_train_df[market_train_df["time"].dt.date == prv_time])
 prv_news_data.append(news_train_df[news_train_df["time"].dt.date == prv_time] )
del data
gc.collect() ;<train_model>
def fixassetopenclose(in_asset_df):
in_asset_df["final_close"] = in_asset_df["close"]
in_asset_df["final_open"] = in_asset_df["open"]
in_asset_df["adjusted_close"] = in_asset_df["close"].shift(-1)/(1 + in_asset_df["returnsClosePrevRaw1"].shift(-1))
in_asset_df["ratio"] = in_asset_df["close"]/in_asset_df["adjusted_close"]
idx_list = in_asset_df[(in_asset_df["ratio"].round(1)<0.9)|(in_asset_df["ratio"].round(1)>1.1)].index
for idx in idx_list:
affected_idx = in_asset_df[in_asset_df.index <= idx].index
ratio = in_asset_df.loc[idx,"ratio"]
in_asset_df.loc[affected_idx,["final_close","final_open"]] = in_asset_df.loc[affected_idx,["final_close","final_open"]] / ratio
del affected_idx
changed_recs = in_asset_df[in_asset_df["final_close"] != in_asset_df["close"]].index
return in_asset_df.loc[changed_recs,["final_close","final_open"]]
def fixopenclose(in_mkt_df):
assetlist = []
changedlist = []
df_group = in_mkt_df.groupby(["assetCode"])
 for group in df_group:
assetlist.append(group[1])
 pool = Pool(4)
 changedlist = pool.map(fixassetopenclose, assetlist)
 pool.close()
changed_df = pd.concat(changedlist)
 changed_df.dropna(inplace=True)
 in_mkt_df.loc[changed_df.index,["close"]] = list(changed_df["final_close"])
 in_mkt_df.loc[changed_df.index,["open"]] = list(changed_df["final_open"])
 fixopenclose(market_train_df )
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
def findnulls(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 for col in in_mkt_df:
cols = in_mkt_df[col]
if fname in debug_func:
print("Market df: Num nulls in " + col + " is " + str(cols[cols.isna() ].shape[0])+ " out of " + str(cols.shape[0]))
for col in in_news_df:
cols = in_news_df[col]
if fname in debug_func:
print("Market df: Num nulls in " + col + " is " + str(cols[cols.isna() ].shape[0])+ " out of " + str(cols.shape[0]))
if fname in debug_func:
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<import_modules>
dic_time = {}
def fix_time_col(in_mkt_df, in_news_df):
global dic_time
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 if "datetime" in str(in_mkt_df["time"].dtype):
in_mkt_df["time"] = in_mkt_df["time"].dt.date
if "datetime" in str(in_news_df["time"].dtype):
in_news_df["time"] = in_news_df["time"].dt.date
set_time = set(in_mkt_df["time"].unique())| set(in_news_df["time"].unique())
 lst_time = list(set_time)
 if len(dic_time)== 0:
dic_time= {k:v for v,k in enumerate(lst_time)}
else:
set_time_new = set_time - set(dic_time.keys())
 if fname in debug_func:
print("Need to get sno for " + str(len(set_time_new)) + " news time.")
 time_max_SNo = len(dic_time)
 dic_time_new = {k:v+time_max_SNo for v,k in enumerate(list(set_time_new)) }
dic_time.update(dic_time_new)
 in_mkt_df["timeSNo"] = in_mkt_df["time"].map(dic_time)
 in_news_df["timeSNo"] = in_news_df["time"].map(dic_time)
 if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def clean_assetName_col(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.lower()
in_mkt_df["assetName"].replace({"inc":"","llc":"","ltd":""}, inplace=True)
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.strip()
in_news_df["assetName"] = in_news_df["assetName"].str.lower()
in_news_df["assetName"].replace({"inc":"","llc":"","ltd":""}, inplace=True)
 in_news_df["assetName"] = in_news_df["assetName"].str.strip()
if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )
plt.style.use('ggplot')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<set_options>
def get_final_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( not math.isnan(min_mkt_assetNameSNo)) &(math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( not math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
else:
return -1
def get_replace_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( math.isnan(min_mkt_assetNameSNo)) &(not math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(not math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return min_assetNameSNo
else:
return -1
def replace_assetName(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 in_news_df["num_rows"] = 1
assetName_df = in_news_df[["assetCodesSNo","assetNameSNo","num_rows"]].groupby(\
["assetCodesSNo","assetNameSNo"], as_index=False ).count()
if fname in debug_func:
print("Distinct assetcodes, assetname combination: " + str(assetName_df.shape[0])+ " out of " + str(in_news_df.shape[0]))
assetCode_df = assetName_df[["assetCodesSNo", \
"assetNameSNo","num_rows"]].groupby(["assetCodesSNo"], \
as_index=False ).agg({"assetNameSNo": ["max","min"], "num_rows": "sum"})
 assetCode_df.columns = ["assetCodesSNo", "max_assetNameSNo", "min_assetNameSNo", "count_num_rows"]
assetCode_df = assetCode_df[assetCode_df["max_assetNameSNo"] != assetCode_df["min_assetNameSNo"]]
num_with_duplicate_assetName = assetCode_df.shape[0]
if fname in debug_func:
print("Rows with different assetname for same assetcode : " + str(num_with_duplicate_assetName)+ " out of " + str(assetName_df.shape[0]))
in_news_df.drop(["num_rows"], axis=1, inplace=True)
 if assetCode_df.shape[0] > 0:
assetCode_df["max_assetName"] = assetCode_df["max_assetNameSNo"].map(dic_assetNameSNo)
 assetCode_df["min_assetName"] = assetCode_df["min_assetNameSNo"].map(dic_assetNameSNo)
 list_mkt_asset = list(in_mkt_df["assetName"].unique())
 dic_mkt_assetName= {k:v for v,k in enumerate(list_mkt_asset)}
assetCode_df["min_mkt_assetNameSNo"] = assetCode_df["min_assetName"].map(dic_mkt_assetName)
 assetCode_df["max_mkt_assetNameSNo"] = assetCode_df["max_assetName"].map(dic_mkt_assetName)
 assetCode_df["final_assetNameSNo"] = assetCode_df.apply(get_final_assetName, axis=1)
 assetCode_df["replace_assetNameSNo"] = assetCode_df.apply(get_replace_assetName, axis=1)
 missing_in_mkt = assetCode_df[assetCode_df["final_assetNameSNo"]==-1].shape[0]
if fname in debug_func:
print("None of the asset names exist in market df for " + str(missing_in_mkt)+ " entries")
 if num_with_duplicate_assetName == missing_in_mkt:
df1 = assetCode_df[["assetCodesSNo","max_assetNameSNo"]]
df1.columns = ["assetCodesSNo","assetNameSNo"]
df2 = assetCode_df[["assetCodesSNo","min_assetNameSNo"]]
df2.columns = ["assetCodesSNo","assetNameSNo"]
tot_df = pd.concat([df1,df2])
 rows_to_be_dropped = pd.merge(in_news_df.reset_index() , tot_df, how="inner", on=["assetCodesSNo", "assetNameSNo"])
 print(rows_to_be_dropped.index.unique())
 print("dropping rows:" + str(rows_to_be_dropped.shape[0]))
in_news_df.drop(rows_to_be_dropped["index"].unique() , axis=0, inplace=True)
 assetCode_df["final_assetName"] = assetCode_df["final_assetNameSNo"].map(dic_assetNameSNo)
 assetCode_df = assetCode_df[assetCode_df["final_assetNameSNo"]!=-1]
df_temp = pd.merge(in_news_df.reset_index() [["index","assetCodesSNo","assetNameSNo"]], \
assetCode_df[["replace_assetNameSNo", "final_assetNameSNo","assetCodesSNo","final_assetName"]], \
how="inner", left_on=["assetCodesSNo","assetNameSNo"], right_on=["assetCodesSNo","replace_assetNameSNo"], left_index=True ).set_index("index")
 in_news_df.loc[df_temp.index, "assetNameSNo"] = list(df_temp["final_assetNameSNo"])
 in_news_df.loc[df_temp.index, "assetName"] = list(df_temp["final_assetName"])
 in_news_df["num_rows"] = 1
assetName_df = in_news_df[["assetCodesSNo","assetNameSNo","num_rows"]].groupby(\
["assetCodesSNo","assetNameSNo"], as_index=False ).count()
if fname in debug_func:
print("After replacement, Distinct assetcodes, assetname combination: " + str(assetName_df.shape[0])+ " out of " + str(in_news_df.shape[0]))
assetCode_df = assetName_df[["assetCodesSNo", \
"assetNameSNo","num_rows"]].groupby(["assetCodesSNo"], \
as_index=False ).agg({"assetNameSNo": ["max","min"], "num_rows": "sum"})
 assetCode_df.columns = ["assetCodesSNo", "max_assetNameSNo", "min_assetNameSNo", "count_num_rows"]
assetCode_df = assetCode_df[assetCode_df["max_assetNameSNo"] != assetCode_df["min_assetNameSNo"]]
if fname in debug_func:
print("After replacement, Rows with different assetname for same assetcode : " + str(assetCode_df.shape[0])+ " out of " + str(assetName_df.shape[0]))
in_news_df.drop(["num_rows"], axis=1, inplace=True)
 in_news_df.drop(["assetCodes","assetCodesSNo"], inplace=True, axis=1)
 if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
def seed_all(seed=42):
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all()<compute_test_metric>
def scale_col(in_news_df, col):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 global dic_rng
global dic_min
if col not in dic_rng.keys() :
col_max = in_news_df[col].max()
col_min = in_news_df[col].min()
col_rng = col_max - col_min
dic_rng[col] = col_rng
dic_min[col] = col_min
else:
col_min = dic_min[col]
col_rng = dic_rng[col]
in_news_df[col] =(in_news_df[col]-col_min)/col_rng
if col_min < 0:
in_news_df[col] = 2*in_news_df[col] - 1
print(col, in_news_df[col].min() , in_news_df[col].max())
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def drop_extra_news_cols(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 news_drop_cols = ["sourceTimestamp","firstCreated","sourceId","headline","urgency","takeSequence","provider","audiences","bodySize",\
"headlineTag","sentenceCount","wordCount","firstMentionSentence","sentimentClass","sentimentWordCount","companyCount", "marketCommentary",
"volumeCounts12H","volumeCounts24H","volumeCounts3D","volumeCounts5D","volumeCounts7D"]
news_drop_cols = [col for col in news_drop_cols if col in in_news_df.columns ]
in_news_df.drop(news_drop_cols, axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def drop_irrelevant_news(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 if "relevance" in in_news_df.columns:
irrelevant_news = in_news_df[in_news_df["relevance"] < 0.3].index
print("Num low relevance news:", len(irrelevant_news))
in_news_df.drop(irrelevant_news, axis=0, inplace=True)
 del irrelevant_news
gc.collect()
if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def drop_neutral_news(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 if "sentimentPositive" in in_news_df.columns:
in_news_df["netSentimentPositive"] =(in_news_df["sentimentPositive"] - in_news_df["sentimentNegative"])
 neutral_news = in_news_df[in_news_df["netSentimentPositive"].abs() < 0.05].index
in_news_df.drop(neutral_news, axis=0, inplace=True)
 in_news_df["netSentimentPositive"] = in_news_df["netSentimentPositive"] *(1 - in_news_df["sentimentNeutral"])
 print("Num neutral news:", len(neutral_news))
del neutral_news
gc.collect()
in_news_df.drop(["sentimentNegative","sentimentNeutral","sentimentPositive"], axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def gen_netnovelty_col(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 if "noveltyCount12H" in in_news_df.columns:
arr_novelty = np.array(in_news_df[["noveltyCount12H","noveltyCount24H","noveltyCount3D","noveltyCount5D","noveltyCount7D"]])
 in_news_df["inv_netNovelty"] = list(1/(np.argmax(arr_novelty, axis=1)+2))
col = "inv_netNovelty"
scale_col(in_news_df, col)
 in_news_df.drop(["noveltyCount12H","noveltyCount24H","noveltyCount3D","noveltyCount5D","noveltyCount7D"], axis=1, inplace=True)
 in_news_df["SenSR"] = in_news_df["netSentimentPositive"] * in_news_df["relevance"] *(in_news_df["inv_netNovelty"])
 in_news_df.drop(["relevance","netSentimentPositive","inv_netNovelty"],axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
def calc_metric_loss(pred_fvc,sigma,true_fvc):
true_fvc=np.reshape(true_fvc,pred_fvc.shape)
 sigma[sigma<70]=70
delta=np.abs(pred_fvc-true_fvc)
 delta[delta>1000]=1000
metric=-(np.sqrt(2)*delta/sigma)-np.log(np.sqrt(2)*sigma)
 return -metric<compute_train_metric>
def create_sector(in_asset_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_asset_df.columns)
 global dic_asset_sector
global set_sector
global sector_col
global dic_sectorSNo
in_asset_df["sector"] = in_asset_df["subjects"].apply(lambda x: str(eval(x)& set_sector))
for col in set_sector:
in_asset_df[col] = in_asset_df["sector"].apply(lambda x: col in eval(x))
asset_df = in_asset_df[sector_col].groupby(["assetNameSNo"] ).sum()
asset_df1 = asset_df.sum(axis=1)
 asset_unknown_sector = list(asset_df1[asset_df1==0].index)
 if len(asset_unknown_sector)> 0:
print("Could not find sector of ", len(asset_unknown_sector), " assets")
 asset_df["sectorNameSNo"] = asset_df.values.argmax(axis=1)
 asset_df["sectorName"] = asset_df["sectorNameSNo"].map(dic_sectorSNo)
 asset_df.loc[asset_unknown_sector,["sectorNameSNo","sectorName"]] = [-1,"Unknown"]
asset_df = asset_df[["sectorName","sectorNameSNo"]]
asset_df.reset_index(inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_asset_df.columns)))
 print("new news cols added: " + str(set(in_asset_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 return asset_df
def get_sector_name(in_news_df, forpred=False):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 global dic_asset_sector
if "sectorName" not in in_news_df.columns:
in_news_df["sectorName"] = ""
in_news_df["sectorNameSNo"] = -1
if len(dic_asset_sector)> 0:
in_news_df["sectorName"] = in_news_df["assetNameSNo"].map(dic_asset_sector)
 in_news_df["sectorName"].fillna("", inplace=True)
 in_news_df["sectorNameSNo"] = in_news_df["sectorName"].map(dic_sector)
 if 1==1:
rec_index = in_news_df[in_news_df["sectorName"] == ""].index
lst_subjects = in_news_df.loc[rec_index]["subjects"].unique()
dic_subjects = {k:v for v,k in enumerate(list(lst_subjects)) }
dic_subjectsSNo = {v:k for v,k in enumerate(list(lst_subjects)) }
in_news_df["subjectsSNo"] = -1
in_news_df.loc[rec_index,["subjectsSNo"]] = in_news_df.loc[rec_index]["subjects"].map(dic_subjects)
 in_asset_df = in_news_df.loc[rec_index]
in_asset_df["tempCol"] = 1
in_asset_df = in_asset_df[["assetNameSNo","subjectsSNo","tempCol"]].groupby(["assetNameSNo","subjectsSNo"], as_index=False ).first()
in_asset_df.drop(["tempCol"], inplace=True, axis=1)
 in_asset_df["subjects"] = in_asset_df["subjectsSNo"].map(dic_subjectsSNo)
 if fname in debug_func:
print("New asset records in news : " + str(in_asset_df.shape[0])+ " out of " + str(in_news_df.shape[0]))
print("***************************")
 all_news = []
if in_asset_df.shape[0] > 0:
if forpred==False:
maxSNo = in_asset_df["assetNameSNo"].max()
partition_size =(maxSNo // 4)+ 1
for i in range(4):
minr = i*partition_size
maxr =(i+1)*partition_size
all_news.append(in_asset_df[(in_asset_df["assetNameSNo"] >= minr)&(in_asset_df["assetNameSNo"] < maxr)])
 pool = Pool(4)
 all_df = pool.map(create_sector, all_news)
 asset_df = pd.concat(all_df)
 pool.close()
else:
asset_df = create_sector(in_asset_df)
 asset_df1 = asset_df.set_index(["assetNameSNo"])[["sectorName"]]
new_dict = asset_df1.to_dict() ["sectorName"]
dic_asset_sector.update(new_dict)
 merged_df = pd.merge(in_news_df.reset_index() [["assetNameSNo","index"]], asset_df[["assetNameSNo","sectorName","sectorNameSNo"]], \
on=["assetNameSNo"], how="left" ).set_index("index")
 in_news_df.loc[merged_df.index,["sectorName","sectorNameSNo"]] = merged_df.loc[merged_df.index,["sectorName","sectorNameSNo"]]
if 1==1:
if "subjects" in in_news_df.columns:
in_news_df.drop("subjects", axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )
class main_model(nn.Module):
def __init__(self, n_additional_features, n_outputs):
super(main_model, self ).__init__()
self.fc1 = nn.Linear(n_additional_features, 100)
 self.fc2 = nn.Linear(100, 100)
 self.fc3 = nn.Linear(100, n_outputs)
 def forward(self, additional_features):
out=additional_features
out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
out = self.fc3(out)
 return out
def metric_loss(self,pred_fvc,true_fvc):
sigma = pred_fvc[:, 2] - pred_fvc[:, 0]
true_fvc=torch.reshape(true_fvc,pred_fvc[:,1].shape)
 sigma_clipped=torch.clamp(sigma,min=70)
 delta=torch.clamp(torch.abs(pred_fvc[:,1]-true_fvc),max=1000)
 metric=torch.div(-torch.sqrt(torch.tensor([2.0] ).to(device)) *delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0] ).to(device)) *sigma_clipped)
 return -metric
def fvc_loss(self,pred_fvc,true_fvc):
true_fvc=torch.reshape(true_fvc,pred_fvc[:,1].shape)
 fvc_err=torch.abs(pred_fvc-true_fvc)
 return fvc_err
def quantile_loss(self,preds, target, quantiles):
assert not target.requires_grad
assert preds.size(0)== target.size(0)
 losses = []
for i, q in enumerate(quantiles):
errors = target - preds[:, i]
losses.append(torch.max(( q - 1)* errors, q * errors ).unsqueeze(1))
loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))
return loss<normalization>
def merge_mkt_news(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 global dic_asset_sector
in_mkt_df["sectorName"] = in_mkt_df["assetNameSNo"].map(dic_asset_sector)
 df_unknown_sector = in_mkt_df[in_mkt_df["sectorName"].isna() ]
in_mkt_df["sectorName"].fillna("Unknown", inplace=True)
 print("no of market recs with unresolved sector: " + str(len(df_unknown_sector["assetNameSNo"].unique())))
 in_mkt_df["sectorNameSNo"] = in_mkt_df["sectorName"].map(dic_sector)
 in_mkt_df["mkt_cap"] =(( in_mkt_df["close"] + in_mkt_df["open"])/2)* in_mkt_df["volume"]
in_news_df1 = in_news_df[["timeSNo","assetNameSNo","SenSR"]].groupby(["assetNameSNo","timeSNo"], as_index=False ).sum()
if "SenSR" in in_mkt_df.columns:
in_mkt_df.drop(["SenSR"], axis=1, inplace=True)
 merged_df = pd.merge(in_mkt_df.reset_index() , in_news_df1, on= ["assetNameSNo","timeSNo"], how="inner" ).set_index("index")
 in_mkt_df["SenSR"] = 0
in_mkt_df.loc[merged_df.index, ["SenSR"]] = merged_df.loc[merged_df.index, ["SenSR"]]
in_mkt_df["weighted_SenSR"] = in_mkt_df["SenSR"] * in_mkt_df["mkt_cap"]
if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )
def load_scan(path):
slices = [pydicom.read_file(path / s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
try:
slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
 except:
slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
 if slice_thickness==0:
slice_thickness=slices[0].SliceThickness
for s in slices:
s.SliceThickness = slice_thickness
return slices
def get_pixels_hu(slices):
image = np.stack([np.array(s.pixel_array,dtype=np.int16)for s in slices])
 image = image.astype(np.int16)
 image[image == -2000] = 0
for slice_number in range(len(slices)) :
intercept = slices[slice_number].RescaleIntercept
slope = slices[slice_number].RescaleSlope
if slope != 1:
image[slice_number] = slope * image[slice_number].astype(np.float64)
 image[slice_number] = image[slice_number].astype(np.int16)
 image[slice_number] += np.int16(intercept)
 return np.array(image, dtype=np.int16)
 def resample(image, scan, new_spacing=[1,1,1]):
spacing = np.array([scan[0].SliceThickness] + list(scan[0].PixelSpacing), dtype=np.float32)
 resize_factor = spacing / new_spacing
new_real_shape = image.shape * resize_factor
new_shape = np.round(new_real_shape)
 real_resize_factor = new_shape / image.shape
new_spacing = spacing / real_resize_factor
image = scipy.ndimage.interpolation.zoom(image, real_resize_factor, mode='nearest')
 return image, new_spacing
def get_segmented_lungs(im, plot=False):

 if plot == True:
f, plots = plt.subplots(8, 1, figsize=(5, 40))

 binary = im < -200
if plot == True:
plots[0].axis('off')
 plots[0].imshow(binary, cmap=plt.cm.bone)

cleared = clear_border(binary)
 if plot == True:
plots[1].axis('off')
 plots[1].imshow(cleared, cmap=plt.cm.bone)

label_image = label(cleared)
 if plot == True:
plots[2].axis('off')
 plots[2].imshow(label_image, cmap=plt.cm.bone)

areas = [r.area for r in regionprops(label_image)]
areas.sort()
if len(areas)> 2:
for region in regionprops(label_image):
if region.area < areas[-2]:
for coordinates in region.coords:
label_image[coordinates[0], coordinates[1]] = 0
binary = label_image > 0
if plot == True:
plots[3].axis('off')
 plots[3].imshow(binary, cmap=plt.cm.bone)

selem = disk(2)
 binary = binary_erosion(binary, selem)
 if plot == True:
plots[4].axis('off')
 plots[4].imshow(binary, cmap=plt.cm.bone)

selem = disk(10)
 binary = binary_closing(binary, selem)
 if plot == True:
plots[5].axis('off')
 plots[5].imshow(binary, cmap=plt.cm.bone)

edges = roberts(binary)
 binary = ndi.binary_fill_holes(edges)
 if plot == True:
plots[6].axis('off')
 plots[6].imshow(binary, cmap=plt.cm.bone)

get_high_vals = binary == 0
im[get_high_vals] = 0
if plot == True:
plots[7].axis('off')
 plots[7].imshow(im, cmap=plt.cm.bone)
 return im
def get_kurtosis_stats(ids,ctscans_dir):
kurt=[]
std=[]
mean=[]
median=[]
for i in ids:
print(i)
 patient_path= ctscans_dir / i
scan = load_scan(patient_path)
 image=get_pixels_hu(scan)
 image, new_spacing = resample(image, scan, new_spacing=[2,2,2])
 image=np.asarray([get_segmented_lungs(slice)for slice in image])
 kurt_i=kurtosis(image.ravel() [image.ravel() < -200])
 std_i=image.ravel() [image.ravel() < -200].std()
mean_i=image.ravel() [image.ravel() < -200].mean()
median_i=np.median(image.ravel() [image.ravel() < -200])
 print('Kurtosis: ', kurt_i)
 print('Standard Deviation: ', std_i)
 kurt.append(kurt_i)
 std.append(std_i)
 mean.append(mean_i)
 median.append(median_i)
 ax=sns.kdeplot(image.ravel() [(image.ravel() < 0)&(image.ravel() > -1200)], bw=0.5)
 ax.set(xlabel='HU', ylabel='% voxels',title='Histogram of voxel characteristics')
 plt.show()
plt.imshow(image[round(image.shape[0]/2),:,:])
 plt.show()
return kurt,std,mean,median<load_from_csv>
def create_sector_lag(df_code,n_lag=[3,7,14,],shift_size=1):
for col in sector_return_features:
for window in n_lag:
if df_code["sectorNameSNo"].unique() [0] == -1:
df_code['%s_lag_%s_mean'%(col,window)] = -1
df_code['%s_lag_%s_max'%(col,window)] = -1
df_code['%s_lag_%s_min'%(col,window)] = -1
else:
rolled = df_code[[col,"mkt_cap"]].shift(shift_size ).rolling(window=window)
 lag_mean = rolled[col].sum() / rolled["mkt_cap"].sum()
lag_max = rolled[col].max() / rolled["mkt_cap"].sum()
lag_min = rolled[col].mean() / rolled["mkt_cap"].sum()
df_code['%s_lag_%s_mean'%(col,window)] = lag_mean
df_code['%s_lag_%s_max'%(col,window)] = lag_max
df_code['%s_lag_%s_min'%(col,window)] = lag_min
return df_code.fillna(-1)
 def generate_sector_lag_features(df,n_lag = [3,7,14]):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(df.columns)
 all_df = []
df_codes = df[['timeSNo','sectorNameSNo',"mkt_cap"] + sector_return_features].groupby(["timeSNo",'sectorNameSNo'], as_index=False ).sum()
df_codes = df_codes.groupby('sectorNameSNo')
 df_codes = [df_code[1][['timeSNo','sectorNameSNo',"mkt_cap"] + sector_return_features] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(create_sector_lag, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(sector_return_features + ["mkt_cap"],axis=1,inplace=True)
 pool.close()
if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(df.columns)))
 print("new mkt cols added: " + str(set(df.columns)- org_mkt_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 return new_df.reset_index()
train=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<merge>
fix_time_col(market_train_df, news_train_df)
 market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]
news_train_df = news_train_df.loc[news_train_df['time']>=date(2010, 1, 1)]
clean_assetName_col(market_train_df, news_train_df)
 dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo = set_assetNameCodes_SNo(market_train_df, news_train_df,dic_assetName, \
dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo)
 replace_assetName(market_train_df, news_train_df)
 drop_extra_news_cols(news_train_df)
 drop_irrelevant_news(news_train_df)
 drop_neutral_news(news_train_df)
 gen_netnovelty_col(news_train_df)
train['base_Weeks']=train.groupby(['Patient'])['Weeks'].transform('min')
 base=train[train.Weeks==train.base_Weeks]
base = base.rename(columns={'FVC': 'base_FVC','Percent': 'base_Percent'})
 base.drop_duplicates(subset=['Patient', 'Weeks'], keep='first',inplace=True)
 train=train.merge(base[['Patient','base_FVC','base_Percent']],on='Patient',how='left')
 train['Week_passed'] = train['Weeks'] - train['base_Weeks']<load_from_csv>
get_sector_name(news_train_df)
 merge_mkt_news(market_train_df, news_train_df)
test = test.rename(columns={'Weeks': 'base_Weeks', 'FVC': 'base_FVC','Percent': 'base_Percent'})
 submission = pd.read_csv(".. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv")
 submission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission['Weeks'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1] ).astype(int)
 test = submission.drop(columns = ["FVC", "Confidence"] ).merge(test, on = 'Patient')
 test['Week_passed'] = test['Weeks'] - test['base_Weeks']
test=test[train.columns.drop(['FVC','Percent'])]<data_type_conversions>
del news_train_df
gc.collect()
COLS = ['Sex','SmokingStatus']
for col in COLS:
for mod in train[col].unique() :
train[mod] =(train[col] == mod ).astype(int)
 test[mod] =(test[col] == mod ).astype(int)
 train.drop(col,axis=1,inplace=True)
 test.drop(col,axis=1,inplace=True )<load_from_csv>
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['timeSNo','assetCodeSNo'])
pixel_stats=pd.read_csv('.. /input/osic-histogram-features/train_pixel_stats.csv')
 train=train.merge(pixel_stats[['Patient','kurtosis','std','mean','median']],how='left',on='Patient' )<normalization>
new_df = generate_sector_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['timeSNo','sectorNameSNo'])
robust_scaler = preprocessing.RobustScaler()
train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.fit_transform(train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])] )<define_variables>
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )
epochs=300
batch_size=64
num_workers=3
quantiles =(0.2, 0.5, 0.8 )<split>
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', "timeSNo",
'universe','sourceTimestamp', "sectorName","index","mkt_cap","open","close"]]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
del market_train_df
gc.collect()
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
X_train, X_test, up_train, up_test, r_train, r_test,u_train,u_test,d_train,d_test = \
model_selection.train_test_split(X, up, r,universe,d,test_size=0.25, random_state=99)
 print(X_train.shape)
 train_data = lgb.Dataset(X, label=up.astype(int), feature_name=fcol, categorical_feature=["assetCodeSNo","assetNameSNo","sectorNameSNo"], free_raw_data=False)
 test_data = lgb.Dataset(X_test, label=up_test.astype(int), feature_name=fcol, categorical_feature=["assetCodeSNo","assetNameSNo","sectorNameSNo"], free_raw_data=False )
ids=train.Patient.unique()
index = np.argwhere(ids=='ID00011637202177653955184')
 ids = list(np.delete(ids, index))
random.shuffle(ids)
 ids=np.array(ids)
 train_ids,val_ids=np.split(ids, [int(round(0.9 * len(ids), 0)) ])
 train_dataset = OSIC(train_ids,train)
 train_dataloader = DataLoader(train_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers)
 val_dataset = OSIC(val_ids,train)
 val_dataloader = DataLoader(val_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers )<load_pretrained>
del X, X_train
gc.collect()
model = main_model(train_dataset.data.shape[1], 3 ).to(device)
 print('Number of parameters:')
 print(sum(p.numel() for p in model.parameters() if p.requires_grad))<train_on_grid>
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
optimizer = optim.Adam(model.parameters())
 epoch_train_metric=[]
epoch_val_metric=[]
epoch_train_loss=[]
epoch_val_loss=[]
epoch=0
min_val_loss = 1e+100
early_stop = False
patience=10
while epoch<epochs and not early_stop:
epoch+=1
train_loss=0
train_metric=0
model.train()
for batch_idx, data in enumerate(train_dataloader):
optimizer.zero_grad()
model_output = model(data['data'].float().to(device))
quantile_loss = model.quantile_loss(model_output, data['fvc'].to(device), quantiles)
 metric_loss = model.metric_loss(model_output,data['fvc'].to(device)).mean()
loss=quantile_loss
loss.backward()
train_loss += loss.item()
train_metric += metric_loss.item()
optimizer.step()
print('====> Epoch: {} Average train quantile loss: {:.4f}'.format(
 epoch, train_loss / len(train_dataloader)))
 print('====> Epoch: {} Average train metric: {:.4f}'.format(
 epoch, train_metric / len(train_dataloader)))
 val_loss=0
val_metric=0
model.eval()
with torch.no_grad() :
for batch_idx, data in enumerate(val_dataloader):
model_output = model(data['data'].float().to(device))
quantile_loss = model.quantile_loss(model_output, data['fvc'].to(device), quantiles)
 metric_loss = model.metric_loss(model_output,data['fvc'].to(device)).mean()
loss=quantile_loss
val_loss += loss.item()
val_metric += metric_loss.item()
print('====> Epoch: {} Average val quantile loss: {:.4f}'.format(
 epoch, val_loss / len(val_dataloader)))
 print('====> Epoch: {} Average val metric: {:.4f}'.format(
 epoch, val_metric / len(val_dataloader)))
 epoch_train_loss.append(train_loss/ len(train_dataloader))
epoch_val_loss.append(val_loss / len(val_dataloader))
epoch_train_metric.append(train_metric/ len(train_dataloader))
epoch_val_metric.append(val_metric / len(val_dataloader))
if val_loss < min_val_loss:
min_val_loss = val_loss
best_model = copy.deepcopy(model.state_dict())
 print('Min loss %0.2f' % min_val_loss)
 epochs_no_improve = 0
else:
epochs_no_improve += 1
if epochs_no_improve == patience:
print('Early stopping!')
 early_stop = True
model.load_state_dict(best_model)
<load_from_csv>
del X_test, d_train
gc.collect()
submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<categorify>
def process_mkt_news_data(market_obs_df, news_obs_df):
global dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo
fix_time_col(market_obs_df, news_obs_df)
 curtime = market_obs_df["time"].unique()
print(curtime)
 found=1
if len(total_market_obs_df)>1:
while found==1:
prev_df = total_market_obs_df[-1]
if prev_df["time"].unique() == curtime:
print("removing------------------------")
 total_market_obs_df.pop(-1)
 total_news_obs_df.pop(-1)
 else:
found=0
clean_assetName_col(market_obs_df, news_obs_df)
 dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo = set_assetNameCodes_SNo(market_obs_df, news_obs_df,dic_assetName, dic_assetNameSNo, \
dic_assetCodes,dic_assetCode, dic_assetCodeSNo)
 drop_extra_news_cols(news_obs_df)
 drop_irrelevant_news(news_obs_df)
 drop_neutral_news(news_obs_df)
 gen_netnovelty_col(news_obs_df)
 get_sector_name(news_obs_df, True)
 return dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo, curtime
test_ids=test.Patient.unique()
test_dataset = OSIC(test_ids,test,train=False)
 test_dataloader = DataLoader(test_dataset, batch_size=batch_size,shuffle=False,num_workers=num_workers)
 model.eval()
fvc_pred = []
sigma_pred = []
with torch.no_grad() :
for batch_idx, data in enumerate(test_dataloader):
model_output = model(data['data'].float().to(device))
fvc_pred.append(model_output[:,1])
 sigma_pred.append(model_output[:,2]-model_output[:,0])
 fvc_pred=torch.cat(fvc_pred, dim=0)
 sigma_pred=torch.cat(sigma_pred, dim=0)
 test['FVC']=fvc_pred.cpu().numpy()
test['Confidence']=sigma_pred.cpu().numpy()<feature_engineering>
if len(total_market_obs_df)== 0:
for i in range(len(prv_mkt_data)) :
mkt_data = prv_mkt_data[i]
news_data = prv_news_data[i]
mkt_data.drop(["universe","returnsOpenNextMktres10"], inplace=True, axis=1)
 dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo, curtime = process_mkt_news_data(mkt_data, news_data)
 total_market_obs_df.append(mkt_data)
 total_news_obs_df.append(news_data)
 def make_random_predictions_new(market_obs_df, news_obs_df,predictions_template_df):
global n_days
global prep_time
global prediction_time
global packaging_time
global total_market_obs_df
global total_news_obs_df
global dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo, curtime = process_mkt_news_data(market_obs_df, news_obs_df)
 total_market_obs_df.append(market_obs_df)
 total_news_obs_df.append(news_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
history_news_df = total_news_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):], sort=False)
 history_news_df = pd.concat(total_news_obs_df[-(np.max(n_lag)+1):], sort=False)
 history_df.sort_values(["timeSNo"], inplace=True)
 history_df.index = list(range(history_df.shape[0]))
history_news_df.sort_values(["timeSNo"], inplace=True)
 history_news_df.index = list(range(history_news_df.shape[0]))
fixopenclose(history_df)
 merge_mkt_news(history_df, history_news_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 history_df = pd.merge(history_df,new_df,how='left',on=['timeSNo','assetCodeSNo'])
 new_df = generate_sector_lag_features(history_df,n_lag=[3,7,14])
 history_df = pd.merge(history_df,new_df,how='left',on=['timeSNo','sectorNameSNo'])
 history_df = mis_impute(history_df)
 history_df.dropna(inplace=True)
 market_obs_df = history_df[history_df["time"].isin(curtime)]
X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
return market_obs_df
test['Patient_Week']=test["Patient"] + '_' + test['Weeks'].apply(str )<merge>
market_obs_df, news_obs_df, predictions_template_df = next(days )
submission=submission[['Patient_Week']].merge(test[['Patient_Week','FVC','Confidence']],on='Patient_Week' )<save_to_csv>
market_obs_df = make_random_predictions_new(market_obs_df, news_obs_df,predictions_template_df )
submission.to_csv('submission.csv', index=False, float_format='%.1f' )<train_model>
for(market_obs_df, news_obs_df, predictions_template_df)in days:
market_obs_df = make_random_predictions_new(market_obs_df, news_obs_df,predictions_template_df)
 print('Done!' )
<compute_train_metric><EOS> 
env.write_submission_file()
sub = pd.read_csv("submission.csv" )