Kaggle competition: Two Sigma: Using News to Predict Stock Movements
train_df=pd.read_csv('/kaggle/input/DontGetKicked/training.csv')
 train_df.head()<load_from_csv>
pd.options.mode.chained_assignment = None
%matplotlib inline
init_notebook_mode(connected=True)
 cf.go_offline()
test_df=pd.read_csv('/kaggle/input/DontGetKicked/test.csv')
 test_df.head()<count_missing_values>
env = twosigmanews.make_env()
print('Done!' )
train_df.isnull().sum()<count_missing_values>
( market_train_df, news_train_df)= env.get_training_data()
print(market_train_df['time'].head)
 print(market_train_df.shape )
test_df.isnull().sum()<count_values>
market_train_df['date'] = market_train_df['time'].dt.strftime('%Y-%m-%d')
 print(market_train_df['time'].dtype)
 print(market_train_df['date'].dtype)
 market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]
print(market_train_df.shape )
train_df['IsBadBuy'].value_counts()<count_values>
def sampleAssetData(assetCode, date, numDays):
d = datetime.strptime(date,'%Y-%m-%d')
 start = d - timedelta(days=numDays)
 end = d + timedelta(days=numDays)
 return market_train_df[(market_train_df['assetCode'] == assetCode)
 &(market_train_df['time'] >= start.date())
 &(market_train_df['time'] <= end.date())].copy()
def updateRawReturns(assetData, indices):
rowsToUpdate1 = assetData[(assetData.index >= indices[0][0])&(assetData.index <= indices[0][1])]
for index, row in rowsToUpdate1.iterrows() :
market_train_df.loc[[index],['returnsClosePrevRaw1']] = assetData['close'].pct_change()
market_train_df.loc[[index],['returnsOpenPrevRaw1']] = assetData['open'].pct_change()
rowsToUpdate2 = assetData[(assetData.index >= indices[1][0])&(assetData.index <= indices[1][1])]
for index, row in rowsToUpdate2.iterrows() :
market_train_df.loc[[index],['returnsClosePrevRaw10']] = assetData['close'].pct_change(periods=10)
 market_train_df.loc[[index],['returnsOpenPrevRaw10']] = assetData['open'].pct_change(periods=10)
 def estimateMktresReturn(sampleData, mktresCol, index):
sampleData['ones'] = 1
sampleData.dropna(inplace=True)
 rawCol = mktresCol.replace('Mktres','Raw')
 A = sampleData[[rawCol,'ones']]
y = sampleData[mktresCol]
m, c = np.linalg.lstsq(A,y,rcond=-1)[0]
return c + m * market_train_df.loc[index,rawCol]
def updateMktresReturns(assetCode, assetData, indices):
sample1 = assetData[(assetData.index < indices[2][0])|(assetData.index > indices[2][1])]
rowsToUpdate1 = assetData[(assetData.index >= indices[2][0])&(assetData.index <= indices[2][1])]
for index, row in rowsToUpdate1.iterrows() :
market_train_df.loc[[index],['returnsClosePrevMktres1']] = estimateMktresReturn(sample1,'returnsClosePrevMktres1',index)
 market_train_df.loc[[index],['returnsOpenPrevMktres1']] = estimateMktresReturn(sample1,'returnsOpenPrevMktres1',index)
 sample2 = assetData[(assetData.index < indices[3][0])|(assetData.index > indices[3][1])]
rowsToUpdate2 = assetData[(assetData.index >= indices[3][0])&(assetData.index <= indices[3][1])]
l = []
for index, row in rowsToUpdate2.iterrows() :
market_train_df.loc[[index],['returnsClosePrevMktres10']] = estimateMktresReturn(sample2,'returnsClosePrevMktres10',index)
 est = estimateMktresReturn(sample2,'returnsOpenPrevMktres10',index)
 l.append(est)
 market_train_df.loc[[index],['returnsOpenPrevMktres10']] = est
rowsToUpdate3 = assetData[(assetData.index >= indices[4][0])&(assetData.index <= indices[4][1])]
i = 0
for index, row in rowsToUpdate3.iterrows() :
market_train_df.loc[[index],['returnsOpenNextMktres10']] = l[i]
i += 1
def fixBadReturnData(assetCode, badDate, badIndex, badReturnDataRanges, dayWindow):
badDataWindow = sampleAssetData(assetCode,badDate,dayWindow)
 badDataWindow.reset_index(inplace=True)
 newIdx = badDataWindow[badDataWindow['index'] == badIndex].index[0]
indices = [
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[0],'index']],
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[1],'index']],
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[2],'index']],
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[3],'index']],
[badDataWindow.loc[newIdx+badReturnDataRanges[4],'index'],badDataWindow.loc[newIdx+badReturnDataRanges[5],'index']]
]
badDataWindow.set_index('index',inplace=True)
 updateRawReturns(badDataWindow,indices)
 updateMktresReturns(assetCode,badDataWindow,indices)
train_df['Model'].value_counts()<drop_column>
assetCode = 'ZNGA.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 19213100
market_train_df.loc[[badIndex],['open']] = 2.64
market_train_df.loc[[badIndex],['close']] = 2.75
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'FLEX.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 5406600
market_train_df.loc[[badIndex],['open']] = 11.580
market_train_df.loc[[badIndex],['close']] = 11.750
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'SHLD.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 279300
market_train_df.loc[[badIndex],['open']] = 12.8900
market_train_df.loc[[badIndex],['close']] = 13.1400
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'MAT.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 3242100
market_train_df.loc[[badIndex],['open']] = 32.13
market_train_df.loc[[badIndex],['close']] = 31.52
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'BBBY.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 4205500
market_train_df.loc[[badIndex],['open']] = 42.23
market_train_df.loc[[badIndex],['close']] = 43.55
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'DISH.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 2303300
market_train_df.loc[[badIndex],['open']] = 50.06
market_train_df.loc[[badIndex],['close']] = 51.33
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'NDAQ.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 733400
market_train_df.loc[[badIndex],['open']] = 64.64
market_train_df.loc[[badIndex],['close']] = 64.74
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'PCAR.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 2394300
market_train_df.loc[[badIndex],['open']] = 50.16
market_train_df.loc[[badIndex],['close']] = 50.79
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'PZZA.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 185100
market_train_df.loc[[badIndex],['open']] = 67.86
market_train_df.loc[[badIndex],['close']] = 67.91
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow )
train_df.drop('Model',axis=1,inplace=True)
 test_df.drop('Model',axis=1,inplace=True )<count_values>
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'] )
train_df['Trim'].value_counts()<drop_column>
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )
train_df.drop('Trim',axis=1,inplace=True)
 test_df.drop('Trim',axis=1,inplace=True )<count_values>
def data_prep(market_train):
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train_df = data_prep(market_train_df)
 print(market_train_df.shape )
train_df['SubModel'].value_counts()<drop_column>
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'date' , 'universe','sourceTimestamp']]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
X_train, X_test, up_train, up_test, r_train, r_test,u_train,u_test,d_train,d_test = model_selection.train_test_split(X, up, r,universe,d, test_size=0.25, random_state=99)
 train_data = lgb.Dataset(X, label=up.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))
train_df.drop('SubModel',axis=1,inplace=True)
 test_df.drop('SubModel',axis=1,inplace=True )<count_values>
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
train_df['Color'].value_counts()<count_values>
del X_train,X_test
gc.collect()
<data_type_conversions><EOS> 
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = mis_impute(market_obs_df)
 market_obs_df = data_prep(market_obs_df)
 X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
sub = pd.read_csv("submission.csv" )