Kaggle competition: CareerCon 2019 - Help Navigate Robots
u_valid =(universe.iloc[n_train:] > 0)
 t_valid = time.iloc[n_train:]
X_valid = X_valid[u_valid]
y_valid = y_valid[u_valid]
t_valid = t_valid[u_valid]
d_valid = t_valid.dt.date
del u_valid<feature_engineering>
data = pd.read_csv('.. /input/X_train.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 test = pd.read_csv('.. /input/X_test.csv' )
def get_cat_num_split(df):
X = {}
X['num'] = df.loc[:, num_cols].values
X['num'] = np.reshape(X['num'],(X['num'].shape[0], 1, X['num'].shape[1]))
for cat in categorical_cols:
X[cat] = df.loc[:, cat].values
return X<split>
data = data[['series_id', 'measurement_number', 'orientation_X',
'orientation_Y', 'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z']]
X_train_split = get_cat_num_split(X_train)
 X_valid_split = get_cat_num_split(X_valid)
 y_train_bin =(y_train >= 0 ).values
y_valid_bin =(y_valid >= 0 ).values<define_variables>
f_data = pd.merge(data, target, how='inner')
 full_data = f_data[['series_id', 'measurement_number', 'orientation_X',
'orientation_Y', 'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z', 'surface']]
encoding_len = {k: len(encodings[k])+ 1 for k in encodings}<choose_model_class>
from sklearn.ensemble import BaggingClassifier, RandomForestClassifier
DROPOUT_RATE = 0.2
cat_inputs = [Input(shape=[1], name=cat)for cat in categorical_cols]
embeddings = [Embedding(encoding_len[cat], embedding_lengths[i] )(cat_inputs[i])for i, cat in enumerate(categorical_cols)]
categorical_logits = Concatenate()([(cat_emb)for cat_emb in embeddings])
 categorical_logits = LSTM(128, activation='relu', input_shape=(1, len(categorical_cols)) , return_sequences=True,
kernel_regularizer=l2(1e-5), kernel_initializer='random_uniform' )(categorical_logits)
 numerical_inputs = Input(shape=(1, len(num_cols)) , name='num')
 numerical_logits = LSTM(256, activation='relu', input_shape=(1, len(num_cols)) , return_sequences=True,
kernel_regularizer=l2(1e-5), kernel_initializer='random_uniform' )(numerical_inputs)
 numerical_logits = Dropout(DROPOUT_RATE )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = LSTM(256, activation='relu', kernel_initializer='random_uniform' )(logits)
 out = Dense(1, activation='sigmoid', name='confidence_level' )(logits)
 model = Model(inputs = cat_inputs + [numerical_inputs], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer='adam' )<train_model>
colum = ['series_id','orientation_X', 'orientation_Y',
'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z']
tr_dat = full_data[colum+['surface']].groupby(['series_id'] ).max()
tr_dat_1 = full_data[colum].groupby(['series_id'] ).min()
tr_dat_2 = full_data[colum].groupby(['series_id'] ).median()
tr_dat_3 = full_data[colum].groupby(['series_id'] ).mean()
tr_dat_4 = full_data[colum].groupby(['series_id'] ).std()
tr_dat_5 = full_data[colum].groupby(['series_id'] ).quantile(0.25)
 tr_dat_6 = full_data[colum].groupby(['series_id'] ).quantile(0.5)
 tr_dat_7 = full_data[colum].groupby(['series_id'] ).quantile(0.75)
 tr_dat_8 = full_data[colum].groupby(['series_id'] ).max() /full_data[colum].groupby(['series_id'] ).min()
check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=1,verbose=True)
 model.fit(X_train_split, y_train_bin,
validation_data=(X_valid_split, y_valid_bin),
epochs=1,
verbose=True,
callbacks=[early_stop,check_point] )<compute_test_metric>
tr_dat_5 = full_data[colum].groupby(['series_id'] ).quantile(0.25)
 tr_dat_5.head()
x_t_i = confidence_valid * y_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<predict_on_test>
ss = [tr_dat,tr_dat_1,tr_dat_2,tr_dat_3,tr_dat_4,tr_dat_5,tr_dat_6,tr_dat_7,tr_dat_8]
df_final = reduce(lambda left,right: pd.merge(left,right,on='series_id'), ss)
 df_final.head()
def make_predictions(predictions_template_df, market_obs_df, news_obs_df):
inp = get_x(market_obs_df, news_obs_df, False)
 scale_numeric(inp)
 inp_split = get_cat_num_split(inp)
 scaled_pred = model.predict(inp_split)* 2 - 1
predictions_template_df.confidenceValue = np.clip(scaled_pred, -1, 1 )<predict_on_test>
ts_dat = test[colum].groupby(['series_id'] ).max()
ts_dat_1 = test[colum].groupby(['series_id'] ).min()
ts_dat_2 = test[colum].groupby(['series_id'] ).median()
ts_dat_3 = test[colum].groupby(['series_id'] ).mean()
ts_dat_4 = test[colum].groupby(['series_id'] ).std()
ts_dat_5 = test[colum].groupby(['series_id'] ).quantile(0.25)
 ts_dat_6 = test[colum].groupby(['series_id'] ).quantile(0.5)
 ts_dat_7 = test[colum].groupby(['series_id'] ).quantile(0.75)
 ts_dat_8 = test[colum].groupby(['series_id'] ).max() /test[colum].groupby(['series_id'] ).min()
ts_ss = [ts_dat,ts_dat_1,ts_dat_2,ts_dat_3,ts_dat_4,ts_dat_5,ts_dat_6,ts_dat_7,ts_dat_8]
ts_final = reduce(lambda left,right: pd.merge(left,right,on='series_id'), ts_ss)
 cols = ['orientation_X_x', 'orientation_Y_x', 'orientation_Z_x',
'orientation_W_x', 'angular_velocity_X_x', 'angular_velocity_Y_x',
'angular_velocity_Z_x', 'linear_acceleration_X_x',
'linear_acceleration_Y_x', 'linear_acceleration_Z_x',
'orientation_X_y', 'orientation_Y_y', 'orientation_Z_y',
'orientation_W_y', 'angular_velocity_X_y', 'angular_velocity_Y_y',
'angular_velocity_Z_y', 'linear_acceleration_X_y',
'linear_acceleration_Y_y', 'linear_acceleration_Z_y', 'orientation_X_x',
'orientation_Y_x', 'orientation_Z_x', 'orientation_W_x',
'angular_velocity_X_x', 'angular_velocity_Y_x', 'angular_velocity_Z_x',
'linear_acceleration_X_x', 'linear_acceleration_Y_x',
'linear_acceleration_Z_x', 'orientation_X_y', 'orientation_Y_y',
'orientation_Z_y', 'orientation_W_y', 'angular_velocity_X_y',
'angular_velocity_Y_y', 'angular_velocity_Z_y',
'linear_acceleration_X_y', 'linear_acceleration_Y_y',
'linear_acceleration_Z_y', 'orientation_X_x', 'orientation_Y_x',
'orientation_Z_x', 'orientation_W_x', 'angular_velocity_X_x',
'angular_velocity_Y_x', 'angular_velocity_Z_x',
'linear_acceleration_X_x', 'linear_acceleration_Y_x',
'linear_acceleration_Z_x', 'angular_velocity_X_y',
'angular_velocity_Y_y', 'angular_velocity_Z_y',
'linear_acceleration_X_y', 'linear_acceleration_Y_y',
'linear_acceleration_Z_y', 'orientation_W_y', 'orientation_X_y',
'orientation_Y_y', 'orientation_Z_y', 'angular_velocity_X_x',
'angular_velocity_Y_x', 'angular_velocity_Z_x',
'linear_acceleration_X_x', 'linear_acceleration_Y_x',
'linear_acceleration_Z_x', 'orientation_W_x', 'orientation_X_x',
'orientation_Y_x', 'orientation_Z_x', 'angular_velocity_X_y',
'angular_velocity_Y_y', 'angular_velocity_Z_y',
'linear_acceleration_X_y', 'linear_acceleration_Y_y',
'linear_acceleration_Z_y', 'orientation_W_y', 'orientation_X_y',
'orientation_Y_y', 'orientation_Z_y', 'orientation_X', 'orientation_Y',
'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z']
days = env.get_prediction_days()
x1,y1,z1 = None, None, None
for(market_obs_df, news_obs_df, predictions_template_df)in days:
x1,y1,z1 = predictions_template_df, market_obs_df, news_obs_df
make_predictions(x1,y1,z1)
 env.predict(predictions_template_df)
 print('Done!' )<save_to_csv>
trains = df_final[cols]
targs = df_final['surface']
tests = ts_final[cols]
env.write_submission_file()<train_model>
new_feat_x = []
new_feat_y = []
new_feat_z = []
new_feat_w = []
skm = LinearRegression()
for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_X'].index))
skm.fit(x, qq['orientation_X'].values)
 new_feat_x.append([skm.intercept_, skm.coef_[0]])
 for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Y'].index))
skm.fit(x, qq['orientation_Y'].values)
 new_feat_y.append([skm.intercept_, skm.coef_[0]])
 for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Z'].index))
skm.fit(x, qq['orientation_Z'].values)
 new_feat_z.append([skm.intercept_, skm.coef_[0]])
 for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_W'].index))
skm.fit(x, qq['orientation_W'].values)
 new_feat_w.append([skm.intercept_, skm.coef_[0]])
 new_feat_x = np.array(new_feat_x)
 new_feat_y = np.array(new_feat_y)
 new_feat_z = np.array(new_feat_z)
 new_feat_w = np.array(new_feat_w)
env = twosigmanews.make_env() 
(marketdf, newsdf)= env.get_training_data() 
<categorify>
trains['intercept_x'] = new_feat_x[:,0]
trains['coef_x'] = new_feat_x[:,1]
trains['intercept_y'] = new_feat_y[:,0]
trains['coef_y'] = new_feat_y[:,1]
trains['intercept_z'] = new_feat_z[:,0]
trains['coef_z'] = new_feat_z[:,1]
trains['intercept_w'] = new_feat_w[:,0]
trains['coef_w'] = new_feat_w[:,1]
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close' ]
add_features=['pricevolume' , 'bartrend' , 'average' , 'returnsOpenNextMktres10']
def create_lag(df_code,n_lag=[3,7,10], shift_size=1):
code = df_code['assetCode'].unique()
for col in return_features:
for window in n_lag:
rolled = df_code[col].shift(shift_size ).rolling(window=window)
 lag_mean = rolled.mean()
df_code['%s_lag_%s_mean'%(col,window)] = lag_mean
return df_code.fillna(-1)
 def generate_lag_features(df,n_lag = [3,7,10]):
features = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10',
'returnsOpenNextMktres10', 'universe']
assetCodes = df['assetCode'].unique()
all_df = []
df['pricevolume'] = df['volume'] * df['close']
df['bartrend'] = df['close'] / df['open']
df['average'] =(df['close'] + df['open'])/2
df_codes = df.groupby('assetCode')
 df_codes = [df_code[1][['time','assetCode'] + add_features + return_features ] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(create_lag, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(return_features,axis=1,inplace=True)
 pool.close()
return new_df<merge>
new_feat_x = []
new_feat_y = []
new_feat_z = []
new_feat_w = []
new_feat_ax = []
new_feat_ay = []
new_feat_az = []
for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_X'].index))
skm.fit(x, qq['orientation_X'].values)
 new_feat_x.append([skm.intercept_, skm.coef_[0]])
 for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Y'].index))
skm.fit(x, qq['orientation_Y'].values)
 new_feat_y.append([skm.intercept_, skm.coef_[0]])
 for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Z'].index))
skm.fit(x, qq['orientation_Z'].values)
 new_feat_z.append([skm.intercept_, skm.coef_[0]])
 for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_W'].index))
skm.fit(x, qq['orientation_Z'].values)
 new_feat_w.append([skm.intercept_, skm.coef_[0]])
 new_feat_x = np.array(new_feat_x)
 new_feat_y = np.array(new_feat_y)
 new_feat_z = np.array(new_feat_z)
 new_feat_w = np.array(new_feat_w)
 tests['intercept_x'] = new_feat_x[:,0]
tests['coef_x'] = new_feat_x[:,1]
tests['intercept_y'] = new_feat_y[:,0]
tests['coef_y'] = new_feat_y[:,1]
tests['intercept_z'] = new_feat_z[:,0]
tests['coef_z'] = new_feat_z[:,1]
tests['intercept_w'] = new_feat_w[:,0]
tests['coef_w'] = new_feat_w[:,1]
<data_type_conversions>
clf = RandomForestClassifier(n_estimators = 2000,
max_depth=20,
min_samples_split=5,
class_weight='balanced')
 clf.fit(trains, targs)
 ress = clf.predict(tests )
def prepare_data(marketdf, newsdf):
n_lag = [3,7,10]
marketdf = generate_lag_features(marketdf, n_lag = n_lag)
 marketdf['time'] = marketdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['time'] = newsdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['assetCode'] = newsdf['assetCodes'].map(lambda x: list(eval(x)) [0])
 newsdf['position'] = newsdf['firstMentionSentence'] / newsdf['sentenceCount']
newsdf['coverage'] = newsdf['sentimentWordCount'] / newsdf['wordCount']
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'assetCodes','urgency','wordCount','sentimentWordCount']
newsdf.drop(droplist, axis=1, inplace=True)
 newsgp = newsdf.groupby(['time','assetCode'], sort=False ).aggregate(np.mean ).reset_index()
return pd.merge(marketdf, newsgp, how='left', on=['time', 'assetCode'], copy=False)
 cdf = prepare_data(marketdf, newsdf )<prepare_x_and_y>
submission = pd.read_csv(".. /input/sample_submission.csv")
 print(len(submission))
print(test.shape)
 submission['surface'] = ress
submission.to_csv("NEW_submission.csv", index = False)
 submission.head()