Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<feature_engineering>
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
def prepare_data(marketdf, newsdf):
marketdf['time'] = marketdf.time.dt.strftime("%Y%m%d" ).astype(int)
 marketdf['bartrend'] = marketdf['close'] / marketdf['open']
marketdf['average'] =(marketdf['close'] + marketdf['open'])/2
marketdf['pricevolume'] = marketdf['volume'] * marketdf['close']
newsdf['time'] = newsdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['assetCode'] = newsdf['assetCodes'].map(lambda x: list(eval(x)) [0])
 newsdf['position'] = newsdf['firstMentionSentence'] / newsdf['sentenceCount']
newsdf['coverage'] = newsdf['sentimentWordCount'] / newsdf['wordCount']
marketdf = marketdf.loc[marketdf['time'] > 20120000]
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'assetCodes','urgency','wordCount','sentimentWordCount']
newsdf.drop(droplist, axis=1, inplace=True)
 marketdf.drop(['assetName', 'volume'], axis=1, inplace=True)
 newsgp = newsdf.groupby(['time','assetCode'], sort=False ).aggregate(np.mean ).reset_index()
return pd.merge(marketdf, newsgp, how='left', on=['time', 'assetCode'], copy=False)
 cdf = prepare_data(marketdf, newsdf)
 del marketdf, newsdf<prepare_x_and_y>
x = pd.read_csv('.. /input/X_train.csv')
 y = pd.read_csv('.. /input/y_train.csv')
 test = pd.read_csv('.. /input/X_test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv' )
print('building training set...')
 targetcols = ['returnsOpenNextMktres10']
traincols = [col for col in cdf.columns if col not in ['time', 'assetCode', 'universe'] + targetcols]
dates = cdf['time'].unique()
train = range(len(dates)) [:int(0.85*len(dates)) ]
val = range(len(dates)) [int(0.85*len(dates)) :]
cdf[targetcols[0]] =(cdf[targetcols[0]] > 0 ).astype(int)
 Xt = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Yt = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Xv = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
Yv = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
print(Xt.shape, Xv.shape )<train_model>
x['series_id'].nunique()
print('Training lightgbm')
 params = {"objective" : "binary",
"metric" : "binary_logloss",
"num_leaves" : 125,
"max_depth": -1,
"learning_rate" : 0.0005,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
"bagging_freq" : 5,
"bagging_seed" : 2018,
"verbosity" : -1 }
lgtrain, lgval = lgb.Dataset(Xt, Yt[:,0]), lgb.Dataset(Xv, Yv[:,0])
 lgbmodel = lgb.train(params, lgtrain, 2000, valid_sets=[lgtrain, lgval], early_stopping_rounds=100, verbose_eval=200)
<normalization>
x[x.isnull().any(axis=1)]
def post_scaling(df):
mean, std = np.mean(df), np.std(df)
 df =(df - mean)/(std * 8)
 return np.clip(df,-1,1 )<predict_on_test>
y[y.isnull().any(axis=1)]
print("generating predictions...")
 preddays = env.get_prediction_days()
for marketdf, newsdf, predtemplatedf in preddays:
cdf = prepare_data(marketdf, newsdf)
 Xp = cdf[traincols].fillna(0 ).values
preds = lgbmodel.predict(Xp, num_iteration=lgbmodel.best_iteration)* 2 - 1
predsdf = pd.DataFrame({'ast':cdf['assetCode'],'conf':post_scaling(preds)})
 predtemplatedf['confidenceValue'][predtemplatedf['assetCode'].isin(predsdf.ast)] = predsdf['conf'].values
env.predict(predtemplatedf)
 env.write_submission_file() 
<set_options>
test[test.isnull().any(axis=1)]
pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 %matplotlib inline<split>
le = LabelEncoder()
le.fit(y['surface'])
 y['surface'] = le.transform(y['surface'] )
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()<define_variables>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def fe(df):
df['total_angular_velocity'] =(df['angular_velocity_X'] ** 2 + df['angular_velocity_Y'] ** 2 + df['angular_velocity_Z'] ** 2)** 0.5
df['total_linear_acceleration'] =(df['linear_acceleration_X'] ** 2 + df['linear_acceleration_Y'] ** 2 + df['linear_acceleration_Z'] ** 2)** 0.5
df['total_xyz'] =(df['orientation_X']**2 + df['orientation_Y']**2 +df['orientation_Z'])**0.5
df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
temp_df = pd.DataFrame()
for col in df.columns[3:]:
temp_df[col + '_mean'] = df.groupby(['series_id'])[col].mean()
temp_df[col + '_median'] = df.groupby(['series_id'])[col].median()
temp_df[col + '_max'] = df.groupby(['series_id'])[col].max()
temp_df[col + '_min'] = df.groupby(['series_id'])[col].min()
temp_df[col + '_std'] = df.groupby(['series_id'])[col].std()
temp_df[col + '_range'] = temp_df[col + '_max'] - temp_df[col + '_min']
temp_df[col + '_maxtoMin'] = temp_df[col + '_max'] / temp_df[col + '_min']
temp_df[col + '_mean_abs_chg'] = df.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
temp_df[col + '_abs_min'] = df.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 temp_df[col + '_abs_max'] = df.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 temp_df[col + '_abs_avg'] =(temp_df[col + '_abs_min'] + temp_df[col + '_abs_max'])/2
return temp_df
news_cols_agg = {
'bodySize': ['min', 'max', 'mean', 'std'],
'sentimentNegative': ['min', 'max', 'mean', 'std'],
'sentimentNeutral': ['min', 'max', 'mean', 'std'],
'sentimentPositive': ['min', 'max', 'mean', 'std'],
}
categorical_cols = ['assetName', 'dayofweek', 'month', 'year']
embedding_lengths = [100, 2, 2, 3]
encodings = {}<merge>
train=fe(x)
 train.head()
def join_market_news(market_train_df, news_train_df):
news_train_df['assetCodes'] = news_train_df['assetCodes'].str.findall(f"'([\w\./]+)'")
 assetCodes_expanded = list(chain(*news_train_df['assetCodes']))
assetCodes_index = news_train_df.index.repeat(news_train_df['assetCodes'].apply(len))
assert len(assetCodes_index)== len(assetCodes_expanded)
 df_assetCodes = pd.DataFrame({'level_0': assetCodes_index, 'assetCode': assetCodes_expanded})
 news_cols = ['time', 'assetCodes'] + sorted(news_cols_agg.keys())
 news_train_df_expanded = pd.merge(df_assetCodes, news_train_df[news_cols], left_on='level_0', right_index=True, suffixes=(['','_old']))
del news_train_df, df_assetCodes
news_train_df_aggregated = news_train_df_expanded.groupby(['time', 'assetCode'] ).agg(news_cols_agg)
 del news_train_df_expanded
news_train_df_aggregated = news_train_df_aggregated.apply(np.float32)
 news_train_df_aggregated.columns = ['_'.join(col ).strip() for col in news_train_df_aggregated.columns.values]
market_train_df = market_train_df.join(news_train_df_aggregated, on=['time', 'assetCode'])
 del news_train_df_aggregated
return market_train_df<categorify>
test_df=fe(test)
 test_df.head()
def get_xy(market_train_df, news_train_df):
x = get_x(market_train_df, news_train_df)
 y = market_train_df['returnsOpenNextMktres10'].clip(-1, 1)
 return x, y
def label_encode(series, min_counts=2):
vc = series.value_counts()
le = {c : i+1 for i, c in enumerate(vc.index[vc > min_counts])}
le['UNKN'] = 0
return le
def get_encodings(df, cat_cols):
if len(encodings)== 0:
for col in cat_cols:
encodings[col] = label_encode(df[col])
 return encodings
def map_encodings(df, cat_cols, encs):
for col in cat_cols:
df[col] = df[col].map(encs[col] ).fillna(0 ).astype(int)
 def get_x(market_train_df, news_train_df, isTrain=True):
news_train_df['time'] =(news_train_df['time'] - np.timedelta64(22,'h')).dt.ceil('1D')
 market_train_df['time'] = market_train_df['time'].dt.floor('1D')
 x = join_market_news(market_train_df, news_train_df)
 x['dayofweek'], x['day'], x['month'], x['year'] = x.time.dt.dayofweek, x.time.dt.day, x.time.dt.month, x.time.dt.year
encodings = get_encodings(x, categorical_cols)
 map_encodings(x, categorical_cols, encodings)
 if isTrain:
cols_to_drop = ['returnsOpenNextMktres10', 'universe', 'time']
else:
cols_to_drop = ['time']
x.drop(columns=cols_to_drop, inplace=True)
 return x<prepare_x_and_y>
train.fillna(0, inplace = True)
 train.replace(-np.inf, 0, inplace = True)
 train.replace(np.inf, 0, inplace = True)
 test_df.fillna(0, inplace = True)
 test_df.replace(-np.inf, 0, inplace = True)
 test_df.replace(np.inf, 0, inplace = True )
X, y = get_xy(market_train_df, news_train_df )<drop_column>
skf = StratifiedKFold(n_splits=50, shuffle=True, random_state=2019 )
universe = market_train_df['universe']
time = market_train_df['time']
del market_train_df, news_train_df<define_variables>
preds = np.zeros(( test_df.shape[0],9))
measured= np.zeros(( train.shape[0]))
total_score = 0
for n_fold,(train_idx, valid_idx)in enumerate(skf.split(train.values, y['surface'].values)) :
print(n_fold)
 model = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 model.fit(train.iloc[train_idx],y['surface'][train_idx])
 measured[valid_idx] = model.predict(train.iloc[valid_idx])
 preds += model.predict_proba(test_df)/ 50
total_score += model.score(train.iloc[valid_idx],y['surface'][valid_idx])
 print("Fold: {} score: {}".format(n_fold,model.score(train.iloc[valid_idx],y['surface'][valid_idx])))
 print('Avg Accuracy', total_score / 50)
num_cols = [x for x in X.columns if x not in categorical_cols]
num_cols = [x for x in num_cols if x not in ['assetCode']]<data_type_conversions>
sub['surface'] = le.inverse_transform(preds.argmax(axis=1))
sub
def scale_numeric(df):
df[num_cols] = df[num_cols].fillna(0)
 scaler = StandardScaler()
for i in range(0, len(num_cols), 4):
cols = num_cols[i:i + 3]
df[cols] = scaler.fit_transform(df[cols].astype(float))
scale_numeric(X )<split>
sub.to_csv('submission.csv', index=False )
n_train = int(X.shape[0] * 0.8)
 X_train, y_train = X.iloc[:n_train], y.iloc[:n_train]
X_valid, y_valid = X.iloc[n_train:], y.iloc[n_train:]<prepare_x_and_y>
sub.to_csv('submission.csv', index=False )