Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<train_model>
%%time
sns.set()
%matplotlib inline
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )
print("Training data starts at",market_data['time'].dt.date.min() ,
"and it ends at",market_data['time'].dt.date.max() )<feature_engineering>
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )
market_data=market_data.loc[market_data['time'].dt.date>=pd.datetime(2009,1,1 ).date() ]
news_data=news_data.loc[news_data['time'].dt.date>=pd.datetime(2009,1,1 ).date() ]
print(market_data.shape,news_data.shape )<categorify>
gblabel_encoder_asset_code={name:idx for idx,name in enumerate(market_data['assetCode'].unique())}
gblabel_encoder_asset_name={name:idx for idx,name in enumerate(market_data['assetName'].unique())}<categorify>
mapper_ord_1 = {'Novice': 1,
'Contributor': 2,
'Expert': 3,
'Master': 4,
'Grandmaster': 5}
traintest = pd.concat([train, test])
 train['ord_1_count'] = train['ord_1'].map(traintest['ord_1'].value_counts().to_dict())
 test['ord_1_count'] = test['ord_1'].map(traintest['ord_1'].value_counts().to_dict())
 mapper_ord_2 = {'Freezing': 1,
'Cold': 2,
'Warm': 3,
'Hot': 4,
'Boiling Hot': 5,
'Lava Hot': 6}
mapper_ord_3 = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8,
'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15}
mapper_ord_4 = {'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7, 'H': 8,
'I': 9, 'J': 10, 'K': 11, 'L': 12, 'M': 13, 'N': 14, 'O': 15,
'P': 16, 'Q': 17, 'R': 18, 'S': 19, 'T': 20, 'U': 21, 'V': 22,
'W': 23, 'X': 24, 'Y': 25, 'Z': 26}
for col, mapper in zip(['ord_1', 'ord_2', 'ord_3', 'ord_4'], [mapper_ord_1, mapper_ord_2, mapper_ord_3, mapper_ord_4]):
train[col+'_oe'] = train[col].replace(mapper)
 test[col+'_oe'] = test[col].replace(mapper)
 train.drop(col, axis=1, inplace=True)
 test.drop(col, axis=1, inplace=True )
def make_the_dataframe(df,assetcode_encoder,assetname_encoder,is_train):
unique_values_asset_code=df['assetCode'].unique()
for name in unique_values_asset_code:
if name not in assetcode_encoder.keys() :
assetcode_encoder[name]=max(assetcode_encoder.values())+1
df['assetCode_encoding']=df['assetCode'].map(assetcode_encoder)
 unique_values_asset_name=df['assetName'].unique()
for name in unique_values_asset_name:
if name not in assetname_encoder.keys() :
assetname_encoder[name]=max(assetname_encoder.values())+1
df['assetName_encoding']=df['assetName'].map(assetname_encoder)
 df['weekofyear']=df['time'].dt.weekofyear
df['dayofweek']=df['time'].dt.dayofweek
df['close_to_open_ratio']=df['close']/df['open']
if is_train:
df['label']=(df['returnsOpenNextMktres10']>0 ).astype(int)
 df['time']=df['time'].dt.date
necessary_df=df[['time','assetCode','assetName','returnsOpenNextMktres10','universe']].copy()
df.drop(columns=['time','assetName','assetCode','returnsOpenNextMktres10','universe'],inplace=True)
 return df,necessary_df
else:
df.drop(columns=['time','assetName','assetCode'],inplace=True)
 return df
def make_the_numpy_array(market_data_train,market_data_val):
X_train=market_data_train.iloc[:,:-1].values
X_val=market_data_val.iloc[:,:-1].values
y_train=market_data_train.iloc[:,-1].values
y_val=market_data_val.iloc[:,-1].values
return X_train,X_val,y_train,y_val
def competitoin_metric(extract,probs):
extract['confidence']=2*probs-1
extract['score']=extract['universe']*extract['returnsOpenNextMktres10']*extract['confidence']
x=extract.groupby('time')['score'].sum().values
return np.mean(x)/np.std(x )<create_dataframe>
train['ord_5_oe1'] = train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))
test['ord_5_oe1'] = test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))
train['ord_5_oe2'] = train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))
test['ord_5_oe2'] = test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))
ord_5 = sorted(list(set(train['ord_5'].values)))
 ord_5 = dict(zip(ord_5, range(len(ord_5))))
train.loc[:, 'ord_5'] = train['ord_5'].apply(lambda x: ord_5[x] ).astype(float)
 test.loc[:, 'ord_5'] = test['ord_5'].apply(lambda x: ord_5[x] ).astype(float)
splits=[2012,2013,2014,2015]
models=[]
for idx,year in enumerate(splits):
print("Year = ",year)
 yr,mo,dte=year,1,1
print("Splitting the data with respect to time")
 market_data_val=market_data.loc[(market_data['time'].dt.date>=pd.datetime(yr,mo+2,dte ).date())]
market_data_train=market_data.loc[market_data['time'].dt.date<pd.datetime(yr,mo,dte ).date() ]
market_data_train,extract_train=make_the_dataframe(market_data_train.copy() ,gblabel_encoder_asset_code.copy() ,
gblabel_encoder_asset_name.copy() ,is_train=True)
 market_data_val,extract_val=make_the_dataframe(market_data_val.copy() ,gblabel_encoder_asset_code.copy() ,
gblabel_encoder_asset_name.copy() ,is_train=True)
 X_train,X_val,y_train,y_val=make_the_numpy_array(market_data_train,market_data_val)
 print("The training data shape is",X_train.shape,y_train.shape)
 print("The validation data shape is",X_val.shape,y_val.shape)
 gc.enable()
del market_data_train,market_data_val
gc.collect()
clf=xgb.XGBClassifier(n_jobs=4,max_depth=7)
 clf.fit(X_train,y_train,eval_set=[(X_train,y_train),(X_val,y_val)],eval_metric='logloss',
early_stopping_rounds=25,verbose=10)
 print("The Training Competition metric is:",competitoin_metric(extract_train,
clf.predict_proba(X_train)[:,1]))
print("The validation competition metric is:",competitoin_metric(extract_val,
clf.predict_proba(X_val)[:,1]))
models.append(clf)
 gc.enable()
del X_train,X_val,y_train,y_val
gc.collect()<create_dataframe>
for col in ['ord_0', 'ord_1_oe', 'ord_2_oe', 'ord_3_oe', 'ord_4_oe', 'ord_5_oe1', 'ord_5_oe2', 'ord_1_count']:
train[col]= train[col].astype('float64')
 test[col]= test[col].astype('float64' )
<predict_on_test>
def make_the_predictions(models,test_market_data,test_news_data,predictions_template_df):
test=make_the_dataframe(test_market_data,gblabel_encoder_asset_code.copy() ,
gblabel_encoder_asset_name.copy() ,is_train=False)
 X_test=test.iloc[:,:].values
gc.enable()
del test
gc.collect()
predictions_template_df['confidenceValue']=0
for clf in models:
confidences=2*clf.predict_proba(X_test)[:,1]-1
predictions_template_df['confidenceValue']=predictions_template_df['confidenceValue']+confidences
predictions_template_df['confidenceValue']=predictions_template_df['confidenceValue']/len(models)
 return predictions_template_df<predict_on_test>
categorical_features_indices = np.where(test.dtypes != np.float)[0]
print('Categorial Feature Indices: ', categorical_features_indices )
for day,(market_obs_df, news_obs_df, predictions_template_df)in enumerate(env.get_prediction_days()):
if(day==0):
print("The columns of market are",market_obs_df.columns)
 predictions=make_the_predictions(models,market_obs_df, news_obs_df, predictions_template_df)
 env.predict(predictions)
 if(( day+1)%20==0):
print("Predictions for",day+1,"days done" )<save_to_csv>
%%time
def run_cv_model(categorical_indices, train, test, target, model_fn, params={}, eval_fn=None, label='model', n_folds=5):
kf = KFold(n_splits=n_folds)
 fold_splits = kf.split(train, target)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0]))
feature_importances = pd.DataFrame()
feature_importances['feature'] = test.columns
i = 1
for dev_index, val_index in fold_splits:
print('-------------------------------------------')
 print('Started ' + label + ' fold ' + str(i)+ f'/{n_folds}')
 dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target.iloc[dev_index], target.iloc[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, fi = model_fn(categorical_indices, dev_X, dev_y, val_X, val_y, test, params2)
 feature_importances[f'fold_{i}'] = fi
pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 print(label + ' cv score {}: {}'.format(i, cv_score), '
')
 i += 1
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std score : {}'.format(label, np.std(cv_scores)))
 pred_full_test = pred_full_test / n_folds
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'fi': feature_importances}
return results
def runCAT(categorical_indices, train_X, train_y, test_X, test_y, test_X2, params):
print('Pool Data')
 _train = Pool(train_X, label=train_y, cat_features = categorical_indices)
 _valid = Pool(test_X, label=test_y, cat_features = categorical_indices)
 print('Train CAT')
 model = CatBoostClassifier(**params)
 fit_model = model.fit(_train,
eval_set=_valid,
use_best_model=True,
verbose=1000,
plot=False)
 feature_im = fit_model.feature_importances_
print('Predict 1/2')
 pred_test_y = fit_model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = fit_model.predict_proba(test_X2)[:, 1]
return pred_test_y, pred_test_y2, feature_im
cat_params = {'loss_function': 'CrossEntropy',
'eval_metric': "AUC",
'task_type': "GPU",
'learning_rate': 0.01,
'iterations': 10000,
'random_seed': 42,
'od_type': "Iter",
'early_stopping_rounds': 500,
}
n_folds = 5
results = run_cv_model(categorical_features_indices, train, test, target, runCAT, cat_params, auc, 'cat', n_folds=n_folds )
env.write_submission_file()<compute_test_metric>
%%time
submission = pd.DataFrame({'id': test_id, 'target': results['test']})
 submission.to_csv('submission.csv', index=False )
<drop_column><EOS> 
feature_importances = results['fi']
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(n_folds)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 feature_importances.sort_values(by='average', ascending=False ).head()