Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<define_variables>
%%time
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )
UpOrDown_market = market_train_df.returnsOpenNextMktres10 >= 0
UpOrDown_market = UpOrDown_market.values
returns_market = market_train_df.returnsOpenNextMktres10.values
X_market=market_train_df[features_col_market].values
mins_X_market = np.min(X_market, axis=0)
 maxs_X_market = np.max(X_market, axis=0)
 range_X_market = maxs_X_market - mins_X_market
X_market = 1 -(( maxs_X_market - X_market)/ range_X_market)
<split>
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )
X_train, X_test, UpOrDown_train, UpOrDown_test, returns_train, returns_test = model_selection.train_test_split(X, UpOrDown, returns, test_size=0.1, random_state=99 )<train_model>
def count_transform(column):
return column.map(column.value_counts().to_dict() )
params = {'learning_rate': 0.05, 'max_depth': 5, 'boosting': 'gbdt', 'objective': 'binary', 'metric': 'auc', 'is_training_metric': True, 'seed': 42}
model = lgb.train(params, train_set=lgb.Dataset(X_train, label=UpOrDown_train), num_boost_round=2000,
valid_sets=[lgb.Dataset(X_train, label=UpOrDown_train), lgb.Dataset(X_test, label=UpOrDown_test)],
verbose_eval=50, early_stopping_rounds=30)
<train_model>
rang = {"Grandmaster" : 4, "Master" : 3, "Expert" : 2, "Contributor" : 1, "Novice" : 0}
temperature = {"Freezing" : 0, "Cold": 1, "Warm" : 2, "Hot": 3, "Boiling Hot" : 4, "Lava Hot" : 5}
X_train_market, X_test_market, UpOrDown_train_market, UpOrDown_test_market, returns_train_market, returns_test_market = model_selection.train_test_split(X_market, UpOrDown_market, returns_market, test_size=0.1, random_state=99)
 params = {'learning_rate': 0.05, 'max_depth': 5, 'boosting': 'gbdt', 'objective': 'binary', 'metric': 'auc', 'is_training_metric': True, 'seed': 42}
model_market = lgb.train(params, train_set=lgb.Dataset(X_train_market, label=UpOrDown_train_market), num_boost_round=2000,
valid_sets=[lgb.Dataset(X_train_market, label=UpOrDown_train_market), lgb.Dataset(X_test_market, label=UpOrDown_test_market)],
verbose_eval=50, early_stopping_rounds=30 )<groupby>
traintest = pd.concat([train, test] )
days = env.get_prediction_days()
n_days = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if n_days % 50 == 0:
print(n_days,end=' ')
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
market_obs_df=prep_market(market_obs_df)
 news_obs_df=prep_news(news_obs_df)
 news_obs_df=group_news(news_obs_df)
 market_news_obs_df = merge_market_news(market_obs_df, news_obs_df)
 assetcode_set=set(market_news_obs_df['assetCode'].values)
 X_live = market_news_obs_df[features_col].values
X_live = 1 -(( maxs_X - X_live)/ range_X)
 lp = model.predict(X_live)
 confidence = 2 * lp -1
preds = pd.DataFrame({'assetCode':market_news_obs_df['assetCode'],'confidence':confidence})
 market_only_obs_df=market_obs_df[~market_obs_df['assetCode'].isin(assetcode_set)]
X_live_market_only=market_only_obs_df[features_col_market].values
X_live_market_only=1-(( maxs_X_market - X_live_market_only)/ range_X_market)
 lp_market_only = model_market.predict(X_live_market_only)
 confidence_market_only = 2 * lp_market_only -1
preds_market_only= pd.DataFrame({'assetCode':market_only_obs_df['assetCode'],'confidence':confidence_market_only})
 preds_all=pd.concat([preds,preds_market_only],sort='False')
 predictions_template_df = predictions_template_df.merge(preds_all,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 env.write_submission_file()<set_options>
%%time
traintest['ord_1_new'] = traintest['ord_1'].map(rang)
 traintest['ord_2_new'] = traintest['ord_2'].map(temperature)
 traintest['ord_3_new'] = traintest['ord_3'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_3'])) })
 traintest['ord_4_new'] = traintest['ord_4'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_4'])) })
 traintest['ord_5_new_1'] = traintest['ord_5'].apply(lambda x: x[0])
 traintest['ord_5_new_1'] = traintest['ord_5_new_1'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_5_new_1'])) })
 traintest['ord_5_new_2'] = traintest['ord_5'].apply(lambda x: x[1])
 traintest['ord_5_new_2'] = traintest['ord_5_new_2'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_5_new_2'])) })
 traintest['ord_5_new_2'] = traintest['ord_5_new_2'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_5_new_2'])) })
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True ).to_sparse().to_coo()
traintest_new = csr_matrix(traintest[list(filter(lambda x: 'new' in x, traintest.columns)) ] )
env=twosigmanews.make_env()<import_modules>
feature_df = hstack([dummies, traintest_new] ).tocsr()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgbm
import xgboost as xgb
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split<categorify>
%%time
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = KFold(n_splits=5)
 fold_splits = kf.split(train, target)
 cv_scores = []
pred_full_test = 0
coefs = []
pred_train = np.zeros(( train.shape[0]))
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
trn_res = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_val_y, pred_test_y = trn_res['pred_val_y'], trn_res['pred_test_y']
pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 coefs.append(trn_res['coef'])
 print(label + ' cv score {}: {}'.format(i, cv_score))
i += 1
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std score : {}'.format(label, np.std(cv_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores,
'coefs' : coefs}
return results
def runLR(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = LogisticRegression(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = model.predict_proba(test_X2)[:, 1]
return {'pred_val_y' : pred_test_y, 'pred_test_y' : pred_test_y2, 'coef' : model.coef_}
def runRLR(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = Ridge(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2)
 return pred_test_y, pred_test_y2
rr_params = {'alpha' : 1, 'solver': 'lsqr', "fit_intercept" : False}
lr_params = {'solver': 'lbfgs', 'C': 0.1,'max_iter' : 1000}
results = run_cv_model(train_ohe, test_ohe, target, runLR, lr_params, auc, 'Ridge' )
<feature_engineering><EOS> 
%%time
submission = pd.DataFrame({'id': test_id, 'target': results['test']})
 submission.to_csv('submission.csv', index=False )