Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<correct_missing_values>
import numpy as np
import pandas as pd
test_merged['Open'].fillna(1,inplace=True )<count_missing_values>
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape)
 train['ord_5_a'] = train['ord_5'].str[0]
train['ord_5_b'] = train['ord_5'].str[1]
test['ord_5_a'] = test['ord_5'].str[0]
test['ord_5_b'] = test['ord_5'].str[1]
train.drop(['ord_5'],inplace=True,axis=1)
 test.drop(['ord_5'],inplace=True,axis=1)
 test.head()
test_merged.isna().sum()<feature_engineering>
ord_3 = CategoricalDtype(categories=['a', 'b', 'c', 'd', 'e', 'f', 'g',
'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'], ordered=True)
 ord_4 = CategoricalDtype(categories=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'], ordered=True )
test_merged['Date']=pd.to_datetime(test_merged['Date'],format='%Y-%m-%d')
 test_merged['day']=test_merged['Date'].dt.day
test_merged['month']=test_merged['Date'].dt.month
test_merged['year']=test_merged['Date'].dt.year 
<drop_column>
train.ord_3 = train.ord_3.astype(ord_3 ).cat.codes
train.ord_4 = train.ord_4.astype(ord_4 ).cat.codes
test.ord_3 = test.ord_3.astype(ord_3 ).cat.codes
test.ord_4 = test.ord_4.astype(ord_4 ).cat.codes
train.head()
test_merged=test_merged.drop('Date',1 )<feature_engineering>
def date_cyc_enc(df, col, max_vals):
df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)
 df[col + '_cos'] = np.cos(2 * np.pi * df[col]/max_vals)
 return df
train = date_cyc_enc(train, 'day', 7)
 test = date_cyc_enc(test, 'day', 7)
 train = date_cyc_enc(train, 'month', 12)
 test = date_cyc_enc(test, 'month', 12)
test_merged['StateHoliday']=test_merged['StateHoliday'].apply(lambda x:'0' if x==0 or x=='0' else x )<count_values>
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id','bin_0'], axis=1, inplace=True)
 test.drop(['id','bin_0'], axis=1, inplace=True)
 print(train.shape)
 print(test.shape )
test_merged['StateHoliday'].value_counts()<count_values>
one_hot_cols = ['bin_3','bin_4','nom_0', 'nom_1', 'nom_2','ord_0', 'ord_1','ord_2',
'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9', 'ord_5_a', 'ord_5_b','day','month']
traintest = pd.concat([train, test])
 cols_to_scale = ['day_sin','day_cos','month_sin','month_cos','ord_3','ord_4']
scaler = StandardScaler()
traintest[cols_to_scale] = scaler.fit_transform(traintest[cols_to_scale])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe = dummies.iloc[:train.shape[0], :]
test_ohe = dummies.iloc[train.shape[0]:, :]
print(train_ohe.shape)
 print(test_ohe.shape )
test_merged['StoreType'].value_counts()<count_values>
%%time
train_ohe = train_ohe.sparse.to_coo().tocsr()
test_ohe = test_ohe.sparse.to_coo().tocsr()
test_merged['Assortment'].value_counts()<count_values>
test_merged['PromoInterval'].value_counts()<categorify>
test_merged['StateHoliday']=test_merged['StateHoliday'].map({'0':0,'a':1})
 test_merged['StateHoliday']=test_merged['StateHoliday'].astype(int)
 test_merged['StoreType']=test_merged['StoreType'].map({'a':0,'b':1,'c':2,'d':3})
 test_merged['StoreType']=test_merged['StoreType'].astype(int)
 test_merged['Assortment']=test_merged['Assortment'].map({'a':0,'b':1,'c':2})
 test_merged['Assortment']=test_merged['Assortment'].astype(int)
 test_merged['PromoInterval']=test_merged['PromoInterval'].map({'Jan,Apr,Jul,Oct':0,'Feb,May,Aug,Nov':1,'Mar,Jun,Sept,Dec':2})
 test_merged['PromoInterval']=test_merged['PromoInterval'].astype(int )<drop_column>
%%time
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = KFold(n_splits=5)
 fold_splits = kf.split(train, target)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0]))
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 print(label + ' cv score {}: {}'.format(i, cv_score))
i += 1
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std score : {}'.format(label, np.std(cv_scores)))
 ',pred_full_test,'
')
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores}
return results
def runLR(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = LogisticRegression(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = model.predict_proba(test_X2)[:, 1]
return pred_test_y, pred_test_y2
def runSGD(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = SGDClassifier(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = model.predict_proba(test_X2)[:, 1]
return pred_test_y, pred_test_y2
lr_params = {'solver': 'lbfgs', 'C': 0.096, 'max_iter':500, 'class_weight':'balanced'}
sgd_params = {'loss': 'log', 'penalty': 'l2', 'max_iter':1000}
results = run_cv_model(train_ohe, test_ohe, target, runLR, lr_params, auc, 'lr' )
<predict_on_test><EOS> 
submission = pd.DataFrame({'id': test_id, 'target': results['test']})
 submission.to_csv('submission_11.csv', index=False )