Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<train_model>
warnings.filterwarnings('ignore')
 pd.options.display.max_columns = 50
models = [ 
("Lasso", Lasso, {"alpha": [0.1, 0.5],
"random_state": [0],
"max_iter": [1000],
"tol": [0.001],
"selection": ["random"],
"fit_intercept": [False]}), 
("Ridge", Ridge, {"solver": ["lsqr", "sparse_cg"],
"alpha": [0.1, 0.5, 1],
"random_state": [0],
"tol": [0.001],
"fit_intercept": [False]}), 
("ElasticNet", ElasticNet, {"alpha": [0.1, 0.5],
"l1_ratio": [0.1, 0.5, 0.9],
"random_state": [0],
"max_iter": [1000],
"tol": [0.001],
"selection": ["random"],
"fit_intercept": [False]}), 
("DecisionTreeRegressor", DecisionTreeRegressor, {"max_depth": [3, 7],
"random_state": [0]})
 ]<train_on_grid>
%%time
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv')
 BIN_COL = [s for s in train.columns if 'bin_' in s]
NOM_COL = [s for s in train.columns if 'nom_' in s]
ORD_COL = [s for s in train.columns if 'ord_' in s]
NOM_5_9 = ['nom_5','nom_6','nom_7','nom_8','nom_9']
DATE_COL = ['day','month']
best_models = []
for name, model_class, params in models:
gs = GridSearchCV(model_class() , params, scoring=scorer, cv=5, n_jobs=1, refit=True)
 gs.fit(X, log_y)
 best_models.append(( name, model_class, gs.best_estimator_, gs.best_params_, gs.best_score_))<save_to_csv>
feats = test.drop(columns=['id'] ).columns
print('1.null data size --')
 tmp_df = pd.concat([train[feats].isnull().sum() ,
test[feats].isnull().sum() ],axis=1 ).rename(columns={0:'train',1:'test'})
 print(tmp_df)
 print()
print('2.null data rate -------')
 tmp_df['train'] = tmp_df['train']/len(train)*100
tmp_df['test'] = tmp_df['test']/len(test)*100
print(tmp_df)
 del tmp_df;gc.collect()
for model_name, _, estimator, _, _ in best_models:
predicted = np.expm1(estimator.predict(X_test))
pd.DataFrame({"price": predicted}, index=df_test.index ).to_csv("baseline_{}.csv".format(model_name), sep="," )<train_model>
%%time
train['null_count'] = train.isnull().sum(axis=1)
 test['null_count'] = test.isnull().sum(axis=1)
 print('train null_count value_counts-----------')
 print(train['null_count'].value_counts())
 print('test null_count value_counts-----------')
 print(test['null_count'].value_counts())
 train.null_count = np.clip(train.null_count,0,6)
 for col in test.drop(columns=['id','null_count'] ).columns.tolist() :
train[f'{col}_missing'] = train[col].isnull().astype(int)
 test[f'{col}_missing'] = test[col].isnull().astype(int )
lgr = LGBMRegressor(n_jobs=-1, n_estimators=100)
 lgr.fit(X, log_y, eval_metric=rmsle )<choose_model_class>
best_models.append(( "LGBMRegressor", LGBMRegressor, lgr, lgr.get_params() , lgr.best_score_))<predict_on_test>
all_models_preds_train = np.zeros(( X.shape[0], len(best_models)))
 for i in range(len(best_models)) :
_, _, estimator, _, _ = best_models[i]
all_models_preds_train[..., i] = estimator.predict(X )<predict_on_test>
gc.collect()
all_models_preds_test = np.zeros(( X_test.shape[0], len(best_models)))
 for i in range(len(best_models)) :
_, _, estimator, _, _ = best_models[i]
all_models_preds_test[..., i] = estimator.predict(X_test )<choose_model_class>
%time
tmp_df = pd.concat([train.drop(columns=['target'] ).nunique() ,
test.nunique() ],axis=1)
 tmp_df = pd.concat([tmp_df, pd.concat([train,test], axis=0 ).drop(columns=['target'] ).nunique() ], axis=1)
 tmp_df = tmp_df.reset_index()
tmp_df.columns = ['feature','train','test','all']
tmp_df = tmp_df.loc[tmp_df.feature!='id'].reset_index(drop=True)
 print(tmp_df )
lr = LinearRegression()
lr.fit(all_models_preds_train, log_y)
 final_preds = np.expm1(lr.predict(all_models_preds_test))<save_to_csv>
for col in test.drop(columns='id' ).columns:
if len(set(train[col].dropna().unique().tolist())^ set(test[col].dropna().unique().tolist())) >0:
print(col,
'(train only)', set(train[col].dropna().unique().tolist())- set(test[col].dropna().unique().tolist()),
'(test only)', set(test[col].dropna().unique().tolist())- set(train[col].dropna().unique().tolist()))
print(f'train only nom_5 count:', len(train[train['nom_5'].isin(['b3ad70fcb'])]))
print(f'train only nom_6 count:', len(train[train['nom_6'].isin(['f0732a795', 'ee6983c6d', '3a121fefb'])]))
print(f'test only nom_6 count:', len(test[test['nom_6'].isin(['a885aacec'])]))
print(f'train only nom_9 count:', len(train[train['nom_9'].isin(['3d19cd31d', '1065f10dd'])]))
pd.DataFrame({"price": final_preds}, index=df_test.index ).to_csv("ensemble.csv", sep="," )<import_modules>
%%time
for df in [train, test]:
df.bin_3.replace({'F':0, 'T':1}, inplace=True)
 df.bin_4.replace({'N':0, 'Y':1}, inplace=True )
import pandas as pd<import_modules>
%%time
ord_1_map = {'Novice':1,'Contributor':2,'Expert':3,'Master':4,'Grandmaster':5}
ord_2_map = {'Freezing':1, 'Cold':2,'Warm':3,'Hot':4, 'Boiling Hot':5,'Lava Hot':6}
train.loc[train['ord_1'].notnull() ,'ord_1'] = train.loc[train['ord_1'].notnull() ,'ord_1'].map(ord_1_map)
 train.loc[train['ord_2'].notnull() ,'ord_2'] = train.loc[train['ord_2'].notnull() ,'ord_2'].map(ord_2_map)
 train.loc[train['ord_3'].notnull() ,'ord_3'] = train.loc[train['ord_3'].notnull() ,'ord_3'].apply(lambda c: ord(c)- ord('a')+ 1)
 train.loc[train['ord_4'].notnull() ,'ord_4'] = train.loc[train['ord_4'].notnull() ,'ord_4'].apply(lambda c: ord(c)- ord('A')+ 1)
 test.loc[test['ord_1'].notnull() ,'ord_1'] = test.loc[test['ord_1'].notnull() ,'ord_1'].map(ord_1_map)
 test.loc[test['ord_2'].notnull() ,'ord_2'] = test.loc[test['ord_2'].notnull() ,'ord_2'].map(ord_2_map)
 test.loc[test['ord_3'].notnull() ,'ord_3'] = test.loc[test['ord_3'].notnull() ,'ord_3'].apply(lambda c: ord(c)- ord('a')+ 1)
 test.loc[test['ord_4'].notnull() ,'ord_4'] = test.loc[test['ord_4'].notnull() ,'ord_4'].apply(lambda c: ord(c)- ord('A')+ 1)
 for i in range(1,5):
train[f'ord_{i}'] = train[f'ord_{i}'].astype(float)
 test[f'ord_{i}'] = test[f'ord_{i}'].astype(float )
import pandas as pd<load_from_csv>
result = pd.read_csv('.. /input/gruresult/kernel_submission.csv' )<feature_engineering>
%%time
for df in [train, test]:
df.loc[df.ord_5.notnull() , 'ord_5_1'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[0])
 df.loc[df.ord_5.notnull() , 'ord_5_2'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[1] )
result['price'] =result['price'].apply(abs )<data_type_conversions>
%%time
for col in ['ord_5_1', 'ord_5_2']:
_map = pd.concat([train[col], test[col]] ).value_counts().rank().to_dict()
train[f'{col}_freq'] = train[col].map(_map)
 test[f'{col}_freq'] = test[col].map(_map )
result['test_id'] = result['test_id'].astype(int )<save_to_csv>
%%time
train.loc[train['ord_5_1'].notnull() ,'ord_5_1'] = train.loc[train['ord_5_1'].notnull() ,'ord_5_1'].apply(lambda c: ord(c)- ord('a')+ 33 ).astype(float)
 test.loc[test['ord_5_1'].notnull() ,'ord_5_1'] = test.loc[test['ord_5_1'].notnull() ,'ord_5_1'].apply(lambda c: ord(c)- ord('a')+ 33 ).astype(float)
 train.loc[train['ord_5_2'].notnull() ,'ord_5_2'] = train.loc[train['ord_5_2'].notnull() ,'ord_5_2'].apply(lambda c: ord(c)- ord('a')+ 33 ).astype(float)
 test.loc[test['ord_5_2'].notnull() ,'ord_5_2'] = test.loc[test['ord_5_2'].notnull() ,'ord_5_2'].apply(lambda c: ord(c)- ord('a')+ 33 ).astype(float )
result.to_csv('submission_gru_1230.csv',index=False )<set_options>
%%time
for i in range(5):
_map = pd.concat([train[f'nom_{i}'], test[f'nom_{i}']] ).value_counts().rank().to_dict()
train[f'nom_{i}_freq'] = train[f'nom_{i}'].map(_map)
 test[f'nom_{i}_freq'] = test[f'nom_{i}'].map(_map )
%matplotlib inline
def timer(start_time=None):
if not start_time:
start_time = datetime.now()
return start_time
elif start_time:
thour, temp_sec = divmod(
(datetime.now() - start_time ).total_seconds() , 3600)
 tmin, tsec = divmod(temp_sec, 60)
 print('
Time taken: %i hours %i minutes and %s seconds.' % 
(thour, tmin, round(tsec, 2)))
 def rmse_min_func(weights):
final_prediction = 0
for weight, prediction in zip(weights, blend_train):
final_prediction += weight * prediction
return np.sqrt(mean_squared_error(y_train, final_prediction))
start_time = timer(None)
 NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000
print('Reading in Data')
 df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 y_train = np.log1p(df_train['price'])
 tr_ids = df_train['train_id'].values.astype(np.int32)
 df_train.drop(['price', 'train_id'], axis=1, inplace=True)
 te_ids = df_test['test_id'].values.astype(np.int32)
 df_test.drop(['test_id'], axis=1, inplace=True)
 df = pd.concat([df_train, df_test])
 nrow_train = df_train.shape[0]
del df_train, df_test
gc.collect()
df['category_name'] = df['category_name'].fillna('Other' ).astype('category')
 df['brand_name'] = df['brand_name'].fillna('unknown')
 pop_brands = df['brand_name'].value_counts().index[:NUM_BRANDS]
df.loc[~df['brand_name'].isin(pop_brands), 'brand_name'] = 'Other'
df['item_description'] = df['item_description'].fillna('None')
 df['item_condition_id'] = df['item_condition_id'].astype('category')
 df['brand_name'] = df['brand_name'].astype('category')
 print('Encodings')
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df['name'])
 print('Category Encoders')
 unique_categories = pd.Series('/'.join(df['category_name'].unique().astype(
 'str')).split('/')).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df['category_name'])
 print('Descp encoders')
 count_descp = TfidfVectorizer(max_features=MAX_FEAT_DESCP,
ngram_range=(1, 3),
stop_words='english')
 X_descp = count_descp.fit_transform(df['item_description'])
 print('Brand encoders')
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df['brand_name'])
 print('Dummy Encoders')
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
'item_condition_id', 'shipping'
]], sparse=True ).values)
 X = scipy.sparse.hstack(( X_dummies, X_descp, X_brand, X_category,
X_name)).tocsr()
print([X_dummies.shape, X_category.shape, X_name.shape, X_descp.shape,
X_brand.shape])
 X_train = X[:nrow_train]
X_test = X[nrow_train:]
timer(start_time )<define_variables>
%%time
for i in range(5,10):
col_name = f'nom_{i}'
print(f'{col_name} train data ---------------')
 print(train[col_name].value_counts(dropna=False, normalize=False)[:20])
 print(f'{col_name} test data ---------------')
 print(test[col_name].value_counts(dropna=False, normalize=False)[:20] )
folds = 3
sgd_cv_sum = 0
ridge_cv_sum = 0
lgb_cv_sum = 0
lgb_pred = []
sgd_pred = []
ridge_pred = []
lgb_fpred = []
sgd_fpred = []
ridge_fpred = []
avreal = y_train
lgb_avpred = np.zeros(X_train.shape[0])
 sgd_avpred = np.zeros(X_train.shape[0])
 ridge_avpred = np.zeros(X_train.shape[0])
 idpred = tr_ids
blend_train = []
blend_test = []
train_time = timer(None)
 kf = KFold(n_splits=folds, random_state=1001)
 for i,(train_index, val_index)in enumerate(kf.split(X_train, y_train)) :
start_time = timer(None)
 Xtrain, Xval = X_train[train_index], X_train[val_index]
ytrain, yval = y_train[train_index], y_train[val_index]
model = SGDRegressor(penalty='l2',
loss='squared_epsilon_insensitive',
max_iter=200,
tol=0.00001,
epsilon=0.0001,
learning_rate='invscaling',
fit_intercept=False,
alpha=1e-10,
l1_ratio=0.09,
shuffle=True,
verbose=0,
random_state=1001)
 model.fit(Xtrain, ytrain)
 sgd_scores_val = model.predict(Xval)
 sgd_RMSLE = np.sqrt(mean_squared_error(yval, sgd_scores_val))
print('
Fold %02d SGD RMSLE: %.6f' %(( i + 1), sgd_RMSLE))
sgd_y_pred = model.predict(X_test)
 model = Ridge(alpha=4.75,
solver='sag',
fit_intercept=False,
random_state=1001,
max_iter=1000)
 model.fit(Xtrain, ytrain)
 ridge_scores_val = model.predict(Xval)
 ridge_RMSLE = np.sqrt(mean_squared_error(yval, ridge_scores_val))
print(' Fold %02d Ridge RMSLE: %.6f' %(( i + 1), ridge_RMSLE))
ridge_y_pred = model.predict(X_test)
 params = {
'boosting': 'gbdt',
'max_depth': 7,
'min_data_in_leaf': 80,
'num_leaves': 40,
'learning_rate': 0.75,
'objective': 'regression',
'metric': 'rmse',
'nthread': 4,
'bagging_freq': 1,
'subsample': 0.94,
'colsample_bytree': 0.7,
'min_child_weight': 17,
'is_unbalance': False,
'verbose': -1,
'seed': 1001
}
dtrain = lgb.Dataset(Xtrain, label=ytrain, max_bin=4095)
 dval = lgb.Dataset(Xval, label=yval, max_bin=4095)
 watchlist = [dtrain, dval]
watchlist_names = ['train', 'val']
model = lgb.train(params,
train_set=dtrain,
num_boost_round=1800,
valid_sets=watchlist,
valid_names=watchlist_names,
early_stopping_rounds=100,
verbose_eval=50)
 lgb_scores_val = model.predict(Xval)
 lgb_RMSLE = np.sqrt(mean_squared_error(yval, lgb_scores_val))
print(' Fold %02d LightGBM RMSLE: %.6f' %(( i + 1), lgb_RMSLE))
lgb_y_pred = model.predict(X_test)
 del Xtrain, Xval
gc.collect()
timer(start_time)
 sgd_avpred[val_index] = sgd_scores_val
ridge_avpred[val_index] = ridge_scores_val
lgb_avpred[val_index] = lgb_scores_val
if i > 0:
sgd_fpred = sgd_pred + sgd_y_pred
ridge_fpred = ridge_pred + ridge_y_pred
lgb_fpred = lgb_pred + lgb_y_pred
else:
sgd_fpred = sgd_y_pred
ridge_fpred = ridge_y_pred
lgb_fpred = lgb_y_pred
sgd_pred = sgd_fpred
ridge_pred = ridge_fpred
lgb_pred = lgb_fpred
sgd_cv_sum = sgd_cv_sum + sgd_RMSLE
ridge_cv_sum = ridge_cv_sum + ridge_RMSLE
lgb_cv_sum = lgb_cv_sum + lgb_RMSLE
timer(train_time)
 sgd_cv_score =(sgd_cv_sum / folds)
 ridge_cv_score =(ridge_cv_sum / folds)
 lgb_cv_score =(lgb_cv_sum / folds)
 sgd_oof_RMSLE = np.sqrt(mean_squared_error(avreal, sgd_avpred))
ridge_oof_RMSLE = np.sqrt(mean_squared_error(avreal, ridge_avpred))
lgb_oof_RMSLE = np.sqrt(mean_squared_error(avreal, lgb_avpred))
print('
Average SGD RMSLE:\t%.6f' % sgd_cv_score)
 print(' Out-of-fold SGD RMSLE:\t%.6f' % sgd_oof_RMSLE)
 print('
Average Ridge RMSLE:\t%.6f' % ridge_cv_score)
 print(' Out-of-fold Ridge RMSLE:\t%.6f' % ridge_oof_RMSLE)
 print('
Average LightGBM RMSLE:\t%.6f' % lgb_cv_score)
 print(' Out-of-fold LightGBM RMSLE:\t%.6f' % lgb_oof_RMSLE)
 sgd_score = round(sgd_oof_RMSLE, 6)
 ridge_score = round(ridge_oof_RMSLE, 6)
 lgb_score = round(lgb_oof_RMSLE, 6)
 sgd_mpred = sgd_pred / folds
ridge_mpred = ridge_pred / folds
lgb_mpred = lgb_pred / folds<define_variables>
%%time
for i in range(5, 10):
col_name = f'nom_{i}'
print(f'{col_name} train data ---------------')
 print(train[col_name].value_counts(dropna=False, normalize=True)[:20])
 print(f'{col_name} test data ---------------')
 print(test[col_name].value_counts(dropna=False, normalize=True)[:20] )
blend_time = timer(None)
 blend_train.append(sgd_avpred)
 blend_train.append(ridge_avpred)
 blend_train.append(lgb_avpred)
 blend_train = np.array(blend_train)
 blend_test.append(sgd_mpred)
 blend_test.append(ridge_mpred)
 blend_test.append(lgb_mpred)
 blend_test = np.array(blend_test)
 print('
Finding Blending Weights...')
 res_list = []
weights_list = []
for k in range(20):
starting_values = np.random.uniform(size=len(blend_train))
bounds = [(-1, 1)] * len(blend_train)
 res = minimize(rmse_min_func,
starting_values,
method='L-BFGS-B',
bounds=bounds,
options={'disp': False,
'maxiter': 100000})
 res_list.append(res['fun'])
 weights_list.append(res['x'])
 print('{iter}\tScore: {score}\tWeights: {weights}'.format(
 iter=(k + 1),
score=res['fun'],
weights='\t'.join([str(item)for item in res['x']])))
 bestSC = np.min(res_list)
 bestWght = weights_list[np.argmin(res_list)]
weights = bestWght
blend_score = round(bestSC, 6)
 print('
Ensemble Score: {best_score}'.format(best_score=bestSC))
print('
Best Weights: {weights}'.format(weights=bestWght))
train_prices = np.zeros(len(blend_train[0]))
test_prices = np.zeros(len(blend_test[0]))
print('
Your final model:')
 for k in range(len(blend_test)) :
print(' %.6f * model-%d' %(weights[k],(k + 1)))
 test_prices += blend_test[k] * weights[k]
for k in range(len(blend_train)) :
train_prices += blend_train[k] * weights[k]
timer(blend_time )<save_to_csv>
now = datetime.now()
oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(sgd_avpred)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_SGDRegressor-3fold-00-v1-oof_' + str(sgd_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(sgd_mpred), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your SGD Regressor prediction:')
 print(result.head(10))
sub_file = 'submission_SGDRegressor-3fold-00-v1_' + str(sgd_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')
 oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(ridge_avpred)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_Ridge-3fold-00-v1-oof_' + str(ridge_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(ridge_mpred), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your Ridge Regressor prediction:')
 print(result.head(10))
sub_file = 'submission_Ridge-3fold-00-v1_' + str(ridge_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')
 oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(lgb_avpred)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_LightGBM-3fold-00-v1-oof_' + str(lgb_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(lgb_mpred), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your LightGBM prediction:')
 print(result.head(10))
sub_file = 'submission_LightGBM-3fold-00-v1_' + str(lgb_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')
 oof_result = pd.DataFrame(np.expm1(avreal), columns=['price'])
 oof_result['prediction'] = np.expm1(train_prices)
 oof_result['train_id'] = idpred
oof_result['train_id'] = oof_result['train_id'].astype(np.int32)
 oof_result.sort_values('train_id', inplace=True)
 oof_result = oof_result[['train_id', 'price', 'prediction']]
sub_file = 'train_blend-3fold-00-v1-oof_' + str(blend_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing out-of-fold train file:: %s' % sub_file)
 oof_result.to_csv(sub_file, index=False, float_format='%.6f')
 result = pd.DataFrame(np.expm1(test_prices), columns=['price'])
 result['test_id'] = te_ids
result['test_id'] = result['test_id'].astype(np.int32)
 result = result.set_index('test_id')
 print('
First 10 lines of your blended prediction:')
 print(result.head(10))
sub_file = 'submission_blend-3fold-00-v1_' + str(blend_score)+ '_' + str(
 now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 result.to_csv(sub_file, index=True, index_label='test_id', float_format='%.6f')
<load_from_csv>
train.day = train.day.replace({3:5,2:6,1:7})
 test.day = test.day.replace({3:5,2:6,1:7} )
np.random.seed(0)
 df_train = pd.read_csv('.. /input/train_users_2.csv')
 df_test = pd.read_csv('.. /input/test_users.csv')
 labels = df_train['country_destination'].values
df_train = df_train.drop(['country_destination'], axis=1)
 id_test = df_test['id']
piv_train = df_train.shape[0]
df_all = pd.concat(( df_train, df_test), axis=0, ignore_index=True)
 df_all = df_all.drop(['id', 'date_first_booking'], axis=1)
 df_all = df_all.fillna(-1)
 dac = np.vstack(df_all.date_account_created.astype(str ).apply(lambda x: list(map(int, x.split('-')))).values)
 df_all['dac_year'] = dac[:,0]
df_all['dac_month'] = dac[:,1]
df_all['dac_day'] = dac[:,2]
df_all = df_all.drop(['date_account_created'], axis=1)
 tfa = np.vstack(df_all.timestamp_first_active.astype(str ).apply(lambda x: list(map(int, [x[:4],x[4:6],x[6:8],x[8:10],x[10:12],x[12:14]])) ).values)
 df_all['tfa_year'] = tfa[:,0]
df_all['tfa_month'] = tfa[:,1]
df_all['tfa_day'] = tfa[:,2]
df_all = df_all.drop(['timestamp_first_active'], axis=1)
 av = df_all.age.values
df_all['age'] = np.where(np.logical_or(av<14, av>100), -1, av)
 ohe_feats = ['gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel', 'affiliate_provider', 'first_affiliate_tracked', 'signup_app', 'first_device_type', 'first_browser']
for f in ohe_feats:
df_all_dummy = pd.get_dummies(df_all[f], prefix=f)
 df_all = df_all.drop([f], axis=1)
 df_all = pd.concat(( df_all, df_all_dummy), axis=1)
 vals = df_all.values
X = vals[:piv_train]
le = LabelEncoder()
y = le.fit_transform(labels)
 X_test = vals[piv_train:]
xgb = XGBClassifier(max_depth=6, learning_rate=0.3, n_estimators=25,
objective='multi:softprob', subsample=0.5, colsample_bytree=0.5, seed=0)
 xgb.fit(X, y)
 y_pred = xgb.predict_proba(X_test)
 ids = []
cts = []
for i in range(len(id_test)) :
idx = id_test[i]
ids += [idx] * 5
cts += le.inverse_transform(np.argsort(y_pred[i])[::-1])[:5].tolist()
sub = pd.DataFrame(np.column_stack(( ids, cts)) , columns=['id', 'country'])
 sub.to_csv('sub.csv',index=False )<import_modules>
from sklearn.metrics import confusion_matrix<predict_on_test>
display(train.day.value_counts(dropna=False))
display(test.day.value_counts(dropna=False))
x_pred = xgb.predict_proba(X )<feature_engineering>
train:', train[col].value_counts().head(1))
for i in range(len(id_test)) :
idx = id_test[i]
ids += [idx] * 5
cts += le.inverse_transform(np.argsort(x_pred[i])[::-1])[:5].tolist()<compute_test_metric>
%%time
for col in NOM_5_9:
train.loc[train[col].notnull() , col] = train.loc[train[col].notnull() , col].astype(str ).apply(int, base=16)
 test.loc[test[col].notnull() , col] = test.loc[test[col].notnull() , col].astype(str ).apply(int, base=16)
 for col in NOM_5_9:
train[col] = train[col].astype(float)
 test[col] = test[col].astype(float)
 feats = train.select_dtypes(float ).columns.drop([] ).tolist()
for col in feats:
if col in ['bin_0']:
train[col].fillna(0, inplace=True)
 test[col].fillna(0, inplace=True)
 else:
train[col].fillna(-1, inplace=True)
 test[col].fillna(-1, inplace=True)
 for col in feats:
train[col] = train[col].astype(int)
 test[col] = test[col].astype(int )
confusion_matrix(y, x_pred )<set_options>
%%time
for col in NOM_COL+['ord_5']:
_map = train.groupby(col ).mean() ['target'].to_dict()
train[f'{col}_mean'] = train[col].map(_map)
 test[f'{col}_mean'] = test[col].map(_map )
%matplotlib inline 
<load_from_csv>
feats = train.select_dtypes(int ).columns.drop([
'id','target',
'nom_0_freq', 'nom_1_freq', 'nom_2_freq', 'nom_3_freq', 'nom_4_freq',
] ).tolist() +['nom_0_mean','nom_1_mean','nom_2_mean','nom_3_mean', 'nom_4_mean']
islgb, isxgb, isctb = False, False, True
X = train[feats]
y = train.target
X_test = test[feats] 
(X_train,X_val, y_train, y_val)= train_test_split(X, y, stratify=y, random_state=42)
 print(X_train.shape,X_val.shape, y_train.shape, y_val.shape)
 feats
df_train_raw = pd.read_csv('.. /input/train_users_2.csv')
 df_test = pd.read_csv('.. /input/test_users.csv')
 labels = df_train_raw['country_destination'].values
df_train = df_train_raw.drop(['country_destination'], axis=1)
 id_test = df_test['id']
piv_train = df_train.shape[0]<feature_engineering>
def create_categorical_feats(df, category_columns):
index_df = pd.DataFrame([df.columns, df.dtypes] ).T
index_df = index_df.rename(columns={0:"column_name", 1:'dtype',})
 print(categorical_feats)
 return categorical_feats
df_all = pd.concat(( df_train, df_test), axis=0, ignore_index=True)
 df_all = df_all.drop(['id', 'date_first_booking'], axis=1)
 df_all = df_all.fillna(-1)
 df_all.date_account_created = pd.to_datetime(df_all.date_account_created)
 df_all['dac_year'] = df_all.date_account_created.apply(lambda x: x.year)
 df_all['dac_month'] = df_all.date_account_created.apply(lambda x: x.month)
 df_all['dac_day'] = df_all.date_account_created.apply(lambda x: x.day)
 df_all['dac_weekday'] = df_all.date_account_created.apply(lambda x: x.weekday())
 df_all['dac_week'] = df_all.date_account_created.apply(lambda x: x.week)
 df_all['dac_log_elapsed'] = np.log(( datetime.date(2016, 1, 1)- df_all.date_account_created ).astype('timedelta64[D]'))
df_all = df_all.drop(['date_account_created'], axis=1)
 df_all.timestamp_first_active = pd.to_datetime(df_all.timestamp_first_active, format='%Y%m%d%H%M%S')
 df_all['tfa_year'] = df_all.timestamp_first_active.apply(lambda x: x.year)
 df_all['tfa_month'] = df_all.timestamp_first_active.apply(lambda x: x.month)
 df_all['tfa_day'] = df_all.timestamp_first_active.apply(lambda x: x.day)
 df_all['tfa_weekday'] = df_all.timestamp_first_active.apply(lambda x: x.weekday())
 df_all['tfa_week'] = df_all.timestamp_first_active.apply(lambda x: x.week)
 df_all['tfa_log_elapsed'] = np.log(( datetime.date(2016, 1, 1)- df_all.timestamp_first_active ).astype('timedelta64[D]'))
df_all = df_all.drop(['timestamp_first_active'], axis=1)
 av = df_all.age.values
df_all['age'] = np.where(np.logical_or(av<14, av>90), -1, av)
 df_all['age_year'] = np.where(av > 1900, -1, av)
 ohe_feats = ['gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel',
'affiliate_provider', 'first_affiliate_tracked', 'signup_app', 'first_device_type',
'first_browser']
for f in ohe_feats:
df_all_dummy = pd.get_dummies(df_all[f], prefix=f)
 df_all = df_all.drop([f], axis=1)
 df_all = pd.concat(( df_all, df_all_dummy), axis=1 )<categorify>
categorical_feats = [
'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',
'day',
'month',
]
lgb_params = {
'boosting_type':'gbdt',
'num_leaves':2**4-1,
'learning_rate':0.05,
'n_estimators':3000,
'objective':'binary',
'metrics':'auc',
'feature_fraction':0.8,
'reg_alpha':0.9,
'reg_lambda':0.5,
'random_state':42,
'early_stopping_rounds':100
}
xgb_params = {
'learning_rate':0.1,
'n_estimators':3000,
'objective':'binary:logistic',
'eval_metric':'auc',
'random_state':42,
}
ctb_params = {
'task_type':'GPU',
'learning_rate':0.1,
'n_estimators':10000,
'objective':'Logloss',
'eval_metric':'AUC',
'random_state':10372,
'use_best_model':True,
'verbose':1000,
'early_stopping_rounds':100,
'l2_leaf_reg':0.9,
'cat_features':create_categorical_feats(X, categorical_feats)
 }
vals = df_all.values
X_train = vals[:piv_train]
le = LabelEncoder()
y_train = le.fit_transform(labels)
 X_test = vals[piv_train:]<prepare_x_and_y>
def set_importance(model, feature_importance_df, fold_idx, feats, modeltype):
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
if modeltype=='lgb':
fold_importance_df["importance"] = np.log1p(model.feature_importance(
 importance_type='gain',iteration=model.best_iteration))
else:
fold_importance_df["importance"] = np.log1p(model.feature_importances_)
 fold_importance_df["fold"] = fold_idx + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 return feature_importance_df
np.random.seed(42)
 samples = np.random.choice(piv_train, 50000)
 X_train = vals[samples]
y_train = le.fit_transform(labels)[samples]<train_model>
%%time
kf = StratifiedKFold(n_splits=10,
shuffle=True, random_state=42)
 oof_preds = np.zeros(len(X)).astype(np.float32)
 sub_preds = np.zeros(len(X_test)).astype(np.float32)
 feature_importance_df = pd.DataFrame()
for fold_,(train_idx, val_idx)in enumerate(kf.split(X,y=y)) :
X_train = X.loc[train_idx]
y_train = y.loc[train_idx]
X_val, y_val = X.loc[val_idx], y.loc[val_idx]
if islgb:
model = lgb.LGBMClassifier(**lgb_params)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)],
verbose=100)
 oof_preds[val_idx] = model.predict_proba(
 X_val, num_iteration=model.best_iteration_)[:,1]
sub_preds += model.predict_proba(
 X_test, num_iteration=model.best_iteration_)[:,1] / kf.n_splits
plt.figure(figsize=(12,10))
lgb.plot_importance(model)
 feature_importance_df = set_importance(model, feature_importance_df, fold_, feats, 'lgb')
 if isctb:
model = ctb.CatBoostClassifier(**ctb_params)
 model.fit(X_train, y_train,
eval_set=[ 
(X_val, y_val)],
plot=True)
 oof_preds[val_idx] = model.predict_proba(X_val)[:,1]
sub_preds += model.predict_proba(X_test)[:,1] / kf.n_splits
feature_importance_df = set_importance(
 model, feature_importance_df, fold_, feats, 'ctb')
 plt.title(f'auc_score:{roc_auc_score(y, oof_preds)}')
 sns.distplot(oof_preds)
 sns.distplot(sub_preds)
 plt.legend(['train','test'])
 plt.show()
xgb = XGBClassifier(max_depth=6, learning_rate=0.25, n_estimators=42,
objective='multi:softprob', subsample=0.6, colsample_bytree=0.6, seed=42)
 xgb.fit(X_train, y_train)
 y_pred = xgb.predict_proba(X_test)
 n_class = 5
ids = []
cts = []
for i in range(len(id_test)) :
idx = id_test[i]
ids += [idx] * n_class
cts += le.inverse_transform(np.argsort(y_pred[i])[::-1])[:n_class].tolist()
sub = pd.DataFrame(np.column_stack(( ids, cts)) , columns=['id', 'country'])
 sub.to_csv('sub.csv',index=False )<set_options>
submission = pd.read_csv('.. /input/cat-in-the-dat-ii/sample_submission.csv')
 submission['target'] = sub_preds
print('loading libs...')
 warnings.filterwarnings("ignore")
 print('done' )<load_from_csv>
%%time
submission.to_csv('submission.csv', index=False )
<define_variables><EOS> 
gc.collect()