Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<predict_on_test>
%reset -f
warnings.filterwarnings('ignore' )
y_pred = ce.predict(df_test )<save_to_csv>
import sys
import random
import gc
df_submission = pd.concat([df_test['id'], pd.Series(y_pred, name='target')], axis=1)
 df_submission.to_csv('submission.csv', index=False )<set_options>
import numpy as np
import pandas as pd
warnings.filterwarnings('ignore' )<init_hyperparams>
%matplotlib inline
matplotlib.rc('figure', figsize=(14, 11))
set_matplotlib_formats('retina' )
n_models = 4
n_splits = 12
seed_split = 1111
seed_gm = 2222
cov_type = 'SC_0.5'
n_init = 4
init_params = 'random'
n_clusters_per_class = 3<load_from_csv>
init_notebook_mode(connected=True )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train = train.reset_index(drop=True)
 print('train:', train.shape)
 print('test:', test.shape )<compute_train_metric>
import scipy
from scipy import stats
from scipy.special import boxcox1p
from scipy.special import inv_boxcox1p
def get_cov_estimator(cov_type):
if cov_type == 'LW':
model = LedoitWolf()
elif cov_type == 'OAS':
model = OAS()
elif cov_type == 'MCD':
model = MinCovDet()
elif cov_type[:2] == 'SC':
shrinkage = float(cov_type.split('_')[1])
 model = ShrunkCovariance(shrinkage=shrinkage)
 elif cov_type[:2] == 'GL':
alpha = float(cov_type.split('_')[1])
 model = GraphicalLasso(alpha=alpha)
 return model
def get_mean_cov(x, y, n_clusters_per_class=1, cov_type='LW'):
model = get_cov_estimator(cov_type)
 ones =(y == 1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y == 0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1] * n_clusters_per_class + [m2] * n_clusters_per_class)
 ps = np.stack([p1] * n_clusters_per_class + [p2] * n_clusters_per_class)
 return ms, ps<define_variables>
import sklearn
import sklearn.decomposition
import sklearn.model_selection
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof_avg = np.zeros(len(train))
preds = np.zeros(len(test))
for j in range(n_models):
t1 = time.time()
oof = np.zeros(len(train))
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=n_splits,
random_state=seed_split + j,
shuffle=True)
 for train_idx, valid_idx in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_idx, :],
train2.loc[train_idx]['target'].values,
n_clusters_per_class=n_clusters_per_class,
cov_type=cov_type)
 gm = GaussianMixture(n_components=2 * n_clusters_per_class,
init_params=init_params,
covariance_type='full',
max_iter=100,
n_init=n_init,
precisions_init=ps,
random_state=seed_gm + i)
 gm.fit(np.concatenate([train3[train_idx, :], test3]))
oof[idx1[valid_idx]] += gm.predict_proba(train3[valid_idx, :])[:, :2].sum(1)
 preds[idx2] += gm.predict_proba(test3)[:, :2].sum(1)
 oof_avg += oof
auc_one = roc_auc_score(train['target'], oof)
 auc_avg = roc_auc_score(train['target'], oof_avg)
 print(f' model:{j} AUC_avg:{auc_avg:.5f} AUC:{auc_one:.5f} time:{time.time() - t1:.1f}s' )<save_to_csv>
import lightgbm as lgb
from catboost import CatBoostRegressor, Pool
preds /= n_models * n_splits
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv', index=False )<import_modules>
random.seed(321)
 np.random.seed(321)
 np.set_printoptions(suppress=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 9999
import pandas as pd
import numpy as np
import gc<load_from_csv>
def getDatasetInformation(csv_filepath):

 dataset_tmp = pd.read_csv(csv_filepath)
 dataset_tmp_shape = pd.DataFrame(list(dataset_tmp.shape), index=['No of Rows', 'No of Columns'], columns=['Total'])
 dataset_tmp_shape = dataset_tmp_shape.reset_index()
dataset_tmp_dtypes = dataset_tmp.dtypes.reset_index()
dataset_tmp_dtypes.columns = ['Column Names', 'Column Data Types']
dataset_tmp_desc = pd.DataFrame(dataset_tmp.describe())
 dataset_tmp_desc = dataset_tmp_desc.transpose()
dataset_tmp_unique = dataset_tmp.nunique().reset_index()
dataset_tmp_unique.columns = ["Column Name", "Unique Value(s)Count"]
dataset_tmp_missing = dataset_tmp.isnull().sum(axis=0 ).reset_index()
dataset_tmp_missing.columns = ['Column Names', 'NULL value count per Column']
dataset_tmp_missing = dataset_tmp_missing.sort_values(by='NULL value count per Column', ascending=False)
 dataset_tmp_corr = pd.DataFrame()
return [dataset_tmp, dataset_tmp_shape, dataset_tmp_dtypes, dataset_tmp_desc, dataset_tmp_unique, dataset_tmp_missing, dataset_tmp_corr]
nrows = 10000000
path = '.. /input/quora-question-pairs/'
train = pd.read_csv(path+"train.csv",nrows=nrows ).astype(str)
 test = pd.read_csv(path+"test.csv",nrows=nrows ).astype(str )<import_modules>
dataset_santandervp_train, df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr = getDatasetInformation('.. /input/train.csv' )
from sklearn.metrics import log_loss<statistical_test>
del(df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr )
def try_divide(x, y, val=0.0):

 if y != 0.0:
val = float(x)/ y
return val
def get_jaccard(seq1, seq2):

 set1, set2 = set(seq1), set(seq2)
 return 1 - try_divide(len(set1 & set2),float(len(set1 | set2)))
 def get_dice(A,B):
A, B = set(A), set(B)
 intersect = len(A.intersection(B))
union = float(len(A)+ len(B))
d = try_divide(2*intersect, union)
 return d
def get_sorensen(seq1, seq2):

 set1, set2 = set(seq1), set(seq2)
 return 1- try_divide(2 * len(set1 & set2),float(len(set1)+ len(set2)))
 def get_count_q1_in_q2(seq1,seq2):
set1, set2 = set(seq1), set(seq2)
 return len(set1 & set2)
 def get_ratio_q1_in_q2(seq1,seq2):
set1, set2 = set(seq1), set(seq2)
 try:
return len(set1 & set2)/float(len(set1))
except:
return 0.0
def get_count_of_question(seq1):
return len(seq1)
 def get_count_of_unique_question(seq1):
set1 = set(seq1)
 return len(set1)
 def get_ratio_of_unique_question(seq1):
set1 = set(seq1)
 try:
return len(set1)/float(len(seq1))
except:
return 0.0
def get_count_of_digit(seq1):
return sum([1.for w in seq1 if w.isdigit() ])
 def get_ratio_of_digit(seq1):
try:
return sum([1.for w in seq1 if w.isdigit() ])/float(len(seq1))
except:
return 0.0
def get_sim_feature(X_batch_1,X_batch_2):
X_jaccard = np.array([ get_jaccard(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_dice = np.array([ get_dice(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_count = np.array([ get_count_q1_in_q2(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_ratio = np.array([ get_ratio_q1_in_q2(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_len1 = np.array([ get_count_of_question(x1)for x1 in X_batch_1] ).reshape(-1,1)
 X_len2 = np.array([ get_count_of_question(x2)for x2 in X_batch_2] ).reshape(-1,1)
 X_len1_unique = np.array([ get_count_of_unique_question(x1)for x1 in X_batch_1] ).reshape(-1,1)
 X_len2_unique = np.array([ get_count_of_unique_question(x2)for x2 in X_batch_2] ).reshape(-1,1)
 X_len_diff = np.abs(X_len2-X_len1)
 X_batch_sim = np.hstack([X_jaccard,X_dice,X_count,X_ratio,X_len1,X_len2,X_len1_unique,X_len2_unique,X_len_diff])
 return X_batch_sim 
<prepare_x_and_y>
dataset_santandervp_train_target = dataset_santandervp_train['target']
X = get_sim_feature(train['question1'].apply(lambda x:x.split(' ')) ,train['question2'].apply(lambda x:x.split(' ')))
 X_test = get_sim_feature(test['question1'].apply(lambda x:x.split(' ')) ,test['question2'].apply(lambda x:x.split(' ')))
 y = train['is_duplicate'].values
scaler = StandardScaler()
X = scaler.fit_transform(X)
 X_test = scaler.transform(X_test)
 print(X.shape,X_test.shape)
<categorify>
getContinuousVariableDistributionGraph(tuple(range(dataset_santandervp_train.shape[0])) , dataset_santandervp_train_target, 'Target - Distribution' )
corpus = []
feats = ['question1','question2']
for f in feats:
train[f] = train[f].astype(str)
 test[f] = test[f].astype(str)
 corpus+=train[f].values.tolist()
ngram_range =(1,2)
 max_features = 50000
min_df = 3
tfidf = TfidfVectorizer(max_features=max_features,ngram_range=ngram_range, min_df=min_df)
 tfidf.fit(corpus)
 X_tfidf_q1 = tfidf.transform(train['question1'])
 X_tfidf_q2 = tfidf.transform(train['question2'])
 X_tfidf_test_q1 = tfidf.transform(test['question1'])
 X_tfidf_test_q2 = tfidf.transform(test['question2'])
 X_tfidf = ssp.hstack([X_tfidf_q1,X_tfidf_q2] ).tocsr()
X_tfidf_test = ssp.hstack([X_tfidf_test_q1,X_tfidf_test_q2] ).tocsr()
del X_tfidf_q1
del X_tfidf_q2
del X_tfidf_test_q1
del X_tfidf_test_q2
gc.collect()<normalization>
getContinuousVariableDistributionGraph(tuple(range(dataset_santandervp_train.shape[0])) , np.log1p(dataset_santandervp_train_target), 'Target - log1p - Distribution' )
svd = TruncatedSVD(100)
 svd.fit(ssp.vstack([X_tfidf,X_tfidf_test]))
X_svd = svd.transform(X_tfidf)
 X_test_svd = svd.transform(X_tfidf_test )<concatenate>
dataset_santandervp_train_target = \
np.log1p(dataset_santandervp_train_target )
X = np.hstack([X,X_svd])
 X_test = np.hstack([X_test,X_test_svd])
<split>
dataset_santandervp_train.drop(['ID', 'target'], axis=1, inplace=True )
X_train,X_val,y_train,y_val = train_test_split(X,y,train_size=0.8,random_state=1024)
 print(X_train.shape,X_val.shape )<train_model>
santandervp_columns_one_unique_value = \
dataset_santandervp_train.columns[dataset_santandervp_train.nunique()
== 1].tolist()
dataset_santandervp_train.drop(santandervp_columns_one_unique_value,
axis=1, inplace=True)
 dataset_santandervp_train = dataset_santandervp_train.replace([0, 1563411.76],
np.nan )
lgb_params = {"boosting": "gbdt", 'learning_rate': 0.05,
"feature_fraction": 0.6, "bagging_freq": 1, "bagging_fraction": 0.8 , 'n_estimators': 100000,
"metric": 'mae', "lambda_l1": 0.1, 'num_leaves': 32, 'min_data_in_leaf': 50, "verbose": 1, "num_threads": 8,
"bagging_seed" : 1024,
"seed": 1024,
'feature_fraction_seed': 1024,
}
clf = lgb.LGBMClassifier(**lgb_params)
 clf.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_val, y_val)], verbose = 100, eval_metric ='logloss',early_stopping_rounds=200)
 y_pred = clf.predict_proba(X_val)[:,1]
eval_score = log_loss(y_val,y_pred)
 print(eval_score)
<save_to_csv>
dataset_santandervp_train = np.log1p(dataset_santandervp_train)
 dataset_santandervp_train.head()
y_pred_test = clf.predict_proba(X_test)[:,1]
submission = pd.DataFrame()
submission['test_id'] = test['test_id']
submission['is_duplicate'] = y_pred_test
submission.to_csv('submission_lgb.csv',index=False)
<set_options>
dataset_santandervp_train_tmp = pd.DataFrame()
dataset_santandervp_train_tmp['svp_quantile'] = \
dataset_santandervp_train[dataset_santandervp_train
> 0].quantile(0.5, axis=1)
 dataset_santandervp_train_tmp['svp_max'] = \
dataset_santandervp_train.max(axis=1)
 dataset_santandervp_train_tmp['svp_sum_nonzeroval'] = \ 
(dataset_santandervp_train != 0 ).sum(axis=1)
 dataset_santandervp_train_tmp['svp_skew'] = \
dataset_santandervp_train.skew(axis=1)
 dataset_santandervp_train_tmp['svp_kurtosis'] = \
dataset_santandervp_train.kurtosis(axis=1)
 dataset_santandervp_train_tmp['svp_sum_all'] = \
dataset_santandervp_train.sum(axis=1)
 dataset_santandervp_train_tmp['svp_variance'] = \
dataset_santandervp_train.var(axis=1)
 dataset_santandervp_train_tmp.head()
warnings.filterwarnings('ignore' )<load_from_csv>
dataset_santandervp_train = pd.concat([dataset_santandervp_train,
dataset_santandervp_train_tmp], axis=1)
 dataset_santandervp_train.head(15 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )<concatenate>
( X_train, X_test, y_train, y_test)= \
sklearn.model_selection.train_test_split(dataset_santandervp_train,
dataset_santandervp_train_target, test_size=0.20 )
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}' )<feature_engineering>
catboost_params = {
'iterations': 10000,
'learning_rate': 0.05,
'depth': 11,
'l2_leaf_reg': 40,
'bootstrap_type': 'Bernoulli',
'eval_metric': 'RMSE',
'metric_period': 100,
'od_type': 'Iter',
'od_wait': 80,
'allow_writing_files': False,
'task_type': 'CPU',
'thread_count': 4,
}
catboost_model = CatBoostRegressor(**catboost_params)
 catboost_model.fit(X_train, y_train, eval_set=(X_test, y_test), cat_features=[], use_best_model=True )
for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.94, 'target'] = 1
test.loc[test['target'] < 0.06, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.98, 'target'] = 1
new_train.loc[oof < 0.02, 'target'] = 0
oof2 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof2[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof2)
 print(f'AUC: {auc:.5}' )<save_to_csv>
dataset_transformed_train_lgb = lgb.Dataset(data=X_train, label=y_train)
 dataset_transformed_test_lgb = lgb.Dataset(data=X_test, label=y_test)
 watchlist = [dataset_transformed_train_lgb,
dataset_transformed_test_lgb]
evaluation_results = {}
lightgbm_params = {
'objective': 'regression_l2',
'metric': 'rmse',
'boosting': 'gbdt',
'num_leaves': 144,
'learning_rate': 0.004,
'bagging_fraction': 0.4,
'feature_fraction': 0.6,
'bagging_freq': 4,
'max_depth': 4,
'reg_alpha': 1,
'reg_lambda': 0.1,
'min_child_weight': 10,
'zero_as_missing': True,
'is_training_metric': True,
'verbosity': -1,
'device': 'cpu',
'nthread': 4,
}
lightgbm_model = lgb.train(
 lightgbm_params,
train_set=dataset_transformed_train_lgb,
num_boost_round=10000,
valid_sets=watchlist,
early_stopping_rounds=100,
evals_result=evaluation_results,
verbose_eval=100,
)
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False )<load_from_csv>
max_num_features=37
importance = lightgbm_model.feature_importance()
feature_name = lightgbm_model.feature_name()
tuples = sorted(zip(feature_name, importance), key=lambda x: x[1])
 tuples = tuples[-max_num_features:]
labels, values = zip(*tuples)
(X_train, X_test)=(X_train.loc[ : , labels], X_test.loc[ : , labels] )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<compute_test_metric>
dataset_transformed_train_lgb = lgb.Dataset(data=X_train, label=y_train)
 dataset_transformed_test_lgb = lgb.Dataset(data=X_test, label=y_test)
 watchlist = [dataset_transformed_train_lgb,
dataset_transformed_test_lgb]
evaluation_results = {}
lightgbm_params = {
'objective': 'regression_l2',
'metric': 'rmse',
'boosting': 'gbdt',
'num_leaves': 144,
'learning_rate': 0.004,
'bagging_fraction': 0.4,
'feature_fraction': 0.6,
'bagging_freq': 4,
'max_depth': 4,
'reg_alpha': 1,
'reg_lambda': 0.1,
'min_child_weight': 10,
'zero_as_missing': True,
'is_training_metric': True,
'verbosity': -1,
'device': 'cpu',
'nthread': 4,
}
lightgbm_model = lgb.train(
 lightgbm_params,
train_set=dataset_transformed_train_lgb,
num_boost_round=10000,
valid_sets=watchlist,
early_stopping_rounds=100,
evals_result=evaluation_results,
verbose_eval=100,
)
def get_mv(x,y):
ones =(y==1 ).astype(bool)
 x2 = x[ones]
cov1 = np.cov(x2.T)
 m1 = np.mean(x2, axis = 0)
 zeros =(y==0 ).astype(bool)
 x2b = x[zeros]
cov2 = np.cov(x2b.T)
 m2 = np.mean(x2b, axis = 0)
 mv1 = multivariate_normal(mean=m1, cov=cov1)
 mv2 = multivariate_normal(mean=m2, cov=cov2)
 return mv1, mv2 
<compute_test_metric>
del(dataset_santandervp_train, dataset_santandervp_train_target, dataset_santandervp_train_tmp, dataset_transformed_train_lgb, dataset_transformed_test_lgb, watchlist, X_train, X_test, y_train, y_test)
 gc.collect()
def calc_prob(x,mv1,mv2):
y_pred2 = np.zeros(( len(x),))
for i in range(len(x)) :
a = mv1.pdf(x[i])
 b = mv2.pdf(x[i])
 y_pred2[i] = a/(a+b)
 return y_pred2<statistical_test>
dataset_santandervp_test, df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr = getDatasetInformation('.. /input/test.csv')
 dataset_santandervp_test.head()
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
mv1, mv2 = get_mv(train3[train_index,:],train2.loc[train_index]['target'].values)
 oof[idx1[test_index]] = calc_prob(train3[test_index,:],mv1,mv2)
 preds[idx2] += calc_prob(test3,mv1,mv2)/ skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))<filter>
del(df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr )
test['target'] = preds
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for k in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
mv1, mv2 = get_mv(train3p[train_index,:],train2p.loc[train_index]['target'].values)
 oof[idx1[test_index3]] = calc_prob(train3[test_index3,:],mv1,mv2)
 preds[test2.index] += calc_prob(test3,mv1,mv2)/ skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))<save_to_csv>
dataset_santandervp_test_ID = dataset_santandervp_test['ID']
dataset_santandervp_test.drop(['ID'], axis=1, inplace=True)
 dataset_santandervp_test.drop(santandervp_columns_one_unique_value,
axis=1, inplace=True)
 dataset_santandervp_test = dataset_santandervp_test.replace([0, 1563411.76],
np.nan )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<load_from_csv>
dataset_santandervp_test = np.log1p(dataset_santandervp_test)
 dataset_santandervp_test.head()
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )<count_missing_values>
dataset_santandervp_test_tmp = pd.DataFrame()
dataset_santandervp_test_tmp['svp_quantile'] = \
dataset_santandervp_test[dataset_santandervp_test
> 0].quantile(0.5, axis=1)
 dataset_santandervp_test_tmp['svp_max'] = \
dataset_santandervp_test.max(axis=1)
 dataset_santandervp_test_tmp['svp_sum_nonzeroval'] = \ 
(dataset_santandervp_test != 0 ).sum(axis=1)
 dataset_santandervp_test_tmp['svp_skew'] = \
dataset_santandervp_test.skew(axis=1)
 dataset_santandervp_test_tmp['svp_kurtosis'] = \
dataset_santandervp_test.kurtosis(axis=1)
 dataset_santandervp_test_tmp['svp_sum_all'] = \
dataset_santandervp_test.sum(axis=1)
 dataset_santandervp_test_tmp['svp_variance'] = \
dataset_santandervp_test.var(axis=1)
 dataset_santandervp_test_tmp.head()
null_cnt = train.isnull().sum().sort_values()
print('null count:', null_cnt[null_cnt > 0] )<count_unique_values>
dataset_santandervp_test = pd.concat([dataset_santandervp_test,
dataset_santandervp_test_tmp], axis=1)
 dataset_santandervp_test.head(15 )
print(train['wheezy-copper-turtle-magic'].describe())
 print()
print('unique value count:', train['wheezy-copper-turtle-magic'].nunique() )<drop_column>
catboost_model_predictions_test = \
catboost_model.predict(dataset_santandervp_test)
 catboost_model_predictions_test
numcols = train.drop(['id','target','wheezy-copper-turtle-magic'],axis=1 ).select_dtypes(include='number' ).columns.values<train_model>
catboost_model_predictions_test = \
np.expm1(catboost_model_predictions_test)
 catboost_model_predictions_test
pca = PCA()
pca.fit(train[numcols])
 plt.xlabel('components')
 plt.plot(np.add.accumulate(pca.explained_variance_ratio_))
plt.show()<train_model>
dataset_santandervp_test = dataset_santandervp_test.loc[ : , labels]
X_subset = train[train['wheezy-copper-turtle-magic'] == 1][numcols]
Y_subset = train[train['wheezy-copper-turtle-magic'] == 1]['target']
for k in range(2, 10):
knc = KNeighborsClassifier(n_neighbors=k)
 knc.fit(X_subset, Y_subset)
 score = knc.score(X_subset, Y_subset)
 print("[{}] score: {:.2f}".format(k, score))<train_on_grid>
lightgbm_model_predictions_test = \
lightgbm_model.predict(dataset_santandervp_test,
num_iteration=lightgbm_model.best_iteration)
 lightgbm_model_predictions_test
filters = [('StandardScaler', StandardScaler()), 
('MinMaxScaler', MinMaxScaler()), 
('PCA', PCA(n_components=0.98)) , 
('KernelPCA(poly)', KernelPCA(kernel="poly", degree=3, gamma=15)) , 
('KernelPCA(rbf)', KernelPCA(kernel="rbf")) ,
]
for name, f in filters:
X_subset2 = f.fit_transform(X_subset)
 knc = KNeighborsClassifier(n_neighbors=3)
 knc.fit(X_subset2, Y_subset)
 score = knc.score(X_subset2, Y_subset)
 print("[{}] score: {:.2f}".format(name, score))<train_model>
lightgbm_model_predictions_test = \
np.expm1(lightgbm_model_predictions_test)
 lightgbm_model_predictions_test
knn3 = KNeighborsClassifier(n_neighbors=3)
 knnb = BaggingClassifier(base_estimator=knn3, n_estimators=20)
 knnb.fit(X_subset, Y_subset)
 score = knnb.score(X_subset, Y_subset)
 print("score: {:.2f}".format(score))<concatenate>
dataset_submission = pd.DataFrame()
dataset_submission['ID'] = dataset_santandervp_test_ID
dataset_submission['target'] =(( lightgbm_model_predictions_test * 0.5)+(catboost_model_predictions_test * 0.5))
dataset_submission.to_csv('submission.csv', index=False,
float_format='%.2f' )
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()
all_data.head()<count_unique_values>
dataset_submission_lightgbm = pd.DataFrame()
dataset_submission_lightgbm['ID'] = dataset_santandervp_test_ID
dataset_submission_lightgbm['target'] =(( lightgbm_model_predictions_test))
dataset_submission_lightgbm.to_csv('submission_lightgbm.csv', index=False,
float_format='%.2f' )
<data_type_conversions><EOS> 
dataset_submission_catboost = pd.DataFrame()
dataset_submission_catboost['ID'] = dataset_santandervp_test_ID
dataset_submission_catboost['target'] =(( catboost_model_predictions_test))
dataset_submission_catboost.to_csv('submission_catboost.csv', index=False,
float_format='%.2f' )