Kaggle competition: 2019 Data Science Bowl
<SOS> metric: QuadraticWeightedKappa Kaggle data source: data-science-bowl-2019<save_to_csv>
import os
import pandas as pd
import seaborn as sns
import numpy as np
import datetime
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegressionCV
from sklearn import model_selection
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix,classification_report
import matplotlib.pyplot as plt
from pprint import pprint
from bayes_opt import BayesianOptimization
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import cohen_kappa_score
import xgboost as xgb
from xgboost import XGBClassifier
from xgboost import plot_importance
from sklearn.model_selection import StratifiedKFold, GroupKFold,GroupShuffleSplit,StratifiedShuffleSplit
from sklearn.preprocessing import LabelEncoder
from sklearn import model_selection
from sklearn.cross_decomposition import PLSRegression, PLSSVD
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import scale
from sklearn.utils import class_weight
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import hashlib
from ml_metrics import quadratic_weighted_kappa
from catboost import CatBoostClassifier,Pool,CatBoostRegressor
import shap
import statistics
from functools import partial
import scipy as sp
from lightgbm import LGBMRegressor
import lightgbm as lgb
from xgboost import XGBRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import AdaBoostRegressor
from mlxtend.regressor import StackingCVRegressor
import pickle
from sklearn.model_selection import cross_val_score
from sklearn.utils import shuffle
sub.to_csv('submission.csv', index = False)
 sub.head()<predict_on_test>
df_train=pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv' )
def permute_predict(y):
_y = y.copy()
_c1 = _y < 0.00001
_c2 = _y > 0.99999
_y[_c1] = _y[_c1].max() - _y[_c1] + _y[_c1].min()
_y[_c2] = _y[_c2].max() - _y[_c2] + _y[_c2].min()
return _y<set_options>
spec=pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv' )
warnings.filterwarnings('ignore' )<load_from_csv>
df_test=pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv' )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<drop_column>
spec['info']=spec['info'].str.upper()
spec['hashed_info']=spec['info'].transform(hash)
 spec_unique=pd.DataFrame(spec[['hashed_info']].drop_duplicates())
 spec_unique['deduped_event_id']=np.arange(len(spec_unique))
spec=pd.merge(spec,spec_unique,on='hashed_info',how='left')
 z=dict(zip(spec.event_id,spec.deduped_event_id))
df_train['event_id']=df_train['event_id'].map(z)
 df_test['event_id']=df_test['event_id'].map(z)
 df_train=df_train[df_train['event_id']!=137]
df_event_id_train=pd.pivot_table(df_train.loc[:,['installation_id','game_session','event_id']],aggfunc=len,columns=['event_id'],index=['installation_id','game_session'] ).add_prefix('event_id_' ).rename_axis(None,axis=1 ).reset_index()
df_event_id_test=pd.pivot_table(df_test.loc[:,['installation_id','game_session','event_id']],aggfunc=len,columns=['event_id'],index=['installation_id','game_session'] ).add_prefix('event_id_' ).rename_axis(None,axis=1 ).reset_index()
df_event_id_train=df_event_id_train.fillna(0)
 df_event_id_train=df_event_id_train.fillna(0)
 df_event_id_test=df_event_id_test.fillna(0 )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic' )<create_dataframe>
def create_features(df):
df['timestamp']=pd.to_datetime(df['timestamp'])
 df['Incorrect_Game_Attempt']=np.where(( df['event_data'].str.contains('"correct":false')&(df['type']=='Game')) ,1,0)
 df['Correct_Game_Attempt']=np.where(( df['event_data'].str.contains('"correct":true')&(df['type']=='Game')) ,1,0)
 df['Is_Weekend']=np.where(((df['timestamp'].dt.day_name() =='Sunday')|(df['timestamp'].dt.day_name() =='Saturday')) ,1,0)
 df['Phase_Of_Day']=np.where(df['timestamp'].dt.hour.isin(range(6,12)) ,'Morning',np.where(df['timestamp'].dt.hour.isin(range(13,19)) ,'Evening','Night'))
df_world=pd.pivot_table(df.loc[df['world']!='NONE',['installation_id','game_session','world']].drop_duplicates() ,index=['installation_id','game_session'],columns=['world'],aggfunc=len ).add_prefix('rolling_' ).rename_axis(None, axis=1 ).reset_index()
df_type_world=pd.merge(df_world,pd.pivot_table(df.loc[:,['installation_id','game_session','type']].drop_duplicates() ,index=['installation_id','game_session'],columns=['type'],fill_value=0,aggfunc=len ).rename_axis(None, axis=1 ).reset_index() ,on=['installation_id','game_session'],how='right')
 df_type_world_title=pd.merge(df_type_world,pd.pivot_table(df.loc[:,['installation_id','game_session','title']].drop_duplicates() ,index=['installation_id','game_session'],columns=['title'],fill_value=0,aggfunc=len ).add_prefix('rolling_' ).rename_axis(None, axis=1 ).reset_index() ,on=['installation_id','game_session'],how='right')
 df_activity_weekend=pd.merge(df_type_world_title,pd.DataFrame(pd.pivot_table(df.loc[:,['installation_id','game_session','Is_Weekend']].drop_duplicates() ,index=['installation_id','game_session'],columns=['Is_Weekend'],fill_value=0,aggfunc=len)).add_prefix('Weekend_' ).rename_axis(None, axis=1 ).reset_index() ,on=['installation_id','game_session'],how='right')
 df_activity_weekend_phase_of_day=pd.merge(pd.DataFrame(pd.pivot_table(df.loc[:,['installation_id','game_session','Phase_Of_Day']].drop_duplicates() ,index=['installation_id','game_session'],columns=['Phase_Of_Day'],fill_value=0,aggfunc=len)).rename_axis(None, axis=1 ).reset_index() ,df_activity_weekend,on=['installation_id','game_session'],how='left')
 df_train_Assessed=df.copy()
df_train_Assessed['Incorrect_Attempt']=np.where(( df['event_data'].str.contains('"correct":false')) &(((df['title'] != "Bird Measurer(Assessment)")&(df['event_code']==4100)) |(( df['title'] == "Bird Measurer(Assessment)")&(df['event_code']==4110))),1,0)
 df_train_Assessed['Correct_Attempt']=np.where(( df['event_data'].str.contains('"correct":true')) &(((df['title'] != "Bird Measurer(Assessment)")&(df['event_code']==4100)) |(( df['title'] == "Bird Measurer(Assessment)")&(df['event_code']==4110))),1,0)
 df_train_acc=df_train_Assessed[df_train_Assessed['title'].isin(['Bird Measurer(Assessment)','Mushroom Sorter(Assessment)','Cauldron Filler(Assessment)','Chest Sorter(Assessment)','Cart Balancer(Assessment)'])].groupby(['installation_id','title','game_session'])['Incorrect_Attempt','Correct_Attempt'].sum().rename_axis(None, axis=1 ).reset_index()
df_train_acc['Total_Attempts']=df_train_acc.apply(lambda x: x['Incorrect_Attempt'] + x['Correct_Attempt'], axis=1)
 df_train_acc['accuracy']=np.where(df_train_acc['Total_Attempts']>0,df_train_acc['Correct_Attempt']/ df_train_acc['Total_Attempts'],0)
 df_train_acc['accuracy_group']=np.where(df_train_acc['accuracy']==1,3,np.where(df_train_acc['accuracy']==.5,2,np.where(df_train_acc['accuracy']==0,0,1)))
 df_game_attempt=df.groupby(['installation_id','game_session'])['Incorrect_Game_Attempt','Correct_Game_Attempt'].sum().rename_axis(None, axis=1 ).reset_index()
df_event_codes=pd.pivot_table(df_train_Assessed.loc[:,['installation_id','game_session','event_code']],index=['installation_id','game_session'],columns=['event_code'],fill_value=0,aggfunc=len ).add_prefix('event_code_' ).rename_axis(None, axis=1 ).reset_index()
df_final=pd.merge(pd.merge(df_train_acc,df_activity_weekend_phase_of_day,on=['installation_id','game_session'],how='right'),df_event_codes,on=['installation_id','game_session'],how='right')
 df_gametime=df.groupby(['installation_id','game_session'])['game_time','timestamp','event_count'].max().reset_index()
df_final=pd.merge(df_final,df_gametime,on=['installation_id','game_session'],how='left')
 df_final=df_final.fillna(value=0)
 df_final=pd.merge(df_final,df.loc[df['world']!="NONE",['installation_id','game_session','world']].drop_duplicates() ,on=['installation_id','game_session'],how='left')
 df_final=pd.merge(df_final,df_game_attempt,on=['installation_id','game_session'],how='left')
 df_final=df_final.fillna(value=0)
 return(df_final )
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(2.3 ).fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
for c in range(train3.shape[1]):
low_=np.quantile(train3[:,c] , 0.001)
 up_=np.quantile(train3[:,c], 0.999)
 train3[:,c]=np.clip(train3[:,c],low_, up_)
 test3[:,c]=np.clip(test3[:,c],low_, up_)
 skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
gmm=GMM(n_components=5, random_state=42, covariance_type='full')
 gmm.fit(np.vstack([train3[train_index], test3]))
gmm_1_train=gmm.predict_proba(train3[train_index])
 gmm_1_val=gmm.predict_proba(train3[test_index])
 gmm_1_test=gmm.predict_proba(test3)
 gmm=GMM(n_components=4, random_state=42, covariance_type='full')
 gmm.fit(np.vstack([train3[train_index], test3]))
gmm_2_train=gmm.predict_proba(train3[train_index])
 gmm_2_val=gmm.predict_proba(train3[test_index])
 gmm_2_test=gmm.predict_proba(test3)
 gmm=GMM(n_components=6, random_state=42, covariance_type='full')
 gmm.fit(np.vstack([train3[train_index], test3]))
gmm_3_train=gmm.predict_proba(train3[train_index])
 gmm_3_val=gmm.predict_proba(train3[test_index])
 gmm_3_test=gmm.predict_proba(test3)
 bgm=BGM(n_components=5, random_state=42)
 bgm.fit(np.vstack([train3[train_index], test3]))
bgm_1_train=bgm.predict_proba(train3[train_index])
 bgm_1_val=bgm.predict_proba(train3[test_index])
 bgm_1_test=bgm.predict_proba(test3)
 bgm=BGM(n_components=4, random_state=42)
 bgm.fit(np.vstack([train3[train_index], test3]))
bgm_2_train=bgm.predict_proba(train3[train_index])
 bgm_2_val=bgm.predict_proba(train3[test_index])
 bgm_2_test=bgm.predict_proba(test3)
 bgm=BGM(n_components=6, random_state=42)
 bgm.fit(np.vstack([train3[train_index], test3]))
bgm_3_train=bgm.predict_proba(train3[train_index])
 bgm_3_val=bgm.predict_proba(train3[test_index])
 bgm_3_test=bgm.predict_proba(test3)
 _train = np.hstack(( train3[train_index],
gmm_1_train, gmm_2_train, gmm_3_train,
bgm_1_train, bgm_2_train, bgm_3_train))
_val = np.hstack(( train3[test_index],
gmm_1_val, gmm_2_val, gmm_3_val,
bgm_1_val, bgm_2_val, bgm_3_val))
_test = np.hstack(( test3,
gmm_1_test, gmm_2_test, gmm_3_test,
bgm_1_test, bgm_2_test, bgm_3_test))
clf = QuadraticDiscriminantAnalysis(reg_param=0.04, tol=0.01)
 clf.fit(_train,train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(_val)[:,1]
preds[idx2] += clf.predict_proba(_test)[:,1] / skf.n_splits
print(i, roc_auc_score(train2['target'], oof[idx1]))
print(roc_auc_score(train['target'], oof))<save_to_csv>
def rolling_exponential_average(df):
col=list(df.select_dtypes(include=['float64','int64'] ).columns)
 df=df.sort_values(['installation_id','timestamp'])
 df_rolling_avg=df.groupby(['installation_id'])[col].apply(lambda x:x.ewm(alpha=0.1,min_periods=1 ).mean())
 df_rolling_avg=df_rolling_avg.rename(columns={'accuracy_group':'rolling_accuracy_group','CRYSTALCAVES':'rolling_CRYSTALCAVES','MAGMAPEAK':'rolling_MAGMAPEAK', 'TREETOPCITY':'rolling_TREETOPCITY'})
 df_rolling_avg.index.names=['level_1']
return(df_rolling_avg )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = permute_predict(preds)
 sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<import_modules>
def create_final_dataset(df,Is_Test=0):
f=create_features(df)
 df_assessed=set(df.loc[df['type']=='Assessment'] 
.loc[(( df['title'] != "Bird Measurer(Assessment)")&(df['event_code']==4100)) | 
(( df['title'] == "Bird Measurer(Assessment)")&(df['event_code']==4110)) ] 
.loc[:,'game_session'].unique())
 if(Is_Test==0):
f=pd.merge(f,df_event_id_train,on=['installation_id','game_session'],how='left')
 else:
f=pd.merge(f,df_event_id_test,on=['installation_id','game_session'],how='left')
 df_rolling_avg=rolling_exponential_average(f)
 df_rolling_avg=df_rolling_avg.fillna(0)
 f.index.names=['level_1']
t=pd.merge(f[['installation_id','game_session','timestamp','world','title']],df_rolling_avg,left_index=True,right_index=True,how='left')
 t=t.reset_index(drop=True ).sort_values(['installation_id','timestamp'])
 tm=pd.merge(t[['installation_id','game_session','title','world']],t.groupby(['installation_id'] ).shift(1 ).drop(['game_session','title','world'],axis=1),left_index=True,right_index=True,how='left')
 tm=tm.dropna()
tm=tm[tm['game_session'].isin(df_assessed)]
if(Is_Test==0):
tm['last_attempt_time']=tm.groupby(['installation_id'] ).shift(1)['timestamp']
tm['last_world']=tm.groupby(['installation_id'] ).shift(1)['world']
tm['last_title']=tm.groupby(['installation_id'] ).shift(1)['title']
tm['time_since_last_attempt']=np.where(tm['last_attempt_time'].isna() ,0,(tm['timestamp']-tm['last_attempt_time'] ).dt.seconds)
 else:
df_latest_Attempt=t.sort_values(['installation_id','timestamp'] ).groupby(['installation_id'],as_index=False ).last()
tm=tm.sort_values(['installation_id','timestamp','title','world'] ).groupby(['installation_id'],as_index=False ).last().rename(columns={'timestamp':'last_attempt_time','world':'last_world','title':'last_title'} ).drop_duplicates()
df_latest_Attempt=pd.merge(tm[['installation_id','last_attempt_time','last_world','last_title']],df_latest_Attempt,on=['installation_id'],how='right')
 df_latest_Attempt['time_since_last_attempt']=np.where(df_latest_Attempt['last_attempt_time'].isna() ,0,(pd.to_datetime(df_latest_Attempt['timestamp'].dt.tz_localize(None)) -pd.to_datetime(df_latest_Attempt['last_attempt_time'])).dt.seconds)
 tm=df_latest_Attempt.copy()
tm1=pd.merge(tm,f[['accuracy_group','game_session','installation_id']],on=['game_session','installation_id'],how='left')
 tm1=tm1.fillna(0)
 return(tm1 )
%matplotlib inline<load_from_csv>
f_train=create_final_dataset(df_train,0)
 f_test=create_final_dataset(df_test,1)
 f_train = f_train.reindex(sorted(f_train.columns), axis=1)
 f_test = f_test.reindex(sorted(f_test.columns), axis=1 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<concatenate>
f_train_1=shuffle(f_train )
def data_augmentation(X, y):
mean_train2_0 = X[y==0].mean()
mean_train2_1 = X[y==1].mean()
train2_0 = 2*mean_train2_0.reshape(1,-1)- X[y==0]
train2_1 = 2*mean_train2_1.reshape(1,-1)- X[y==1]
tmp_train2_0 = np.vstack([X[y==0], train2_0])
 tmp_train2_1 = np.vstack([X[y==1], train2_1])
 train2 = np.vstack([tmp_train2_0, tmp_train2_1])
 y2 = np.array([0]*len(tmp_train2_0)+ [1]*len(tmp_train2_1))
return np.vstack([X, train2]), np.concatenate([y, y2])
<train_on_grid>
f_train_1.reset_index(inplace=True, drop=True )
def get_kmeans_clusters(x, y, k_pos=1, k_neg=1):
x_zeros = x[y==0]
x_ones = x[y==1]
model_0 = KMeans(n_clusters=k_neg)
 model_1 = KMeans(n_clusters=k_pos)
 model_0.fit(x_zeros)
 model_1.fit(x_ones)
 model_0_clus = [x_zeros[model_0.labels_==k] for k in range(model_0.n_clusters)]
model_1_clus = [x_ones[model_1.labels_==k] for k in range(model_1.n_clusters)]
return model_1_clus + model_0_clus
def fit_multicluster_gmm(x, y, xt, k_pos, k_neg, max_iter=100):

 clusters = get_kmeans_clusters(x, y, k_pos=k_pos, k_neg=k_neg)
 for i in range(len(clusters)) :
x_cluster = clusters[i]
model = ShrunkCovariance()
model.fit(x_cluster)
 if(i==0):
ps = np.expand_dims(model.precision_, axis=0)
 ms = np.expand_dims(model.location_, axis=0)
 else:
ps = np.concatenate([ps, np.expand_dims(model.precision_, axis=0)], axis=0)
 ms = np.concatenate([ms, np.expand_dims(model.location_, axis=0)], axis=0)
 gm = mixture.GaussianMixture(n_components=k_pos+k_neg,
init_params='random',
covariance_type='full',
tol=0.001,
reg_covar=0.001,
max_iter=100,
n_init=5,
means_init=ms,
precisions_init=ps)
 gm.fit(np.vstack(( x.astype(np.float), xt.astype(np.float))))
preds = gm.predict_proba(x.astype(np.float)) [:,0]
score = roc_auc_score(y, preds)
 return score, gm, k_pos, k_neg
def get_mean_cov(x,y, model=GraphicalLasso() , max_iter=100):
try:
model.set_params(**{'max_iter':200})
 except:
pass
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps<prepare_x_and_y>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return -cohen_kappa_score(y, preds, weights = 'quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method = 'nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return preds
def coefficients(self):
return self.coef_['x']
def extract_wheezy_copper_turtle_magic(train, i):
train2 = train[train['wheezy-copper-turtle-magic']==i].copy()
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
target = train2['target'].astype(np.int ).values
train2.reset_index(drop=True, inplace=True)
 return train2.drop(['id', 'target'], axis=1), test2.drop(['id'], axis=1), idx1, idx2, target<define_variables>
def model_classification(train_X,train_y,val_X,val_y,cat_feat):
CatBoost=CatBoostClassifier(
 learning_rate=0.2,
max_depth=6,
bootstrap_type='Bernoulli',
l2_leaf_reg= 0.2,
iterations=400,
subsample=.7,
loss_function='MultiClass',
eval_metric="WKappa",
random_seed=4,cat_features=cat_feat,class_weights=[1,2,2,.5],
classes_count=4)
 clf = CatBoost.fit(train_X,train_y,eval_set=(val_X,val_y), verbose_eval=False,use_best_model=True,plot=True,early_stopping_rounds=100)
 return(clf)
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
test_preds = np.zeros(len(test))
oof_gmm = np.zeros(len(train))
test_preds_gmm = np.zeros(len(test))
oof_gmm_2 = np.zeros(len(train))
test_preds_gmm_2 = np.zeros(len(test))
trials = 3
cat_dict = dict()
cluster_report = list()
for i in tqdm(range(512)) :
train2, test2, idx1, idx2, target = extract_wheezy_copper_turtle_magic(train, i)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ])
 train3 = pipe.fit_transform(train2[cols])
 test3 = pipe.fit_transform(test2[cols])
 data2 = StandardScaler().fit_transform(PCA(n_components=40, random_state=4 ).fit_transform(data[cols]))
train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
cat_dict[i] = train3.shape[1]
try:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=3)
 except:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=1)
 oof_gmm[idx1] = gm.predict_proba(train3)[:,0]
test_preds_gmm[idx2] += gm.predict_proba(test3)[:,0]
try:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=3, k_neg=3)
 except:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=1)
 clusters = gm.predict_proba(train3 ).shape[1]
oof_gmm_2[idx1] = np.sum(gm.predict_proba(train3)[:,:clusters//2], axis=1)
 test_preds_gmm_2[idx2] += np.sum(gm.predict_proba(test3)[:,:clusters//2], axis=1 )<compute_test_metric>
def model_regressor(train_X,train_y,val_X,val_y,fit):
lgb_model=LGBMRegressor(objective='regression',
max_depth=13,num_leaves=6,
n_estimators=1200,
bagging_fraction=0.8,
random_state=42,verbose=0,reg_alpha=.3,reg_lambda=.3,callbacks=[lgb.reset_parameter(learning_rate=[0.2] * 200 + [0.15] * 250+[.1]*250+[.05]*250+[.01]*250)])
 if fit==0:
reg=lgb_model.fit(train_X,train_y,eval_set=(val_X,val_y), verbose=False)
 else:
reg=lgb_model.fit(train_X,train_y,verbose=False)
 return(reg)
oof_auc_gmm = roc_auc_score(train['target'], oof_gmm)
 print('OOF AUC: =',round(oof_auc_gmm, 5))
oof_auc_gmm_2 = roc_auc_score(train['target'], oof_gmm_2)
 print('OOF AUC: =',round(oof_auc_gmm_2, 5))
oof_auc_blend = roc_auc_score(train['target'],(0.3*oof_gmm+0.7*oof_gmm_2))
print('OOF AUC: =',round(oof_auc_blend, 5))<save_to_csv>
def fit(X_train,y_train,classifier):
train_X = X_train
train_y=y_train
train_X=train_X.drop('installation_id',axis=1)
 if(classifier==1):
clf=model_classification(train_X,train_y,0,0,cat_feat)
 pred_val=clf.predict(val_X)
 print(classification_report(val_y,pred_val))
score=cohen_kappa_score(pred_val,val_y,weights='quadratic')
 scores.append(score)
 print('choen_kappa_score :',score)
 coeff=0
else:
clf=model_regressor(train_X,train_y,0,0,1)
 optR = OptimizedRounder()
optR.fit(clf.predict(train_X), train_y)
 coeff = optR.coefficients()
return(clf,coeff )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = test_preds_gmm
sub.to_csv('submission_gmm.csv',index=False)
 sub['target'] = test_preds_gmm_2
sub.to_csv('submission_gmm_2.csv',index=False)
 sub['target'] =(test_preds_gmm + test_preds_gmm_2)/2
sub.to_csv('submission_blend.csv',index=False )<import_modules>
def cv(X_train,y_train,classifier=1,n_splits=5):
scores=[]
Gf=GroupKFold(n_splits=n_splits)
 i=0
for train_index, val_index in Gf.split(X_train, y_train,X_train['installation_id']):
train_X = X_train.iloc[train_index]
train_X=train_X.drop('installation_id',axis=1)
 val=X_train.iloc[val_index].join(y_train[val_index])
 train_y = y_train[train_index]
val=val.groupby('installation_id',as_index=False ).apply(lambda x :x.iloc[np.random.randint(0,len(x)) ])
 val_X=val.drop(['accuracy_group','installation_id'],axis=1)
 val_y=val['accuracy_group']
if(classifier==1):
clf=model_classification(train_X,train_y,val_X,val_y,0)
 pred_val=clf.predict(val_X)
 print(classification_report(val_y,pred_val))
score=cohen_kappa_score(pred_val,val_y,weights='quadratic')
 scores.append(score)
 print('choen_kappa_score :',score)
 else:
clf=model_regressor(train_X,train_y,val_X,val_y,0)
 print(clf.predict(train_X))
optR = OptimizedRounder()
optR.fit(clf.predict(train_X), train_y)
 coeff = optR.coefficients()
print(coeff)
 val_pred=clf.predict(val_X)
 pred_val=[regression_class(val_pred_i,coeff)for val_pred_i in val_pred]
print(classification_report(val_y,pred_val))
score=cohen_kappa_score(pred_val,val_y,weights='quadratic')
 scores.append(score)
 print('choen_kappa_score :',score)
 print(statistics.mean(scores),statistics.stdev(scores))
return(clf)
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import roc_auc_score
from tqdm import tqdm
from sklearn.covariance import EmpiricalCovariance
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import sympy 
<load_from_csv>
def regression_class(pred,coeff):
if(pred<=coeff[0]):
class_=0
elif(coeff[0]<pred<=coeff[1]):
class_=1
elif(coeff[1]<pred<=coeff[2]):
class_=2
else:
class_=3
return(class_ )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<data_type_conversions>
f_train_1['accuracy_group']=f_train_1['accuracy_group'].astype('int' )
def get_mean_cov(x,y):
model = OAS()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps 
<define_variables>
X_train=f_train_1.drop(['accuracy_group','game_session','last_attempt_time','timestamp'],axis=1)
 y_train=f_train_1['accuracy_group']
X_test=f_test.drop(['accuracy_group','game_session','last_attempt_time','timestamp'],axis=1)
 y_test=f_test['accuracy_group']
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
x_train, y_train = train3[train_index,:], train2.loc[train_index]['target'].values
x_train_0 = x_train[y_train==0]
x_train_1 = x_train[y_train==1]
brc = Birch(branching_factor=50, n_clusters=3, threshold=0.4, compute_labels=True)
 labels_0 = brc.fit_predict(x_train_0)
 labels_1 = brc.fit_predict(x_train_1)
 zero_mean = []
zero_cov = []
for l in np.unique(labels_0):
model = OAS()
model.fit(x_train_0[labels_0==l])
 p = model.precision_
m = model.location_
zero_mean.append(m)
 zero_cov.append(p)
 one_mean = []
one_cov = []
for l in np.unique(labels_1):
model = OAS()
model.fit(x_train_1[labels_1==l])
 p = model.precision_
m = model.location_
one_mean.append(m)
 one_cov.append(p)
 ms = np.stack(zero_mean + one_mean)
 ps = np.stack(zero_cov + one_cov)
 gm = GaussianMixture(n_components=6, init_params='random',
covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1, means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:, 0:3].mean(axis=1)
 preds[idx2] += gm.predict_proba(test3)[:, 0:3].mean(axis=1)/ skf.n_splits
print('AUC ', i, roc_auc_score(1- train2['target'], oof[idx1]))
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5)) 
<compute_test_metric>
c=['last_world','last_title','world','title']
cat_feat=[X_train.columns.get_loc(col)for col in c]
X_train=X_train.join(pd.get_dummies(X_train[c])).drop(c,axis=1 )
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5))<load_from_csv>
X_test=X_test.join(pd.get_dummies(X_test[c])).drop(c,axis=1 )
x_test_0 = pd.read_csv('.. /input/test.csv')
 x_test_0['target']=preds<drop_column>
reg_cv=cv(X_train,y_train,0,5 )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic' )<set_options>
reg_model,coeff=fit(X_train,y_train,0 )
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for k in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = x_test_0[x_test_0['wheezy-copper-turtle-magic']==k]
test2p = test2
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = KFold(n_splits=17, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p):
test_index3 = test_index[ test_index<len(train3)]
clf = neighbors.KNeighborsRegressor(n_neighbors=9, weights='distance')
 clf.fit(train3p[train_index,:], train2p.loc[train_index]['target'])
 oof[idx1[test_index3]] = clf.predict(train3[test_index3,:])
 preds[test2.index] += clf.predict(test3)/ skf.n_splits
if k%64==0: print(k)
 auc = roc_auc_score(train['target'], oof)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))<save_to_csv>
f_test['accuracy_group']=[regression_class(pred_i,coeff)for pred_i in reg_model.predict(X_test.drop('installation_id',axis=1)) ]
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<define_variables>
Submission_file=f_test[['installation_id','accuracy_group']]
PATH_BASE = Path('.. /input')
 PATH_WORKING = Path('.. /working' )<load_from_csv>
Submission_file['accuracy_group'].value_counts()
<data_type_conversions><EOS> 
Submission_file.to_csv('submission.csv',index=False )