Kaggle competition: 2019 Data Science Bowl
<SOS> metric: QuadraticWeightedKappa Kaggle data source: data-science-bowl-2019<concatenate>
%matplotlib inline
warnings.filterwarnings("ignore")
 pd.set_option('max_rows', 500)
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()
all_data.head()<count_unique_values>
def read_data() :
print('Reading train.csv file.... ')
 train = pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv')
 print('Training.csv file have {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Reading test.csv file.... ')
 test = pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv')
 print('Test.csv file have {} rows and {} columns'.format(test.shape[0], test.shape[1]))
print('Reading train_labels.csv file.... ')
 train_labels = pd.read_csv('/kaggle/input/data-science-bowl-2019/train_labels.csv')
 print('Train_labels.csv file have {} rows and {} columns'.format(train_labels.shape[0], train_labels.shape[1]))
print('Reading specs.csv file.... ')
 specs = pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv')
 print('Specs.csv file have {} rows and {} columns'.format(specs.shape[0], specs.shape[1]))
print('Reading sample_submission.csv file.... ')
 sample_submission = pd.read_csv('/kaggle/input/data-science-bowl-2019/sample_submission.csv')
 print('Sample_submission.csv file have {} rows and {} columns'.format(sample_submission.shape[0], sample_submission.shape[1]))
return train, test, train_labels, specs, sample_submission
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )<data_type_conversions>
def encode_title(train, test, train_labels):
train['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), train['title'], train['event_code']))
test['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), test['title'], test['event_code']))
all_title_event_code = list(set(train["title_event_code"].unique() ).union(test["title_event_code"].unique()))
list_of_user_activities = list(set(train['title'].unique() ).union(set(test['title'].unique())))
 list_of_event_code = list(set(train['event_code'].unique() ).union(set(test['event_code'].unique())))
 list_of_event_id = list(set(train['event_id'].unique() ).union(set(test['event_id'].unique())))
 list_of_worlds = list(set(train['world'].unique() ).union(set(test['world'].unique())))
 activities_map = dict(zip(list_of_user_activities, np.arange(len(list_of_user_activities))))
activities_labels = dict(zip(np.arange(len(list_of_user_activities)) , list_of_user_activities))
activities_world = dict(zip(list_of_worlds, np.arange(len(list_of_worlds))))
assess_titles = list(set(train[train['type'] == 'Assessment']['title'].value_counts().index ).union(set(test[test['type'] == 'Assessment']['title'].value_counts().index)))
 train['title'] = train['title'].map(activities_map)
 test['title'] = test['title'].map(activities_map)
 train['world'] = train['world'].map(activities_world)
 test['world'] = test['world'].map(activities_world)
 train_labels['title'] = train_labels['title'].map(activities_map)
 win_code = dict(zip(activities_map.values() ,(4100*np.ones(len(activities_map)) ).astype('int')))
 win_code[activities_map['Bird Measurer(Assessment)']] = 4110
train['timestamp'] = pd.to_datetime(train['timestamp'])
 test['timestamp'] = pd.to_datetime(test['timestamp'])
 return train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code
corr_matrix = all_data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [c for c in upper.columns if any(upper[c] > 0.95)]
del upper
drop_column = all_data.columns[to_drop]
print('drop columns:', drop_column)
<prepare_x_and_y>
def get_data(user_sample, test_set=False):

 last_activity = 0
user_activities_count = {'Clip':0, 'Activity': 0, 'Assessment': 0, 'Game':0}
last_session_time_sec = 0
accuracy_groups = {0:0, 1:0, 2:0, 3:0}
all_assessments = []
accumulated_accuracy_group = 0
accumulated_accuracy = 0
accumulated_correct_attempts = 0
accumulated_uncorrect_attempts = 0
accumulated_actions = 0
counter = 0
time_first_activity = float(user_sample['timestamp'].values[0])
 durations = []
last_accuracy_title = {'acc_' + title: -1 for title in assess_titles}
event_code_count: Dict[str, int] = {ev: 0 for ev in list_of_event_code}
event_id_count: Dict[str, int] = {eve: 0 for eve in list_of_event_id}
title_count: Dict[str, int] = {eve: 0 for eve in activities_labels.values() }
title_event_code_count: Dict[str, int] = {t_eve: 0 for t_eve in all_title_event_code}
for i, session in user_sample.groupby('game_session', sort=False):
session_type = session['type'].iloc[0]
session_title = session['title'].iloc[0]
session_title_text = activities_labels[session_title]
if(session_type == 'Assessment')&(test_set or len(session)>1):
all_attempts = session.query(f'event_code == {win_code[session_title]}')
 true_attempts = all_attempts['event_data'].str.contains('true' ).sum()
false_attempts = all_attempts['event_data'].str.contains('false' ).sum()
features = user_activities_count.copy()
features.update(last_accuracy_title.copy())
 features.update(event_code_count.copy())
 features.update(event_id_count.copy())
 features.update(title_count.copy())
 features.update(title_event_code_count.copy())
 features.update(last_accuracy_title.copy())
 features['installation_id'] = session['installation_id'].iloc[-1]
features['session_title'] = session['title'].iloc[0]
features['accumulated_correct_attempts'] = accumulated_correct_attempts
features['accumulated_uncorrect_attempts'] = accumulated_uncorrect_attempts
accumulated_correct_attempts += true_attempts
accumulated_uncorrect_attempts += false_attempts
if durations == []:
features['duration_mean'] = 0
else:
features['duration_mean'] = np.mean(durations)
 durations.append(( session.iloc[-1, 2] - session.iloc[0, 2] ).seconds)
 features['accumulated_accuracy'] = accumulated_accuracy/counter if counter > 0 else 0
accuracy = true_attempts/(true_attempts+false_attempts)if(true_attempts+false_attempts)!= 0 else 0
accumulated_accuracy += accuracy
last_accuracy_title['acc_' + session_title_text] = accuracy
if accuracy == 0:
features['accuracy_group'] = 0
elif accuracy == 1:
features['accuracy_group'] = 3
elif accuracy == 0.5:
features['accuracy_group'] = 2
else:
features['accuracy_group'] = 1
features.update(accuracy_groups)
 accuracy_groups[features['accuracy_group']] += 1
features['accumulated_accuracy_group'] = accumulated_accuracy_group/counter if counter > 0 else 0
accumulated_accuracy_group += features['accuracy_group']
features['accumulated_actions'] = accumulated_actions
if test_set:
all_assessments.append(features)
 elif true_attempts+false_attempts > 0:
all_assessments.append(features)
 counter += 1
def update_counters(counter: dict, col: str):
num_of_session_count = Counter(session[col])
 for k in num_of_session_count.keys() :
x = k
if col == 'title':
x = activities_labels[k]
counter[x] += num_of_session_count[k]
return counter
event_code_count = update_counters(event_code_count, "event_code")
 event_id_count = update_counters(event_id_count, "event_id")
 title_count = update_counters(title_count, 'title')
 title_event_code_count = update_counters(title_event_code_count, 'title_event_code')
 accumulated_actions += len(session)
 if last_activity != session_type:
user_activities_count[session_type] += 1
last_activitiy = session_type
if test_set:
return all_assessments[-1]
return all_assessments
X_train = all_data[all_data['target'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['target'].isnull() ].drop(['target'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
X_train.drop(['id'], axis=1, inplace=True)
 X_test_ID = X_test.pop('id')
 Y_train = X_train.pop('target')
 print(X_train.shape, X_test.shape )<prepare_x_and_y>
def get_train_and_test(train, test):

 compiled_train = []
compiled_test = []
for i,(ins_id, user_sample)in tqdm(enumerate(train.groupby('installation_id', sort = False)) , total = 17000):
compiled_train += get_data(user_sample)
 for ins_id, user_sample in tqdm(test.groupby('installation_id', sort = False), total = 1000):
test_data = get_data(user_sample, test_set = True)
 compiled_test.append(test_data)
 reduce_train = pd.DataFrame(compiled_train)
 reduce_test = pd.DataFrame(compiled_test)
 categoricals = ['session_title']
return reduce_train, reduce_test, categoricals
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 splits = 11
for i in range(512):
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[X_train['wheezy-copper-turtle-magic'] == i]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = sel.fit_transform(train2)
 test2 = sel.transform(test2)
 skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train2[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] = clf.predict_proba(train2[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits<filter>
def preprocess(reduce_train, reduce_test):

 for df in [reduce_train, reduce_test]:
df['installation_session_count'] = df.groupby(['installation_id'])['Clip'].transform('count')
 df['installation_duration_mean'] = df.groupby(['installation_id'])['duration_mean'].transform('mean')
 df['installation_title_nunique'] = df.groupby(['installation_id'])['session_title'].transform('nunique')
 df['sum_event_code_count'] = df[[2050, 4100, 4230, 5000, 4235, 2060, 4110, 5010, 2070, 2075, 2080, 2081, 2083, 3110, 4010, 3120, 3121, 4020, 4021,
4022, 4025, 4030, 4031, 3010, 4035, 4040, 3020, 3021, 4045, 2000, 4050, 2010, 2020, 4070, 2025, 2030, 4080, 2035,
2040, 4090, 4220, 4095]].sum(axis = 1)
 df['installation_event_code_count_mean'] = df.groupby(['installation_id'])['sum_event_code_count'].transform('mean')
 return reduce_train, reduce_test
len(X_train[(oof_preds > 0.3)&(oof_preds < 0.7)] )<prepare_x_and_y>
train, test, train_labels, specs, sample_submission = read_data()
train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code = encode_title(train, test, train_labels)
 reduce_train = None
reduce_test = None
reduce_train, reduce_test, categoricals = get_train_and_test(train, test)
 reduce_train, reduce_test = preprocess(reduce_train, reduce_test )
X_train = X_train[(oof_preds <= 0.3)|(oof_preds >= 0.7)]
Y_train = Y_train[(oof_preds <= 0.3)|(oof_preds >= 0.7)]<concatenate>
@jit
def qwk(a1, a2):

 max_rat = 3
a1 = np.asarray(a1, dtype=int)
 a2 = np.asarray(a2, dtype=int)
 hist1 = np.zeros(( max_rat + 1,))
hist2 = np.zeros(( max_rat + 1,))
o = 0
for k in range(a1.shape[0]):
i, j = a1[k], a2[k]
hist1[i] += 1
hist2[j] += 1
o +=(i - j)*(i - j)
 e = 0
for i in range(max_rat + 1):
for j in range(max_rat + 1):
e += hist1[i] * hist2[j] *(i - j)*(i - j)
 e = e / a1.shape[0]
return 1 - o / e
def eval_qwk_lgb(y_true, y_pred):

 y_pred = y_pred.reshape(len(np.unique(y_true)) , -1 ).argmax(axis=0)
 return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_lgb_regr(y_true, y_pred):

 y_pred[y_pred <= 1.12232214] = 0
y_pred[np.where(np.logical_and(y_pred > 1.12232214, y_pred <= 1.73925866)) ] = 1
y_pred[np.where(np.logical_and(y_pred > 1.73925866, y_pred <= 2.22506454)) ] = 2
y_pred[y_pred > 2.22506454] = 3
return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_xgb(y_pred, y_true):

 y_true = y_true.get_label()
y_pred = y_pred.argmax(axis=1)
 return 'cappa', -qwk(y_true, y_pred)
 class MainTransformer(BaseEstimator, TransformerMixin):
def __init__(self, convert_cyclical: bool = False, create_interactions: bool = False, n_interactions: int = 20):

 self.convert_cyclical = convert_cyclical
self.create_interactions = create_interactions
self.feats_for_interaction = None
self.n_interactions = n_interactions
def fit(self, X, y=None):
if self.create_interactions:
self.feats_for_interaction = [col for col in X.columns if 'sum' in col
or 'mean' in col or 'max' in col or 'std' in col
or 'attempt' in col]
self.feats_for_interaction1 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 self.feats_for_interaction2 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 if self.create_interactions:
for col1 in self.feats_for_interaction1:
for col2 in self.feats_for_interaction2:
data[f'{col1}_int_{col2}'] = data[col1] * data[col2]
if self.convert_cyclical:
data['timestampHour'] = np.sin(2 * np.pi * data['timestampHour'] / 23.0)
 data['timestampMonth'] = np.sin(2 * np.pi * data['timestampMonth'] / 23.0)
 data['timestampWeek'] = np.sin(2 * np.pi * data['timestampWeek'] / 23.0)
 data['timestampMinute'] = np.sin(2 * np.pi * data['timestampMinute'] / 23.0)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
 class FeatureTransformer(BaseEstimator, TransformerMixin):
def __init__(self, main_cat_features: list = None, num_cols: list = None):

 self.main_cat_features = main_cat_features
self.num_cols = num_cols
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
X_test_p1 = X_test[(sub_preds <= 0.01)].copy()
X_test_p2 = X_test[(sub_preds >= 0.99)].copy()
X_test_p1['target'] = 0
X_test_p2['target'] = 1
print(X_test_p1.shape, X_test_p2.shape)
 Y_train = pd.concat([Y_train, X_test_p1.pop('target'), X_test_p2.pop('target')], axis=0)
 X_train = pd.concat([X_train, X_test_p1, X_test_p2], axis=0)
 Y_train.reset_index(drop=True, inplace=True)
 X_train.reset_index(drop=True, inplace=True )<categorify>
class LGBWrapper_regr(object):

 def __init__(self):
self.model = lgb.LGBMRegressor()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
if params['objective'] == 'regression':
eval_metric = eval_qwk_lgb_regr
else:
eval_metric = 'auc'
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
print("fit : ", " eval_names=", eval_names, " eval_metric=", eval_metric, " categorical_feature=", categorical_columns)
 self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_metric,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict(self, X_test):
return self.model.predict(X_test, num_iteration=self.model.best_iteration_)
_=
for i in range(512):
train_f =(X_train['wheezy-copper-turtle-magic'] == i)
 test_f =(X_test['wheezy-copper-turtle-magic'] == i)
 X_train_sub = X_train[train_f][numcols]
Y_train_sub = Y_train[train_f]
X_test_sub = X_test[test_f][numcols]
lda = LDA(n_components=1)
 lda.fit(X_train_sub, Y_train_sub)
 X_train.loc[train_f, 'lda'] = lda.transform(X_train_sub ).reshape(-1)
 X_test.loc[test_f, 'lda'] = lda.transform(X_test_sub ).reshape(-1)
 knc = KNeighborsClassifier(n_neighbors=3)
 knc.fit(X_train_sub, Y_train_sub)
 X_train.loc[train_f, 'knc'] = knc.predict_proba(X_train_sub)[:,1]
X_test.loc[test_f, 'knc'] = knc.predict_proba(X_test_sub)[:,1] 
<prepare_x_and_y>
class LGBWrapper(object):

 def __init__(self):
self.model = lgb.LGBMClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_qwk_lgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_ )
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 splits = 11
for i in range(512):
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[X_train['wheezy-copper-turtle-magic'] == i]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = sel.fit_transform(train2)
 test2 = sel.transform(test2)
 skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train2[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] = clf.predict_proba(train2[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits<save_to_csv>
class RegressorModel(object):

 def __init__(self, columns: list = None, model_wrapper=None):

 self.columns = columns
self.model_wrapper = model_wrapper
self.result_dict = {}
self.train_one_fold = False
self.preprocesser = None
def fit(self, X: pd.DataFrame, y,
X_holdout: pd.DataFrame = None, y_holdout=None,
folds=None,
params: dict = None,
eval_metric='rmse',
cols_to_drop: list = None,
preprocesser=None,
transformers: dict = None,
adversarial: bool = False,
plot: bool = True):

 if folds is None:
folds = KFold(n_splits=3, random_state=42)
 self.train_one_fold = True
self.columns = X.columns if self.columns is None else self.columns
self.feature_importances = pd.DataFrame(columns=['feature', 'importance'])
 self.trained_transformers = {k: [] for k in transformers}
self.transformers = transformers
self.models = []
self.folds_dict = {}
self.eval_metric = eval_metric
n_target = 1
self.oof = np.zeros(( len(X), n_target))
self.n_target = n_target
X = X[self.columns]
if X_holdout is not None:
X_holdout = X_holdout[self.columns]
if preprocesser is not None:
self.preprocesser = preprocesser
self.preprocesser.fit(X, y)
 X = self.preprocesser.transform(X, y)
 self.columns = X.columns.tolist()
if X_holdout is not None:
X_holdout = self.preprocesser.transform(X_holdout)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y, X['installation_id'])) :
if X_holdout is not None:
X_hold = X_holdout.copy()
else:
X_hold = None
self.folds_dict[fold_n] = {}
if params['verbose']:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 self.folds_dict[fold_n] = {}
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if self.train_one_fold:
X_train = X[self.original_columns]
y_train = y
X_valid = None
y_valid = None
datasets = {'X_train': X_train, 'X_valid': X_valid, 'X_holdout': X_hold, 'y_train': y_train}
X_train, X_valid, X_hold = self.transform_(datasets, cols_to_drop)
 self.folds_dict[fold_n]['columns'] = X_train.columns.tolist()
model = copy.deepcopy(self.model_wrapper)
 if adversarial:
X_new1 = X_train.copy()
if X_valid is not None:
X_new2 = X_valid.copy()
elif X_holdout is not None:
X_new2 = X_holdout.copy()
X_new = pd.concat([X_new1, X_new2], axis=0)
 y_new = np.hstack(( np.zeros(( X_new1.shape[0])) , np.ones(( X_new2.shape[0]))))
X_train, X_valid, y_train, y_valid = train_test_split(X_new, y_new)
 model.fit(X_train, y_train, X_valid, y_valid, X_hold, y_holdout, params=params)
 self.folds_dict[fold_n]['scores'] = model.best_score_
if self.oof.shape[0] != len(X):
self.oof = np.zeros(( X.shape[0], self.oof.shape[1]))
if not adversarial:
self.oof[valid_index] = model.predict(X_valid ).reshape(-1, n_target)
 fold_importance = pd.DataFrame(list(zip(X_train.columns, model.feature_importances_)) ,
columns=['feature', 'importance'])
 self.feature_importances = self.feature_importances.append(fold_importance)
 self.models.append(model)
 self.feature_importances['importance'] = self.feature_importances['importance'].astype(int)
 self.calc_scores_()
if plot:
fig, ax = plt.subplots(figsize=(16, 12))
plt.subplot(2, 2, 1)
 self.plot_feature_importance(top_n=20)
 plt.subplot(2, 2, 2)
 self.plot_metric()
plt.subplot(2, 2, 3)
 plt.hist(y.values.reshape(-1, 1)- self.oof)
 plt.title('Distribution of errors')
 plt.subplot(2, 2, 4)
 plt.hist(self.oof)
 plt.title('Distribution of oof predictions');
def transform_(self, datasets, cols_to_drop):
for name, transformer in self.transformers.items() :
transformer.fit(datasets['X_train'], datasets['y_train'])
 datasets['X_train'] = transformer.transform(datasets['X_train'])
 if datasets['X_valid'] is not None:
datasets['X_valid'] = transformer.transform(datasets['X_valid'])
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = transformer.transform(datasets['X_holdout'])
 self.trained_transformers[name].append(transformer)
 if cols_to_drop is not None:
cols_to_drop = [col for col in cols_to_drop if col in datasets['X_train'].columns]
datasets['X_train'] = datasets['X_train'].drop(cols_to_drop, axis=1)
 if datasets['X_valid'] is not None:
datasets['X_valid'] = datasets['X_valid'].drop(cols_to_drop, axis=1)
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = datasets['X_holdout'].drop(cols_to_drop, axis=1)
 self.cols_to_drop = cols_to_drop
return datasets['X_train'], datasets['X_valid'], datasets['X_holdout']
def calc_scores_(self):
datasets = [k for k, v in [v['scores'] for k, v in self.folds_dict.items() ][0].items() if len(v)> 0]
self.scores = {}
for d in datasets:
scores = [v['scores'][d][self.eval_metric] for k, v in self.folds_dict.items() ]
self.scores[d] = np.mean(scores)
 def predict(self, X_test, averaging: str = 'usual'):

 full_prediction = np.zeros(( X_test.shape[0], self.oof.shape[1]))
if self.preprocesser is not None:
X_test = self.preprocesser.transform(X_test)
 for i in range(len(self.models)) :
X_t = X_test.copy()
for name, transformers in self.trained_transformers.items() :
X_t = transformers[i].transform(X_t)
 if self.cols_to_drop is not None:
cols_to_drop = [col for col in self.cols_to_drop if col in X_t.columns]
X_t = X_t.drop(cols_to_drop, axis=1)
 y_pred = self.models[i].predict(X_t[self.folds_dict[i]['columns']] ).reshape(-1, full_prediction.shape[1])
 if full_prediction.shape[0] != len(y_pred):
full_prediction = np.zeros(( y_pred.shape[0], self.oof.shape[1]))
if averaging == 'usual':
full_prediction += y_pred
elif averaging == 'rank':
full_prediction += pd.Series(y_pred ).rank().values
return full_prediction / len(self.models)
 def plot_feature_importance(self, drop_null_importance: bool = True, top_n: int = 10):

 top_feats = self.get_top_features(drop_null_importance, top_n)
 feature_importances = self.feature_importances.loc[self.feature_importances['feature'].isin(top_feats)]
feature_importances['feature'] = feature_importances['feature'].astype(str)
 top_feats = [str(i)for i in top_feats]
sns.barplot(data=feature_importances, x='importance', y='feature', orient='h', order=top_feats)
 plt.title('Feature importances')
 def get_top_features(self, drop_null_importance: bool = True, top_n: int = 10):

 grouped_feats = self.feature_importances.groupby(['feature'])['importance'].mean()
if drop_null_importance:
grouped_feats = grouped_feats[grouped_feats != 0]
return list(grouped_feats.sort_values(ascending=False ).index)[:top_n]
def plot_metric(self):

 full_evals_results = pd.DataFrame()
for model in self.models:
evals_result = pd.DataFrame()
for k in model.model.evals_result_.keys() :
evals_result[k] = model.model.evals_result_[k][self.eval_metric]
evals_result = evals_result.reset_index().rename(columns={'index': 'iteration'})
 full_evals_results = full_evals_results.append(evals_result)
 full_evals_results = full_evals_results.melt(id_vars=['iteration'] ).rename(columns={'value': self.eval_metric,
'variable': 'dataset'})
 sns.lineplot(data=full_evals_results, x='iteration', y=self.eval_metric, hue='dataset')
 plt.title('Training progress' )
submission = pd.DataFrame({
'id': X_test_ID,
'target': sub_preds
})
 submission.to_csv("submission.csv", index=False )<set_options>
y = reduce_train['accuracy_group']
cols_to_drop = ['game_session', 'installation_id', 'timestamp', 'accuracy_group', 'timestampDate']
n_fold = 6
folds = GroupKFold(n_splits=n_fold )
%matplotlib inline
np.random.seed(1111)
 warnings.filterwarnings('ignore' )<load_from_csv>
def LGB_bayesian(max_depth,
lambda_l1,
lambda_l2,
bagging_fraction,
bagging_freq,
colsample_bytree,
learning_rate):
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'eval_metric': 'cappa',
'n_jobs': -1,
'seed': 42,
'early_stopping_rounds': 100,
'n_estimators': 2000,
'learning_rate': learning_rate,
'max_depth': int(max_depth),
'lambda_l1': lambda_l1,
'lambda_l2': lambda_l2,
'bagging_fraction': bagging_fraction,
'bagging_freq': int(bagging_freq),
'colsample_bytree': colsample_bytree,
'verbose': 0
}
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
model = RegressorModel(model_wrapper=LGBWrapper_regr())
 model.fit(X=reduce_train,
y=y,
folds=folds,
params=params,
preprocesser=mt,
transformers=transformers,
eval_metric='cappa',
cols_to_drop=cols_to_drop,
plot=False)
 return model.scores['valid']
train = pd.read_csv('.. /input/instant-gratification/train.csv')
 test = pd.read_csv('.. /input/instant-gratification/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]<load_pretrained>
gc.collect()
gm_list = pickle.load(open('.. /input/models-v5/gm_models_v5.pkl', 'rb'))<choose_model_class>
init_points = 16
n_iter = 16
bounds_LGB = {
'max_depth':(8, 12),
'lambda_l1':(0, 5),
'lambda_l2':(0, 5),
'bagging_fraction':(0.4, 0.6),
'bagging_freq':(1, 10),
'colsample_bytree':(0.4, 0.6),
'learning_rate':(0.025, 0.1)
 }
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=1029)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6)
 print(LGB_BO.max )
class MyGM(GaussianMixture):
def __init__(self, n_components=1, covariance_type='full', tol=1e-3,
reg_covar=1e-6, max_iter=100, n_init=1, init_params='kmeans',
weights_init=None, means_init=None, precisions_init=None,
random_state=None, warm_start=False,
verbose=0, verbose_interval=10, init_clusters=None, y=None):
super().__init__(
 n_components=n_components, tol=tol, reg_covar=reg_covar,
max_iter=max_iter, n_init=n_init, init_params=init_params,
random_state=random_state, warm_start=warm_start,
verbose=verbose, verbose_interval=verbose_interval)
 self.init_clusters_ = np.asarray(init_clusters ).astype('int')
 self.y = y
def _initialize_parameters(self, X, random_state):

 n_samples, _ = X.shape
if self.init_params == 'kmeans':
resp = np.zeros(( n_samples, self.n_components))
label = cluster.KMeans(n_clusters=self.n_components, n_init=1,
random_state=random_state ).fit(X ).labels_
resp[np.arange(n_samples), label] = 1
elif self.init_params == 'random':
resp = random_state.rand(n_samples, self.n_components)
 resp /= resp.sum(axis=1)[:, np.newaxis]
elif self.init_params == 'clusters':
resp = np.zeros(( n_samples, self.n_components))
resp[np.arange(self.init_clusters_.shape[0]), self.init_clusters_] = 1
else:
raise ValueError("Unimplemented initialization method '%s'"
% self.init_params)
 self._initialize(X, resp)
 def estimate_log_ratio(self, X):
weighted_log_prob = self._estimate_weighted_log_prob(X)
 return logsumexp(weighted_log_prob[:, 1::2], axis=1)- logsumexp(weighted_log_prob[:, 0::2], axis=1)
<choose_model_class>
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'eval_metric': 'cappa',
'n_jobs': -1,
'seed': 42,
'early_stopping_rounds': 100,
'n_estimators': 2000,
'learning_rate': LGB_BO.max['params']['learning_rate'],
'max_depth': int(LGB_BO.max['params']['max_depth']),
'lambda_l1': LGB_BO.max['params']['lambda_l1'],
'lambda_l2': LGB_BO.max['params']['lambda_l2'],
'bagging_fraction': LGB_BO.max['params']['bagging_fraction'],
'bagging_freq': int(LGB_BO.max['params']['bagging_freq']),
'colsample_bytree': LGB_BO.max['params']['colsample_bytree'],
'verbose': 100
}
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model.fit(X=reduce_train,
y=y,
folds=folds,
params=params,
preprocesser=mt,
transformers=transformers,
eval_metric='cappa',
cols_to_drop=cols_to_drop)
 preds_1 = regressor_model.predict(reduce_test)

w_1 = LGB_BO.max['target']
class MyBGM(BayesianGaussianMixture):
def __init__(self, n_components=1, covariance_type='full', tol=1e-3,
reg_covar=1e-6, max_iter=100, n_init=1, init_params='kmeans',
weight_concentration_prior_type='dirichlet_process',
weight_concentration_prior=None,
mean_precision_prior=None, mean_prior=None,
degrees_of_freedom_prior=None, covariance_prior=None,
random_state=None, warm_start=False, verbose=0,
verbose_interval=10, init_clusters=None):
super().__init__(
 n_components=n_components, covariance_type=covariance_type, tol=tol,
reg_covar=reg_covar, max_iter=max_iter, n_init=n_init, init_params=init_params,
weight_concentration_prior_type=weight_concentration_prior_type,
weight_concentration_prior=weight_concentration_prior,
mean_precision_prior=mean_precision_prior, mean_prior=mean_prior,
degrees_of_freedom_prior=degrees_of_freedom_prior, covariance_prior=covariance_prior,
random_state=random_state, warm_start=warm_start, verbose=verbose,
verbose_interval=verbose_interval)
 self.init_clusters_ = np.asarray(init_clusters ).astype('int')
 def _initialize_parameters(self, X, random_state):

 n_samples, _ = X.shape
if self.init_params == 'kmeans':
resp = np.zeros(( n_samples, self.n_components))
label = cluster.KMeans(n_clusters=self.n_components, n_init=1,
random_state=random_state ).fit(X ).labels_
resp[np.arange(n_samples), label] = 1
elif self.init_params == 'random':
resp = random_state.rand(n_samples, self.n_components)
 resp /= resp.sum(axis=1)[:, np.newaxis]
elif self.init_params == 'clusters':
resp = np.zeros(( n_samples, self.n_components))
resp[np.arange(self.init_clusters_.shape[0]), self.init_clusters_] = 1
elif self.init_params == 'proba':
resp = self.init_proba_.copy()
resp[np.arange(self.init_clusters_.shape[0]), self.init_clusters_] = 1
resp /= resp.sum(axis=1)[:, np.newaxis]
else:
raise ValueError("Unimplemented initialization method '%s'"
% self.init_params)
 self._initialize(X, resp )<compute_test_metric>
del bounds_LGB, LGB_BO, params, mt, ft, transformers, regressor_model
gc.collect()
roc_auc_score(train["target"], oof_preds )<save_to_csv>
bounds_LGB = {
'max_depth':(10, 14),
'lambda_l1':(0, 10),
'lambda_l2':(0, 10),
'bagging_fraction':(0.7, 1),
'bagging_freq':(1, 10),
'colsample_bytree':(0.7, 1),
'learning_rate':(0.06, 0.24)
 }
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=1030)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6)
 print(LGB_BO.max )
sub = pd.read_csv('.. /input/instant-gratification/sample_submission.csv')
 sub['target'] = test_preds
sub.to_csv('submission.csv',index=False )<load_from_csv>
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'eval_metric': 'cappa',
'n_jobs': -1,
'seed': 42,
'early_stopping_rounds': 100,
'n_estimators': 2000,
'learning_rate': LGB_BO.max['params']['learning_rate'],
'max_depth': int(LGB_BO.max['params']['max_depth']),
'lambda_l1': LGB_BO.max['params']['lambda_l1'],
'lambda_l2': LGB_BO.max['params']['lambda_l2'],
'bagging_fraction': LGB_BO.max['params']['bagging_fraction'],
'bagging_freq': int(LGB_BO.max['params']['bagging_freq']),
'colsample_bytree': LGB_BO.max['params']['colsample_bytree'],
'verbose': 100
}
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model.fit(X=reduce_train,
y=y,
folds=folds,
params=params,
preprocesser=mt,
transformers=transformers,
eval_metric='cappa',
cols_to_drop=cols_to_drop)
 preds_2 = regressor_model.predict(reduce_test)
 w_2 = LGB_BO.max['target']
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape)
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 print(new_train.head(100))<load_from_csv>
del bounds_LGB, LGB_BO, params, mt, ft, transformers, regressor_model
gc.collect()
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape)
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.94, 'target'] = 1
test.loc[test['target'] < 0.06, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.98, 'target'] = 1
new_train.loc[oof < 0.02, 'target'] = 0
oof3 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof3[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof3)
 print(f'AUC: {auc:.5}')
 sub2 = pd.read_csv('.. /input/sample_submission.csv')
 sub2['target'] = preds 
<load_from_csv>
preds =(w_1/(w_1+w_2)) * preds_1 +(w_2/(w_1+w_2)) * preds_2
del preds_1, preds_2
gc.collect()
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape)
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
params = [{'reg_param': [0.1, 0.2, 0.3, 0.4, 0.5]}]
reg_params = np.zeros(512)
 for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
qda = QuadraticDiscriminantAnalysis()
clf = GridSearchCV(qda, params, cv=4)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 reg_params[i] = clf.best_params_['reg_param']
oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 for itr in range(10):
test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0
oof4 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=time.time)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof4[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof4)
 print(f'AUC: {auc:.5}')
 sub3 = pd.read_csv('.. /input/sample_submission.csv')
 sub3['target'] = preds 
<compute_test_metric>
coefficients = [1.12232214, 1.73925866, 2.22506454]
preds[preds <= coefficients[0]] = 0
preds[np.where(np.logical_and(preds > coefficients[0], preds <= coefficients[1])) ] = 1
preds[np.where(np.logical_and(preds > coefficients[1], preds <= coefficients[2])) ] = 2
preds[preds > coefficients[2]] = 3
print('CV score ensemble=',round(roc_auc_score(train['target'],oof2*0.35 + oof3*0.25+ oof4*0.4),5))<load_from_csv>
sample_submission['accuracy_group'] = preds.astype(int)
 sample_submission['accuracy_group'].value_counts(normalize=True )
<feature_engineering><EOS> 
sample_submission.to_csv('submission.csv', index=False )