Kaggle competition: 2019 Data Science Bowl
<SOS> metric: QuadraticWeightedKappa Kaggle data source: data-science-bowl-2019<define_variables>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
pd.set_option('max_rows', 500)
oof_features = ['y_0', 'y_1', 'y_2']<split>
a0 = 1.12
a1 = 1.74
a2 = 2.23
for m in sorted(train['wheezy-copper-turtle-magic'].unique()):
idx_tr =(train['wheezy-copper-turtle-magic']==m)
 idx_te =(test['wheezy-copper-turtle-magic']==m)
 oofs = []
preds = []
kf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 oof_preds = np.zeros(( len(train[idx_tr][oof_features]), 1))
test_preds = np.zeros(( len(test[idx_te][oof_features]), 1))
for idx,(train_index, valid_index)in enumerate(kf.split(train[idx_tr][oof_features], train[idx_tr]['target'])) :
y_train, y_valid = train[idx_tr]['target'].iloc[train_index], train[idx_tr]['target'].iloc[valid_index]
x_train, x_valid = train[idx_tr][oof_features].iloc[train_index,:], train[idx_tr][oof_features].iloc[valid_index,:]
model = linear_model.Ridge(alpha=3)
 model.fit(x_train, y_train)
 oof_preds[valid_index, :] = model.predict(x_valid ).reshape(( -1, 1))
test_preds += model.predict(test[idx_te][oof_features] ).reshape(( -1, 1)) / 11
print('OOF AUC ', m, ' ', roc_auc_score(train[idx_tr]['target'], oof_preds))
oofs.append(( idx_tr, oof_preds))
preds.append(( idx_te, test_preds))
for ids, target in oofs:
train.loc[ids,'target_final'] = target
for ids, target in preds:
test.loc[ids,'target_final'] = target
print('OOF AUC ', roc_auc_score(train['target'], train['target_final']))<create_dataframe>
def add_datepart(df: pd.DataFrame, field_name: str,
prefix: str = None, drop: bool = True, time: bool = True, date: bool = True):

 field = df[field_name]
prefix = ifnone(prefix, re.sub('[Dd]ate$', '', field_name))
attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Is_month_end', 'Is_month_start']
if date:
attr.append('Date')
 if time:
attr = attr + ['Hour', 'Minute']
for n in attr:
df[prefix + n] = getattr(field.dt, n.lower())
 if drop:
df.drop(field_name, axis=1, inplace=True)
 return df
def ifnone(a: Any, b: Any)-> Any:
test_preds = pd.DataFrame(np.concatenate([preds_0.reshape(-1, 1), preds_1.reshape(-1, 1), preds_2.reshape(-1, 1), 1 - test['target_final'].values.reshape(-1, 1)], axis=1))<prepare_x_and_y>
@jit
def qwk(a1, a2):

 max_rat = 3
a1 = np.asarray(a1, dtype=int)
 a2 = np.asarray(a2, dtype=int)
 hist1 = np.zeros(( max_rat + 1,))
hist2 = np.zeros(( max_rat + 1,))
o = 0
for k in range(a1.shape[0]):
i, j = a1[k], a2[k]
hist1[i] += 1
hist2[j] += 1
o +=(i - j)*(i - j)
 e = 0
for i in range(max_rat + 1):
for j in range(max_rat + 1):
e += hist1[i] * hist2[j] *(i - j)*(i - j)
 e = e / a1.shape[0]
return 1 - o / e
def eval_qwk_lgb(y_true, y_pred):

 y_pred = y_pred.reshape(len(np.unique(y_true)) , -1 ).argmax(axis=0)
 return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_lgb_regr(y_true, y_pred):

 y_pred[y_pred <= a0] = 0
y_pred[np.where(np.logical_and(y_pred > a0, y_pred <= a1)) ] = 1
y_pred[np.where(np.logical_and(y_pred > a1, y_pred <= a2)) ] = 2
y_pred[y_pred > a2] = 3
return 'cappa', qwk(y_true, y_pred), True
class LGBWrapper_regr(object):

 def __init__(self):
self.model = lgb.LGBMRegressor()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
if params['objective'] == 'regression':
eval_metric = eval_qwk_lgb_regr
else:
eval_metric = 'auc'
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_metric,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict(self, X_test):
return self.model.predict(X_test, num_iteration=self.model.best_iteration_)
 def eval_qwk_xgb(y_pred, y_true):

 y_true = y_true.get_label()
y_pred = y_pred.argmax(axis=1)
 return 'cappa', -qwk(y_true, y_pred)
 class LGBWrapper(object):

 def __init__(self):
self.model = lgb.LGBMClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_qwk_lgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)
 class CatWrapper(object):

 def __init__(self):
self.model = cat.CatBoostClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**{k: v for k, v in params.items() if k != 'cat_cols'})
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = None
else:
categorical_columns = None
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
cat_features=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if 'MultiClass' not in self.model.get_param('loss_function'):
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)
 class XGBWrapper(object):

 def __init__(self):
self.model = xgb.XGBClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_metric=eval_qwk_xgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'])
 scores = self.model.evals_result()
self.best_score_ = {k: {m: m_v[-1] for m, m_v in v.items() } for k, v in scores.items() }
self.best_score_ = {k: {m: n if m != 'cappa' else -n for m, n in v.items() } for k, v in self.best_score_.items() }
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)
 class MainTransformer(BaseEstimator, TransformerMixin):
def __init__(self, convert_cyclical: bool = False, create_interactions: bool = False, n_interactions: int = 20):

 self.convert_cyclical = convert_cyclical
self.create_interactions = create_interactions
self.feats_for_interaction = None
self.n_interactions = n_interactions
def fit(self, X, y=None):
if self.create_interactions:
self.feats_for_interaction = [col for col in X.columns if 'sum' in col
or 'mean' in col or 'max' in col or 'std' in col
or 'attempt' in col]
self.feats_for_interaction1 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 self.feats_for_interaction2 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 if self.create_interactions:
for col1 in self.feats_for_interaction1:
for col2 in self.feats_for_interaction2:
data[f'{col1}_int_{col2}'] = data[col1] * data[col2]
if self.convert_cyclical:
data['timestampHour'] = np.sin(2 * np.pi * data['timestampHour'] / 23.0)
 data['timestampMonth'] = np.sin(2 * np.pi * data['timestampMonth'] / 23.0)
 data['timestampWeek'] = np.sin(2 * np.pi * data['timestampWeek'] / 23.0)
 data['timestampMinute'] = np.sin(2 * np.pi * data['timestampMinute'] / 23.0)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
 class FeatureTransformer(BaseEstimator, TransformerMixin):
def __init__(self, main_cat_features: list = None, num_cols: list = None):

 self.main_cat_features = main_cat_features
self.num_cols = num_cols
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )
y_hat = test_preds.rank(axis=0, method='min' ).mul(test_preds.shape[1] * [1 / test_preds.shape[1]] ).sum(1)/ test_preds.shape[0] 
<save_to_csv>
class RegressorModel(object):

 def __init__(self, columns: list = None, model_wrapper=None):

 self.columns = columns
self.model_wrapper = model_wrapper
self.result_dict = {}
self.train_one_fold = False
self.preprocesser = None
def fit(self, X: pd.DataFrame, y,
X_holdout: pd.DataFrame = None, y_holdout=None,
folds=None,
params: dict = None,
eval_metric='rmse',
cols_to_drop: list = None,
preprocesser=None,
transformers: dict = None,
adversarial: bool = False,
plot: bool = True):

 if folds is None:
folds = KFold(n_splits=3, random_state=42)
 self.train_one_fold = True
self.columns = X.columns if self.columns is None else self.columns
self.feature_importances = pd.DataFrame(columns=['feature', 'importance'])
 self.trained_transformers = {k: [] for k in transformers}
self.transformers = transformers
self.models = []
self.folds_dict = {}
self.eval_metric = eval_metric
n_target = 1
self.oof = np.zeros(( len(X), n_target))
self.n_target = n_target
X = X[self.columns]
if X_holdout is not None:
X_holdout = X_holdout[self.columns]
if preprocesser is not None:
self.preprocesser = preprocesser
self.preprocesser.fit(X, y)
 X = self.preprocesser.transform(X, y)
 self.columns = X.columns.tolist()
if X_holdout is not None:
X_holdout = self.preprocesser.transform(X_holdout)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y, X['installation_id'])) :
if X_holdout is not None:
X_hold = X_holdout.copy()
else:
X_hold = None
self.folds_dict[fold_n] = {}
if params['verbose']:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 self.folds_dict[fold_n] = {}
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if self.train_one_fold:
X_train = X[self.original_columns]
y_train = y
X_valid = None
y_valid = None
datasets = {'X_train': X_train, 'X_valid': X_valid, 'X_holdout': X_hold, 'y_train': y_train}
X_train, X_valid, X_hold = self.transform_(datasets, cols_to_drop)
 self.folds_dict[fold_n]['columns'] = X_train.columns.tolist()
model = copy.deepcopy(self.model_wrapper)
 if adversarial:
X_new1 = X_train.copy()
if X_valid is not None:
X_new2 = X_valid.copy()
elif X_holdout is not None:
X_new2 = X_holdout.copy()
X_new = pd.concat([X_new1, X_new2], axis=0)
 y_new = np.hstack(( np.zeros(( X_new1.shape[0])) , np.ones(( X_new2.shape[0]))))
X_train, X_valid, y_train, y_valid = train_test_split(X_new, y_new)
 model.fit(X_train, y_train, X_valid, y_valid, X_hold, y_holdout, params=params)
 self.folds_dict[fold_n]['scores'] = model.best_score_
if self.oof.shape[0] != len(X):
self.oof = np.zeros(( X.shape[0], self.oof.shape[1]))
if not adversarial:
self.oof[valid_index] = model.predict(X_valid ).reshape(-1, n_target)
 fold_importance = pd.DataFrame(list(zip(X_train.columns, model.feature_importances_)) ,
columns=['feature', 'importance'])
 self.feature_importances = self.feature_importances.append(fold_importance)
 self.models.append(model)
 self.feature_importances['importance'] = self.feature_importances['importance'].astype(int)
 self.calc_scores_()
if plot:
fig, ax = plt.subplots(figsize=(16, 12))
plt.subplot(2, 2, 1)
 self.plot_feature_importance(top_n=20)
 plt.subplot(2, 2, 2)
 self.plot_metric()
plt.subplot(2, 2, 3)
 plt.hist(y.values.reshape(-1, 1)- self.oof)
 plt.title('Distribution of errors')
 plt.subplot(2, 2, 4)
 plt.hist(self.oof)
 plt.title('Distribution of oof predictions');
def transform_(self, datasets, cols_to_drop):
for name, transformer in self.transformers.items() :
transformer.fit(datasets['X_train'], datasets['y_train'])
 datasets['X_train'] = transformer.transform(datasets['X_train'])
 if datasets['X_valid'] is not None:
datasets['X_valid'] = transformer.transform(datasets['X_valid'])
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = transformer.transform(datasets['X_holdout'])
 self.trained_transformers[name].append(transformer)
 if cols_to_drop is not None:
cols_to_drop = [col for col in cols_to_drop if col in datasets['X_train'].columns]
datasets['X_train'] = datasets['X_train'].drop(cols_to_drop, axis=1)
 if datasets['X_valid'] is not None:
datasets['X_valid'] = datasets['X_valid'].drop(cols_to_drop, axis=1)
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = datasets['X_holdout'].drop(cols_to_drop, axis=1)
 self.cols_to_drop = cols_to_drop
return datasets['X_train'], datasets['X_valid'], datasets['X_holdout']
def calc_scores_(self):
print()
datasets = [k for k, v in [v['scores'] for k, v in self.folds_dict.items() ][0].items() if len(v)> 0]
self.scores = {}
for d in datasets:
scores = [v['scores'][d][self.eval_metric] for k, v in self.folds_dict.items() ]
print(f"CV mean score on {d}: {np.mean(scores):.4f} +/- {np.std(scores):.4f} std.")
 self.scores[d] = np.mean(scores)
 def predict(self, X_test, averaging: str = 'usual'):

 full_prediction = np.zeros(( X_test.shape[0], self.oof.shape[1]))
if self.preprocesser is not None:
X_test = self.preprocesser.transform(X_test)
 for i in range(len(self.models)) :
X_t = X_test.copy()
for name, transformers in self.trained_transformers.items() :
X_t = transformers[i].transform(X_t)
 if self.cols_to_drop is not None:
cols_to_drop = [col for col in self.cols_to_drop if col in X_t.columns]
X_t = X_t.drop(cols_to_drop, axis=1)
 y_pred = self.models[i].predict(X_t[self.folds_dict[i]['columns']] ).reshape(-1, full_prediction.shape[1])
 if full_prediction.shape[0] != len(y_pred):
full_prediction = np.zeros(( y_pred.shape[0], self.oof.shape[1]))
if averaging == 'usual':
full_prediction += y_pred
elif averaging == 'rank':
full_prediction += pd.Series(y_pred ).rank().values
return full_prediction / len(self.models)
 def plot_feature_importance(self, drop_null_importance: bool = True, top_n: int = 10):

 top_feats = self.get_top_features(drop_null_importance, top_n)
 feature_importances = self.feature_importances.loc[self.feature_importances['feature'].isin(top_feats)]
feature_importances['feature'] = feature_importances['feature'].astype(str)
 top_feats = [str(i)for i in top_feats]
sns.barplot(data=feature_importances, x='importance', y='feature', orient='h', order=top_feats)
 plt.title('Feature importances')
 def get_top_features(self, drop_null_importance: bool = True, top_n: int = 10):

 grouped_feats = self.feature_importances.groupby(['feature'])['importance'].mean()
if drop_null_importance:
grouped_feats = grouped_feats[grouped_feats != 0]
return list(grouped_feats.sort_values(ascending=False ).index)[:top_n]
def plot_metric(self):

 full_evals_results = pd.DataFrame()
for model in self.models:
evals_result = pd.DataFrame()
for k in model.model.evals_result_.keys() :
evals_result[k] = model.model.evals_result_[k][self.eval_metric]
evals_result = evals_result.reset_index().rename(columns={'index': 'iteration'})
 full_evals_results = full_evals_results.append(evals_result)
 full_evals_results = full_evals_results.melt(id_vars=['iteration'] ).rename(columns={'value': self.eval_metric,
'variable': 'dataset'})
 sns.lineplot(data=full_evals_results, x='iteration', y=self.eval_metric, hue='dataset')
 plt.title('Training progress')
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = 1 - test['target_final'].values
sub.to_csv('submission.csv',index=False)
 plt.hist(1 - test['target_final'].values ,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<import_modules>
class CategoricalTransformer(BaseEstimator, TransformerMixin):
def __init__(self, cat_cols=None, drop_original: bool = False, encoder=OrdinalEncoder()):

 self.cat_cols = cat_cols
self.drop_original = drop_original
self.encoder = encoder
self.default_encoder = OrdinalEncoder()
def fit(self, X, y=None):
if self.cat_cols is None:
kinds = np.array([dt.kind for dt in X.dtypes])
 is_cat = kinds == 'O'
self.cat_cols = list(X.columns[is_cat])
 self.encoder.set_params(cols=self.cat_cols)
 self.default_encoder.set_params(cols=self.cat_cols)
 self.encoder.fit(X[self.cat_cols], y)
 self.default_encoder.fit(X[self.cat_cols], y)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 new_cat_names = [f'{col}_encoded' for col in self.cat_cols]
encoded_data = self.encoder.transform(data[self.cat_cols])
 if encoded_data.shape[1] == len(self.cat_cols):
data[new_cat_names] = encoded_data
else:
pass
if self.drop_original:
data = data.drop(self.cat_cols, axis=1)
 else:
data[self.cat_cols] = self.default_encoder.transform(data[self.cat_cols])
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.metrics import roc_auc_score
from tqdm import tqdm
from sklearn.covariance import LedoitWolf, OAS
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import RobustScaler
from sklearn.cluster import KMeans, Birch
from sklearn.pipeline import Pipeline
from scipy.stats import multivariate_normal
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from scipy.stats import rankdata
from scipy.optimize import minimize 
<install_modules>
def read_data() :
print('Reading train.csv file.... ')
 train = pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv')
 print('Training.csv file have {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Reading test.csv file.... ')
 test = pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv')
 print('Test.csv file have {} rows and {} columns'.format(test.shape[0], test.shape[1]))
print('Reading train_labels.csv file.... ')
 train_labels = pd.read_csv('/kaggle/input/data-science-bowl-2019/train_labels.csv')
 print('Train_labels.csv file have {} rows and {} columns'.format(train_labels.shape[0], train_labels.shape[1]))
print('Reading specs.csv file.... ')
 specs = pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv')
 print('Specs.csv file have {} rows and {} columns'.format(specs.shape[0], specs.shape[1]))
print('Reading sample_submission.csv file.... ')
 sample_submission = pd.read_csv('/kaggle/input/data-science-bowl-2019/sample_submission.csv')
 print('Sample_submission.csv file have {} rows and {} columns'.format(sample_submission.shape[0], sample_submission.shape[1]))
return train, test, train_labels, specs, sample_submission
def encode_title(train, test, train_labels):
train['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), train['title'], train['event_code']))
test['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), test['title'], test['event_code']))
all_title_event_code = list(set(train["title_event_code"].unique() ).union(test["title_event_code"].unique()))
list_of_user_activities = list(set(train['title'].unique() ).union(set(test['title'].unique())))
 list_of_event_code = list(set(train['event_code'].unique() ).union(set(test['event_code'].unique())))
 list_of_event_id = list(set(train['event_id'].unique() ).union(set(test['event_id'].unique())))
 list_of_worlds = list(set(train['world'].unique() ).union(set(test['world'].unique())))
 activities_map = dict(zip(list_of_user_activities, np.arange(len(list_of_user_activities))))
activities_labels = dict(zip(np.arange(len(list_of_user_activities)) , list_of_user_activities))
activities_world = dict(zip(list_of_worlds, np.arange(len(list_of_worlds))))
assess_titles = list(set(train[train['type'] == 'Assessment']['title'].value_counts().index ).union(set(test[test['type'] == 'Assessment']['title'].value_counts().index)))
 train['title'] = train['title'].map(activities_map)
 test['title'] = test['title'].map(activities_map)
 train['world'] = train['world'].map(activities_world)
 test['world'] = test['world'].map(activities_world)
 train_labels['title'] = train_labels['title'].map(activities_map)
 win_code = dict(zip(activities_map.values() ,(4100*np.ones(len(activities_map)) ).astype('int')))
 win_code[activities_map['Bird Measurer(Assessment)']] = 4110
train['timestamp'] = pd.to_datetime(train['timestamp'])
 test['timestamp'] = pd.to_datetime(test['timestamp'])
 return train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code
def get_data(user_sample, test_set=False):

 last_activity = 0
user_activities_count = {'Clip':0, 'Activity': 0, 'Assessment': 0, 'Game':0}
last_session_time_sec = 0
accuracy_groups = {0:0, 1:0, 2:0, 3:0}
all_assessments = []
accumulated_accuracy_group = 0
accumulated_accuracy = 0
accumulated_correct_attempts = 0
accumulated_uncorrect_attempts = 0
accumulated_actions = 0
counter = 0
time_first_activity = float(user_sample['timestamp'].values[0])
 durations = []
last_accuracy_title = {'acc_' + title: -1 for title in assess_titles}
event_code_count: Dict[str, int] = {ev: 0 for ev in list_of_event_code}
event_id_count: Dict[str, int] = {eve: 0 for eve in list_of_event_id}
title_count: Dict[str, int] = {eve: 0 for eve in activities_labels.values() }
title_event_code_count: Dict[str, int] = {t_eve: 0 for t_eve in all_title_event_code}
for i, session in user_sample.groupby('game_session', sort=False):
session_type = session['type'].iloc[0]
session_title = session['title'].iloc[0]
session_title_text = activities_labels[session_title]
if(session_type == 'Assessment')&(test_set or len(session)>1):
all_attempts = session.query(f'event_code == {win_code[session_title]}')
 true_attempts = all_attempts['event_data'].str.contains('true' ).sum()
false_attempts = all_attempts['event_data'].str.contains('false' ).sum()
features = user_activities_count.copy()
features.update(last_accuracy_title.copy())
 features.update(event_code_count.copy())
 features.update(event_id_count.copy())
 features.update(title_count.copy())
 features.update(title_event_code_count.copy())
 features.update(last_accuracy_title.copy())
 features['installation_id'] = session['installation_id'].iloc[-1]
features['session_title'] = session['title'].iloc[0]
features['accumulated_correct_attempts'] = accumulated_correct_attempts
features['accumulated_uncorrect_attempts'] = accumulated_uncorrect_attempts
accumulated_correct_attempts += true_attempts
accumulated_uncorrect_attempts += false_attempts
if durations == []:
features['duration_mean'] = 0
else:
features['duration_mean'] = np.mean(durations)
 durations.append(( session.iloc[-1, 2] - session.iloc[0, 2] ).seconds)
 features['accumulated_accuracy'] = accumulated_accuracy/counter if counter > 0 else 0
accuracy = true_attempts/(true_attempts+false_attempts)if(true_attempts+false_attempts)!= 0 else 0
accumulated_accuracy += accuracy
last_accuracy_title['acc_' + session_title_text] = accuracy
if accuracy == 0:
features['accuracy_group'] = 0
elif accuracy == 1:
features['accuracy_group'] = 3
elif accuracy == 0.5:
features['accuracy_group'] = 2
else:
features['accuracy_group'] = 1
features.update(accuracy_groups)
 accuracy_groups[features['accuracy_group']] += 1
features['accumulated_accuracy_group'] = accumulated_accuracy_group/counter if counter > 0 else 0
accumulated_accuracy_group += features['accuracy_group']
features['accumulated_actions'] = accumulated_actions
if test_set:
all_assessments.append(features)
 elif true_attempts+false_attempts > 0:
all_assessments.append(features)
 counter += 1
def update_counters(counter: dict, col: str):
num_of_session_count = Counter(session[col])
 for k in num_of_session_count.keys() :
x = k
if col == 'title':
x = activities_labels[k]
counter[x] += num_of_session_count[k]
return counter
event_code_count = update_counters(event_code_count, "event_code")
 event_id_count = update_counters(event_id_count, "event_id")
 title_count = update_counters(title_count, 'title')
 title_event_code_count = update_counters(title_event_code_count, 'title_event_code')
 accumulated_actions += len(session)
 if last_activity != session_type:
user_activities_count[session_type] += 1
last_activitiy = session_type
if test_set:
return all_assessments[-1]
return all_assessments
def get_train_and_test(train, test):
compiled_train = []
compiled_test = []
for i,(ins_id, user_sample)in tqdm(enumerate(train.groupby('installation_id', sort = False)) , total = 17000):
compiled_train += get_data(user_sample)
 for ins_id, user_sample in tqdm(test.groupby('installation_id', sort = False), total = 1000):
test_data = get_data(user_sample, test_set = True)
 compiled_test.append(test_data)
 reduce_train = pd.DataFrame(compiled_train)
 reduce_test = pd.DataFrame(compiled_test)
 categoricals = ['session_title']
return reduce_train, reduce_test, categoricals
train, test, train_labels, specs, sample_submission = read_data()
train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code = encode_title(train, test, train_labels)
 reduce_train, reduce_test, categoricals = get_train_and_test(train, test )
!pip install MulticoreTSNE<import_modules>
def preprocess(reduce_train, reduce_test):
for df in [reduce_train, reduce_test]:
df['installation_session_count'] = df.groupby(['installation_id'])['Clip'].transform('count')
 df['installation_duration_mean'] = df.groupby(['installation_id'])['duration_mean'].transform('mean')
 df['installation_title_nunique'] = df.groupby(['installation_id'])['session_title'].transform('nunique')
 df['sum_event_code_count'] = df[[2050, 4100, 4230, 5000, 4235, 2060, 4110, 5010, 2070, 2075, 2080, 2081, 2083, 3110, 4010, 3120, 3121, 4020, 4021,
4022, 4025, 4030, 4031, 3010, 4035, 4040, 3020, 3021, 4045, 2000, 4050, 2010, 2020, 4070, 2025, 2030, 4080, 2035,
2040, 4090, 4220, 4095]].sum(axis = 1)
 df['installation_event_code_count_mean'] = df.groupby(['installation_id'])['sum_event_code_count'].transform('mean')
 features = reduce_train.loc[(reduce_train.sum(axis=1)!= 0),(reduce_train.sum(axis=0)!= 0)].columns
features = [x for x in features if x not in ['accuracy_group', 'installation_id']] + ['acc_' + title for title in assess_titles]
return reduce_train, reduce_test, features
reduce_train, reduce_test, features = preprocess(reduce_train, reduce_test )
from MulticoreTSNE import MulticoreTSNE as TSNE<load_from_csv>
params = {'n_estimators':2000,
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'subsample': 0.75,
'subsample_freq': 1,
'learning_rate': 0.04,
'feature_fraction': 0.9,
'max_depth': 15,
'lambda_l1': 1,
'lambda_l2': 1,
'verbose': 100,
'early_stopping_rounds': 100, 'eval_metric': 'cappa'
}
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
y = reduce_train['accuracy_group']
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<define_variables>
n_fold = 5
folds = GroupKFold(n_splits=n_fold )
modif_list = [7, 24, 27,28, 36, 42,49,52,54, 71,72, 79,
81, 85,87,89,92, 93, 94, 95, 96, 102, 103,
104, 107,109, 114, 125, 129, 130, 131, 143,
149,151, 152, 162,166, 177, 182,187, 193,194, 195,
197, 198, 206,207, 208,211, 213, 220,
221, 222, 230,233, 234, 235, 242,244,251,
252,258, 262, 263, 264, 266, 268, 270, 271, 273,
287, 289,293,308, 310, 312,313, 315,320, 321,
328, 329, 332, 333, 335, 342,344,345,346,
349, 335, 356, 357, 369, 372, 373,374, 375, 377,
380, 381, 383, 384, 386, 387, 388, 393, 394, 404,
411, 412, 415, 419, 422, 425, 426,427, 431,433,
435, 436, 437, 443, 449, 451, 457, 458, 465, 471,
486,487, 489, 490, 491,495, 498, 499, 501, 508, 509, 510]<init_hyperparams>
cols_to_drop = ['game_session', 'installation_id', 'timestamp', 'accuracy_group', 'timestampDate']
params = {
7: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
24: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
27: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
28: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
36: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
42: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
49: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
52: {'init_params': 'random', 'n_init': 1, 'reg_covar': 1e-06},
54: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
71: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
72: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
79: {'init_params': 'kmeans', 'n_init': 5, 'reg_covar': 1e-08},
81: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
85: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
87: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
89: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
92: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
93: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
94: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
95: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
96: {'init_params': 'random', 'n_init': 10, 'reg_covar': 0.01},
102: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
103: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
104: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 1e-08},
107: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 1e-06},
109: {'init_params': 'kmeans', 'n_init': 1, 'reg_covar': 0.0001},
114: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
125: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
129: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
130: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
131: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
143: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
149: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
151: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
152: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
162: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
166: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 1e-08},
177: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
182: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
187: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
193: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
194: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
195: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-08},
197: {'init_params': 'kmeans', 'n_init': 1, 'reg_covar': 0.0001},
198: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 1e-08},
206: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 1e-08},
207: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
208: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
211: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 1e-06},
213: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 1e-06},
220: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
221: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.1},
222: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
230: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
233: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
234: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
235: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
242: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-08},
244: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
251: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.0001},
252: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
258: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-06},
262: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
263: {'init_params': 'kmeans', 'n_init': 5, 'reg_covar': 0.01},
264: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
266: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
268: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
270: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
271: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
273: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.0001},
287: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
289: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 1e-06},
293: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
308: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
310: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
312: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-06},
313: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
315: {'init_params': 'random', 'n_init': 30, 'reg_covar': 0.0001},
320: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
321: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
328: {'init_params': 'random', 'n_init': 1, 'reg_covar': 0.0001},
329: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
332: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
333: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
335: {'init_params': 'random', 'n_init': 20, 'reg_covar': 0.01},
342: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
344: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
345: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
346: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
349: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
356: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-06},
357: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
369: {'init_params': 'random', 'n_init': 30, 'reg_covar': 0.0001},
372: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 1e-06},
373: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
374: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
375: {'init_params': 'kmeans', 'n_init': 5, 'reg_covar': 0.0001},
377: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
380: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-06},
381: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
383: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.01},
384: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
386: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 1e-08},
387: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-08},
388: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.1},
393: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-06},
394: {'init_params': 'kmeans', 'n_init': 1, 'reg_covar': 1e-06},
404: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
411: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.0001},
412: {'init_params': 'kmeans', 'n_init': 1, 'reg_covar': 0.0001},
415: {'init_params': 'random', 'n_init': 1, 'reg_covar': 0.0001},
419: {'init_params': 'random', 'n_init': 30, 'reg_covar': 0.0001},
422: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
425: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
426: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
427: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
431: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-08},
433: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
435: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
436: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-08},
437: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-06},
443: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 1e-06},
449: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
451: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
457: {'init_params': 'kmeans', 'n_init': 1, 'reg_covar': 0.1},
458: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.01},
465: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.1},
471: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.1},
486: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01},
487: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
489: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.0001},
490: {'init_params': 'kmeans', 'n_init': 1, 'reg_covar': 1e-06},
491: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
495: {'init_params': 'kmeans', 'n_init': 5, 'reg_covar': 0.01},
498: {'init_params': 'kmeans', 'n_init': 1, 'reg_covar': 0.0001},
499: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.0001},
501: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.0001},
508: {'init_params': 'kmeans', 'n_init': 20, 'reg_covar': 0.1},
509: {'init_params': 'kmeans', 'n_init': 10, 'reg_covar': 0.1},
510: {'init_params': 'kmeans', 'n_init': 30, 'reg_covar': 0.01}}<define_variables>
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model1 = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model1.fit(X=reduce_train, y=y, folds=folds, params=params, preprocesser=mt, transformers=transformers,
eval_metric='cappa', cols_to_drop=cols_to_drop )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(( len(train),4))
preds = np.zeros(( len(test),4))
oof2 = np.zeros(( len(train),4))
preds2 = np.zeros(( len(test),4))
oof3 = np.zeros(( len(train),1))
preds3 = np.zeros(( len(test),1))
ooff = np.zeros(( len(train),4))
predsf = np.zeros(( len(test),4))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ,('scaler', RobustScaler(quantile_range=(35, 65)))])
 sel = pipe.fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 print(train3.shape)
 NK = 2
skf = StratifiedKFold(n_splits=10, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
x = train3[train_index]
y = train2.loc[train_index,'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = Birch(threshold=0.5, branching_factor=150, n_clusters=NK, compute_labels=True ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = Birch(threshold=0.5, branching_factor=150, n_clusters=NK, compute_labels=True ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
ms = np.stack([m11,m12,m21,m22])
 ps = np.stack([p11,p12,p21,p22])
 gm = GaussianMixture(n_components=4,random_state=42, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=1000)
 gm.fit(np.concatenate([train3[train_index],test3],axis = 0))
oof[idx1[oof_test_index]] = gm.predict_proba(train3[oof_test_index,:])
 preds[idx2] += gm.predict_proba(test3)/ skf.n_splits
x = train3[train_index[:int(0.95*len(train_index)) ]]
y = train2.loc[train_index[:int(0.95*len(train_index)) ],'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = Birch(threshold=0.5, branching_factor=150, n_clusters=NK, compute_labels=True ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = Birch(threshold=0.5, branching_factor=150, n_clusters=NK, compute_labels=True ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
ms = np.stack([m11,m12,m21,m22])
 ps = np.stack([p11,p12,p21,p22])
 gm = GaussianMixture(n_components=4,random_state=43, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=500)
 gm.fit(np.concatenate([train3[train_index[:int(0.95*len(train_index)) ]],test3[:int(0.95*test3.shape[0])]],axis = 0))
oof[idx1[oof_test_index]] += gm.predict_proba(train3[oof_test_index,:])
 preds[idx2] += gm.predict_proba(test3)/ skf.n_splits
x = train3[train_index[int(0.15*len(train_index)) :]]
y = train2.loc[train_index[int(0.15*len(train_index)) :],'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = Birch(threshold=0.5, branching_factor=150, n_clusters=NK, compute_labels=True ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = Birch(threshold=0.5, branching_factor=150, n_clusters=NK, compute_labels=True ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
ms = np.stack([m11,m12,m21,m22])
 ps = np.stack([p11,p12,p21,p22])
 gm = GaussianMixture(n_components=4,random_state=44, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=500)
 gm.fit(np.concatenate([train3[train_index[int(0.15*len(train_index)) :]],test3[int(0.15*test3.shape[0]):]],axis = 0))
oof[idx1[oof_test_index]] += gm.predict_proba(train3[oof_test_index,:])
 preds[idx2] += gm.predict_proba(test3)/ skf.n_splits
train_index2 = train_index[ :int(0.95*len(train_index)) ]
clf = KNeighborsClassifier(n_neighbors=50, metric='braycurtis')
 clf.fit(train3[train_index2,:],train2.loc[train_index2]['target'])
 oof3[idx1[test_index],0] += clf.predict_proba(train3[oof_test_index,:])[:,1]
preds3[idx2,0] += clf.predict_proba(test3)[:,1] / skf.n_splits
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]))
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]+0.10*oof3[idx1,0]))
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]+0.20*oof3[idx1,0]))
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]+0.40*oof3[idx1,0]))
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]+0.50*oof3[idx1,0]))
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]+0.60*oof3[idx1,0]))
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]+0.80*oof3[idx1,0]))
print(roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1]+1.00*oof3[idx1,0]))
px = np.where(train['wheezy-copper-turtle-magic']<=i)[0]
print(roc_auc_score(train['target'].values[px], oof[px,0]+oof[px,1]+0.10*oof3[px,0])) 
<define_variables>
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):

 X_p = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return -qwk(y, X_p)
 def fit(self, X, y):

 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 return pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 def coefficients(self):

 return self.coef_['x']
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(( len(train),4))
preds = np.zeros(( len(test),4))
oof2 = np.zeros(( len(train),4))
preds2 = np.zeros(( len(test),4))
oof3 = np.zeros(( len(train),1))
preds3 = np.zeros(( len(test),1))
ooff = np.zeros(( len(train),4))
predsf = np.zeros(( len(test),4))
K2=[]
K3=[]
K4=[]
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ,('scaler', RobustScaler(quantile_range=(35, 65)))])
 sel = pipe.fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 print(train3.shape)
 NK = 2
skf = StratifiedKFold(n_splits=16, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
x = train3[train_index]
y = train2.loc[train_index,'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = Birch(threshold=0.5, branching_factor=100, n_clusters=NK, compute_labels=True ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = Birch(threshold=0.5, branching_factor=100, n_clusters=NK, compute_labels=True ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
ms = np.stack([m11,m12,m21,m22])
 ps = np.stack([p11,p12,p21,p22])
 gm = GaussianMixture(n_components=4,random_state=42, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=1000)
 gm.fit(np.concatenate([train3[train_index],test3],axis = 0))
oof[idx1[oof_test_index]] = gm.predict_proba(train3[oof_test_index,:])
 sc2 = roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1])
 NK = 3
for train_index, test_index in skf.split(train3, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
x = train3[train_index]
y = train2.loc[train_index,'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = Birch(threshold=0.5, branching_factor=100, n_clusters=NK, compute_labels=True ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = Birch(threshold=0.5, branching_factor=100, n_clusters=NK, compute_labels=True ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
ms = np.stack([m11,m12,m21,m22])
 ps = np.stack([p11,p12,p21,p22])
 gm = GaussianMixture(n_components=4,random_state=42, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=1000)
 gm.fit(np.concatenate([train3[train_index],test3],axis = 0))
oof[idx1[oof_test_index]] = gm.predict_proba(train3[oof_test_index,:])
 sc3 = roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1])
 NK = 4
for train_index, test_index in skf.split(train3, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
x = train3[train_index]
y = train2.loc[train_index,'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = Birch(threshold=0.5, branching_factor=100, n_clusters=NK, compute_labels=True ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = Birch(threshold=0.5, branching_factor=100, n_clusters=NK, compute_labels=True ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
ms = np.stack([m11,m12,m21,m22])
 ps = np.stack([p11,p12,p21,p22])
 gm = GaussianMixture(n_components=4,random_state=42, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=1000)
 gm.fit(np.concatenate([train3[train_index],test3],axis = 0))
oof[idx1[oof_test_index]] = gm.predict_proba(train3[oof_test_index,:])
 sc4 = roc_auc_score(train2['target'], oof[idx1,0]+oof[idx1,1])
 K2.append(sc2)
 K3.append(sc3)
 K4.append(sc4)
 print(sc2,sc3,sc4)
<string_transform>
opt_preds = optR.predict(pr1.reshape(-1,), coefficients)
 qwk(y, opt_preds )
print(','.join(K2))
print(','.join(K3))
print(','.join(K4))<set_options>
pr1 = regressor_model1.predict(reduce_test)
 pr1[pr1 <= a0] = 0
pr1[np.where(np.logical_and(pr1 > a0, pr1 <= a1)) ] = 1
pr1[np.where(np.logical_and(pr1 > a1, pr1 <= a2)) ] = 2
pr1[pr1 > a2] = 3
warnings.filterwarnings('ignore')
 warnings.filterwarnings('ignore')
<load_from_csv>
sample_submission['accuracy_group'] = pr1.astype(int)
 sample_submission.to_csv('submission.csv', index=False )
<feature_engineering><EOS> 
sample_submission['accuracy_group'].value_counts(normalize=True )