Kaggle competition: 2019 Data Science Bowl
<SOS> metric: QuadraticWeightedKappa Kaggle data source: data-science-bowl-2019<statistical_test>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
pd.set_option('max_rows', 500)
def projectMeans(means):
means[means>0]=1
means[means<=0]=-1
return means
def _compute_precision_cholesky(covariances, covariance_type):
estimate_precision_error_message =("Hell no")
 if covariance_type in 'full':
n_components, n_features, _ = covariances.shape
precisions_chol = np.empty(( n_components, n_features, n_features))
for k, covariance in enumerate(covariances):
try:
cov_chol = linalg.cholesky(covariance, lower=True)
 except linalg.LinAlgError:
raise ValueError(estimate_precision_error_message)
 precisions_chol[k] = linalg.solve_triangular(cov_chol,
np.eye(n_features),
lower=True ).T
return precisions_chol
def _estimate_gaussian_covariances_full(resp, X, nk, means, reg_covar):
n_components, n_features = means.shape
covariances = np.empty(( n_components, n_features, n_features))
for k in range(n_components):
diff = X - means[k]
covariances[k] = np.dot(resp[:, k] * diff.T, diff)/ nk[k]
covariances[k].flat[::n_features + 1] += reg_covar
return covariances
def _estimate_gaussian_parameters2(X, resp, reg_covar, covariance_type):
nk = resp.sum(axis=0)+ 10 * np.finfo(resp.dtype ).eps
means = np.dot(resp.T, X)/ nk[:, np.newaxis]
means = projectMeans(means)
 covariances = {"full": _estimate_gaussian_covariances_full}[covariance_type](resp, X, nk, means, reg_covar)
 return nk, means, covariances
class GaussianMixture2(GaussianMixture):
def _m_step(self, X, log_resp):
resp = np.exp(log_resp)
 sums = resp.sum(0)
 if sums.max() - sums.min() > 2:
for i in range(3):
resp = len(X)* resp / resp.sum(0)/ len(sums)
 resp = resp/resp.sum(1)[:,None]
n_samples, _ = X.shape
self.weights_, self.means_, self.covariances_ =(
 _estimate_gaussian_parameters2(X, resp, self.reg_covar,
self.covariance_type))
self.weights_ /= n_samples
self.precisions_cholesky_ = _compute_precision_cholesky(
 self.covariances_, self.covariance_type)
 random.seed(1234)
 np.random.seed(1234)
 os.environ['PYTHONHASHSEED'] = str(1234 )<define_variables>
def add_datepart(df: pd.DataFrame, field_name: str,
prefix: str = None, drop: bool = True, time: bool = True, date: bool = True):

 field = df[field_name]
prefix = ifnone(prefix, re.sub('[Dd]ate$', '', field_name))
attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Is_month_end', 'Is_month_start']
if date:
attr.append('Date')
 if time:
attr = attr + ['Hour', 'Minute']
for n in attr:
df[prefix + n] = getattr(field.dt, n.lower())
 if drop:
df.drop(field_name, axis=1, inplace=True)
 return df
def ifnone(a: Any, b: Any)-> Any:
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
N_RAND_INIT = 2
N_CLUST_OPT = 3
N_TEST = 1
all_acc = np.zeros(( 512, N_CLUST_OPT, N_RAND_INIT))
all_roc = np.zeros(( 512, N_CLUST_OPT, N_RAND_INIT))
cluster_cnt = np.zeros(( 512, N_CLUST_OPT, N_RAND_INIT))
j_selection = np.zeros(N_CLUST_OPT)
 for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 test_index = range(len(train3))
yf = train2['target']
ms, ps = get_mean_cov(train3,yf)
 cc_list = []
nc_list = 2*(np.array(range(N_CLUST_OPT)) + 2)
 for j in range(N_CLUST_OPT):
cc_list.append(['cluster_' + str(i)for i in range(nc_list[j])])
 gm_list = []
acc = np.zeros(( N_CLUST_OPT, N_RAND_INIT))
res_list = []
ctc_list = []
for j in range(N_CLUST_OPT):
gm_list.append([])
 res_list.append([])
 ctc_list.append([])
 nc = nc_list[j]
cl = int(0.5*nc)
 for k in range(N_RAND_INIT):
ps_list = np.concatenate([ps]*cl, axis=0)
 th_step = 100/(cl+1)
 th_p = np.arange(th_step,99,th_step)+ 0.5*(np.random.rand(cl)- 0.5)*th_step
th = np.percentile(ms,th_p)
 ms_list = []
for t in range(cl):
ms_new = ms.copy()
ms_new[ms>=th[t]]=1
ms_new[ms<th[t]]=-1
ms_list.append(ms_new)
 ms_list = np.concatenate(ms_list, axis=0)
 perm = np.random.permutation(nc)
 ps_list = ps_list[perm]
ms_list = ms_list[perm]
gm = GaussianMixture2(n_components=nc, init_params='random', covariance_type='full', tol=0.0001,reg_covar=0.001,
max_iter=5000, n_init=1, means_init=ms_list, precisions_init=ps_list, random_state=1234)
 gm.fit(np.concatenate([train3,test3],axis = 0))
res = pd.concat([pd.DataFrame(gm.predict_proba(train3), columns = cc_list[j]),
yf.to_frame().reset_index(drop=True)], sort=False, axis=1)
 cluster_to_class = res.groupby('target' ).agg('mean' ).values.argmax(0)
 cluster_cnt[i,j,k] = cluster_to_class.sum()
res = pd.concat([pd.DataFrame(gm.predict_proba(train3), columns = cc_list[j]),
pd.DataFrame(cluster_to_class, index=cc_list[j],
columns=['target'] ).transpose() ], sort=False, axis=0 ).\
transpose().groupby('target' ).agg(sum ).transpose()
res_list[j].append(res[1])
 gm_list[j].append(gm)
 ctc_list[j].append(cluster_to_class)
 acc[j,k] =(res.values.argmax(1)== yf.values ).mean()
all_acc[i,j,k] = acc[j,k]
all_roc[i,j,k] = roc_auc_score(yf.values, res[1])
 best_j = acc.mean(1 ).argmax()
j_selection[best_j] += 1
for k in np.argsort(acc[best_j,:])[-N_TEST:]:
res2 = pd.concat([pd.DataFrame(gm_list[best_j][k].predict_proba(test3), columns = cc_list[best_j]),
pd.DataFrame(ctc_list[best_j][k], index=cc_list[best_j],
columns=['target'] ).transpose() ], sort=False, axis=0 ).\
transpose().groupby('target' ).agg(sum ).transpose()
oof[idx1] += res_list[best_j][k]/N_TEST
preds[idx2] += res2[1]/N_TEST
if i%10==0: print('QMM scores CV =',round(roc_auc_score(train['target'],oof),5))
auc = roc_auc_score(train['target'],oof)
 print(j_selection)
 print('Final QMM scores CV =',round(auc,5))<define_variables>
@jit
def qwk(a1, a2):

 max_rat = 3
a1 = np.asarray(a1, dtype=int)
 a2 = np.asarray(a2, dtype=int)
 hist1 = np.zeros(( max_rat + 1,))
hist2 = np.zeros(( max_rat + 1,))
o = 0
for k in range(a1.shape[0]):
i, j = a1[k], a2[k]
hist1[i] += 1
hist2[j] += 1
o +=(i - j)*(i - j)
 e = 0
for i in range(max_rat + 1):
for j in range(max_rat + 1):
e += hist1[i] * hist2[j] *(i - j)*(i - j)
 e = e / a1.shape[0]
return 1 - o / e
def eval_qwk_lgb(y_true, y_pred):

 y_pred = y_pred.reshape(len(np.unique(y_true)) , -1 ).argmax(axis=0)
 return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_lgb_regr(y_true, y_pred):

 y_pred[y_pred <= 1.12119976] = 0
y_pred[np.where(np.logical_and(y_pred > 1.12119976, y_pred <= 1.73094009)) ] = 1
y_pred[np.where(np.logical_and(y_pred > 1.73094009, y_pred <= 2.17857588)) ] = 2
y_pred[y_pred > 2.17857588] = 3
return 'cappa', qwk(y_true, y_pred), True
class LGBWrapper_regr(object):

 def __init__(self):
self.model = lgb.LGBMRegressor()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
if params['objective'] == 'regression':
eval_metric = eval_qwk_lgb_regr
else:
eval_metric = 'auc'
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_metric,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict(self, X_test):
return self.model.predict(X_test, num_iteration=self.model.best_iteration_)
 def eval_qwk_xgb(y_pred, y_true):

 y_true = y_true.get_label()
y_pred = y_pred.argmax(axis=1)
 return 'cappa', -qwk(y_true, y_pred)
 class LGBWrapper(object):

 def __init__(self):
self.model = lgb.LGBMClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_qwk_lgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)
 class CatWrapper(object):

 def __init__(self):
self.model = cat.CatBoostClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**{k: v for k, v in params.items() if k != 'cat_cols'})
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = None
else:
categorical_columns = None
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
cat_features=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if 'MultiClass' not in self.model.get_param('loss_function'):
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)
 class XGBWrapper(object):

 def __init__(self):
self.model = xgb.XGBClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_metric=eval_qwk_xgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'])
 scores = self.model.evals_result()
self.best_score_ = {k: {m: m_v[-1] for m, m_v in v.items() } for k, v in scores.items() }
self.best_score_ = {k: {m: n if m != 'cappa' else -n for m, n in v.items() } for k, v in self.best_score_.items() }
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)
 class MainTransformer(BaseEstimator, TransformerMixin):
def __init__(self, convert_cyclical: bool = False, create_interactions: bool = False, n_interactions: int = 20):

 self.convert_cyclical = convert_cyclical
self.create_interactions = create_interactions
self.feats_for_interaction = None
self.n_interactions = n_interactions
def fit(self, X, y=None):
if self.create_interactions:
self.feats_for_interaction = [col for col in X.columns if 'sum' in col
or 'mean' in col or 'max' in col or 'std' in col
or 'attempt' in col]
self.feats_for_interaction1 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 self.feats_for_interaction2 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 if self.create_interactions:
for col1 in self.feats_for_interaction1:
for col2 in self.feats_for_interaction2:
data[f'{col1}_int_{col2}'] = data[col1] * data[col2]
if self.convert_cyclical:
data['timestampHour'] = np.sin(2 * np.pi * data['timestampHour'] / 23.0)
 data['timestampMonth'] = np.sin(2 * np.pi * data['timestampMonth'] / 23.0)
 data['timestampWeek'] = np.sin(2 * np.pi * data['timestampWeek'] / 23.0)
 data['timestampMinute'] = np.sin(2 * np.pi * data['timestampMinute'] / 23.0)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
 class FeatureTransformer(BaseEstimator, TransformerMixin):
def __init__(self, main_cat_features: list = None, num_cols: list = None):

 self.main_cat_features = main_cat_features
self.num_cols = num_cols
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )
for j in range(N_CLUST_OPT):
print(np.all(cluster_cnt[:,j,:] == 0.5*nc_list[j]))<save_to_csv>
class RegressorModel(object):

 def __init__(self, columns: list = None, model_wrapper=None):

 self.columns = columns
self.model_wrapper = model_wrapper
self.result_dict = {}
self.train_one_fold = False
self.preprocesser = None
def fit(self, X: pd.DataFrame, y,
X_holdout: pd.DataFrame = None, y_holdout=None,
folds=None,
params: dict = None,
eval_metric='rmse',
cols_to_drop: list = None,
preprocesser=None,
transformers: dict = None,
adversarial: bool = False,
plot: bool = True):

 if folds is None:
folds = KFold(n_splits=3, random_state=42)
 self.train_one_fold = True
self.columns = X.columns if self.columns is None else self.columns
self.feature_importances = pd.DataFrame(columns=['feature', 'importance'])
 self.trained_transformers = {k: [] for k in transformers}
self.transformers = transformers
self.models = []
self.folds_dict = {}
self.eval_metric = eval_metric
n_target = 1
self.oof = np.zeros(( len(X), n_target))
self.n_target = n_target
X = X[self.columns]
if X_holdout is not None:
X_holdout = X_holdout[self.columns]
if preprocesser is not None:
self.preprocesser = preprocesser
self.preprocesser.fit(X, y)
 X = self.preprocesser.transform(X, y)
 self.columns = X.columns.tolist()
if X_holdout is not None:
X_holdout = self.preprocesser.transform(X_holdout)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y, X['installation_id'])) :
if X_holdout is not None:
X_hold = X_holdout.copy()
else:
X_hold = None
self.folds_dict[fold_n] = {}
if params['verbose']:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 self.folds_dict[fold_n] = {}
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if self.train_one_fold:
X_train = X[self.original_columns]
y_train = y
X_valid = None
y_valid = None
datasets = {'X_train': X_train, 'X_valid': X_valid, 'X_holdout': X_hold, 'y_train': y_train}
X_train, X_valid, X_hold = self.transform_(datasets, cols_to_drop)
 self.folds_dict[fold_n]['columns'] = X_train.columns.tolist()
model = copy.deepcopy(self.model_wrapper)
 if adversarial:
X_new1 = X_train.copy()
if X_valid is not None:
X_new2 = X_valid.copy()
elif X_holdout is not None:
X_new2 = X_holdout.copy()
X_new = pd.concat([X_new1, X_new2], axis=0)
 y_new = np.hstack(( np.zeros(( X_new1.shape[0])) , np.ones(( X_new2.shape[0]))))
X_train, X_valid, y_train, y_valid = train_test_split(X_new, y_new)
 model.fit(X_train, y_train, X_valid, y_valid, X_hold, y_holdout, params=params)
 self.folds_dict[fold_n]['scores'] = model.best_score_
if self.oof.shape[0] != len(X):
self.oof = np.zeros(( X.shape[0], self.oof.shape[1]))
if not adversarial:
self.oof[valid_index] = model.predict(X_valid ).reshape(-1, n_target)
 fold_importance = pd.DataFrame(list(zip(X_train.columns, model.feature_importances_)) ,
columns=['feature', 'importance'])
 self.feature_importances = self.feature_importances.append(fold_importance)
 self.models.append(model)
 self.feature_importances['importance'] = self.feature_importances['importance'].astype(int)
 self.calc_scores_()
if plot:
fig, ax = plt.subplots(figsize=(16, 12))
plt.subplot(2, 2, 1)
 self.plot_feature_importance(top_n=20)
 plt.subplot(2, 2, 2)
 self.plot_metric()
plt.subplot(2, 2, 3)
 plt.hist(y.values.reshape(-1, 1)- self.oof)
 plt.title('Distribution of errors')
 plt.subplot(2, 2, 4)
 plt.hist(self.oof)
 plt.title('Distribution of oof predictions');
def transform_(self, datasets, cols_to_drop):
for name, transformer in self.transformers.items() :
transformer.fit(datasets['X_train'], datasets['y_train'])
 datasets['X_train'] = transformer.transform(datasets['X_train'])
 if datasets['X_valid'] is not None:
datasets['X_valid'] = transformer.transform(datasets['X_valid'])
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = transformer.transform(datasets['X_holdout'])
 self.trained_transformers[name].append(transformer)
 if cols_to_drop is not None:
cols_to_drop = [col for col in cols_to_drop if col in datasets['X_train'].columns]
datasets['X_train'] = datasets['X_train'].drop(cols_to_drop, axis=1)
 if datasets['X_valid'] is not None:
datasets['X_valid'] = datasets['X_valid'].drop(cols_to_drop, axis=1)
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = datasets['X_holdout'].drop(cols_to_drop, axis=1)
 self.cols_to_drop = cols_to_drop
return datasets['X_train'], datasets['X_valid'], datasets['X_holdout']
def calc_scores_(self):
print()
datasets = [k for k, v in [v['scores'] for k, v in self.folds_dict.items() ][0].items() if len(v)> 0]
self.scores = {}
for d in datasets:
scores = [v['scores'][d][self.eval_metric] for k, v in self.folds_dict.items() ]
print(f"CV mean score on {d}: {np.mean(scores):.4f} +/- {np.std(scores):.4f} std.")
 self.scores[d] = np.mean(scores)
 def predict(self, X_test, averaging: str = 'usual'):

 full_prediction = np.zeros(( X_test.shape[0], self.oof.shape[1]))
if self.preprocesser is not None:
X_test = self.preprocesser.transform(X_test)
 for i in range(len(self.models)) :
X_t = X_test.copy()
for name, transformers in self.trained_transformers.items() :
X_t = transformers[i].transform(X_t)
 if self.cols_to_drop is not None:
cols_to_drop = [col for col in self.cols_to_drop if col in X_t.columns]
X_t = X_t.drop(cols_to_drop, axis=1)
 y_pred = self.models[i].predict(X_t[self.folds_dict[i]['columns']] ).reshape(-1, full_prediction.shape[1])
 if full_prediction.shape[0] != len(y_pred):
full_prediction = np.zeros(( y_pred.shape[0], self.oof.shape[1]))
if averaging == 'usual':
full_prediction += y_pred
elif averaging == 'rank':
full_prediction += pd.Series(y_pred ).rank().values
return full_prediction / len(self.models)
 def plot_feature_importance(self, drop_null_importance: bool = True, top_n: int = 10):

 top_feats = self.get_top_features(drop_null_importance, top_n)
 feature_importances = self.feature_importances.loc[self.feature_importances['feature'].isin(top_feats)]
feature_importances['feature'] = feature_importances['feature'].astype(str)
 top_feats = [str(i)for i in top_feats]
sns.barplot(data=feature_importances, x='importance', y='feature', orient='h', order=top_feats)
 plt.title('Feature importances')
 def get_top_features(self, drop_null_importance: bool = True, top_n: int = 10):

 grouped_feats = self.feature_importances.groupby(['feature'])['importance'].mean()
if drop_null_importance:
grouped_feats = grouped_feats[grouped_feats != 0]
return list(grouped_feats.sort_values(ascending=False ).index)[:top_n]
def plot_metric(self):

 full_evals_results = pd.DataFrame()
for model in self.models:
evals_result = pd.DataFrame()
for k in model.model.evals_result_.keys() :
evals_result[k] = model.model.evals_result_[k][self.eval_metric]
evals_result = evals_result.reset_index().rename(columns={'index': 'iteration'})
 full_evals_results = full_evals_results.append(evals_result)
 full_evals_results = full_evals_results.melt(id_vars=['iteration'] ).rename(columns={'value': self.eval_metric,
'variable': 'dataset'})
 sns.lineplot(data=full_evals_results, x='iteration', y=self.eval_metric, hue='dataset')
 plt.title('Training progress')
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<set_options>
class CategoricalTransformer(BaseEstimator, TransformerMixin):
def __init__(self, cat_cols=None, drop_original: bool = False, encoder=OrdinalEncoder()):

 self.cat_cols = cat_cols
self.drop_original = drop_original
self.encoder = encoder
self.default_encoder = OrdinalEncoder()
def fit(self, X, y=None):
if self.cat_cols is None:
kinds = np.array([dt.kind for dt in X.dtypes])
 is_cat = kinds == 'O'
self.cat_cols = list(X.columns[is_cat])
 self.encoder.set_params(cols=self.cat_cols)
 self.default_encoder.set_params(cols=self.cat_cols)
 self.encoder.fit(X[self.cat_cols], y)
 self.default_encoder.fit(X[self.cat_cols], y)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 new_cat_names = [f'{col}_encoded' for col in self.cat_cols]
encoded_data = self.encoder.transform(data[self.cat_cols])
 if encoded_data.shape[1] == len(self.cat_cols):
data[new_cat_names] = encoded_data
else:
pass
if self.drop_original:
data = data.drop(self.cat_cols, axis=1)
 else:
data[self.cat_cols] = self.default_encoder.transform(data[self.cat_cols])
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )
warnings.filterwarnings('ignore' )<load_from_csv>
def read_data() :
print('Reading train.csv file.... ')
 train = pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv')
 print('Training.csv file have {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Reading test.csv file.... ')
 test = pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv')
 print('Test.csv file have {} rows and {} columns'.format(test.shape[0], test.shape[1]))
print('Reading train_labels.csv file.... ')
 train_labels = pd.read_csv('/kaggle/input/data-science-bowl-2019/train_labels.csv')
 print('Train_labels.csv file have {} rows and {} columns'.format(train_labels.shape[0], train_labels.shape[1]))
print('Reading specs.csv file.... ')
 specs = pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv')
 print('Specs.csv file have {} rows and {} columns'.format(specs.shape[0], specs.shape[1]))
print('Reading sample_submission.csv file.... ')
 sample_submission = pd.read_csv('/kaggle/input/data-science-bowl-2019/sample_submission.csv')
 print('Sample_submission.csv file have {} rows and {} columns'.format(sample_submission.shape[0], sample_submission.shape[1]))
return train, test, train_labels, specs, sample_submission
def encode_title(train, test, train_labels):
train['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), train['title'], train['event_code']))
test['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), test['title'], test['event_code']))
all_title_event_code = list(set(train["title_event_code"].unique() ).union(test["title_event_code"].unique()))
list_of_user_activities = list(set(train['title'].unique() ).union(set(test['title'].unique())))
 list_of_event_code = list(set(train['event_code'].unique() ).union(set(test['event_code'].unique())))
 list_of_event_id = list(set(train['event_id'].unique() ).union(set(test['event_id'].unique())))
 list_of_worlds = list(set(train['world'].unique() ).union(set(test['world'].unique())))
 activities_map = dict(zip(list_of_user_activities, np.arange(len(list_of_user_activities))))
activities_labels = dict(zip(np.arange(len(list_of_user_activities)) , list_of_user_activities))
activities_world = dict(zip(list_of_worlds, np.arange(len(list_of_worlds))))
assess_titles = list(set(train[train['type'] == 'Assessment']['title'].value_counts().index ).union(set(test[test['type'] == 'Assessment']['title'].value_counts().index)))
 train['title'] = train['title'].map(activities_map)
 test['title'] = test['title'].map(activities_map)
 train['world'] = train['world'].map(activities_world)
 test['world'] = test['world'].map(activities_world)
 train_labels['title'] = train_labels['title'].map(activities_map)
 win_code = dict(zip(activities_map.values() ,(4100*np.ones(len(activities_map)) ).astype('int')))
 win_code[activities_map['Bird Measurer(Assessment)']] = 4110
train['timestamp'] = pd.to_datetime(train['timestamp'])
 test['timestamp'] = pd.to_datetime(test['timestamp'])
 return train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code
def get_data(user_sample, test_set=False):

 last_activity = 0
user_activities_count = {'Clip':0, 'Activity': 0, 'Assessment': 0, 'Game':0}
last_session_time_sec = 0
accuracy_groups = {0:0, 1:0, 2:0, 3:0}
all_assessments = []
accumulated_accuracy_group = 0
accumulated_accuracy = 0
accumulated_correct_attempts = 0
accumulated_uncorrect_attempts = 0
accumulated_actions = 0
counter = 0
time_first_activity = float(user_sample['timestamp'].values[0])
 durations = []
last_accuracy_title = {'acc_' + title: -1 for title in assess_titles}
event_code_count: Dict[str, int] = {ev: 0 for ev in list_of_event_code}
event_id_count: Dict[str, int] = {eve: 0 for eve in list_of_event_id}
title_count: Dict[str, int] = {eve: 0 for eve in activities_labels.values() }
title_event_code_count: Dict[str, int] = {t_eve: 0 for t_eve in all_title_event_code}
for i, session in user_sample.groupby('game_session', sort=False):
session_type = session['type'].iloc[0]
session_title = session['title'].iloc[0]
session_title_text = activities_labels[session_title]
if(session_type == 'Assessment')&(test_set or len(session)>1):
all_attempts = session.query(f'event_code == {win_code[session_title]}')
 true_attempts = all_attempts['event_data'].str.contains('true' ).sum()
false_attempts = all_attempts['event_data'].str.contains('false' ).sum()
features = user_activities_count.copy()
features.update(last_accuracy_title.copy())
 features.update(event_code_count.copy())
 features.update(event_id_count.copy())
 features.update(title_count.copy())
 features.update(title_event_code_count.copy())
 features.update(last_accuracy_title.copy())
 features['meta_len_session'] = len(session)
 features['meta_i'] = i
features['meta_set'] = 1 if test_set else 0
features['installation_id'] = session['installation_id'].iloc[-1]
features['session_title'] = session['title'].iloc[0]
features['accumulated_correct_attempts'] = accumulated_correct_attempts
features['accumulated_uncorrect_attempts'] = accumulated_uncorrect_attempts
accumulated_correct_attempts += true_attempts
accumulated_uncorrect_attempts += false_attempts
if durations == []:
features['duration_mean'] = 0
else:
features['duration_mean'] = np.mean(durations)
 durations.append(( session.iloc[-1, 2] - session.iloc[0, 2] ).seconds)
 features['accumulated_accuracy'] = accumulated_accuracy/counter if counter > 0 else 0
accuracy = true_attempts/(true_attempts+false_attempts)if(true_attempts+false_attempts)!= 0 else 0
accumulated_accuracy += accuracy
last_accuracy_title['acc_' + session_title_text] = accuracy
if accuracy == 0:
features['accuracy_group'] = 0
elif accuracy == 1:
features['accuracy_group'] = 3
elif accuracy == 0.5:
features['accuracy_group'] = 2
else:
features['accuracy_group'] = 1
features.update(accuracy_groups)
 accuracy_groups[features['accuracy_group']] += 1
features['accumulated_accuracy_group'] = accumulated_accuracy_group/counter if counter > 0 else 0
accumulated_accuracy_group += features['accuracy_group']
features['accumulated_actions'] = accumulated_actions
if test_set:
all_assessments.append(features)
 elif true_attempts+false_attempts > 0:
all_assessments.append(features)
 counter += 1
def update_counters(counter: dict, col: str):
num_of_session_count = Counter(session[col])
 for k in num_of_session_count.keys() :
x = k
if col == 'title':
x = activities_labels[k]
counter[x] += num_of_session_count[k]
return counter
event_code_count = update_counters(event_code_count, "event_code")
 event_id_count = update_counters(event_id_count, "event_id")
 title_count = update_counters(title_count, 'title')
 title_event_code_count = update_counters(title_event_code_count, 'title_event_code')
 accumulated_actions += len(session)
 if last_activity != session_type:
user_activities_count[session_type] += 1
last_activitiy = session_type
return all_assessments
def get_train_and_test(train, test):
compiled_train = []
compiled_semi_train = []
for ins_id, user_sample in tqdm(test.groupby('installation_id', sort = False), total = 10000):
compiled_semi_train += get_data(user_sample, test_set=True)
 for i,(ins_id, user_sample)in tqdm(enumerate(train.groupby('installation_id', sort = False)) , total = 17000):
compiled_train += get_data(user_sample, test_set=False)
 reduce_train = pd.DataFrame(compiled_train)
 reduce_semi_train = pd.DataFrame(compiled_semi_train)
 categoricals = ['session_title']
return reduce_train, reduce_semi_train, categoricals
train, test, train_labels, specs, sample_submission = read_data()
train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code = encode_title(train, test, train_labels)
 my_train, my_semi_train, categoricals = get_train_and_test(train, test )
train = pd.read_csv('.. /input/instant-gratification/train.csv')
 test = pd.read_csv('.. /input/instant-gratification/test.csv')
 train.head()<load_pretrained>
def preprocess(reduce_train, reduce_test):
for df in [reduce_train, reduce_test]:
df['sum_event_code_count'] = df[[2050, 4100, 4230, 5000, 4235, 2060, 4110, 5010, 2070, 2075, 2080, 2081, 2083, 3110, 4010, 3120, 3121, 4020, 4021,
4022, 4025, 4030, 4031, 3010, 4035, 4040, 3020, 3021, 4045, 2000, 4050, 2010, 2020, 4070, 2025, 2030, 4080, 2035,
2040, 4090, 4220, 4095]].sum(axis = 1)
 df['installation_event_code_count_mean'] = df.groupby(['installation_id'])['sum_event_code_count']\ 
.apply(lambda x: x.shift().expanding().mean())
 df['installation_session_count'] = df.groupby(['installation_id'])['Clip']\ 
.apply(lambda x: x.shift().expanding().count())
 df['installation_duration_mean'] = df.groupby(['installation_id'])['duration_mean']\ 
.apply(lambda x: x.shift().expanding().mean())
 df['installation_title_nunique'] = df.groupby(['installation_id'])['session_title']\ 
.apply(lambda x: x.shift().expanding().apply(lambda s: len(np.unique(s))))
features = reduce_train.loc[(reduce_train.sum(axis=1)!= 0),(reduce_train.sum(axis=0)!= 0)].columns
features = [x for x in features if x not in ['accuracy_group', 'installation_id']] + ['acc_' + title for title in assess_titles]
return reduce_train, reduce_test, features
print(my_train.shape, my_semi_train.shape)
 my_train, my_semi_train, features = preprocess(my_train, my_semi_train)
 print(my_train.shape, my_semi_train.shape)
 reduce_train = pd.concat([my_train, my_semi_train[my_semi_train["meta_len_session"] > 1]] ).reset_index(drop=True)
 reduce_test = my_semi_train[(my_semi_train["meta_len_session"] == 1)].reset_index(drop=True)
 print(reduce_train.shape, reduce_test.shape )
model = pickle.load(open('.. /input/instant-model-v2/gmm_pretrained.v2.pkl','rb'))<save_to_csv>
params = {'n_estimators':2000,
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'subsample': 0.75,
'subsample_freq': 1,
'learning_rate': 0.04,
'feature_fraction': 0.9,
'max_depth': 15,
'lambda_l1': 1,
'lambda_l2': 1,
'verbose': 100,
'early_stopping_rounds': 100, 'eval_metric': 'cappa'
}
sub = pd.read_csv('.. /input/instant-gratification/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False )<import_modules>
y = reduce_train['accuracy_group']
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import roc_auc_score
from tqdm import tqdm
from sklearn.covariance import EmpiricalCovariance
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn import linear_model
import sympy
import numpy as np
from sklearn.covariance import OAS<load_from_csv>
n_fold = 5
folds = GroupKFold(n_splits=n_fold )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<filter>
cols_to_drop = ['game_session', 'installation_id', 'timestamp', 'accuracy_group', 'timestampDate']
cols_to_drop += [c for c in reduce_train.columns if "meta" in str(c)]
print(cols_to_drop )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
x_train, y_train = train3[train_index,:], train2.loc[train_index]['target'].values
x_train_0 = x_train[y_train==0]
x_train_1 = x_train[y_train==1]
brc = Birch(branching_factor=50, n_clusters=3, threshold=0.6, compute_labels=True)
 labels_0 = brc.fit_predict(x_train_0)
 labels_1 = brc.fit_predict(x_train_1)
 zero_mean = []
zero_cov = []
for l in np.unique(labels_0):
model = OAS()
model.fit(x_train_0[labels_0==l])
 p = model.precision_
m = model.location_
zero_mean.append(m)
 zero_cov.append(p)
 one_mean = []
one_cov = []
for l in np.unique(labels_1):
model = OAS()
model.fit(x_train_1[labels_1==l])
 p = model.precision_
m = model.location_
one_mean.append(m)
 one_cov.append(p)
 ms = np.stack(zero_mean + one_mean)
 ps = np.stack(zero_cov + one_cov)
 gm = GaussianMixture(n_components=6, init_params='kmeans',
covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,
means_init=ms, precisions_init=ps, random_state=666)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:, 0:3].mean(axis=1)
 preds[idx2] += gm.predict_proba(test3)[:, 0:3].mean(axis=1)/ skf.n_splits
print('AUC ', i, roc_auc_score(1- train2['target'], oof[idx1]))
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5))<prepare_output>
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model1 = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model1.fit(X=reduce_train, y=y, folds=folds, params=params, preprocesser=mt, transformers=transformers,
eval_metric='cappa', cols_to_drop=cols_to_drop )
oof_0 = oof
preds_0 = preds<filter>
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):

 X_p = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return -qwk(y, X_p)
 def fit(self, X, y):

 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 return pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 def coefficients(self):

 return self.coef_['x']
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
x_train, y_train = train3[train_index,:], train2.loc[train_index]['target'].values
x_train_0 = x_train[y_train==0]
x_train_1 = x_train[y_train==1]
brc = Birch(branching_factor=50, n_clusters=4, threshold=0.6, compute_labels=True)
 labels_0 = brc.fit_predict(x_train_0)
 labels_1 = brc.fit_predict(x_train_1)
 zero_mean = []
zero_cov = []
for l in np.unique(labels_0):
model = OAS()
model.fit(x_train_0[labels_0==l])
 p = model.precision_
m = model.location_
zero_mean.append(m)
 zero_cov.append(p)
 one_mean = []
one_cov = []
for l in np.unique(labels_1):
model = OAS()
model.fit(x_train_1[labels_1==l])
 p = model.precision_
m = model.location_
one_mean.append(m)
 one_cov.append(p)
 ms = np.stack(zero_mean + one_mean)
 ps = np.stack(zero_cov + one_cov)
 gm = GaussianMixture(n_components=8, init_params='kmeans',
covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,
means_init=ms, precisions_init=ps, random_state=666)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:, 0:4].mean(axis=1)
 preds[idx2] += gm.predict_proba(test3)[:, 0:4].mean(axis=1)/ skf.n_splits
print('AUC ', i, roc_auc_score(1- train2['target'], oof[idx1]))
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5))<prepare_output>
opt_preds = optR.predict(pr1.reshape(-1,), coefficients)
 qwk(y, opt_preds )
oof_1 = oof
preds_1 = preds<filter>
pr1 = regressor_model1.predict(reduce_test)
 pr1[pr1 <= coefficients[0]] = 0
pr1[np.where(np.logical_and(pr1 > coefficients[0], pr1 <= coefficients[1])) ] = 1
pr1[np.where(np.logical_and(pr1 > coefficients[1], pr1 <= coefficients[2])) ] = 2
pr1[pr1 > coefficients[2]] = 3
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
x_train, y_train = train3[train_index,:], train2.loc[train_index]['target'].values
x_train_0 = x_train[y_train==0]
x_train_1 = x_train[y_train==1]
brc = Birch(branching_factor=50, n_clusters=2, threshold=0.6, compute_labels=True)
 labels_0 = brc.fit_predict(x_train_0)
 labels_1 = brc.fit_predict(x_train_1)
 zero_mean = []
zero_cov = []
for l in np.unique(labels_0):
model = OAS()
model.fit(x_train_0[labels_0==l])
 p = model.precision_
m = model.location_
zero_mean.append(m)
 zero_cov.append(p)
 one_mean = []
one_cov = []
for l in np.unique(labels_1):
model = OAS()
model.fit(x_train_1[labels_1==l])
 p = model.precision_
m = model.location_
one_mean.append(m)
 one_cov.append(p)
 ms = np.stack(zero_mean + one_mean)
 ps = np.stack(zero_cov + one_cov)
 gm = GaussianMixture(n_components=4, init_params='kmeans',
covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,
means_init=ms, precisions_init=ps, random_state=666)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:, 0:2].mean(axis=1)
 preds[idx2] += gm.predict_proba(test3)[:, 0:2].mean(axis=1)/ skf.n_splits
print('AUC ', i, roc_auc_score(1- train2['target'], oof[idx1]))
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5))<prepare_output>
sample_submission['accuracy_group'] = pr1.astype(int)
 sample_submission.to_csv('submission.csv', index=False )
<prepare_output><EOS> 
sample_submission['accuracy_group'].value_counts(normalize=True )