Kaggle competition: House Prices - Advanced Regression Techniques
roc_auc = metrics.roc_auc_score(oof_target, oof_prediction)
 print('Our out of folds roc auc score is: ', roc_auc )<set_options>
sns.set(style="darkgrid")
 np.random.seed(42)
 %matplotlib inline
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
pd.set_option('max_rows', 500)
<categorify>
def load_housing_data(filename, house_path):
csv_path = os.path.join(house_path, filename)
 return pd.read_csv(csv_path )
def add_datepart(df: pd.DataFrame, field_name: str,
prefix: str = None, drop: bool = True, time: bool = True, date: bool = True):

 field = df[field_name]
prefix = ifnone(prefix, re.sub('[Dd]ate$', '', field_name))
attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Is_month_end', 'Is_month_start']
if date:
attr.append('Date')
 if time:
attr = attr + ['Hour', 'Minute']
for n in attr:
df[prefix + n] = getattr(field.dt, n.lower())
 if drop:
df.drop(field_name, axis=1, inplace=True)
 return df
def ifnone(a: Any, b: Any)-> Any: 
<categorify>
train_data = load_housing_data('train.csv',".. /input")
 test_data = load_housing_data('test.csv','.. /input' )
@jit
def qwk(a1, a2):

 max_rat = 3
a1 = np.asarray(a1, dtype=int)
 a2 = np.asarray(a2, dtype=int)
 hist1 = np.zeros(( max_rat + 1,))
hist2 = np.zeros(( max_rat + 1,))
o = 0
for k in range(a1.shape[0]):
i, j = a1[k], a2[k]
hist1[i] += 1
hist2[j] += 1
o +=(i - j)*(i - j)
 e = 0
for i in range(max_rat + 1):
for j in range(max_rat + 1):
e += hist1[i] * hist2[j] *(i - j)*(i - j)
 e = e / a1.shape[0]
return 1 - o / e
def eval_qwk_lgb(y_true, y_pred):

 y_pred = y_pred.reshape(len(np.unique(y_true)) , -1 ).argmax(axis=0)
 return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_lgb_regr(y_true, y_pred):

 y_pred[y_pred <= 1.12119976] = 0
y_pred[np.where(np.logical_and(y_pred > 1.12119976, y_pred <= 1.73094009)) ] = 1
y_pred[np.where(np.logical_and(y_pred > 1.73094009, y_pred <= 2.17857588)) ] = 2
y_pred[y_pred > 2.17857588] = 3
return 'cappa', qwk(y_true, y_pred), True
class LGBWrapper_regr(object):

 def __init__(self):
self.model = lgb.LGBMRegressor()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
if params['objective'] == 'regression':
eval_metric = eval_qwk_lgb_regr
else:
eval_metric = 'auc'
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_metric,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict(self, X_test):
return self.model.predict(X_test, num_iteration=self.model.best_iteration_)
 def eval_qwk_xgb(y_pred, y_true):

 y_true = y_true.get_label()
y_pred = y_pred.argmax(axis=1)
 return 'cappa', -qwk(y_true, y_pred)
 class LGBWrapper(object):

 def __init__(self):
self.model = lgb.LGBMClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_qwk_lgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)
 class CatWrapper(object):

 def __init__(self):
self.model = cat.CatBoostClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**{k: v for k, v in params.items() if k != 'cat_cols'})
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = None
else:
categorical_columns = None
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
cat_features=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if 'MultiClass' not in self.model.get_param('loss_function'):
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)
 class XGBWrapper(object):

 def __init__(self):
self.model = xgb.XGBClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_metric=eval_qwk_xgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'])
 scores = self.model.evals_result()
self.best_score_ = {k: {m: m_v[-1] for m, m_v in v.items() } for k, v in scores.items() }
self.best_score_ = {k: {m: n if m != 'cappa' else -n for m, n in v.items() } for k, v in self.best_score_.items() }
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)
 class MainTransformer(BaseEstimator, TransformerMixin):
def __init__(self, convert_cyclical: bool = False, create_interactions: bool = False, n_interactions: int = 20):

 self.convert_cyclical = convert_cyclical
self.create_interactions = create_interactions
self.feats_for_interaction = None
self.n_interactions = n_interactions
def fit(self, X, y=None):
if self.create_interactions:
self.feats_for_interaction = [col for col in X.columns if 'sum' in col
or 'mean' in col or 'max' in col or 'std' in col
or 'attempt' in col]
self.feats_for_interaction1 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 self.feats_for_interaction2 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 if self.create_interactions:
for col1 in self.feats_for_interaction1:
for col2 in self.feats_for_interaction2:
data[f'{col1}_int_{col2}'] = data[col1] * data[col2]
if self.convert_cyclical:
data['timestampHour'] = np.sin(2 * np.pi * data['timestampHour'] / 23.0)
 data['timestampMonth'] = np.sin(2 * np.pi * data['timestampMonth'] / 23.0)
 data['timestampWeek'] = np.sin(2 * np.pi * data['timestampWeek'] / 23.0)
 data['timestampMinute'] = np.sin(2 * np.pi * data['timestampMinute'] / 23.0)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
 class FeatureTransformer(BaseEstimator, TransformerMixin):
def __init__(self, main_cat_features: list = None, num_cols: list = None):

 self.main_cat_features = main_cat_features
self.num_cols = num_cols
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )<train_model>
housing = train_data.copy()
class RegressorModel(object):

 def __init__(self, columns: list = None, model_wrapper=None):

 self.columns = columns
self.model_wrapper = model_wrapper
self.result_dict = {}
self.train_one_fold = False
self.preprocesser = None
def fit(self, X: pd.DataFrame, y,
X_holdout: pd.DataFrame = None, y_holdout=None,
folds=None,
params: dict = None,
eval_metric='rmse',
cols_to_drop: list = None,
preprocesser=None,
transformers: dict = None,
adversarial: bool = False,
plot: bool = True):

 if folds is None:
folds = KFold(n_splits=3, random_state=42)
 self.train_one_fold = True
self.columns = X.columns if self.columns is None else self.columns
self.feature_importances = pd.DataFrame(columns=['feature', 'importance'])
 self.trained_transformers = {k: [] for k in transformers}
self.transformers = transformers
self.models = []
self.folds_dict = {}
self.eval_metric = eval_metric
n_target = 1
self.oof = np.zeros(( len(X), n_target))
self.n_target = n_target
X = X[self.columns]
if X_holdout is not None:
X_holdout = X_holdout[self.columns]
if preprocesser is not None:
self.preprocesser = preprocesser
self.preprocesser.fit(X, y)
 X = self.preprocesser.transform(X, y)
 self.columns = X.columns.tolist()
if X_holdout is not None:
X_holdout = self.preprocesser.transform(X_holdout)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y, X['installation_id'])) :
if X_holdout is not None:
X_hold = X_holdout.copy()
else:
X_hold = None
self.folds_dict[fold_n] = {}
if params['verbose']:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 self.folds_dict[fold_n] = {}
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if self.train_one_fold:
X_train = X[self.original_columns]
y_train = y
X_valid = None
y_valid = None
datasets = {'X_train': X_train, 'X_valid': X_valid, 'X_holdout': X_hold, 'y_train': y_train}
X_train, X_valid, X_hold = self.transform_(datasets, cols_to_drop)
 self.folds_dict[fold_n]['columns'] = X_train.columns.tolist()
model = copy.deepcopy(self.model_wrapper)
 if adversarial:
X_new1 = X_train.copy()
if X_valid is not None:
X_new2 = X_valid.copy()
elif X_holdout is not None:
X_new2 = X_holdout.copy()
X_new = pd.concat([X_new1, X_new2], axis=0)
 y_new = np.hstack(( np.zeros(( X_new1.shape[0])) , np.ones(( X_new2.shape[0]))))
X_train, X_valid, y_train, y_valid = train_test_split(X_new, y_new)
 model.fit(X_train, y_train, X_valid, y_valid, X_hold, y_holdout, params=params)
 self.folds_dict[fold_n]['scores'] = model.best_score_
if self.oof.shape[0] != len(X):
self.oof = np.zeros(( X.shape[0], self.oof.shape[1]))
if not adversarial:
self.oof[valid_index] = model.predict(X_valid ).reshape(-1, n_target)
 fold_importance = pd.DataFrame(list(zip(X_train.columns, model.feature_importances_)) ,
columns=['feature', 'importance'])
 self.feature_importances = self.feature_importances.append(fold_importance)
 self.models.append(model)
 self.feature_importances['importance'] = self.feature_importances['importance'].astype(int)
 self.calc_scores_()
if plot:
fig, ax = plt.subplots(figsize=(16, 12))
plt.subplot(2, 2, 1)
 self.plot_feature_importance(top_n=20)
 plt.subplot(2, 2, 2)
 self.plot_metric()
plt.subplot(2, 2, 3)
 plt.hist(y.values.reshape(-1, 1)- self.oof)
 plt.title('Distribution of errors')
 plt.subplot(2, 2, 4)
 plt.hist(self.oof)
 plt.title('Distribution of oof predictions');
def transform_(self, datasets, cols_to_drop):
for name, transformer in self.transformers.items() :
transformer.fit(datasets['X_train'], datasets['y_train'])
 datasets['X_train'] = transformer.transform(datasets['X_train'])
 if datasets['X_valid'] is not None:
datasets['X_valid'] = transformer.transform(datasets['X_valid'])
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = transformer.transform(datasets['X_holdout'])
 self.trained_transformers[name].append(transformer)
 if cols_to_drop is not None:
cols_to_drop = [col for col in cols_to_drop if col in datasets['X_train'].columns]
datasets['X_train'] = datasets['X_train'].drop(cols_to_drop, axis=1)
 if datasets['X_valid'] is not None:
datasets['X_valid'] = datasets['X_valid'].drop(cols_to_drop, axis=1)
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = datasets['X_holdout'].drop(cols_to_drop, axis=1)
 self.cols_to_drop = cols_to_drop
return datasets['X_train'], datasets['X_valid'], datasets['X_holdout']
def calc_scores_(self):
print()
datasets = [k for k, v in [v['scores'] for k, v in self.folds_dict.items() ][0].items() if len(v)> 0]
self.scores = {}
for d in datasets:
scores = [v['scores'][d][self.eval_metric] for k, v in self.folds_dict.items() ]
print(f"CV mean score on {d}: {np.mean(scores):.4f} +/- {np.std(scores):.4f} std.")
 self.scores[d] = np.mean(scores)
 def predict(self, X_test, averaging: str = 'usual'):

 full_prediction = np.zeros(( X_test.shape[0], self.oof.shape[1]))
if self.preprocesser is not None:
X_test = self.preprocesser.transform(X_test)
 for i in range(len(self.models)) :
X_t = X_test.copy()
for name, transformers in self.trained_transformers.items() :
X_t = transformers[i].transform(X_t)
 if self.cols_to_drop is not None:
cols_to_drop = [col for col in self.cols_to_drop if col in X_t.columns]
X_t = X_t.drop(cols_to_drop, axis=1)
 y_pred = self.models[i].predict(X_t[self.folds_dict[i]['columns']] ).reshape(-1, full_prediction.shape[1])
 if full_prediction.shape[0] != len(y_pred):
full_prediction = np.zeros(( y_pred.shape[0], self.oof.shape[1]))
if averaging == 'usual':
full_prediction += y_pred
elif averaging == 'rank':
full_prediction += pd.Series(y_pred ).rank().values
return full_prediction / len(self.models)
 def plot_feature_importance(self, drop_null_importance: bool = True, top_n: int = 10):

 top_feats = self.get_top_features(drop_null_importance, top_n)
 feature_importances = self.feature_importances.loc[self.feature_importances['feature'].isin(top_feats)]
feature_importances['feature'] = feature_importances['feature'].astype(str)
 top_feats = [str(i)for i in top_feats]
sns.barplot(data=feature_importances, x='importance', y='feature', orient='h', order=top_feats)
 plt.title('Feature importances')
 def get_top_features(self, drop_null_importance: bool = True, top_n: int = 10):

 grouped_feats = self.feature_importances.groupby(['feature'])['importance'].mean()
if drop_null_importance:
grouped_feats = grouped_feats[grouped_feats != 0]
return list(grouped_feats.sort_values(ascending=False ).index)[:top_n]
def plot_metric(self):

 full_evals_results = pd.DataFrame()
for model in self.models:
evals_result = pd.DataFrame()
for k in model.model.evals_result_.keys() :
evals_result[k] = model.model.evals_result_[k][self.eval_metric]
evals_result = evals_result.reset_index().rename(columns={'index': 'iteration'})
 full_evals_results = full_evals_results.append(evals_result)
 full_evals_results = full_evals_results.melt(id_vars=['iteration'] ).rename(columns={'value': self.eval_metric,
'variable': 'dataset'})
 sns.lineplot(data=full_evals_results, x='iteration', y=self.eval_metric, hue='dataset')
 plt.title('Training progress')
<categorify>
class ReduceVIF(BaseEstimator, TransformerMixin):
def __init__(self, thresh=5.0, impute=True, impute_strategy='median'):
self.thresh = thresh
if impute:
self.imputer = SimpleImputer(strategy=impute_strategy)
 def fit(self, X, y=None):
print('ReduceVIF fit')
 print(self.imputer)
 if hasattr(self, 'imputer'):
self.imputer.fit(X)
 return self
def transform(self, X, y=None):
print('ReduceVIF transform')
 columns = X.columns.tolist()
if hasattr(self, 'imputer'):
X = pd.DataFrame(self.imputer.transform(X), columns=columns)
 return ReduceVIF.calculate_vif(X, self.thresh)
 @staticmethod
def calculate_vif(X, thresh=5.0):
dropped=True
while dropped:
variables = X.columns
dropped = False
vif = [variance_inflation_factor(X[variables].values, X.columns.get_loc(var)) for var in X.columns]
max_vif = max(vif)
 if max_vif > thresh:
maxloc = vif.index(max_vif)
 print(f'Dropping {X.columns[maxloc]} with vif={max_vif}')
 X = X.drop([X.columns.tolist() [maxloc]], axis=1)
 dropped=True
print(X.columns)
 return X
class CategoricalTransformer(BaseEstimator, TransformerMixin):
def __init__(self, cat_cols=None, drop_original: bool = False, encoder=OrdinalEncoder()):

 self.cat_cols = cat_cols
self.drop_original = drop_original
self.encoder = encoder
self.default_encoder = OrdinalEncoder()
def fit(self, X, y=None):
if self.cat_cols is None:
kinds = np.array([dt.kind for dt in X.dtypes])
 is_cat = kinds == 'O'
self.cat_cols = list(X.columns[is_cat])
 self.encoder.set_params(cols=self.cat_cols)
 self.default_encoder.set_params(cols=self.cat_cols)
 self.encoder.fit(X[self.cat_cols], y)
 self.default_encoder.fit(X[self.cat_cols], y)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 new_cat_names = [f'{col}_encoded' for col in self.cat_cols]
encoded_data = self.encoder.transform(data[self.cat_cols])
 if encoded_data.shape[1] == len(self.cat_cols):
data[new_cat_names] = encoded_data
else:
pass
if self.drop_original:
data = data.drop(self.cat_cols, axis=1)
 else:
data[self.cat_cols] = self.default_encoder.transform(data[self.cat_cols])
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )<load_from_csv>
transformer = ReduceVIF(thresh=15)
 num_attributes = list(transformer.fit_transform(housing.select_dtypes(include=['int64','float64'])).columns.values )
def read_data() :
print('Reading train.csv file.... ')
 train = pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv')
 print('Training.csv file have {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Reading test.csv file.... ')
 test = pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv')
 print('Test.csv file have {} rows and {} columns'.format(test.shape[0], test.shape[1]))
print('Reading train_labels.csv file.... ')
 train_labels = pd.read_csv('/kaggle/input/data-science-bowl-2019/train_labels.csv')
 print('Train_labels.csv file have {} rows and {} columns'.format(train_labels.shape[0], train_labels.shape[1]))
print('Reading specs.csv file.... ')
 specs = pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv')
 print('Specs.csv file have {} rows and {} columns'.format(specs.shape[0], specs.shape[1]))
print('Reading sample_submission.csv file.... ')
 sample_submission = pd.read_csv('/kaggle/input/data-science-bowl-2019/sample_submission.csv')
 print('Sample_submission.csv file have {} rows and {} columns'.format(sample_submission.shape[0], sample_submission.shape[1]))
return train, test, train_labels, specs, sample_submission
def encode_title(train, test, train_labels):
train['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), train['title'], train['event_code']))
test['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), test['title'], test['event_code']))
all_title_event_code = list(set(train["title_event_code"].unique() ).union(test["title_event_code"].unique()))
list_of_user_activities = list(set(train['title'].unique() ).union(set(test['title'].unique())))
 list_of_event_code = list(set(train['event_code'].unique() ).union(set(test['event_code'].unique())))
 list_of_event_id = list(set(train['event_id'].unique() ).union(set(test['event_id'].unique())))
 list_of_worlds = list(set(train['world'].unique() ).union(set(test['world'].unique())))
 activities_map = dict(zip(list_of_user_activities, np.arange(len(list_of_user_activities))))
activities_labels = dict(zip(np.arange(len(list_of_user_activities)) , list_of_user_activities))
activities_world = dict(zip(list_of_worlds, np.arange(len(list_of_worlds))))
assess_titles = list(set(train[train['type'] == 'Assessment']['title'].value_counts().index ).union(set(test[test['type'] == 'Assessment']['title'].value_counts().index)))
 train['title'] = train['title'].map(activities_map)
 test['title'] = test['title'].map(activities_map)
 train['world'] = train['world'].map(activities_world)
 test['world'] = test['world'].map(activities_world)
 train_labels['title'] = train_labels['title'].map(activities_map)
 win_code = dict(zip(activities_map.values() ,(4100*np.ones(len(activities_map)) ).astype('int')))
 win_code[activities_map['Bird Measurer(Assessment)']] = 4110
train['timestamp'] = pd.to_datetime(train['timestamp'])
 test['timestamp'] = pd.to_datetime(test['timestamp'])
 return train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code
def get_data(user_sample, test_set=False):

 last_activity = 0
user_activities_count = {'Clip':0, 'Activity': 0, 'Assessment': 0, 'Game':0}
last_session_time_sec = 0
accuracy_groups = {0:0, 1:0, 2:0, 3:0}
all_assessments = []
accumulated_accuracy_group = 0
accumulated_accuracy = 0
accumulated_correct_attempts = 0
accumulated_uncorrect_attempts = 0
accumulated_actions = 0
counter = 0
time_first_activity = float(user_sample['timestamp'].values[0])
 durations = []
last_accuracy_title = {'acc_' + title: -1 for title in assess_titles}
event_code_count: Dict[str, int] = {ev: 0 for ev in list_of_event_code}
event_id_count: Dict[str, int] = {eve: 0 for eve in list_of_event_id}
title_count: Dict[str, int] = {eve: 0 for eve in activities_labels.values() }
title_event_code_count: Dict[str, int] = {t_eve: 0 for t_eve in all_title_event_code}
for i, session in user_sample.groupby('game_session', sort=False):
session_type = session['type'].iloc[0]
session_title = session['title'].iloc[0]
session_title_text = activities_labels[session_title]
if(session_type == 'Assessment')&(test_set or len(session)>1):
all_attempts = session.query(f'event_code == {win_code[session_title]}')
 true_attempts = all_attempts['event_data'].str.contains('true' ).sum()
false_attempts = all_attempts['event_data'].str.contains('false' ).sum()
features = user_activities_count.copy()
features.update(last_accuracy_title.copy())
 features.update(event_code_count.copy())
 features.update(event_id_count.copy())
 features.update(title_count.copy())
 features.update(title_event_code_count.copy())
 features.update(last_accuracy_title.copy())
 features['meta_len_session'] = len(session)
 features['meta_i'] = i
features['meta_set'] = 1 if test_set else 0
features['installation_id'] = session['installation_id'].iloc[-1]
features['session_title'] = session['title'].iloc[0]
features['accumulated_correct_attempts'] = accumulated_correct_attempts
features['accumulated_uncorrect_attempts'] = accumulated_uncorrect_attempts
accumulated_correct_attempts += true_attempts
accumulated_uncorrect_attempts += false_attempts
if durations == []:
features['duration_mean'] = 0
else:
features['duration_mean'] = np.mean(durations)
 durations.append(( session.iloc[-1, 2] - session.iloc[0, 2] ).seconds)
 features['accumulated_accuracy'] = accumulated_accuracy/counter if counter > 0 else 0
accuracy = true_attempts/(true_attempts+false_attempts)if(true_attempts+false_attempts)!= 0 else 0
accumulated_accuracy += accuracy
last_accuracy_title['acc_' + session_title_text] = accuracy
if accuracy == 0:
features['accuracy_group'] = 0
elif accuracy == 1:
features['accuracy_group'] = 3
elif accuracy == 0.5:
features['accuracy_group'] = 2
else:
features['accuracy_group'] = 1
features.update(accuracy_groups)
 accuracy_groups[features['accuracy_group']] += 1
features['accumulated_accuracy_group'] = accumulated_accuracy_group/counter if counter > 0 else 0
accumulated_accuracy_group += features['accuracy_group']
features['accumulated_actions'] = accumulated_actions
if test_set:
all_assessments.append(features)
 elif true_attempts+false_attempts > 0:
all_assessments.append(features)
 counter += 1
def update_counters(counter: dict, col: str):
num_of_session_count = Counter(session[col])
 for k in num_of_session_count.keys() :
x = k
if col == 'title':
x = activities_labels[k]
counter[x] += num_of_session_count[k]
return counter
event_code_count = update_counters(event_code_count, "event_code")
 event_id_count = update_counters(event_id_count, "event_id")
 title_count = update_counters(title_count, 'title')
 title_event_code_count = update_counters(title_event_code_count, 'title_event_code')
 accumulated_actions += len(session)
 if last_activity != session_type:
user_activities_count[session_type] += 1
last_activitiy = session_type
return all_assessments
def get_train_and_test(train, test):
compiled_train = []
compiled_semi_train = []
for ins_id, user_sample in tqdm(test.groupby('installation_id', sort = False), total = 10000):
compiled_semi_train += get_data(user_sample, test_set=True)
 for i,(ins_id, user_sample)in tqdm(enumerate(train.groupby('installation_id', sort = False)) , total = 17000):
compiled_train += get_data(user_sample, test_set=False)
 reduce_train = pd.DataFrame(compiled_train)
 reduce_semi_train = pd.DataFrame(compiled_semi_train)
 categoricals = ['session_title']
return reduce_train, reduce_semi_train, categoricals
train, test, train_labels, specs, sample_submission = read_data()
train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code = encode_title(train, test, train_labels)
 my_train, my_semi_train, categoricals = get_train_and_test(train, test )<categorify>
total = housing.isnull().sum().sort_values(ascending=False)
 percent =(housing.isnull().sum() /housing.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
def preprocess(reduce_train, reduce_test):
for df in [reduce_train, reduce_test]:
df['sum_event_code_count'] = df[[2050, 4100, 4230, 5000, 4235, 2060, 4110, 5010, 2070, 2075, 2080, 2081, 2083, 3110, 4010, 3120, 3121, 4020, 4021,
4022, 4025, 4030, 4031, 3010, 4035, 4040, 3020, 3021, 4045, 2000, 4050, 2010, 2020, 4070, 2025, 2030, 4080, 2035,
2040, 4090, 4220, 4095]].sum(axis = 1)
 df['installation_event_code_count_mean'] = df.groupby(['installation_id'])['sum_event_code_count']\ 
.apply(lambda x: x.shift().expanding().mean())
 df['installation_session_count'] = df.groupby(['installation_id'])['Clip']\ 
.apply(lambda x: x.shift().expanding().count())
 df['installation_duration_mean'] = df.groupby(['installation_id'])['duration_mean']\ 
.apply(lambda x: x.shift().expanding().mean())
 df['installation_title_nunique'] = df.groupby(['installation_id'])['session_title']\ 
.apply(lambda x: x.shift().expanding().apply(lambda s: len(np.unique(s))))
features = reduce_train.loc[(reduce_train.sum(axis=1)!= 0),(reduce_train.sum(axis=0)!= 0)].columns
features = [x for x in features if x not in ['accuracy_group', 'installation_id']] + ['acc_' + title for title in assess_titles]
return reduce_train, reduce_test, features
print(my_train.shape, my_semi_train.shape)
 my_train, my_semi_train, features = preprocess(my_train, my_semi_train)
 print(my_train.shape, my_semi_train.shape)
 reduce_train = pd.concat([my_train, my_semi_train[my_semi_train["meta_len_session"] > 1]] ).reset_index(drop=True)
 reduce_test = my_semi_train[(my_semi_train["meta_len_session"] == 1)].reset_index(drop=True)
 print(reduce_train.shape, reduce_test.shape )<init_hyperparams>
missing_data[missing_data.Percent>=0.15]
params = {'n_estimators':2000,
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'subsample': 0.75,
'subsample_freq': 1,
'learning_rate': 0.04,
'feature_fraction': 0.9,
'max_depth': 15,
'lambda_l1': 1,
'lambda_l2': 1,
'verbose': 100,
'early_stopping_rounds': 100, 'eval_metric': 'cappa'
}<prepare_x_and_y>
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attribute_names]
y = reduce_train['accuracy_group']<choose_model_class>
housing = train_data.copy()
n_fold = 5
folds = GroupKFold(n_splits=n_fold )<define_variables>
housing.drop(['PoolQC','MiscFeature','Alley','Fence','FireplaceQu'],axis=1, inplace=True )
cols_to_drop = ['game_session', 'installation_id', 'timestamp', 'accuracy_group', 'timestampDate']
cols_to_drop += [c for c in reduce_train.columns if "meta" in str(c)]
print(cols_to_drop )<train_model>
cat_pipeline = Pipeline([ 
("selector", DataFrameSelector(attribute_names=cat_attributes))
] )
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model1 = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model1.fit(X=reduce_train, y=y, folds=folds, params=params, preprocesser=mt, transformers=transformers,
eval_metric='cappa', cols_to_drop=cols_to_drop )<compute_test_metric>
class CategoricalImputer(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
self.most_frequent_ = pd.Series([X[attr].value_counts().index[0] for attr in X], index=X.columns)
 return self
def transform(self, X, y=None):
return X.fillna(self.most_frequent_ )
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):

 X_p = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return -qwk(y, X_p)
 def fit(self, X, y):

 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 return pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 def coefficients(self):

 return self.coef_['x']<predict_on_test>
cat_pipeline = Pipeline([ 
("selector", DataFrameSelector(attribute_names=cat_attributes)) , 
('imputer', CategoricalImputer())
 ] )
opt_preds = optR.predict(pr1.reshape(-1,), coefficients)
 qwk(y, opt_preds )<feature_engineering>
cat_pipeline.fit_transform(housing ).loc[some_incomplete_rows.index]
pr1 = regressor_model1.predict(reduce_test)
 pr1[pr1 <= coefficients[0]] = 0
pr1[np.where(np.logical_and(pr1 > coefficients[0], pr1 <= coefficients[1])) ] = 1
pr1[np.where(np.logical_and(pr1 > coefficients[1], pr1 <= coefficients[2])) ] = 2
pr1[pr1 > coefficients[2]] = 3<save_to_csv>
cat_pipeline = Pipeline([ 
("selector", DataFrameSelector(attribute_names=cat_attributes)) , 
('imputer', CategoricalImputer()), 
("encoder", OneHotEncoder(sparse=False))
] )
sample_submission['accuracy_group'] = pr1.astype(int)
 sample_submission.to_csv('submission.csv', index=False )<count_values>
cat_pipeline.fit_transform(housing )
sample_submission['accuracy_group'].value_counts(normalize=True )<import_modules>
housing = train_data.copy()
from sklearn.metrics import log_loss
from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestRegressor<load_from_csv>
housing.drop(['Id','SalePrice', 'LotFrontage'],axis=1, inplace=True )
teams = pd.read_csv('.. /input/wdatafiles/WTeams.csv')
 teams2 = pd.read_csv('.. /input/wdatafiles/WTeamSpellings.csv', encoding='latin-1')
 season_cresults = pd.read_csv('.. /input/wdatafiles/WRegularSeasonCompactResults.csv')
 season_dresults = pd.read_csv('.. /input/wdatafiles/WRegularSeasonDetailedResults.csv')
 tourney_cresults = pd.read_csv('.. /input/wdatafiles/WNCAATourneyCompactResults.csv')
 tourney_dresults = pd.read_csv('.. /input/wdatafiles/WNCAATourneyDetailedResults.csv')
 slots = pd.read_csv('.. /input/wdatafiles/WNCAATourneySlots.csv')
 seeds = pd.read_csv('.. /input/wdatafiles/WNCAATourneySeeds.csv')
 seeds = {'_'.join(map(str,[int(k1),k2])) :int(v[1:3])for k1, v, k2 in seeds[['Season', 'Seed', 'TeamID']].values}
seeds = {**seeds, **{k.replace('2018_','2019_'):seeds[k] for k in seeds if '2018_' in k}}
cities = pd.read_csv('.. /input/wdatafiles/WCities.csv')
 gcities = pd.read_csv('.. /input/wdatafiles/WGameCities.csv')
 seasons = pd.read_csv('.. /input/wdatafiles/WSeasons.csv')
 sub = pd.read_csv('.. /input/WSampleSubmissionStage1.csv' )<merge>
transformer = ReduceVIF(thresh=15)
 num_attributes = list(transformer.fit_transform(housing.select_dtypes(include=['int64','float64'])).columns.values )
teams2 = teams2.groupby(by='TeamID', as_index=False)['TeamNameSpelling'].count()
teams2.columns = ['TeamID', 'TeamNameCount']
teams = pd.merge(teams, teams2, how='left', on=['TeamID'])
 del teams2<categorify>
imputer = SimpleImputer(strategy='mean' )
season_cresults['ST'] = 'S'
season_dresults['ST'] = 'S'
tourney_cresults['ST'] = 'T'
tourney_dresults['ST'] = 'T'
games = pd.concat(( season_dresults, tourney_dresults), axis=0, ignore_index=True)
 games.reset_index(drop=True, inplace=True)
 games['WLoc'] = games['WLoc'].map({'A': 1, 'H': 2, 'N': 3} )<feature_engineering>
housing_transform = pd.DataFrame(imputer.fit_transform(housing[num_attributes]), columns=num_attributes )
games['ID'] = games.apply(lambda r: '_'.join(map(str, [r['Season']]+sorted([r['WTeamID'],r['LTeamID']]))), axis=1)
 games['IDTeams'] = games.apply(lambda r: '_'.join(map(str, sorted([r['WTeamID'],r['LTeamID']]))), axis=1)
 games['Team1'] = games.apply(lambda r: sorted([r['WTeamID'],r['LTeamID']])[0], axis=1)
 games['Team2'] = games.apply(lambda r: sorted([r['WTeamID'],r['LTeamID']])[1], axis=1)
 games['IDTeam1'] = games.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team1']])) , axis=1)
 games['IDTeam2'] = games.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team2']])) , axis=1 )<categorify>
housing_transform = housing_transform.apply(lambda x: np.sign(x)* np.log(1 + np.abs(x)) )
games['Team1Seed'] = games['IDTeam1'].map(seeds ).fillna(0)
 games['Team2Seed'] = games['IDTeam2'].map(seeds ).fillna(0 )<feature_engineering>
class LogModulusTransformer(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
return np.apply_along_axis(lambda x: np.sign(x)* np.log(1 + np.abs(x)) , 1, X)
games['ScoreDiff'] = games['WScore'] - games['LScore']
games['Pred'] = games.apply(lambda r: 1.if sorted([r['WTeamID'],r['LTeamID']])[0]==r['WTeamID'] else 0., axis=1)
 games['ScoreDiffNorm'] = games.apply(lambda r: r['ScoreDiff'] * -1 if r['Pred'] == 0.else r['ScoreDiff'], axis=1)
 games['SeedDiff'] = games['Team1Seed'] - games['Team2Seed']
games = games.fillna(-1 )<groupby>
BsmtFullBath_ix = list(num_attributes ).index('BsmtFullBath')
 BsmtHalfBath_ix = list(num_attributes ).index('BsmtHalfBath')
 HalfBath_ix = list(num_attributes ).index('HalfBath')
 extra_attributes = ['TotalBath']
class CombinedNumAttributesAdder(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
total_bath = X[:, BsmtFullBath_ix] + X[:, BsmtHalfBath_ix] + X[:, HalfBath_ix]
return np.c_[X, total_bath]
c_score_col = ['NumOT', 'WFGM', 'WFGA', 'WFGM3', 'WFGA3', 'WFTM', 'WFTA', 'WOR', 'WDR', 'WAst', 'WTO', 'WStl',
'WBlk', 'WPF', 'LFGM', 'LFGA', 'LFGM3', 'LFGA3', 'LFTM', 'LFTA', 'LOR', 'LDR', 'LAst', 'LTO', 'LStl',
'LBlk', 'LPF']
c_score_agg = ['sum', 'mean', 'median', 'max', 'min', 'std', 'skew', 'nunique']
gb = games.groupby(by=['IDTeams'] ).agg({k: c_score_agg for k in c_score_col} ).reset_index()
gb.columns = [''.join(c)+ '_c_score' for c in gb.columns]
games = games[games['ST']=='T']<feature_engineering>
num_pipeline = Pipeline([ 
('selector', DataFrameSelector(num_attributes)) , 
('imputer', SimpleImputer(strategy='mean')) , 
('attr_adder', CombinedNumAttributesAdder()), 
('transformer', LogModulusTransformer()), 
('scaler', StandardScaler()),
] )
sub['WLoc'] = 3
sub['Season'] = sub['ID'].map(lambda x: x.split('_')[0])
 sub['Season'] = sub['ID'].map(lambda x: x.split('_')[0])
 sub['Season'] = sub['Season'].astype(int)
 sub['Team1'] = sub['ID'].map(lambda x: x.split('_')[1])
 sub['Team2'] = sub['ID'].map(lambda x: x.split('_')[2])
 sub['IDTeams'] = sub.apply(lambda r: '_'.join(map(str, [r['Team1'], r['Team2']])) , axis=1)
 sub['IDTeam1'] = sub.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team1']])) , axis=1)
 sub['IDTeam2'] = sub.apply(lambda r: '_'.join(map(str, [r['Season'], r['Team2']])) , axis=1)
 sub['Team1Seed'] = sub['IDTeam1'].map(seeds ).fillna(0)
 sub['Team2Seed'] = sub['IDTeam2'].map(seeds ).fillna(0)
 sub['SeedDiff'] = sub['Team1Seed'] - sub['Team2Seed']
sub = sub.fillna(-1 )<merge>
num_pipeline.fit_transform(housing )
games = pd.merge(games, gb, how='left', left_on='IDTeams', right_on='IDTeams_c_score')
 sub = pd.merge(sub, gb, how='left', left_on='IDTeams', right_on='IDTeams_c_score' )<define_variables>
preprocess_pipeline = FeatureUnion(transformer_list=[ 
("num_pipeline", num_pipeline), 
("cat_pipeline", cat_pipeline)
 ] )
col = [c for c in games.columns if c not in ['ID', 'DayNum', 'ST', 'Team1', 'Team2', 'IDTeams', 'IDTeam1', 'IDTeam2', 'WTeamID', 'WScore', 'LTeamID', 'LScore', 'NumOT', 'Pred', 'ScoreDiff', 'ScoreDiffNorm', 'WLoc'] + c_score_col]<train_model>
X_train = train_data.copy()
forest = RandomForestRegressor(n_estimators=100, bootstrap=True, n_jobs=-1)
 forest.fit(games[col].fillna(-1), games['Pred'] )<predict_on_test>
X_train_prepared = preprocess_pipeline.fit_transform(X_train)
 y_train = X_train['SalePrice'].apply(lambda x: np.log(x))
predictions = forest.predict(games[col].fillna(-1)).clip(0,1)
 print('Log Loss:', log_loss(games['Pred'], predictions))<save_to_csv>
cat_encoder = cat_pipeline.named_steps['encoder']
cat_encoder.categories_[:10]
sub['Pred'] = forest.predict(sub[col].fillna(-1)).clip(0,1)
 sub[['ID', 'Pred']].to_csv('fibal.csv', index=False )<set_options>
cat_one_hot_attribs =[]
for cat in range(len(cat_encoder.categories_)) :
cat_one_hot_attribs = cat_one_hot_attribs + list(cat_encoder.categories_[cat] )
gc.enable()<load_from_csv>
attributes = list(num_attributes)+ extra_attributes + cat_one_hot_attribs
attributes
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )<count_duplicates>
from sklearn.metrics import mean_squared_error
<data_type_conversions>
from sklearn.metrics import mean_squared_error
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )<merge>
lin_reg = LinearRegression()
lin_reg.fit(X_train_prepared, y_train)
 y_pred = lin_reg.predict(X_train_prepared)
 lin_mse = mean_squared_error(y_train, y_pred)
 lin_rmse = np.sqrt(lin_mse)
 lin_rmse
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()<create_dataframe>
sgd_reg = SGDRegressor(penalty=None, random_state=42, max_iter=1000, tol=1e-6)
 sgd_reg.fit(X_train_prepared, y_train)
 y_pred = sgd_reg.predict(X_train_prepared)
 sgd_mse = mean_squared_error(y_train, y_pred)
 sgd_rmse = np.sqrt(sgd_mse)
 sgd_rmse
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user.head() 
<merge>
tree_reg = DecisionTreeRegressor(random_state=42)
 tree_reg.fit(X_train_prepared, y_train)
 y_pred = tree_reg.predict(X_train_prepared)
 tree_mse = mean_squared_error(y_train, y_pred)
 tree_rmse = np.sqrt(tree_mse)
 tree_rmse
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()<prepare_output>
forest_reg = RandomForestRegressor(random_state=42, n_estimators=100)
 forest_reg.fit(X_train_prepared, y_train)
 y_pred = forest_reg.predict(X_train_prepared)
 forest_mse = mean_squared_error(y_train, y_pred)
 forest_rmse = np.sqrt(forest_mse)
 forest_rmse
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd.head()<groupby>
svm_reg = SVR(kernel="linear")
 svm_reg.fit(X_train_prepared, y_train)
 y_pred = svm_reg.predict(X_train_prepared)
 svm_mse = mean_squared_error(y_train, y_pred)
 svm_rmse = np.sqrt(svm_mse)
 svm_rmse
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >120)
 p_reorder.head()<merge>
def display_scores(scores):
print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard deviation:", scores.std() )
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()<feature_engineering>
sgd_scores = cross_val_score(sgd_reg, X_train_prepared, y_train, scoring="neg_mean_squared_error", cv=5)
 sgd_rmse_scores = np.sqrt(-sgd_scores)
 display_scores(sgd_rmse_scores )
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()<prepare_output>
tree_scores = cross_val_score(tree_reg, X_train_prepared, y_train, scoring="neg_mean_squared_error", cv=5)
 tree_rmse_scores = np.sqrt(-tree_scores)
 display_scores(tree_rmse_scores )
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp.head()<groupby>
forest_scores = cross_val_score(forest_reg, X_train_prepared, y_train, scoring="neg_mean_squared_error", cv=5)
 forest_rmse_scores = np.sqrt(-forest_scores)
 display_scores(forest_rmse_scores )
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()<prepare_output>
svm_scores = cross_val_score(svm_reg, X_train_prepared, y_train, scoring="neg_mean_squared_error", cv=5)
 svm_rmse_scores = np.sqrt(-svm_scores)
 display_scores(svm_rmse_scores )
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()<merge>
param_grid = [
{'n_estimators': [30, 50, 100],
'max_features': [30, 50, 100]},
{'bootstrap': [False],
'n_estimators': [30, 50, 100],
'max_features': [30, 50, 100]},
]
forest_reg = RandomForestRegressor(random_state=42)
 grid_search_forest = GridSearchCV(forest_reg, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)
 grid_search_forest.fit(X_train_prepared, y_train )
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()<feature_engineering>
grid_search_forest.best_params_
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()<merge>
grid_search_forest.best_estimator_
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
cvres = grid_search_forest.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(-mean_score), params )
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
param_grid = [
{'kernel': ['linear'], 'C': [0.1, 1.0, 10.]},
{'kernel': ['rbf'], 'C': [0.1, 1.0, 10.],
'gamma': [0.001, 0.01, 0.1]},
]
svm_reg = SVR()
grid_search_svm = GridSearchCV(svm_reg, param_grid, cv=5, scoring='neg_mean_squared_error', verbose=2, n_jobs=4)
 grid_search_svm.fit(X_train_prepared, y_train )
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()<drop_column>
grid_search_svm.best_params_
del [times, first_order_no, span]<merge>
cvres = grid_search_svm.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(-mean_score), params )
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()<merge>
grid_search_svm.best_estimator_
data = uxp.merge(user, on='user_id', how='left')
 data.head()<merge>
feature_importances = grid_search_forest.best_estimator_.feature_importances_
data = data.merge(prd, on='product_id', how='left')
 data.head()<set_options>
indices = np.argsort(feature_importances)[::-1]
ranking = pd.DataFrame({'features':[attributes[indices[f]] for f in range(len(attributes[:10])) ],
'values': [feature_importances[indices[f]] for f in range(len(attributes[:10])) ]})
 ranking
del op, user, prd, uxp
gc.collect()<filter>
final_model = grid_search_svm.best_estimator_
final_model
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
<merge>
X_test=test_data.copy()
data = data.merge(orders_future, on='user_id', how='left')
 data.head(10 )<filter>
X_test_prepared = preprocess_pipeline.transform(X_test )
data_train = data[data.eval_set=='train']
data_train.head()<merge>
y_pred = final_model.predict(X_test_prepared)
 print(y_pred.shape)
 y_pred.ravel()
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )<feature_engineering>
test_data['SalePrice'] = np.exp(y_pred.ravel())
 test_data[['Id','SalePrice']].head()
<drop_column><EOS> 
test_data[['Id','SalePrice']].to_csv('submission.csv', index=False )