Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
sns.set_style('whitegrid')
 warnings.simplefilter(action='ignore')
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews<split>
a = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 b = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data() 
<define_variables>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10'] 
<split>
print('train : %sの要素と%sの特徴量があります。'%(a.shape[0],a.shape[1]))
print('test : %sの要素と%sの特徴量があります'%(b.shape[0],b.shape[1]))
print('testはtrainよりも特徴量が1つ少なくなっており、その残り1つが予測対象であるSalePriceです。' )
market_train = market_train.loc[pd.to_datetime(market_train['time'])>= pd.to_datetime('2009-01-01' ).tz_localize('UTC')]
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
<categorify>
na = a.shape[0]
nb = b.shape[0]
y_train = a['SalePrice'].to_frame()
c1 = pd.concat(( a, b), sort=False ).reset_index(drop=True)
 c1.drop(['SalePrice'], axis=1, inplace=True)
 c1.drop(['Id'], axis=1, inplace=True)
 print(f"データサイズ : {c1.shape}" )
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders] 
<normalization>
msv1(c1, 80, color=('black', 'deeppink'))
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
print(market_train['time'].dtypes)
 market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
<prepare_x_and_y>
c=c1.dropna(thresh=len(c1)*0.2, axis=1)
 print(f"trainとtestの結合データ(combination)から{c1.shape[1] - c.shape[1]}つの特徴量を削除しました。" )
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )<choose_model_class>
msv2(c )
warnings.filterwarnings(action ='ignore',category = DeprecationWarning)
 algo = partial(tpe.suggest, n_startup_jobs=10)
 def auto_turing(args):
model = XGBClassifier(n_jobs = 4, n_estimators = args['n_estimators'],max_depth=6)
 model.fit(X_train['num'],y_train.astype(int))
confidence_valid = model.predict(X_valid['num'])*2 -1
score = accuracy_score(confidence_valid>0,y_valid)
 print(args,score)
 return -score
model = RandomForestClassifier(n_estimators=50, criterion='gini', max_depth=None,min_samples_split=2, min_samples_leaf=1,
min_weight_fraction_leaf=0.0,max_features='auto',max_leaf_nodes=None,min_impurity_split=1e-07,
bootstrap=True, oob_score=False, n_jobs=1,random_state=None, verbose=0,warm_start=False, class_weight=None)
 model.fit(X_train['num'], y_train.astype(int))
confidence_valid = model.predict(X_valid['num'])*2 -1
score = accuracy_score(confidence_valid>0,y_valid)
 print(score)
<compute_test_metric>
NA=c[allna.index.to_list() ]
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<define_variables>
c['MasVnrArea']=c.MasVnrArea.fillna(0)
 c['LotFrontage']=c.LotFrontage.fillna(c.LotFrontage.median())
 c['GarageYrBlt']=c["GarageYrBlt"].fillna(1980 )
days = env.get_prediction_days()<categorify>
ffill_cols = ['KitchenQual', 'Electrical', 'SaleType', 'Exterior1st',
'Exterior2nd', 'Utilities', 'Functional', 'MSZoning']
def filling_NA(data, columns, METHOD='ffill'):
fill_cols = columns
for col in data[fill_cols]:
data[col]= data[col].fillna(method=METHOD)
 return data
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
predicted_confidences = np.array([])
 for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df['assetCode_encoded'] = market_obs_df[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
market_obs_df[num_cols] = market_obs_df[num_cols].fillna(0)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
X_test['assetCode'] = market_obs_df['assetCode_encoded'].values
prep_time += time.time() - t
t = time.time()
market_prediction = model.predict(X_test['num'])*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
prediction_time += time.time() -t
t = time.time()
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )<set_options>
msv2(c )
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
pd.set_option('max_rows', 500)
<categorify>
d=filling_NA(c, ffill_cols )
def add_datepart(df: pd.DataFrame, field_name: str,
prefix: str = None, drop: bool = True, time: bool = True, date: bool = True):

 field = df[field_name]
prefix = ifnone(prefix, re.sub('[Dd]ate$', '', field_name))
attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Is_month_end', 'Is_month_start']
if date:
attr.append('Date')
 if time:
attr = attr + ['Hour', 'Minute']
for n in attr:
df[prefix + n] = getattr(field.dt, n.lower())
 if drop:
df.drop(field_name, axis=1, inplace=True)
 return df
def ifnone(a: Any, b: Any)-> Any: 
<categorify>
msv2(d )
@jit
def qwk(a1, a2):

 max_rat = 3
a1 = np.asarray(a1, dtype=int)
 a2 = np.asarray(a2, dtype=int)
 hist1 = np.zeros(( max_rat + 1,))
hist2 = np.zeros(( max_rat + 1,))
o = 0
for k in range(a1.shape[0]):
i, j = a1[k], a2[k]
hist1[i] += 1
hist2[j] += 1
o +=(i - j)*(i - j)
 e = 0
for i in range(max_rat + 1):
for j in range(max_rat + 1):
e += hist1[i] * hist2[j] *(i - j)*(i - j)
 e = e / a1.shape[0]
return 1 - o / e
def eval_qwk_lgb(y_true, y_pred):

 y_pred = y_pred.reshape(len(np.unique(y_true)) , -1 ).argmax(axis=0)
 return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_lgb_regr(y_true, y_pred):

 y_pred[y_pred <= 1.12232214] = 0
y_pred[np.where(np.logical_and(y_pred > 1.12232214, y_pred <= 1.73925866)) ] = 1
y_pred[np.where(np.logical_and(y_pred > 1.73925866, y_pred <= 2.22506454)) ] = 2
y_pred[y_pred > 2.22506454] = 3
return 'cappa', qwk(y_true, y_pred), True
class LGBWrapper_regr(object):

 def __init__(self):
self.model = lgb.LGBMRegressor()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
if params['objective'] == 'regression':
eval_metric = eval_qwk_lgb_regr
else:
eval_metric = 'auc'
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_metric,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict(self, X_test):
return self.model.predict(X_test, num_iteration=self.model.best_iteration_)
 def eval_qwk_xgb(y_pred, y_true):

 y_true = y_true.get_label()
y_pred = y_pred.argmax(axis=1)
 return 'cappa', -qwk(y_true, y_pred)
 class LGBWrapper(object):

 def __init__(self):
self.model = lgb.LGBMClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_qwk_lgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)
 class CatWrapper(object):

 def __init__(self):
self.model = cat.CatBoostClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**{k: v for k, v in params.items() if k != 'cat_cols'})
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = None
else:
categorical_columns = None
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
cat_features=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if 'MultiClass' not in self.model.get_param('loss_function'):
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_end=self.model.best_iteration_)
 class XGBWrapper(object):

 def __init__(self):
self.model = xgb.XGBClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_metric=eval_qwk_xgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'])
 scores = self.model.evals_result()
self.best_score_ = {k: {m: m_v[-1] for m, m_v in v.items() } for k, v in scores.items() }
self.best_score_ = {k: {m: n if m != 'cappa' else -n for m, n in v.items() } for k, v in self.best_score_.items() }
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)[:, 1]
else:
return self.model.predict_proba(X_test, ntree_limit=self.model.best_iteration)
 class MainTransformer(BaseEstimator, TransformerMixin):
def __init__(self, convert_cyclical: bool = False, create_interactions: bool = False, n_interactions: int = 20):

 self.convert_cyclical = convert_cyclical
self.create_interactions = create_interactions
self.feats_for_interaction = None
self.n_interactions = n_interactions
def fit(self, X, y=None):
if self.create_interactions:
self.feats_for_interaction = [col for col in X.columns if 'sum' in col
or 'mean' in col or 'max' in col or 'std' in col
or 'attempt' in col]
self.feats_for_interaction1 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 self.feats_for_interaction2 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 if self.create_interactions:
for col1 in self.feats_for_interaction1:
for col2 in self.feats_for_interaction2:
data[f'{col1}_int_{col2}'] = data[col1] * data[col2]
if self.convert_cyclical:
data['timestampHour'] = np.sin(2 * np.pi * data['timestampHour'] / 23.0)
 data['timestampMonth'] = np.sin(2 * np.pi * data['timestampMonth'] / 23.0)
 data['timestampWeek'] = np.sin(2 * np.pi * data['timestampWeek'] / 23.0)
 data['timestampMinute'] = np.sin(2 * np.pi * data['timestampMinute'] / 23.0)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
 class FeatureTransformer(BaseEstimator, TransformerMixin):
def __init__(self, main_cat_features: list = None, num_cols: list = None):

 self.main_cat_features = main_cat_features
self.num_cols = num_cols
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )<train_model>
fill_cols = ['KitchenQual', 'Electrical', 'SaleType', 'Exterior1st',
'Exterior2nd', 'Utilities', 'Functional', 'MSZoning']
for col in c[fill_cols]:
c[col] = c[col].fillna(method='ffill' )
class RegressorModel(object):

 def __init__(self, columns: list = None, model_wrapper=None):

 self.columns = columns
self.model_wrapper = model_wrapper
self.result_dict = {}
self.train_one_fold = False
self.preprocesser = None
def fit(self, X: pd.DataFrame, y,
X_holdout: pd.DataFrame = None, y_holdout=None,
folds=None,
params: dict = None,
eval_metric='rmse',
cols_to_drop: list = None,
preprocesser=None,
transformers: dict = None,
adversarial: bool = False,
plot: bool = True):

 if folds is None:
folds = KFold(n_splits=3, random_state=42)
 self.train_one_fold = True
self.columns = X.columns if self.columns is None else self.columns
self.feature_importances = pd.DataFrame(columns=['feature', 'importance'])
 self.trained_transformers = {k: [] for k in transformers}
self.transformers = transformers
self.models = []
self.folds_dict = {}
self.eval_metric = eval_metric
n_target = 1
self.oof = np.zeros(( len(X), n_target))
self.n_target = n_target
X = X[self.columns]
if X_holdout is not None:
X_holdout = X_holdout[self.columns]
if preprocesser is not None:
self.preprocesser = preprocesser
self.preprocesser.fit(X, y)
 X = self.preprocesser.transform(X, y)
 self.columns = X.columns.tolist()
if X_holdout is not None:
X_holdout = self.preprocesser.transform(X_holdout)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y, X['installation_id'])) :
if X_holdout is not None:
X_hold = X_holdout.copy()
else:
X_hold = None
self.folds_dict[fold_n] = {}
if params['verbose']:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 self.folds_dict[fold_n] = {}
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if self.train_one_fold:
X_train = X[self.original_columns]
y_train = y
X_valid = None
y_valid = None
datasets = {'X_train': X_train, 'X_valid': X_valid, 'X_holdout': X_hold, 'y_train': y_train}
X_train, X_valid, X_hold = self.transform_(datasets, cols_to_drop)
 self.folds_dict[fold_n]['columns'] = X_train.columns.tolist()
model = copy.deepcopy(self.model_wrapper)
 if adversarial:
X_new1 = X_train.copy()
if X_valid is not None:
X_new2 = X_valid.copy()
elif X_holdout is not None:
X_new2 = X_holdout.copy()
X_new = pd.concat([X_new1, X_new2], axis=0)
 y_new = np.hstack(( np.zeros(( X_new1.shape[0])) , np.ones(( X_new2.shape[0]))))
X_train, X_valid, y_train, y_valid = train_test_split(X_new, y_new)
 model.fit(X_train, y_train, X_valid, y_valid, X_hold, y_holdout, params=params)
 self.folds_dict[fold_n]['scores'] = model.best_score_
if self.oof.shape[0] != len(X):
self.oof = np.zeros(( X.shape[0], self.oof.shape[1]))
if not adversarial:
self.oof[valid_index] = model.predict(X_valid ).reshape(-1, n_target)
 fold_importance = pd.DataFrame(list(zip(X_train.columns, model.feature_importances_)) ,
columns=['feature', 'importance'])
 self.feature_importances = self.feature_importances.append(fold_importance)
 self.models.append(model)
 self.feature_importances['importance'] = self.feature_importances['importance'].astype(int)
 self.calc_scores_()
if plot:
fig, ax = plt.subplots(figsize=(16, 12))
plt.subplot(2, 2, 1)
 self.plot_feature_importance(top_n=20)
 plt.subplot(2, 2, 2)
 self.plot_metric()
plt.subplot(2, 2, 3)
 plt.hist(y.values.reshape(-1, 1)- self.oof)
 plt.title('Distribution of errors')
 plt.subplot(2, 2, 4)
 plt.hist(self.oof)
 plt.title('Distribution of oof predictions');
def transform_(self, datasets, cols_to_drop):
for name, transformer in self.transformers.items() :
transformer.fit(datasets['X_train'], datasets['y_train'])
 datasets['X_train'] = transformer.transform(datasets['X_train'])
 if datasets['X_valid'] is not None:
datasets['X_valid'] = transformer.transform(datasets['X_valid'])
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = transformer.transform(datasets['X_holdout'])
 self.trained_transformers[name].append(transformer)
 if cols_to_drop is not None:
cols_to_drop = [col for col in cols_to_drop if col in datasets['X_train'].columns]
datasets['X_train'] = datasets['X_train'].drop(cols_to_drop, axis=1)
 if datasets['X_valid'] is not None:
datasets['X_valid'] = datasets['X_valid'].drop(cols_to_drop, axis=1)
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = datasets['X_holdout'].drop(cols_to_drop, axis=1)
 self.cols_to_drop = cols_to_drop
return datasets['X_train'], datasets['X_valid'], datasets['X_holdout']
def calc_scores_(self):
print()
datasets = [k for k, v in [v['scores'] for k, v in self.folds_dict.items() ][0].items() if len(v)> 0]
self.scores = {}
for d in datasets:
scores = [v['scores'][d][self.eval_metric] for k, v in self.folds_dict.items() ]
print(f"CV mean score on {d}: {np.mean(scores):.4f} +/- {np.std(scores):.4f} std.")
 self.scores[d] = np.mean(scores)
 def predict(self, X_test, averaging: str = 'usual'):

 full_prediction = np.zeros(( X_test.shape[0], self.oof.shape[1]))
if self.preprocesser is not None:
X_test = self.preprocesser.transform(X_test)
 for i in range(len(self.models)) :
X_t = X_test.copy()
for name, transformers in self.trained_transformers.items() :
X_t = transformers[i].transform(X_t)
 if self.cols_to_drop is not None:
cols_to_drop = [col for col in self.cols_to_drop if col in X_t.columns]
X_t = X_t.drop(cols_to_drop, axis=1)
 y_pred = self.models[i].predict(X_t[self.folds_dict[i]['columns']] ).reshape(-1, full_prediction.shape[1])
 if full_prediction.shape[0] != len(y_pred):
full_prediction = np.zeros(( y_pred.shape[0], self.oof.shape[1]))
if averaging == 'usual':
full_prediction += y_pred
elif averaging == 'rank':
full_prediction += pd.Series(y_pred ).rank().values
return full_prediction / len(self.models)
 def plot_feature_importance(self, drop_null_importance: bool = True, top_n: int = 10):

 top_feats = self.get_top_features(drop_null_importance, top_n)
 feature_importances = self.feature_importances.loc[self.feature_importances['feature'].isin(top_feats)]
feature_importances['feature'] = feature_importances['feature'].astype(str)
 top_feats = [str(i)for i in top_feats]
sns.barplot(data=feature_importances, x='importance', y='feature', orient='h', order=top_feats)
 plt.title('Feature importances')
 def get_top_features(self, drop_null_importance: bool = True, top_n: int = 10):

 grouped_feats = self.feature_importances.groupby(['feature'])['importance'].mean()
if drop_null_importance:
grouped_feats = grouped_feats[grouped_feats != 0]
return list(grouped_feats.sort_values(ascending=False ).index)[:top_n]
def plot_metric(self):

 full_evals_results = pd.DataFrame()
for model in self.models:
evals_result = pd.DataFrame()
for k in model.model.evals_result_.keys() :
evals_result[k] = model.model.evals_result_[k][self.eval_metric]
evals_result = evals_result.reset_index().rename(columns={'index': 'iteration'})
 full_evals_results = full_evals_results.append(evals_result)
 full_evals_results = full_evals_results.melt(id_vars=['iteration'] ).rename(columns={'value': self.eval_metric,
'variable': 'dataset'})
 sns.lineplot(data=full_evals_results, x='iteration', y=self.eval_metric, hue='dataset')
 plt.title('Training progress')
<categorify>
NAcols=c.columns
for col in NAcols:
if c[col].dtype == "object":
c[col] = c[col].fillna("None" )
class CategoricalTransformer(BaseEstimator, TransformerMixin):
def __init__(self, cat_cols=None, drop_original: bool = False, encoder=OrdinalEncoder()):

 self.cat_cols = cat_cols
self.drop_original = drop_original
self.encoder = encoder
self.default_encoder = OrdinalEncoder()
def fit(self, X, y=None):
if self.cat_cols is None:
kinds = np.array([dt.kind for dt in X.dtypes])
 is_cat = kinds == 'O'
self.cat_cols = list(X.columns[is_cat])
 self.encoder.set_params(cols=self.cat_cols)
 self.default_encoder.set_params(cols=self.cat_cols)
 self.encoder.fit(X[self.cat_cols], y)
 self.default_encoder.fit(X[self.cat_cols], y)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 new_cat_names = [f'{col}_encoded' for col in self.cat_cols]
encoded_data = self.encoder.transform(data[self.cat_cols])
 if encoded_data.shape[1] == len(self.cat_cols):
data[new_cat_names] = encoded_data
else:
pass
if self.drop_original:
data = data.drop(self.cat_cols, axis=1)
 else:
data[self.cat_cols] = self.default_encoder.transform(data[self.cat_cols])
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data )<load_from_csv>
for col in NAcols:
if c[col].dtype != "object":
c[col]= c[col].fillna(0 )
def read_data() :
print('Reading train.csv file.... ')
 train = pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv')
 print('Training.csv file have {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Reading test.csv file.... ')
 test = pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv')
 print('Test.csv file have {} rows and {} columns'.format(test.shape[0], test.shape[1]))
print('Reading train_labels.csv file.... ')
 train_labels = pd.read_csv('/kaggle/input/data-science-bowl-2019/train_labels.csv')
 print('Train_labels.csv file have {} rows and {} columns'.format(train_labels.shape[0], train_labels.shape[1]))
print('Reading specs.csv file.... ')
 specs = pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv')
 print('Specs.csv file have {} rows and {} columns'.format(specs.shape[0], specs.shape[1]))
print('Reading sample_submission.csv file.... ')
 sample_submission = pd.read_csv('/kaggle/input/data-science-bowl-2019/sample_submission.csv')
 print('Sample_submission.csv file have {} rows and {} columns'.format(sample_submission.shape[0], sample_submission.shape[1]))
return train, test, train_labels, specs, sample_submission
def encode_title(train, test, train_labels):
train['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), train['title'], train['event_code']))
test['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), test['title'], test['event_code']))
all_title_event_code = list(set(train["title_event_code"].unique() ).union(test["title_event_code"].unique()))
list_of_user_activities = list(set(train['title'].unique() ).union(set(test['title'].unique())))
 list_of_event_code = list(set(train['event_code'].unique() ).union(set(test['event_code'].unique())))
 list_of_event_id = list(set(train['event_id'].unique() ).union(set(test['event_id'].unique())))
 list_of_worlds = list(set(train['world'].unique() ).union(set(test['world'].unique())))
 activities_map = dict(zip(list_of_user_activities, np.arange(len(list_of_user_activities))))
activities_labels = dict(zip(np.arange(len(list_of_user_activities)) , list_of_user_activities))
activities_world = dict(zip(list_of_worlds, np.arange(len(list_of_worlds))))
assess_titles = list(set(train[train['type'] == 'Assessment']['title'].value_counts().index ).union(set(test[test['type'] == 'Assessment']['title'].value_counts().index)))
 train['title'] = train['title'].map(activities_map)
 test['title'] = test['title'].map(activities_map)
 train['world'] = train['world'].map(activities_world)
 test['world'] = test['world'].map(activities_world)
 train_labels['title'] = train_labels['title'].map(activities_map)
 win_code = dict(zip(activities_map.values() ,(4100*np.ones(len(activities_map)) ).astype('int')))
 win_code[activities_map['Bird Measurer(Assessment)']] = 4110
train['timestamp'] = pd.to_datetime(train['timestamp'])
 test['timestamp'] = pd.to_datetime(test['timestamp'])
 return train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code
def get_data(user_sample, test_set=False):

 last_activity = 0
user_activities_count = {'Clip':0, 'Activity': 0, 'Assessment': 0, 'Game':0}
last_session_time_sec = 0
accuracy_groups = {0:0, 1:0, 2:0, 3:0}
all_assessments = []
accumulated_accuracy_group = 0
accumulated_accuracy = 0
accumulated_correct_attempts = 0
accumulated_uncorrect_attempts = 0
accumulated_actions = 0
counter = 0
time_first_activity = float(user_sample['timestamp'].values[0])
 durations = []
last_accuracy_title = {'acc_' + title: -1 for title in assess_titles}
event_code_count: Dict[str, int] = {ev: 0 for ev in list_of_event_code}
event_id_count: Dict[str, int] = {eve: 0 for eve in list_of_event_id}
title_count: Dict[str, int] = {eve: 0 for eve in activities_labels.values() }
title_event_code_count: Dict[str, int] = {t_eve: 0 for t_eve in all_title_event_code}
for i, session in user_sample.groupby('game_session', sort=False):
session_type = session['type'].iloc[0]
session_title = session['title'].iloc[0]
session_title_text = activities_labels[session_title]
if(session_type == 'Assessment')&(test_set or len(session)>1):
all_attempts = session.query(f'event_code == {win_code[session_title]}')
 true_attempts = all_attempts['event_data'].str.contains('true' ).sum()
false_attempts = all_attempts['event_data'].str.contains('false' ).sum()
features = user_activities_count.copy()
features.update(last_accuracy_title.copy())
 features.update(event_code_count.copy())
 features.update(event_id_count.copy())
 features.update(title_count.copy())
 features.update(title_event_code_count.copy())
 features.update(last_accuracy_title.copy())
 features['installation_id'] = session['installation_id'].iloc[-1]
features['session_title'] = session['title'].iloc[0]
features['accumulated_correct_attempts'] = accumulated_correct_attempts
features['accumulated_uncorrect_attempts'] = accumulated_uncorrect_attempts
accumulated_correct_attempts += true_attempts
accumulated_uncorrect_attempts += false_attempts
if durations == []:
features['duration_mean'] = 0
else:
features['duration_mean'] = np.mean(durations)
 durations.append(( session.iloc[-1, 2] - session.iloc[0, 2] ).seconds)
 features['accumulated_accuracy'] = accumulated_accuracy/counter if counter > 0 else 0
accuracy = true_attempts/(true_attempts+false_attempts)if(true_attempts+false_attempts)!= 0 else 0
accumulated_accuracy += accuracy
last_accuracy_title['acc_' + session_title_text] = accuracy
if accuracy == 0:
features['accuracy_group'] = 0
elif accuracy == 1:
features['accuracy_group'] = 3
elif accuracy == 0.5:
features['accuracy_group'] = 2
else:
features['accuracy_group'] = 1
features.update(accuracy_groups)
 accuracy_groups[features['accuracy_group']] += 1
features['accumulated_accuracy_group'] = accumulated_accuracy_group/counter if counter > 0 else 0
accumulated_accuracy_group += features['accuracy_group']
features['accumulated_actions'] = accumulated_actions
if test_set:
all_assessments.append(features)
 elif true_attempts+false_attempts > 0:
all_assessments.append(features)
 counter += 1
def update_counters(counter: dict, col: str):
num_of_session_count = Counter(session[col])
 for k in num_of_session_count.keys() :
x = k
if col == 'title':
x = activities_labels[k]
counter[x] += num_of_session_count[k]
return counter
event_code_count = update_counters(event_code_count, "event_code")
 event_id_count = update_counters(event_id_count, "event_id")
 title_count = update_counters(title_count, 'title')
 title_event_code_count = update_counters(title_event_code_count, 'title_event_code')
 accumulated_actions += len(session)
 if last_activity != session_type:
user_activities_count[session_type] += 1
last_activitiy = session_type
if test_set:
return all_assessments[-1]
return all_assessments
def get_train_and_test(train, test):
compiled_train = []
compiled_test = []
for i,(ins_id, user_sample)in tqdm(enumerate(train.groupby('installation_id', sort = False)) , total = 17000):
compiled_train += get_data(user_sample)
 for ins_id, user_sample in tqdm(test.groupby('installation_id', sort = False), total = 1000):
test_data = get_data(user_sample, test_set = True)
 compiled_test.append(test_data)
 reduce_train = pd.DataFrame(compiled_train)
 reduce_test = pd.DataFrame(compiled_test)
 categoricals = ['session_title']
return reduce_train, reduce_test, categoricals
train, test, train_labels, specs, sample_submission = read_data()
train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code = encode_title(train, test, train_labels)
 reduce_train, reduce_test, categoricals = get_train_and_test(train, test )<categorify>
c['TotalArea'] = c['TotalBsmtSF'] + c['1stFlrSF'] + c['2ndFlrSF'] + c['GrLivArea'] +c['GarageArea']
c['Bathrooms'] = c['FullBath'] + c['HalfBath']*0.5
c['Year average']=(c['YearRemodAdd']+c['YearBuilt'])/2
def preprocess(reduce_train, reduce_test):
for df in [reduce_train, reduce_test]:
df['installation_session_count'] = df.groupby(['installation_id'])['Clip'].transform('count')
 df['installation_duration_mean'] = df.groupby(['installation_id'])['duration_mean'].transform('mean')
 df['installation_title_nunique'] = df.groupby(['installation_id'])['session_title'].transform('nunique')
 df['sum_event_code_count'] = df[[2050, 4100, 4230, 5000, 4235, 2060, 4110, 5010, 2070, 2075, 2080, 2081, 2083, 3110, 4010, 3120, 3121, 4020, 4021,
4022, 4025, 4030, 4031, 3010, 4035, 4040, 3020, 3021, 4045, 2000, 4050, 2010, 2020, 4070, 2025, 2030, 4080, 2035,
2040, 4090, 4220, 4095]].sum(axis = 1)
 df['installation_event_code_count_mean'] = df.groupby(['installation_id'])['sum_event_code_count'].transform('mean')
 features = reduce_train.loc[(reduce_train.sum(axis=1)!= 0),(reduce_train.sum(axis=0)!= 0)].columns
features = [x for x in features if x not in ['accuracy_group', 'installation_id']] + ['acc_' + title for title in assess_titles]
return reduce_train, reduce_test, features
reduce_train, reduce_test, features = preprocess(reduce_train, reduce_test )<init_hyperparams>
c['MoSold'] = c['MoSold'].astype(str)
 c['MSSubClass'] = c['MSSubClass'].apply(str)
 c['YrSold'] = c['YrSold'].astype(str )
params = {'n_estimators':2000,
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'subsample': 0.75,
'subsample_freq': 1,
'learning_rate': 0.04,
'feature_fraction': 0.9,
'max_depth': 15,
'lambda_l1': 1,
'lambda_l2': 1,
'verbose': 100,
'early_stopping_rounds': 100, 'eval_metric': 'cappa'
}<prepare_x_and_y>
cb=pd.get_dummies(c)
 print(f"the shape of the original dataset {c.shape}")
 print(f"the shape of the encoded dataset {cb.shape}")
 print(f"We have {cb.shape[1]- c.shape[1]} new encoded features" )
y = reduce_train['accuracy_group']<choose_model_class>
Train = cb[:na]
Test = cb[na:]
n_fold = 5
folds = GroupKFold(n_splits=n_fold )<define_variables>
a['GrLivArea'].sort_values(ascending=False ).head(2 )
cols_to_drop = ['game_session', 'installation_id', 'timestamp', 'accuracy_group', 'timestampDate']<train_model>
a['TotalBsmtSF'].sort_values(ascending=False ).head(1 )
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model1 = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model1.fit(X=reduce_train, y=y, folds=folds, params=params, preprocesser=mt, transformers=transformers,
eval_metric='cappa', cols_to_drop=cols_to_drop )<compute_test_metric>
a['MasVnrArea'].sort_values(ascending=False ).head(1 )
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):

 X_p = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return -qwk(y, X_p)
 def fit(self, X, y):

 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 return pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 def coefficients(self):

 return self.coef_['x']<predict_on_test>
a['1stFlrSF'].sort_values(ascending=False ).head(1 )
opt_preds = optR.predict(pr1.reshape(-1,), coefficients)
 qwk(y, opt_preds )<predict_on_test>
a['TotRmsAbvGrd'].sort_values(ascending=False ).head(1 )
pr1 = regressor_model1.predict(reduce_test)
 pr1[pr1 <= 1.12232214] = 0
pr1[np.where(np.logical_and(pr1 > 1.12232214, pr1 <= 1.73925866)) ] = 1
pr1[np.where(np.logical_and(pr1 > 1.73925866, pr1 <= 2.22506454)) ] = 2
pr1[pr1 > 2.22506454] = 3<save_to_csv>
train=Train[(Train['GrLivArea'] < 4676)&(Train['MasVnrArea'] < 1600)]
print(f'{Train.shape[0]- train.shape[0]}つの外れ値を削除しました。' )
sample_submission['accuracy_group'] = pr1.astype(int)
 sample_submission.to_csv('submission.csv', index=False )<count_values>
target=a[['SalePrice']]
target.loc[1298]
sample_submission['accuracy_group'].value_counts(normalize=True )<import_modules>
target.loc[523]
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder<load_from_csv>
pos = [1298,523, 297]
target.drop(target.index[pos], inplace=True )
exclude_test_patient_data_from_trainset = True
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 if exclude_test_patient_data_from_trainset:
train = train[~train['Patient'].isin(test['Patient'].unique())]
train = pd.concat([train, test], axis=0, ignore_index=True)\ 
.drop_duplicates()
le_id = LabelEncoder()
train['PatientID'] = le_id.fit_transform(train['Patient'])
 train.head()<import_modules>
print('外れ値を除去した後、特徴量とターゲットの行番号が同じであることを確認します。')
 print('Train: ',train.shape[0],'列')
 print('Target:', target.shape[0],'列' )
import pymc3 as pm<count_unique_values>
from scipy.stats import skew
n_patients = train['Patient'].nunique()
FVC_obs = train['FVC'].values
Weeks = train['Weeks'].values
PatientID = train['PatientID'].values<prepare_output>
target["SalePrice"] = np.log1p(target["SalePrice"] )
pred_template = []
for i in range(train['Patient'].nunique()):
df = pd.DataFrame(columns=['PatientID', 'Weeks'])
 df['Weeks'] = np.arange(-12, 134)
 df['PatientID'] = i
pred_template.append(df)
 pred_template = pd.concat(pred_template, ignore_index=True )<predict_on_test>
x=train
y=np.array(target )
with model_a:
pm.set_data({
"PatientID_shared": pred_template['PatientID'].values.astype(int),
"Weeks_shared": pred_template['Weeks'].values.astype(int),
"FVC_obs_shared": np.zeros(len(pred_template)).astype(int),
})
 post_pred = pm.sample_posterior_predictive(trace_a )<merge>
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0 )
df = pd.DataFrame(columns=['Patient', 'Weeks', 'FVC_pred', 'sigma'])
 df['Patient'] = le_id.inverse_transform(pred_template['PatientID'])
 df['Weeks'] = pred_template['Weeks']
df['FVC_pred'] = post_pred['FVC_like'].T.mean(axis=1)
 df['sigma'] = post_pred['FVC_like'].T.std(axis=1)
 df['FVC_inf'] = df['FVC_pred'] - df['sigma']
df['FVC_sup'] = df['FVC_pred'] + df['sigma']
df = pd.merge(df, train[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 df = df.rename(columns={'FVC': 'FVC_true'})
 df.head()<load_from_csv>
scaler= RobustScaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test )
tr = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df = pd.merge(df, tr[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 use_only_last_3_measures = True
if use_only_last_3_measures:
y = df.dropna().groupby('Patient' ).tail(3)
 else:
y = df.dropna()
rmse =(( y['FVC_pred'] - y['FVC'])** 2 ).mean() **(1/2)
 print(f'RMSE: {rmse:.1f} ml')
 sigma_c = y['sigma'].values
sigma_c[sigma_c < 70] = 70
delta =(y['FVC_pred'] - y['FVC'] ).abs()
delta[delta > 1000] = 1000
lll = - np.sqrt(2)* delta / sigma_c - np.log(np.sqrt(2)* sigma_c)
 print(f'Laplace Log Likelihood: {lll.mean() :.4f}' )<categorify>
lreg=LinearRegression()
MSEs=ms.cross_val_score(lreg, x, y, scoring='neg_mean_squared_error', cv=5)
 meanMSE=np.mean(MSEs)
 print(meanMSE)
 print('RMSE = '+str(math.sqrt(-meanMSE)) )
train['Male'] = train['Sex'].apply(lambda x: 1 if x == 'Male' else 0)
 train["SmokingStatus"] = train["SmokingStatus"].astype(
 pd.CategoricalDtype(['Ex-smoker', 'Never smoked', 'Currently smokes'])
)
 aux = pd.get_dummies(train["SmokingStatus"], prefix='ss')
 aux.columns = ['ExSmoker', 'NeverSmoked', 'CurrentlySmokes']
train['ExSmoker'] = aux['ExSmoker']
train['CurrentlySmokes'] = aux['CurrentlySmokes']
aux = train[['Patient', 'Weeks', 'Percent']].sort_values(by=['Patient', 'Weeks'])
 aux = train.groupby('Patient' ).head(1)
 aux = aux.rename(columns={'Percent': 'Percent_base'})
 train = pd.merge(train, aux[['Patient', 'Percent_base']], how='left',
on='Patient')
 train.head()<count_unique_values>
def score(y_pred):
return str(math.sqrt(sklm.mean_squared_error(y_test, y_pred)) )
n_patients = train['Patient'].nunique()
FVC_obs = train['FVC'].values
X = train[['Weeks', 'Male', 'ExSmoker', 'CurrentlySmokes',
'Percent_base']].values
PatientID = train['PatientID'].values<normalization>
ridge=Ridge()
parameters= {'alpha':[x for x in range(1,101)]}
ridge_reg=ms.GridSearchCV(ridge, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 ridge_reg.fit(x_train,y_train)
 print(f"ベストパラメータ : {ridge_reg.best_params_}")
 print(f"ベストスコア : {math.sqrt(-ridge_reg.best_score_)}")
 ridge_pred=math.sqrt(-ridge_reg.best_score_ )
with pm.Model() as model_b:
FVC_obs_shared = pm.Data("FVC_obs_shared", FVC_obs)
 X_shared = pm.Data('X_shared', X)
 PatientID_shared = pm.Data('PatientID_shared', PatientID)
 mu_a = pm.Normal('mu_a', mu=1700, sigma=400)
 sigma_a = pm.HalfNormal('sigma_a', 1000.)
mu_b = pm.Normal('mu_b', mu=-4., sigma=1., shape=X.shape[1])
 sigma_b = pm.HalfNormal('sigma_b', 5.)
a = pm.Normal('a', mu=mu_a, sigma=sigma_a, shape=n_patients)
 b = pm.Normal('b', mu=mu_b, sigma=sigma_b, shape=(n_patients, X.shape[1]))
sigma = pm.HalfNormal('sigma', 150.)
FVC_est = a[PatientID_shared] +(b[PatientID_shared] * X_shared ).sum(axis=1)
 FVC_like = pm.Normal('FVC_like', mu=FVC_est,
sigma=sigma, observed=FVC_obs_shared )<feature_engineering>
ridge_mod=Ridge(alpha=17)
 ridge_mod.fit(x_train,y_train)
 y_pred_train=ridge_mod.predict(x_train)
 y_pred_test=ridge_mod.predict(x_test)
 print(f'RMSE : train = {str(math.sqrt(sklm.mean_squared_error(y_train, y_pred_train)))}')
 print(f'RMSE : test = {score(y_pred_test)}' )
aux = train.groupby('Patient' ).first().reset_index()
pred_template = []
for i in range(train['Patient'].nunique()):
df = pd.DataFrame(columns=['PatientID', 'Weeks'])
 df['Weeks'] = np.arange(-12, 134)
 df['PatientID'] = i
df['Male'] = aux[aux['PatientID'] == i]['Male'].values[0]
df['ExSmoker'] = aux[aux['PatientID'] == i]['ExSmoker'].values[0]
df['CurrentlySmokes'] = aux[aux['PatientID'] == i]['CurrentlySmokes'].values[0]
df['Percent_base'] = aux[aux['PatientID'] == i]['Percent_base'].values[0]
pred_template.append(df)
 pred_template = pd.concat(pred_template, ignore_index=True )<predict_on_test>
parameters= {'alpha':[10, 1, 0.1, 0.01, 0.001, 0.0001, 0.0003, 0.0005, 0.0007, 0.00075]}
lasso=Lasso()
lasso_reg=ms.GridSearchCV(lasso, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 lasso_reg.fit(x_train,y_train)
 print(f'ベストパラメータ: {lasso_reg.best_params_}' )
with model_b:
pm.set_data({
"PatientID_shared": pred_template['PatientID'].values.astype(int),
"X_shared": pred_template[['Weeks', 'Male', 'ExSmoker',
'CurrentlySmokes',
'Percent_base']].values.astype(int),
"FVC_obs_shared": np.zeros(len(pred_template)).astype(int),
})
 post_pred = pm.sample_posterior_predictive(trace_b )<merge>
lasso_mod=Lasso(alpha=0.0007)
 lasso_mod.fit(x_train,y_train)
 y_lasso_train=lasso_mod.predict(x_train)
 y_lasso_test=lasso_mod.predict(x_test)
 print(f'RMSE : train = {str(math.sqrt(sklm.mean_squared_error(y_train, y_lasso_train)))}')
 print(f'RMSE : test = {score(y_lasso_test)}' )
df = pd.DataFrame(columns=['Patient', 'Weeks', 'FVC_pred', 'sigma'])
 df['Patient'] = le_id.inverse_transform(pred_template['PatientID'])
 df['Weeks'] = pred_template['Weeks']
df['FVC_pred'] = post_pred['FVC_like'].T.mean(axis=1)
 df['sigma'] = post_pred['FVC_like'].T.std(axis=1)
 df['FVC_inf'] = df['FVC_pred'] - df['sigma']
df['FVC_sup'] = df['FVC_pred'] + df['sigma']
df = pd.merge(df, train[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 df = df.rename(columns={'FVC': 'FVC_true'})
 df.head()<load_from_csv>
alphas = [10, 1, 0.1, 0.01, 0.001, 0.0001, 0.0003, 0.0005, 0.0006, 0.00055]
l1ratio = [0.1, 0.3, 0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print('RMSE : test = '+ str(math.sqrt(sklm.mean_squared_error(y_test, ela_pred))))
print(elastic_cv.alpha_ )
tr = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df = pd.merge(df, tr[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 use_only_last_3_measures = True
if use_only_last_3_measures:
y = df.dropna().groupby('Patient' ).tail(3)
 else:
y = df.dropna()
rmse =(( y['FVC_pred'] - y['FVC_true'])** 2 ).mean() **(1/2)
 print(f'RMSE: {rmse:.1f} ml')
 sigma_c = y['sigma'].values
sigma_c[sigma_c < 70] = 70
delta =(y['FVC_pred'] - y['FVC_true'] ).abs()
delta[delta > 1000] = 1000
lll = - np.sqrt(2)* delta / sigma_c - np.log(np.sqrt(2)* sigma_c)
 print(f'Laplace Log Likelihood: {lll.mean() :.4f}' )<load_from_csv>
alphas = [0.00055]
l1ratio = [0.1, 0.3, 0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print(f'RMSE : test = {score(ela_pred)}')
 print(elastic_cv.alpha_)
 print(elastic_cv.l1_ratio_)
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 train = pd.concat([train, test], axis=0, ignore_index=True)\ 
.drop_duplicates()
le_id = LabelEncoder()
train['PatientID'] = le_id.fit_transform(train['Patient'] )<count_unique_values>
def regularization(x,y,modelo=Ridge, scaler=RobustScaler):
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0)
 scaler= scaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test)
 if modelo != ElasticNetCV:
if modelo == Ridge:
parameters= {'alpha':[x for x in range(1,101)]}
elif modelo == Lasso:
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
model=modelo()
model=ms.GridSearchCV(model, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 model.fit(x_train,y_train)
 y_pred= model.predict(x_test)
 print("best_score : %s, RMSE = %s" %(model.best_params_,
score(y_pred)))
 elif modelo == ElasticNetCV:
alphas = [0.00055]
l1ratio = [0.1, 0.3, 0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print("best_score : alpha %s and l1_ratio %s, RMSE = %s"
%(elastic_cv.alpha_,elastic_cv.l1_ratio_, score(ela_pred)))
n_patients = train['Patient'].nunique()
FVC_obs = train['FVC'].values
Weeks = train['Weeks'].values
PatientID = train['PatientID'].values
with pm.Model() as model_a:
FVC_obs_shared = pm.Data("FVC_obs_shared", FVC_obs)
 Weeks_shared = pm.Data('Weeks_shared', Weeks)
 PatientID_shared = pm.Data('PatientID_shared', PatientID)
 mu_a = pm.Normal('mu_a', mu=1700., sigma=400)
 sigma_a = pm.HalfNormal('sigma_a', 1000.)
mu_b = pm.Normal('mu_b', mu=-4., sigma=1)
 sigma_b = pm.HalfNormal('sigma_b', 5.)
a = pm.Normal('a', mu=mu_a, sigma=sigma_a, shape=n_patients)
 b = pm.Normal('b', mu=mu_b, sigma=sigma_b, shape=n_patients)
 sigma = pm.HalfNormal('sigma', 150.)
FVC_est = a[PatientID_shared] + b[PatientID_shared] * Weeks_shared
FVC_like = pm.Normal('FVC_like', mu=FVC_est,
sigma=sigma, observed=FVC_obs_shared)
 trace_a = pm.sample(2000, tune=2000, target_accept=.9, init="adapt_diag" )<categorify>
regularization(x,y,Ridge )
pred_template = []
for p in test['Patient'].unique() :
df = pd.DataFrame(columns=['PatientID', 'Weeks'])
 df['Weeks'] = np.arange(-12, 134)
 df['Patient'] = p
pred_template.append(df)
 pred_template = pd.concat(pred_template, ignore_index=True)
 pred_template['PatientID'] = le_id.transform(pred_template['Patient'])
 with model_a:
pm.set_data({
"PatientID_shared": pred_template['PatientID'].values.astype(int),
"Weeks_shared": pred_template['Weeks'].values.astype(int),
"FVC_obs_shared": np.zeros(len(pred_template)).astype(int),
})
 post_pred = pm.sample_posterior_predictive(trace_a )<save_to_csv>
regularization(x,y, Lasso )
df = pd.DataFrame(columns=['Patient', 'Weeks', 'Patient_Week', 'FVC', 'Confidence'])
 df['Patient'] = pred_template['Patient']
df['Weeks'] = pred_template['Weeks']
df['Patient_Week'] = df['Patient'] + '_' + df['Weeks'].astype(str)
 df['FVC'] = post_pred['FVC_like'].T.mean(axis=1)
 df['Confidence'] = post_pred['FVC_like'].T.std(axis=1)
 final = df[['Patient_Week', 'FVC', 'Confidence']]
final.to_csv('submission.csv', index=False)
 print(final.shape)
 final.head()<install_modules>
regularization(x,y, ElasticNetCV )
!pip install turicreate<import_modules>
from xgboost.sklearn import XGBRegressor
import os
import numpy as np
import pandas as pd
import turicreate
from zipfile import ZipFile<define_variables>
xgb= XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=3, min_child_weight=0, missing=None, n_estimators=4000,
n_jobs=1, nthread=None, objective='reg:squarederror', random_state=0,
reg_alpha=0.0001, reg_lambda=0.01, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 xgmod=xgb.fit(x_train,y_train)
 xg_pred=xgmod.predict(x_test)
 print(f'RMSE : test = {score(xg_pred)}' )
ds_dir = '.. /input/instacart-market-basket-analysis'<load_pretrained>
vote_mod = VotingRegressor([('Ridge', ridge_mod),('Lasso', lasso_mod),('Elastic', elastic_cv), 
('XGBRegressor', xgb)])
 vote= vote_mod.fit(x_train, y_train.ravel())
 vote_pred=vote.predict(x_test)
 print(f'RMSE : test = {score(vote_pred)}' )
with ZipFile(os.path.join(ds_dir,"aisles.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"departments.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"order_products__prior.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"order_products__train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"orders.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"products.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"sample_submission.csv.zip"), 'r')as zipObj:
zipObj.extractall()<load_from_csv>
stregr = StackingRegressor(regressors=[elastic_cv,ridge_mod, lasso_mod, vote_mod],
meta_regressor=xgb, use_features_in_secondary=True 
)
 stack_mod=stregr.fit(x_train, y_train.ravel())
 stacking_pred=stack_mod.predict(x_test)
 print(f'RMSE : test = {score(stacking_pred)}' )
prior = pd.read_csv('order_products__prior.csv')
 train = pd.read_csv('order_products__train.csv')
 orders = pd.read_csv('orders.csv')
 products = pd.read_csv('products.csv')
 prior.fillna(0,inplace=True)
 train.fillna(0,inplace=True)
 orders.fillna(0,inplace=True )<merge>
final_test=(0.3*vote_pred + 0.5*stacking_pred + 0.2*y_lasso_test)
 print(f'RMSE : test= {score(final_test)}')
data = pd.concat([train,prior])
 ou = orders[['order_id','user_id']]
data = pd.merge(data,ou,on='order_id' ).drop('order_id',axis=1 )<merge>
vote_test = vote_mod.predict(X_test)
 final1=np.expm1(vote_test)
 stack_test = stregr.predict(X_test)
 final2=np.expm1(stack_test)
 lasso_test = lasso_mod.predict(X_test)
 final3=np.expm1(lasso_test)
<filter><EOS> 
final=(0.2*final1 + 0.6*final2 + 0.2*final3)
 submission = pd.DataFrame({
"Id": b["Id"],
"SalePrice": final
})
 submission.to_csv("./submission.csv", index=False)
 submission.head()