Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<compute_test_metric>
%matplotlib inline
sns.set_style('darkgrid')
 plt.rcParams['figure.figsize'] = 7, 5
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
house_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 y = house_data['SalePrice']
X = house_data.drop('SalePrice', axis=1 )
sub['FVC1'] = 0.996 * pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
X_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id' )
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
X.dtypes.value_counts()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
top_ten_corr_features = corr_matrix.nlargest(n=11, columns='SalePrice')['SalePrice'][1:]
top_ten_corr_features
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
def impute(df, column, value):

 df[column] = df[column].fillna(value )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
X.isna().sum() [X.isna().sum() > 0]
import numpy as np
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
X_test.isna().sum() [X_test.isna().sum() > 0]
df_train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )<groupby>
print('Variable with NA in training but not test set: ', set(X.columns[X.isna().any() ].tolist())- set(X_test.columns[X_test.isna().any() ].tolist()))
print('Variables with NA in test but not training set: ', set(X_test.columns[X_test.isna().any() ].tolist())- set(X.columns[X.isna().any() ].tolist()))
training_sample_counts = df_train.rename(columns={'Weeks': 'Samples'} ).groupby('Patient' ).agg('count')['Samples']
{("-")* 41}
{training_sample_counts}' )<load_from_csv>
impute(X, 'MSZoning', X['MSZoning'].value_counts().idxmax())
 impute(X_test, 'MSZoning', X['MSZoning'].value_counts().idxmax() )
df_submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 df_submission.head()<set_options>
X['LotFrontage'] = X['LotFrontage'].fillna(X.groupby('Neighborhood')['LotFrontage'].transform('median'))
lotfrontage_median = pd.DataFrame(X.groupby('Neighborhood')['LotFrontage'].median() ).rename(columns={'LotFrontage':'LotFrontage_median'})
 X_test_tmp = X_test.reset_index().merge(lotfrontage_median, on='Neighborhood' ).set_index('Id')
 X_test_tmp['LotFrontage'] = X_test_tmp['LotFrontage'].fillna(X_test_tmp['LotFrontage_median'])
 X_test = X_test_tmp.drop('LotFrontage_median', axis=1 )
warnings.filterwarnings("ignore" )<define_variables>
impute(X, 'Utilities', X['Utilities'].value_counts().idxmax())
 impute(X_test, 'Utilities', X['Utilities'].value_counts().idxmax() )
def seed_everything(seed=777):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )<define_variables>
impute(X, 'Exterior1st', X['Exterior1st'].value_counts().idxmax())
 impute(X_test, 'Exterior1st', X['Exterior1st'].value_counts().idxmax())
 impute(X, 'Exterior2nd', X['Exterior2nd'].value_counts().idxmax())
 impute(X_test, 'Exterior2nd', X['Exterior2nd'].value_counts().idxmax() )
OUTPUT_DICT = './'
ID = 'Patient_Week'
TARGET = 'FVC'
SEED = 777
seed_everything(seed=SEED)
 N_FOLD = 7<load_from_csv>
impute(X, 'KitchenQual', X['KitchenQual'].value_counts().idxmax())
 impute(X_test, 'KitchenQual', X['KitchenQual'].value_counts().idxmax() )
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )<create_dataframe>
impute(X, 'Functional', X['Functional'].value_counts().idxmax())
 impute(X_test, 'Functional', X['Functional'].value_counts().idxmax() )
train = pd.concat([train,otest])
 output = pd.DataFrame()
gb = train.groupby('Patient')
 tk0 = tqdm(gb, total=len(gb))
for _, usr_df in tk0:
usr_output = pd.DataFrame()
for week, tmp in usr_df.groupby('Weeks'):
rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}
tmp = tmp.rename(columns=rename_cols)
 drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']
_usr_output = usr_df.drop(columns=drop_cols ).rename(columns={'Weeks': 'predict_Week'} ).merge(tmp, on='Patient')
 _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']
usr_output = pd.concat([usr_output, _usr_output])
 print(usr_output)
 output = pd.concat([output, usr_output])
 train = output[output['Week_passed']!=0].reset_index(drop=True )<feature_engineering>
impute(X, 'SaleType', X['SaleType'].value_counts().idxmax())
 impute(X_test, 'SaleType', X['SaleType'].value_counts().idxmax() )
test = otest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})
 submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])
 submission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 test = submission.drop(columns=['FVC', 'Confidence'] ).merge(test, on='Patient')
 test['Week_passed'] = test['predict_Week'] - test['base_Week']
test.set_index('Patient_Week', inplace=True )<define_variables>
impute(X, 'Electrical', X['Electrical'].value_counts().idxmax())
 impute(X_test, 'Electrical', X['Electrical'].value_counts().idxmax() )
OUTPUT_DICT = './'
ID = 'Patient_Week'
TARGET = 'FVC'
SEED = 777
seed_everything(seed=SEED)
 N_FOLD = 7<categorify>
print('Number of NA in training set after imputation: ', X.isna().sum().sum())
 print('Number of NA in test set after imputation: ', X_test.isna().sum().sum() )
folds = train[['Patient', TARGET]].copy()
Fold = GroupKFold(n_splits=N_FOLD)
 groups = folds['Patient'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[TARGET], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int )<prepare_x_and_y>
def create_features(X):
X['TotalSF'] = X['TotalBsmtSF'] + X['GrLivArea']
X['TotalPorchSF'] = X['OpenPorchSF'] + X['EnclosedPorch'] + X['3SsnPorch'] + X['ScreenPorch']
X['TotalBath'] = X['BsmtFullBath'] + 0.5 * X['BsmtHalfBath'] + X['FullBath'] + 0.5 * X['HalfBath']
X['TotalRooms'] = X['TotRmsAbvGrd'] + X['FullBath'] + 0.5 * X['HalfBath']
X['HasGarage'] = X['GarageArea'] > 0
X['HasPool'] = X['PoolArea'] > 0
X['HasFireplace'] = X['Fireplaces'] > 0
X['HasBasement'] = X['TotalBsmtSF'] > 0
X['Has2ndFloor'] = X['2ndFlrSF'] > 0
X['IsRemodeled'] = X['YearBuilt'] != X['YearRemodAdd']
return X
X = create_features(X)
 X_test = create_features(X_test )
def run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):
trn_idx = folds[folds.fold!=fold_num].index
val_idx = folds[folds.fold==fold_num].index
y_tr = target.iloc[trn_idx].values
X_tr = train_df.iloc[trn_idx][features].values
y_val = target.iloc[val_idx].values
X_val = train_df.iloc[val_idx][features].values
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
clf.fit(X_tr, y_tr)
 oof[val_idx] = clf.predict(X_val)
 predictions += clf.predict(test_df[features])
 return oof, predictions
def run_kfold_model(clf, train, test, folds, features, target, n_fold=7):
oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_ in range(n_fold):
_oof, _predictions = run_single_model(clf,
train,
test,
folds,
features,
target,
fold_num=fold_)
 oof += _oof
predictions += _predictions/n_fold
return oof, predictions<categorify>
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']
features = [c for c in features if c not in drop_features]
if cat_features:
ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')
 ce_oe.fit(train)
 train = ce_oe.transform(train)
 test = ce_oe.transform(test )<train_on_grid>
num_cols_to_cat = ['MSSubClass', 'MoSold', 'YrSold']
X[num_cols_to_cat] = X[num_cols_to_cat].astype(str)
 X_test[num_cols_to_cat] = X_test[num_cols_to_cat].astype(str )
scorelist = []
for alpha1 in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]:
for l1s in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]:
print(" For alpha:",alpha1,"& l1_ratio:",l1s)
 clf = ElasticNet(alpha=alpha1, l1_ratio = l1s)
 oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)
 train['FVC_pred'] = oof
test['FVC_pred'] = predictions
train['Confidence'] = 100
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score)
 def loss_func(weight, row):
confidence = weight
sigma_clipped = max(confidence, 70)
 diff = abs(row['FVC'] - row['FVC_pred'])
 delta = min(diff, 1000)
 score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)
 return -score
results = []
tk0 = tqdm(train.iterrows() , total=len(train))
for _, row in tk0:
loss_partial = partial(loss_func, row=row)
 weight = [100]
result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')
 x = result['x']
results.append(x[0])
 train['Confidence'] = results
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
scorelist.append(score)
 print(score )<categorify>
cat_cols = [col for col in X.columns if X.dtypes[col] == 'object']
num_cols = [col for col in X.columns if X.dtypes[col] in ['int64', 'float64']]
TARGET = 'Confidence'
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']
features = [c for c in features if c not in drop_features]
oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD )<feature_engineering>
def ordinal_encoder(X):
X = X.replace({
'Street': {'Pave':1, 'Grvl':2},
'Alley': {'None':0, 'Pave':1, 'Grvl':2},
'LotShape': {'IR3':1, 'IR2':2, 'IR1':3, 'Reg':4},
'Utilities': {'ELO':1, 'NoSeWa':2, 'NoSewr':3, 'AllPub':4},
'LandSlope': {'Sev':1, 'Mod':2, 'Gtl':3},
'ExterQual': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'ExterCond': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'BsmtQual': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'BsmtCond': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'BsmtExposure': {'None':0, 'No':1, 'Mn':2, 'Av':3, 'Gd':4},
'BsmtFinType1': {'None':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6},
'BsmtFinType2': {'None':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6},
'HeatingQC': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'KitchenQual': {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'Functional': {'Sal':1, 'Sev':2, 'Maj2':3, 'Maj1':4, 'Mod':5, 'Min2':6, 'Min1':7, 'Typ':8},
'FireplaceQu': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'GarageFinish': {'None':0, 'Unf':1, 'RFn':2, 'Fin':3},
'GarageQual': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'GarageCond': {'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5},
'PavedDrive': {'N':0, 'P':1, 'Y':2},
'PoolQC': {'None':0, 'Fa':1, 'TA':2, 'Gd':3, 'Ex':4},
'Fence': {'None':0, 'MnWw':1, 'GdWo':2, 'MnPrv':3, 'GdPrv':4},
'Electrical':{'Mix':1, 'FuseP':2, 'FuseF':3, 'FuseA':4, 'SBrkr':5}
})
 return X
X = ordinal_encoder(X)
 X_test = ordinal_encoder(X_test )
train['Confidence'] = oof
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )<feature_engineering>
X = pd.get_dummies(X)
 X_test = pd.get_dummies(X_test )
test['Confidence'] = predictions
test = test.reset_index()<save_to_csv>
skewness = X[num_cols].apply(lambda x: skew(x))
skewed_cols = skewness[abs(skewness)> 0.5].index
X[skewed_cols] = np.log1p(X[skewed_cols])
 X_test[skewed_cols] = np.log1p(X_test[skewed_cols] )
sub = submission[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')
 sub = sub.rename(columns={'FVC_pred': 'FVC'})
 for i in range(len(otest)) :
sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
sub[sub.Confidence<1]
sub.to_csv('submission.csv', index=False, float_format='%.1f' )<set_options>
sel = VarianceThreshold(threshold=0.01)
 sel.fit(X)
 print('Number of features remaining:', sel.get_support().sum() )
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )<import_modules>
X = X[[col for col in X.columns[sel.get_support() ]]]
X_test = X_test[[col for col in X_test.columns[sel.get_support() ]]]
import pdb<define_variables>
scaler = RobustScaler()
X_scaled = scaler.fit_transform(X)
 X_test_scaled = scaler.transform(X_test)
 X_scaled = pd.DataFrame(X_scaled, columns=X.columns, index=X.index)
 X_test_scaled = pd.DataFrame(X_test_scaled, columns=X_test.columns, index=X_test.index )
MODELS_PATH = '.. /input/submission-toxicity-classification'<define_variables>
def CV_rmse(estimator, X, y, cv):
rmse = -cross_val_score(estimator=estimator, X=X, y=y, cv=cv, scoring='neg_root_mean_squared_error')
 return rmse
BERT_MODEL_PATH = '.. /input/submission-toxicity-classification/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'<load_from_csv>
linear_model = LinearRegression()
rmse = CV_rmse(estimator=linear_model, X=X_scaled, y=y, cv=10)
 print('Linear Regression RMSE:', rmse.mean() )
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 test_df['comment_text'] = test_df['comment_text'].astype(str )<predict_on_test>
ridge_cv = Ridge(alpha=25, max_iter=20000, random_state=1)
 ridge_cv.fit(X_scaled, y)
 rmse = CV_rmse(estimator=ridge_cv, X=X_scaled, y=y, cv=10)
 print('Ridge Regression RMSE:', rmse.mean() )
def bert_get_preds(model, x_test):
test_preds = np.zeros(( len(x_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i*32:(i+1)*32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
return test_pred<predict_on_test>
ridge_pred = np.expm1(ridge_cv.predict(X_test_scaled))
def gpt2_get_preds(model, x_test):
test_preds = np.zeros(( len(x_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device))
test_preds[i*32:(i+1)*32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
return test_pred<string_transform>
lasso_cv = Lasso(alpha=0.0008, max_iter=20000, random_state=1)
 lasso_cv.fit(X_scaled, y)
 rmse = CV_rmse(estimator=lasso_cv, X=X_scaled, y=y, cv=10)
 print('Lasso Regression RMSE:', rmse.mean() )
def bert_convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )<string_transform>
lasso_pred = np.expm1(lasso_cv.predict(X_test_scaled))
def gpt2_convert_lines(example, max_seq_length, tokenizer):
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)> max_seq_length:
tokens_a = tokens_a[-max_seq_length:]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(tokens_a)+ [0]*(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )<choose_model_class>
class BertHead(nn.Module):
def __init__(self, config, hidden_units=256, num_aux_targets=6):
super(BertHead, self ).__init__()
self.hidden_units = hidden_units
self.num_aux_targets = num_aux_targets
self.config = config
self.bert_model = BertForSequenceClassification.from_pretrained(config, cache_dir=None, num_labels=hidden_units)
 param_optimizer = list(self.bert_model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
self.fc_out = nn.Linear(768, 1)
 self.fc_aux_out = nn.Linear(768, num_aux_targets)
 self.fc_dp = nn.Dropout(p=0.4)
 self.optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
{'params': [p for p in self.fc_out.parameters() ], 'weight_decay': 0.0},
{'params': [p for p in self.fc_aux_out.parameters() ], 'weight_decay': 0.0}
]
def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):
_, pooled_output = self.bert_model.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
 pooled_output = self.bert_model.dropout(pooled_output)
 logits = self.fc_dp(pooled_output)
 out = self.fc_out(logits)
 out_aux = self.fc_aux_out(logits)
 return torch.cat([out, out_aux], 1 )<load_pretrained>
elasticnet_cv = ElasticNet(alpha=0.0008, l1_ratio=1.0, max_iter=20000, random_state=1)
 elasticnet_cv.fit(X_scaled, y)
 rmse = CV_rmse(estimator=elasticnet_cv, X=X_scaled, y=y, cv=10)
 print('ElasticNet Regression RMSE:', rmse.mean() )
bert_tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None, do_lower_case=True )<define_variables>
elasticnet_pred = np.expm1(elasticnet_cv.predict(X_test_scaled))
bert_base_models = ['bert_epoch_2_lb_093967.bin', 'bert_epoch_2_lb_093916.bin', 'bert_epoch_1_lb_093970.bin']<categorify>
xgb_model = xgb.XGBRegressor(random_state=1)
 rmse = CV_rmse(estimator=xgb_model, X=X, y=y, cv=10)
 print('XGBoost(untuned)RMSE:', rmse.mean() )
x_test = bert_convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), 220, bert_tokenizer )<load_pretrained>
xbg_model_tuned = xgb.XGBRegressor(max_depth=3, min_child_weight=6, learning_rate=0.03, subsample=0.7, colsample_bytree=0.8, n_estimators=727, random_state=1)
 xbg_model_tuned.fit(X, y)
 rmse = CV_rmse(estimator=xbg_model_tuned, X=X, y=y, cv=10)
 print('XGBoost(tuned)RMSE:', rmse.mean() )
bert_base_preds = np.zeros(( x_test.shape[0], len(bert_base_models)))
 for i, bert_model in enumerate(bert_base_models):
model = BertHead('.. /input/bert_uncased')
 model.load_state_dict(torch.load(os.path.join(MODELS_PATH, bert_model)))
 model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
preds = bert_get_preds(model, x_test)
 bert_base_preds[:,i] = preds
del model
gc.collect()<choose_model_class>
xgb_pred = np.expm1(xbg_model_tuned.predict(X_test))
class GPT2ClassificationHeadModel(GPT2PreTrainedModel):
def __init__(self, config, clf_dropout=0.4, num_aux_targets=6):
super(GPT2ClassificationHeadModel, self ).__init__(config)
 self.transformer = GPT2Model(config)
 self.dropout = nn.Dropout(clf_dropout)
 self.fc_out = nn.Linear(config.n_embd * 2, 1)
 self.fc_aux_out = nn.Linear(config.n_embd * 2, num_aux_targets)
 def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None):
hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past)
 avg_pool = torch.mean(hidden_states, 1)
 max_pool, _ = torch.max(hidden_states, 1)
 logits = torch.cat(( avg_pool, max_pool), 1)
 logits = self.dropout(logits)
 out = self.fc_out(logits)
 out_aux = self.fc_aux_out(logits)
 return torch.cat([out, out_aux], 1 )<load_pretrained>
avg_pred = 0.2 * ridge_pred + 0.4 * lasso_pred + 0.4 * xgb_pred
gpt2_tokenizer = GPT2Tokenizer.from_pretrained('.. /input/submission-toxicity-classification/gpt2-models/' )<define_variables>
output = pd.DataFrame({'Id': X_test.index,
'SalePrice': avg_pred})
 output.to_csv('submission.csv', index=False )