Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_pretrained>
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
warnings.simplefilter(action='ignore', category=FutureWarning)
 %matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid' )
model = BanglaHandwrittenGraphemeNN().to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best/model.pth', map_location=device))
model_r18_best = model<load_pretrained>
train_path = ".. /input/eda-and-preprocessing-of-the-datasets/preprocessed_train.csv"
test_path = ".. /input/eda-and-preprocessing-of-the-datasets/preprocessed_test.csv"
train = pd.read_csv(train_path)
 test = pd.read_csv(test_path )
model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True ).to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-lossweight-111.pth', map_location=device))
model_r18_111 = model<load_pretrained>
n_train = train.shape[0]
n_test = test.shape[0]
train_y = train['SalePrice']
train_ids = train['Id']
test_ids = test['Id']
train = train.drop('SalePrice', axis=1)
 train = train.drop('Id', axis=1)
 test = test.drop('Id', axis=1 )
model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True ).to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-lossweight-211.pth', map_location=device))
model_r18_211 = model<load_pretrained>
def reset_weights(model):
session = K.get_session()
for layer in model.layers:
if hasattr(layer, 'kernel_initializer'):
layer.kernel.initializer.run(session=session )
model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True ).to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-80ep.pth', map_location=device))
model_r18_80ep = model<choose_model_class>
def get_callbacks(bm_path):
checkpointer = keras.callbacks.ModelCheckpoint(
 filepath=bm_path,
monitor='val_loss',
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1 
)
 lr_patience = 10
lr_scheduler = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.95,
patience=lr_patience,
verbose=0,
mode='auto',
min_delta=0.0001,
cooldown=25,
min_lr=0 
)
 return lr_scheduler, checkpointer
ens1 = MultiTaskEnsemble([model_r18_best, model_r18_111, model_r18_211, model_r18_80ep] ).to(device )<define_search_model>
def get_model(input_shape, num_neurons):
model = keras.Sequential([
keras.layers.Dense(num_neurons, activation='linear', input_shape=input_shape),
keras.layers.Dropout(0.25),
keras.layers.LeakyReLU(alpha=0.1),
keras.layers.Dense(1)
 ])
 return model
ens2 = MultiTaskEnsemble([model1, model2, model3] ).to(device )<load_pretrained>
X = train.values
Y = train_y.values
BATCH_SIZE = 2
NUM_FOLDS = 5
EPOCHS = 250
HOLDOUT_DICT = {'fold{}'.format(fc): {} for fc in range(NUM_FOLDS)}
if os.path.exists('out'):
shutil.rmtree('out')
 os.makedirs('out')
 BASE_MODELS = {'m0' : 16, 'm1' : 32, 'm2' : 64, 'm3' : 128}
def load_model(name, num_tasks):
base = torch.jit.load(name + '-base.trace')
 task_predictors = [torch.jit.load(name + '-task-predictor-{}.trace'.format(i)) for i in range(num_tasks)]
return MultiTaskNN(base, task_predictors )<load_pretrained>
best_models = {mod_name : [] for mod_name in BASE_MODELS.keys() }
r2_stacked = []
mse_stacked = []
mae_stacked = []
rmsle_stacked = []
kfold = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 fold_counter = 0
for train_idxs, test_idxs in kfold.split(X, Y):
HOLDOUT_DICT['fold{}'.format(fold_counter)]['gt'] = Y[test_idxs]
for model_name, model_config in BASE_MODELS.items() :
model = get_model(( train.shape[1],), num_neurons=model_config)
 reset_weights(model)
 optimizer = 'adam'
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 bm_path = "out/fold{}{}_config{}.h5".format(fold_counter, model_name, model_config)
 lr_scheduler, checkpointer = get_callbacks(bm_path)
 history = model.fit(
 x=X[train_idxs],
y=Y[train_idxs],
validation_data=(X[test_idxs], Y[test_idxs]),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[
checkpointer,
],
verbose=0,
shuffle=True 
)
 model.load_weights(bm_path)
 predict_val = model.predict(X[test_idxs] ).flatten()
HOLDOUT_DICT['fold{}'.format(fold_counter)]['{}'.format(model_name)] = predict_val.copy()
r2 = r2_score(predict_val, Y[test_idxs])
 mse = mean_squared_error(predict_val, Y[test_idxs])
 mae = mean_absolute_error(predict_val, Y[test_idxs])
 rmsle = math.sqrt(mse)
 best_models[model_name].append(bm_path)
 mse_stacked.append(mse)
 mae_stacked.append(mae)
 rmsle_stacked.append(rmsle)
 r2_stacked.append(r2)
 plot_history(history)
 print("{}/{} folds model {}: MSE: {} || MAE: {} || RMSLE: {} || R2: {}".format(fold_counter + 1, NUM_FOLDS, model_name, mse, mae, rmsle, r2))
K.clear_session()
fold_counter += 1
print("AVG_RMSLE: {} || AVG_R2: {}".format(np.mean(rmsle_stacked), np.mean(r2_stacked)) )
model1 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail-small', 3 )<load_pretrained>
from sklearn.linear_model import Lasso
model2 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail', 3 )<load_pretrained>
from sklearn.linear_model import Lasso
model3 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-40ep-rotate-shear', 3 )<load_pretrained>
meta_model = Lasso(alpha=0.0005, random_state=1)
 meta_model.fit(X=holdouts_df[list(BASE_MODELS.keys())], y=holdouts_df['gt'])
 rmsle = math.sqrt(mean_squared_error(meta_model.predict(holdouts_df[list(BASE_MODELS.keys())]), holdouts_df['gt']))
print(rmsle )
model4 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail-small-40ep', 3 )<define_search_model>
def ensemble_models(models, model_input):
yModels = [model(model_input)for model in models]
yAvg = average(yModels)
 modelEns = Model(inputs=model_input, outputs=yAvg, name='ensembleModel')
 return modelEns
ens3 = MultiTaskEnsemble([model1, model2, model3, model4] ).to(device )<define_variables>
def metamodel_predict(ensemble_models_dict, meta_model, data):
res = {}
for model_name, model in ensemble_models_dict.items() :
res[model_name] = model.predict(data ).flatten()
res_df = pd.DataFrame(res)
 meta_res = meta_model.predict(res_df.values)
 res_df['meta'] = meta_res
return res_df, meta_res
models = [ens1, ens2, ens3]<define_variables>
res_df, meta_res = metamodel_predict(ensembles, meta_model, X)
 for col in res_df.columns:
pred = res_df[col]
print("Model {}: RMSLE: {} || R2: {}".format(col, math.sqrt(mean_squared_error(pred, Y)) , r2_score(pred, Y)) )
<create_dataframe><EOS> 
res_submission_df, meta_res_submission = metamodel_predict(ensembles, meta_model, test)
 sub = pd.DataFrame()
sub['Id'] = test_ids
sub['SalePrice'] = np.expm1(meta_res_submission)
 sub.to_csv(os.path.join('submission_sm.csv'),index=False)
 sub.head()