Kaggle competition: LANL Earthquake Prediction
%%time
params = {
'learning_rate': 0.02,
'boosting': 'gbdt',
'objective': 'regression',
'metric': ['rmse'],
'is_training_metric': True,
'seed': 19,
'num_leaves': 31,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'bagging_freq': 5
}
model = lgb.train(params, dtrain, num_boost_round=num_boost_round, valid_sets=valid_sets, valid_names=valid_names,
verbose_eval=num_boost_round//20, early_stopping_rounds=50 if validate else None)
 if validate:
best_num_boost_round = model.best_iteration<set_options>
sns.set()
del df_train, dtrain
gc.collect()<save_to_csv>
warnings.filterwarnings('ignore')
 NUM_EVALS = 1000
N_FOLDS = 5
LGBM_MAX_LEAVES = 2**10
LGBM_MAX_DEPTH = 25
EVAL_METRIC_LGBM_REG = 'mae'
EVAL_METRIC_LGBM_CLASS = 'auc'
XGB_MAX_LEAVES = 2**12
XGB_MAX_DEPTH = 25
EVAL_METRIC_XGB_REG = 'mae'
EVAL_METRIC_XGB_CLASS = 'auc'
CB_MAX_DEPTH = 8
OBJECTIVE_CB_REG = 'MAE'
OBJECTIVE_CB_CLASS = 'Logloss'
BEST_SCORE = 0
def quick_hyperopt(data, labels, package='lgbm', num_evals=NUM_EVALS, diagnostic=False, Class=False):
if package=='lgbm':
print('Running {} rounds of LightGBM parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['max_depth',
'num_leaves',
'max_bin',
'min_data_in_leaf',
'min_data_in_bin']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['boosting']['boosting'] == 'goss':
top_rate = space_params['boosting'].get('top_rate')
 other_rate = space_params['boosting'].get('other_rate')
 top_rate = max(top_rate, 0)
 top_rate = min(top_rate, 0.5)
 other_rate = max(other_rate, 0)
 other_rate = min(other_rate, 0.5)
 space_params['top_rate'] = top_rate
space_params['other_rate'] = other_rate
subsample = space_params['boosting'].get('subsample', 1.0)
 space_params['boosting'] = space_params['boosting']['boosting']
space_params['subsample'] = subsample
if Class:
cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=True,
early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_CLASS, seed=42)
 best_loss = 1 - cv_results['auc-mean'][-1]
else:
cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=False,
early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_REG, seed=42)
 best_loss = cv_results['l1-mean'][-1]
return{'loss':best_loss, 'status': STATUS_OK }
train = lgb.Dataset(data, labels)
 boosting_list = [{'boosting': 'gbdt',
'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting': 'goss',
'subsample': 1.0,
'top_rate': hp.uniform('top_rate', 0, 0.5),
'other_rate': hp.uniform('other_rate', 0, 0.5)}]
if Class:
metric_list = ['auc']
objective_list = ['binary', 'cross_entropy']
else:
metric_list = ['MAE', 'RMSE']
objective_list = ['huber', 'gamma',
'fair', 'tweedie']
space ={'boosting' : hp.choice('boosting', boosting_list),
'num_leaves' : hp.quniform('num_leaves', 2, LGBM_MAX_LEAVES, 1),
'max_depth': hp.quniform('max_depth', 2, LGBM_MAX_DEPTH, 1),
'max_bin': hp.quniform('max_bin', 32, 255, 1),
'min_data_in_leaf': hp.quniform('min_data_in_leaf', 1, 256, 1),
'min_data_in_bin': hp.quniform('min_data_in_bin', 1, 256, 1),
'min_gain_to_split' : hp.quniform('min_gain_to_split', 0.1, 5, 0.01),
'lambda_l1' : hp.uniform('lambda_l1', 0, 5),
'lambda_l2' : hp.uniform('lambda_l2', 0, 5),
'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) ,
'metric' : hp.choice('metric', metric_list),
'objective' : hp.choice('objective', objective_list),
'feature_fraction' : hp.quniform('feature_fraction', 0.5, 1, 0.01),
'bagging_fraction' : hp.quniform('bagging_fraction', 0.5, 1, 0.01)
 }
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['boosting'] = boosting_list[best['boosting']]['boosting']
best['metric'] = metric_list[best['metric']]
best['objective'] = objective_list[best['objective']]
for param in integer_params:
best[param] = int(best[param])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 if package=='xgb':
print('Running {} rounds of XGBoost parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['max_depth']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['tree_method']['tree_method'] == 'hist':
max_bin = space_params['tree_method'].get('max_bin')
 space_params['max_bin'] = int(max_bin)
 if space_params['tree_method']['grow_policy']['grow_policy']['grow_policy'] == 'depthwise':
grow_policy = space_params['tree_method'].get('grow_policy' ).get('grow_policy' ).get('grow_policy')
 space_params['grow_policy'] = grow_policy
space_params['tree_method'] = 'hist'
else:
max_leaves = space_params['tree_method']['grow_policy']['grow_policy'].get('max_leaves')
 space_params['grow_policy'] = 'lossguide'
space_params['max_leaves'] = int(max_leaves)
 space_params['tree_method'] = 'hist'
else:
space_params['tree_method'] = space_params['tree_method'].get('tree_method')
 cv_results = xgb.cv(space_params, train, nfold=N_FOLDS, metrics=[EVAL_METRIC_XGB_REG],
early_stopping_rounds=100, stratified=False, seed=42)
 best_loss = cv_results['test-mae-mean'].iloc[-1]
return{'loss':best_loss, 'status': STATUS_OK }
train = xgb.DMatrix(data, labels)
 boosting_list = ['gbtree', 'gblinear']
metric_list = ['MAE', 'RMSE']
tree_method = [{'tree_method' : 'exact'},
{'tree_method' : 'approx'},
{'tree_method' : 'hist',
'max_bin': hp.quniform('max_bin', 2**3, 2**7, 1),
'grow_policy' : {'grow_policy': {'grow_policy':'depthwise'},
'grow_policy' : {'grow_policy':'lossguide',
'max_leaves': hp.quniform('max_leaves', 32, XGB_MAX_LEAVES, 1)}}}]
objective_list_reg = ['reg:linear', 'reg:gamma', 'reg:tweedie']
objective_list_class = ['reg:logistic', 'binary:logistic']
objective_list = objective_list_reg
space ={'boosting' : hp.choice('boosting', boosting_list),
'tree_method' : hp.choice('tree_method', tree_method),
'max_depth': hp.quniform('max_depth', 2, XGB_MAX_DEPTH, 1),
'reg_alpha' : hp.uniform('reg_alpha', 0, 5),
'reg_lambda' : hp.uniform('reg_lambda', 0, 5),
'min_child_weight' : hp.uniform('min_child_weight', 0, 5),
'gamma' : hp.uniform('gamma', 0, 5),
'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) ,
'eval_metric' : hp.choice('eval_metric', metric_list),
'objective' : hp.choice('objective', objective_list),
'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1, 0.01),
'colsample_bynode' : hp.quniform('colsample_bynode', 0.1, 1, 0.01),
'colsample_bylevel' : hp.quniform('colsample_bylevel', 0.1, 1, 0.01),
'subsample' : hp.quniform('subsample', 0.5, 1, 0.05),
'nthread' : -1
}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['tree_method'] = tree_method[best['tree_method']]['tree_method']
best['boosting'] = boosting_list[best['boosting']]
best['eval_metric'] = metric_list[best['eval_metric']]
best['objective'] = objective_list[best['objective']]
for param in integer_params:
best[param] = int(best[param])
 if 'max_leaves' in best:
best['max_leaves'] = int(best['max_leaves'])
 if 'max_bin' in best:
best['max_bin'] = int(best['max_bin'])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 if package=='cb':
print('Running {} rounds of CatBoost parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['depth',
'min_data_in_leaf',
'max_bin']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['bootstrap_type']['bootstrap_type'] == 'Bayesian':
bagging_temp = space_params['bootstrap_type'].get('bagging_temperature')
 space_params['bagging_temperature'] = bagging_temp
if space_params['grow_policy']['grow_policy'] == 'LossGuide':
max_leaves = space_params['grow_policy'].get('max_leaves')
 space_params['max_leaves'] = int(max_leaves)
 space_params['bootstrap_type'] = space_params['bootstrap_type']['bootstrap_type']
space_params['grow_policy'] = space_params['grow_policy']['grow_policy']
space_params['random_strength'] = max(space_params['random_strength'], 0)
 space_params['fold_len_multiplier'] = max(space_params['fold_len_multiplier'], 1)
 cv_results = cb.cv(train, space_params, fold_count=N_FOLDS,
early_stopping_rounds=25, stratified=False, partition_random_seed=42)
 best_loss = cv_results['test-MAE-mean'].iloc[-1]
return{'loss':best_loss, 'status': STATUS_OK}
train = cb.Pool(data, labels.astype('float32'))
bootstrap_type = [{'bootstrap_type':'Poisson'},
{'bootstrap_type':'Bayesian',
'bagging_temperature' : hp.loguniform('bagging_temperature', np.log(1), np.log(50)) },
{'bootstrap_type':'Bernoulli'}]
LEB = ['No', 'AnyImprovement', 'Armijo']
grow_policy = [{'grow_policy':'SymmetricTree'},
{'grow_policy':'Depthwise'},
{'grow_policy':'Lossguide',
'max_leaves': hp.quniform('max_leaves', 2, 32, 1)}]
eval_metric_list_reg = ['MAE', 'RMSE', 'Poisson']
eval_metric_list_class = ['Logloss', 'AUC', 'F1']
eval_metric_list = eval_metric_list_reg
space ={'depth': hp.quniform('depth', 2, CB_MAX_DEPTH, 1),
'max_bin' : hp.quniform('max_bin', 1, 32, 1),
'l2_leaf_reg' : hp.uniform('l2_leaf_reg', 0, 5),
'min_data_in_leaf' : hp.quniform('min_data_in_leaf', 1, 50, 1),
'random_strength' : hp.loguniform('random_strength', np.log(0.005), np.log(5)) ,
'bootstrap_type' : hp.choice('bootstrap_type', bootstrap_type),
'learning_rate' : hp.uniform('learning_rate', 0.05, 0.25),
'eval_metric' : hp.choice('eval_metric', eval_metric_list),
'objective' : OBJECTIVE_CB_REG,
'leaf_estimation_backtracking' : hp.choice('leaf_estimation_backtracking', LEB),
'grow_policy': hp.choice('grow_policy', grow_policy),
'fold_len_multiplier' : hp.loguniform('fold_len_multiplier', np.log(1.01), np.log(2.5)) ,
'od_type' : 'Iter',
'od_wait' : 25,
'task_type' : 'GPU',
'verbose' : 0
}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['bootstrap_type'] = bootstrap_type[best['bootstrap_type']]['bootstrap_type']
best['grow_policy'] = grow_policy[best['grow_policy']]['grow_policy']
best['eval_metric'] = eval_metric_list[best['eval_metric']]
best['leaf_estimation_backtracking'] = LEB[best['leaf_estimation_backtracking']]
for param in integer_params:
best[param] = int(best[param])
 if 'max_leaves' in best:
best['max_leaves'] = int(best['max_leaves'])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 else:
print('Package not recognised.Please use "lgbm" for LightGBM, "xgb" for XGBoost or "cb" for CatBoost.' )
df_test.index.name = 'item_id'
df_test['deal_probability'] = model.predict(df_test[used_cols], num_iteration=best_num_boost_round ).clip(0., 1.)
df_test[['deal_probability']].to_csv('submission.csv', index=True )<import_modules>
def quick_kfold_imp(X, y, test=None, params=None, n_fold=6, random_state=1127):
folds = KFold(n_splits=n_fold, shuffle=True, random_state=random_state)
 test_preds = np.zeros(len(test))
oof_preds = np.zeros(len(X))
imp_s = np.zeros(X.shape[1])
 imp_g = np.zeros(X.shape[1])
 if type(y)is not np.ndarray:
y = y.values.flatten()
for train_idx, valid_idx in folds.split(y):
X_train, X_valid = X.iloc[train_idx, :], X.iloc[valid_idx, :]
y_train, y_valid = y[train_idx], y[valid_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1, eval_metric='mae', importance_type='split')
 model.fit(X_train, y_train,eval_set=[(X_train, y_train),(X_valid, y_valid)],
verbose=0, early_stopping_rounds=200)
 val_preds = model.predict(X_valid)
 imp_s += model.feature_importances_/n_fold
model = lgb.LGBMRegressor(**params,
n_estimators = 50000,
n_jobs = -1, eval_metric='mae',
importance_type='gain')
 model.fit(X_train, y_train,eval_set=[(X_train, y_train),(X_valid, y_valid)],
verbose=0, early_stopping_rounds=200)
 oof_preds[valid_idx] = model.predict(X_valid)
 imp_g += model.feature_importances_/n_fold
test_preds += model.predict(test)/n_fold
print('OOF MAE: {:.7f}'.format(mean_absolute_error(y[:NUM_SEGMENTS],
oof_preds[:NUM_SEGMENTS])))
 return imp_s, imp_g, test_preds, oof_preds
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt<load_from_csv>
_train = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/train_X.csv')
 _test = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/test_X.csv')
 y = pd.read_csv('.. /input/lanl-features/y.csv' ).values.flatten() [:-1]
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]<define_variables>
train = _train
test = _test
NUM_SEGMENTS = len(y )
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples<define_variables>
refined_set = ['abs_max_roll_mean_1000',
'abs_min',
'abs_q01',
'abs_trend',
'autocorrelation_1000',
'av_change_abs_roll_std_10',
'avg_first_10000',
'avg_first_50000',
'c3_5',
'classic_sta_lta4_mean',
'count_big_50000_threshold_5',
'energy_spectra_22640hz',
'energy_spectra_9306hz',
'energy_spectra_lowest100_denoised',
'energy_spectra_norm_109306hz',
'energy_spectra_norm_149306hz_denoised',
'energy_spectra_norm_15973hz_denoised',
'energy_spectra_norm_169306hz_denoised',
'energy_spectra_norm_22640hz_denoised',
'energy_spectra_norm_29306hz_denoised',
'energy_spectra_norm_49306hz_denoised',
'energy_spectra_norm_62640hz_denoised',
'energy_spectra_norm_69306hz',
'energy_spectra_norm_89306hz_denoised',
'energy_spectra_norm_9306hz_denoised',
'energy_spectra_norm_95973hz_denoised',
'fft_100_roll_std_70',
'fft_mean_change_rate',
'fft_min_roll_mean_100',
'fft_spkt_welch_density_100',
'fft_spkt_welch_density_5',
'fft_time_rev_asym_stat_10',
'iqr',
'kstat_3',
'kurt',
'mad',
'max_first_5000',
'max_last_10000',
'max_to_min',
'mean',
'med',
'mean_change_abs',
'num_crossings',
'num_peaks_10',
'q01_roll_std_1000',
'q05_roll_mean_100',
'q05_roll_std_1000',
'q95_roll_mean_100',
'q99_roll_mean_1000',
'skew',
'std_0_to_10',
'std_neg_10_to_0',
'std_neg_2_to_2']
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=256, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in pneumonia_locations:
for location in pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )<define_search_model>
train = _train[refined_set]
test = _test[refined_set]
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.99 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.99 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.99 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.99 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
ref_params = {'bagging_fraction': 0.71,
'boosting': 'gbdt',
'feature_fraction': 0.76,
'lambda_l1': 3.131216244016188,
'lambda_l2': 2.4124061313905836,
'learning_rate': 0.049886848207269734,
'max_bin': 193,
'max_depth': 15,
'metric': 'MAE',
'min_data_in_bin': 167,
'min_data_in_leaf': 62,
'min_gain_to_split': 2.07,
'num_leaves': 38,
'objective': 'fair',
'subsample': 0.9133120405819966}
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 20
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=20, shuffle=True )<predict_on_test>
lgbm_params = quick_hyperopt(train, y, 'lgbm', 150 )
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=25, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )<import_modules>
train_features = train.columns
imp_split = np.zeros(train.shape[1])
 imp_gain = np.zeros(train.shape[1])
 test_preds = np.zeros(len(test))
train_preds = np.zeros(len(train))
N = 30
for i in tqdm_notebook(range(N)) :
imp_s, imp_g, preds, oof_preds = quick_kfold_imp(train, y, test,
lgbm_params, random_state=i)
 imp_split += imp_s/N
imp_gain += imp_g/N
test_preds += preds/N
train_preds += oof_preds/N
initial_imp = pd.DataFrame({'feature':train_features,
'importance_split':imp_split,
'importance_gain':imp_gain,
'importance_score':np.log(( imp_split*imp_gain)) })
 initial_imp.sort_values('importance_score', ascending=False, inplace=True)
 initial_imp.head(10 )
PATH = ".. /input/rsna-pneumonia-detection-challenge/"
print("Section 1 completed" )<import_modules>
index1 =(y>11)
 index2 = np.logical_and(y<11, y>9)
 index3 = np.logical_and(y<9, y>7)
 index4 = np.logical_and(y<7, y>5)
 index5 = np.logical_and(y<5, y>3)
 index6 = np.logical_and(y<3, y>1)
 index7 =(y<1 )
try:
except ImportError:
class Scale(Layer):

 def __init__(self, weights=None, axis=-1, momentum = 0.9, beta_init='zero', gamma_init='one', **kwargs):
self.momentum = momentum
self.axis = axis
self.beta_init = initializations.get(beta_init)
 self.gamma_init = initializations.get(gamma_init)
 self.initial_weights = weights
super(Scale, self ).__init__(**kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
shape =(int(input_shape[self.axis]),)
 self.gamma = K.variable(self.gamma_init(shape), name='{}_gamma'.format(self.name))
self.beta = K.variable(self.beta_init(shape), name='{}_beta'.format(self.name))
self.trainable_weights = [self.gamma, self.beta]
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
 del self.initial_weights
def call(self, x, mask=None):
input_shape = self.input_spec[0].shape
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis]
out = K.reshape(self.gamma, broadcast_shape)* x + K.reshape(self.beta, broadcast_shape)
 return out
def get_config(self):
config = {"momentum": self.momentum, "axis": self.axis}
base_config = super(Scale, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
print("Section 2 Scale Layer completed" )<load_from_csv>
print("OOF MAE for different time_to_failure:")
 print("MAE for ttf>11 = {:.5f}".format(mean_absolute_error(y[index1], train_preds[index1])))
 print("MAE for 9<ttf<11 = {:.5f}".format(mean_absolute_error(y[index2], train_preds[index2])))
 print("MAE for 7<ttf<9 = {:.5f}".format(mean_absolute_error(y[index3], train_preds[index3])))
 print("MAE for 5<ttf<7 = {:.5f}".format(mean_absolute_error(y[index4], train_preds[index4])))
 print("MAE for 3<ttf<5 = {:.5f}".format(mean_absolute_error(y[index5], train_preds[index5])))
 print("MAE for 1<ttf<3 = {:.5f}".format(mean_absolute_error(y[index6], train_preds[index6])))
 print("MAE for ttf<1 = {:.5f}".format(mean_absolute_error(y[index7], train_preds[index7])) )
pneumonia_locations = {}
with open(os.path.join(PATH+'stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]
folder = PATH+'stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples
print('Total train images:',len(filenames))
print('Images with pneumonia:', len(pneumonia_locations))
ns = [len(value)for value in pneumonia_locations.values() ]
plt.figure()
plt.hist(ns)
 plt.xlabel('Pneumonia per image')
 plt.xticks(range(1, np.max(ns)+1))
plt.show()
heatmap = np.zeros(( 1024, 1024))
ws = []
hs = []
for values in pneumonia_locations.values() :
for value in values:
x, y, w, h = value
heatmap[y:y+h, x:x+w] += 1
ws.append(w)
 hs.append(h)
 plt.figure()
plt.title('Pneumonia location heatmap')
 plt.imshow(heatmap)
 plt.figure()
plt.title('Pneumonia height lengths')
 plt.hist(hs, bins=np.linspace(0,1000,50))
plt.show()
plt.figure()
plt.title('Pneumonia width lengths')
 plt.hist(ws, bins=np.linspace(0,1000,50))
plt.show()
print('Minimum pneumonia height:', np.min(hs))
print('Minimum pneumonia width: ', np.min(ws))
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=8, image_size=256, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in pneumonia_locations:
for location in pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 img=np.stack([img] * 3, axis=-1)
 msk=np.stack([msk]*3, axis=-1)
 img = img.reshape(256,256,-1)
 msk= msk.reshape(256,256,-1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size)
 def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = resnet152_model(img_rows=256, img_cols=256, color_type=3, num_classes=10)

sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
 model.compile(optimizer=sgd, loss= iou_bce_loss, metrics=['accuracy', mean_iou])
 print("Section 4 Original Skeleton completed" )<train_model>
imp_idx = list(initial_imp.feature.values[:50] )
def cosine_annealing(x):
lr = 0.01
epochs = 10
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = PATH+'stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=8, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=8, image_size=256, shuffle=False, predict=False)
 with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=10, shuffle=True)
<define_variables>
np.abs(train[imp_idx].corrwith(train['energy_spectra_norm_89306hz_denoised'])) \ 
.sort_values(ascending=False)[:10]
folder = PATH+'stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=8, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submissionResNet152.csv')
<import_modules>
np.abs(train[imp_idx].corrwith(train['num_peaks_10'])).sort_values(ascending=False)[:10]
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt<load_from_csv>
np.abs(train[imp_idx].corrwith(train['fft_100_roll_std_70'])).sort_values(ascending=False)[:10]
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]<define_variables>
sub_1 = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sub_1['time_to_failure'] = test_preds
sub_1.to_csv('sub_orginal_data.csv', index=False )
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples<define_variables>
a = np.arange(0, train.shape[1])
 train_aug = pd.DataFrame(index=train_select.index, columns=train.columns, dtype='float64')
 for i in tqdm_notebook(range(len(train_select))):
AUG_FEATURE_RATIO = 0.6
AUG_FEATURE_COUNT = np.floor(train_select.shape[1]*AUG_FEATURE_RATIO ).astype('int16')
 aug_feature_index = np.random.choice(train_select.shape[1], AUG_FEATURE_COUNT, replace=False)
 aug_feature_index.sort()
feature_index = np.where(np.logical_not(np.in1d(a, aug_feature_index)))[0]
train_aug.iloc[i, feature_index] = train_select.iloc[i, feature_index]
rand_row_index = np.random.choice(len(train_select), len(aug_feature_index), replace=True)
 for n, j in enumerate(aug_feature_index):
train_aug.iloc[i, j] = train_select.iloc[rand_row_index[n], j]
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=256, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in pneumonia_locations:
for location in pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )<define_search_model>
params_all = quick_hyperopt(train_all, y_all, 'lgbm', 150 )
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
imp_split_all = np.zeros(train_aug.shape[1])
 imp_gain_all = np.zeros(train_aug.shape[1])
 test_preds_aug = np.zeros(len(test))
train_preds_aug = np.zeros(len(train_all))
N = 30
for i in tqdm_notebook(range(N)) :
a = np.arange(0, train_select.shape[1])
 np.random.seed(i)
 train_aug = pd.DataFrame(index=train_select.index, columns=train.columns, dtype='float64')
 for i in range(0, len(train_select)) :
AUG_FEATURE_RATIO = 0.6
AUG_FEATURE_COUNT = np.floor(train_select.shape[1]*AUG_FEATURE_RATIO ).astype('int16')
 aug_feature_index = np.random.choice(train_select.shape[1], AUG_FEATURE_COUNT, replace=False)
 aug_feature_index.sort()
feature_index = np.where(np.logical_not(np.in1d(a, aug_feature_index)))[0]
train_aug.iloc[i, feature_index] = train_select.iloc[i, feature_index]
rand_row_index = np.random.choice(len(train_select), len(aug_feature_index), replace=True)
 for n, j in enumerate(aug_feature_index):
train_aug.iloc[i, j] = train_select.iloc[rand_row_index[n], j]
train_all = pd.concat([train, train_aug])
 imp_s, imp_g, preds, oof_preds = quick_kfold_imp(train_all, y_all, test,
params=params_all, random_state=i)
 imp_split_all += imp_s/N
imp_gain_all += imp_g/N
test_preds_aug += preds/N
train_preds_aug += oof_preds/N
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 20
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=20, shuffle=True )<predict_on_test>
_train_preds_aug = train_preds_aug[:NUM_SEGMENTS]
print("OOF mae for different time_to_failure after data augmentation:")
 print("MAE for ttf>11 = {:.5f}".format(mean_absolute_error(y[index1], _train_preds_aug[index1])))
 print("MAE for 9<ttf<11 = {:.5f}".format(mean_absolute_error(y[index2], _train_preds_aug[index2])))
 print("MAE for 7<ttf<9 = {:.5f}".format(mean_absolute_error(y[index3], _train_preds_aug[index3])))
 print("MAE for 5<ttf<7 = {:.5f}".format(mean_absolute_error(y[index4], _train_preds_aug[index4])))
 print("MAE for 3<ttf<5 = {:.5f}".format(mean_absolute_error(y[index5], _train_preds_aug[index5])))
 print("MAE for 1<ttf<3 = {:.5f}".format(mean_absolute_error(y[index6], _train_preds_aug[index6])))
 print("MAE for ttf<1 = {:.5f}".format(mean_absolute_error(y[index7], _train_preds_aug[index7])) )
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=25, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )<load_from_csv>
aug_imp = pd.DataFrame({'feature':train_features,
'importance_split':imp_split_all,
'importance_gain':imp_gain_all,
'importance_score':np.log(( imp_split_all*imp_gain_all)) } )
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 train<define_variables>
aug_imp.sort_values('importance_score', ascending=False, inplace=True)
 list(aug_imp.feature.values[:20] )
num_col = train.loc[:,'MSSubClass':'SaleCondition'].select_dtypes(exclude=['object'] ).columns
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2, num_col)
 train.loc[Outliers_to_drop]<drop_column>
sub_aug = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sub_aug['time_to_failure'] = test_preds_aug
sub_aug.to_csv('sub_aug_data.csv', index=False )
train = train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<prepare_x_and_y>
from sklearn.preprocessing import MinMaxScaler
test_id = test.Id
train_target = train.SalePrice<concatenate>
initial_imp.loc[initial_imp.importance_score<-100, 'importance_score'] = -50
aug_imp.loc[aug_imp.importance_score<-100,'importance_score'] = -100
scaler = MinMaxScaler()
initial_imp.iloc[:, 1:] = scaler.fit_transform(initial_imp.iloc[:, 1:])
 scaler = MinMaxScaler()
aug_imp.iloc[:, 1:] = scaler.fit_transform(aug_imp.iloc[:, 1:] )
df = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'], test.loc[:,'MSSubClass':'SaleCondition']))<feature_engineering>
aug_cols = ['feature'] + [x + '_all' for x in aug_imp.columns if x != 'feature']
aug_imp.columns = aug_cols
feature_df = initial_imp.merge(aug_imp, on='feature', how='inner')
 feature_df['score_change'] = feature_df['importance_score_all'] - feature_df['importance_score']
feature_df.sort_values('score_change', ascending=False, inplace=True)
 feature_df.head(10 )
def basic_details(df):
b = pd.DataFrame()
b['Missing value, %'] = round(df.isnull().sum() /df.shape[0]*100)
 b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(df )<data_type_conversions>
MIN_SCORE = np.percentile(feature_df.score_change.values.flatten() , 20)
 best_features = feature_df.loc[feature_df.score_change >= MIN_SCORE, :].feature.values.flatten()
train = train[best_features]
test = test[best_features]
train_select = train[index_augment]
for col in df:
if df[col].dtype == 'object':
df[col].fillna('N', inplace=True)
 else: df[col].fillna(df[col].median() , inplace=True )<categorify>
params_final = quick_hyperopt(train, y, 'lgbm', 150 )
columns = [i for i in df.columns]
dummies = pd.get_dummies(df,columns=columns, drop_first=True,sparse=True )<categorify>
train_preds_final = np.zeros(len(train_all))
test_preds_final = np.zeros(len(test))
N = 30
for i in tqdm_notebook(range(N)) :
a = np.arange(0, train_select.shape[1])
 np.random.seed(i)
 train_aug = pd.DataFrame(index=train_select.index, columns=train.columns, dtype='float64')
 for i in range(len(train_select)) :
AUG_FEATURE_RATIO = 0.6
AUG_FEATURE_COUNT = np.floor(train_select.shape[1]*AUG_FEATURE_RATIO ).astype('int16')
 aug_feature_index = np.random.choice(train_select.shape[1], AUG_FEATURE_COUNT, replace=False)
 aug_feature_index.sort()
feature_index = np.where(np.logical_not(np.in1d(a, aug_feature_index)))[0]
train_aug.iloc[i, feature_index] = train_select.iloc[i, feature_index]
rand_row_index = np.random.choice(len(train_select), len(aug_feature_index), replace=True)
 for n, j in enumerate(aug_feature_index):
train_aug.iloc[i, j] = train_select.iloc[rand_row_index[n], j]
train_all = pd.concat([train, train_aug])
 _, _, preds, tr_preds = quick_kfold_imp(train_all, y_all, test,
params=params_final, random_state=i)
 test_preds_final += preds/N
train_preds_final += tr_preds/N
train = dummies.iloc[:train.shape[0], :]
test = dummies.iloc[train.shape[0]:, :]<prepare_x_and_y>
sub_final = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sub_final['time_to_failure'] = test_preds_final
sub_final.to_csv('sub_quintile_features_removed.csv', index=False )
<split><EOS> 
_train_preds_final = train_preds_final[:NUM_SEGMENTS]
print("OOF mae for different time_to_failure for refined data:")
 print("MAE for ttf>11 = {:.5f}".format(mean_absolute_error(y[index1], _train_preds_final[index1])))
 print("MAE for 9<ttf<11 = {:.5f}".format(mean_absolute_error(y[index2], _train_preds_final[index2])))
 print("MAE for 7<ttf<9 = {:.5f}".format(mean_absolute_error(y[index3], _train_preds_final[index3])))
 print("MAE for 5<ttf<7 = {:.5f}".format(mean_absolute_error(y[index4], _train_preds_final[index4])))
 print("MAE for 3<ttf<5 = {:.5f}".format(mean_absolute_error(y[index5], _train_preds_final[index5])))
 print("MAE for 1<ttf<3 = {:.5f}".format(mean_absolute_error(y[index6], _train_preds_final[index6])))
 print("MAE for ttf<1 = {:.5f}".format(mean_absolute_error(y[index7], _train_preds_final[index7])) )