Kaggle competition: LANL Earthquake Prediction
def threeMostCommon(listt):
return(' ' ).join([str(val[0])for val in Counter(listt ).most_common(3)] )<create_dataframe>
warnings.filterwarnings('ignore' )
pivotedTrain = trainDF.pivot_table('place_id','identifier', aggfunc = {'place_id':threeMostCommon} ).reset_index() 
<save_to_csv>
train_data = pd.read_csv('.. /input/train.csv', dtype = {'acoustic_data': np.int16, 'time_to_failure': np.float32} )
out = pd.merge(testDF[['row_id','identifier']], pivotedTrain, how = 'left')
 out.sort('row_id', inplace = True)
 out.drop(['identifier'],axis = 1 ).fillna(0 ).to_csv('output.csv',index = False )<install_modules>
seg_length = 150000
total_samples = int(np.floor(( train_data.shape[0])/ seg_length))
COLUMNS = ['AVERAGE', 'STD', 'MAX', 'MIN', 'SUM',
'AVERAGE_FIRST_10000', 'AVERAGE_LAST_10000', 'AVERAGE_FIRST_50000', 'AVERAGE_LAST_50000',
'STD_FIRST_10000', 'STD_LAST_10000', 'STD_FIRST_50000', 'STD_LAST_50000',
'ABS_AVERAGE', 'ABS_STD', 'ABS_MAX', 'ABS_MIN',
'10Q', '25Q', '50Q', '75Q', '90Q',
'ABS_1Q', 'ABS_5Q', 'ABS_30Q', 'ABS_60Q', 'ABS_95Q', 'ABS_99Q',
'KURTOSIS', 'SKEW', 'MEDIAN',
'HILBERT_MEAN', 'HANN_WINDOW_MEAN']
cols = COLUMNS
x_train = pd.DataFrame(index = range(total_samples), columns = cols, dtype = np.float64)
 y_train = pd.DataFrame(index = range(total_samples), columns = ['time_to_failure'], dtype = np.float64 )
!pip install efficientnet > /dev/null<load_pretrained>
for value in tqdm_notebook(range(total_samples)) :
sample = train_data.iloc[value*seg_length : value*seg_length + seg_length]
x = pd.Series(sample['acoustic_data'].values)
 y = sample['time_to_failure'].values[-1]
y_train.loc[value, 'time_to_failure'] = y
x_train.loc[value, 'AVERAGE'] = x.mean()
x_train.loc[value, 'STD'] = x.std()
x_train.loc[value, 'MAX'] = x.max()
x_train.loc[value, 'MIN'] = x.min()
x_train.loc[value, 'SUM'] = x.sum()
x_train.loc[value, 'AVERAGE_FIRST_10000'] = x[:10000].mean()
x_train.loc[value, 'AVERAGE_LAST_10000'] = x[-10000:].mean()
x_train.loc[value, 'AVERAGE_FIRST_50000'] = x[:50000].mean()
x_train.loc[value, 'AVERAGE_LAST_50000'] = x[-50000:].mean()
x_train.loc[value, 'STD_FIRST_10000'] = x[:10000].std()
x_train.loc[value, 'STD_LAST_10000'] = x[-10000:].std()
x_train.loc[value, 'STD_FIRST_50000'] = x[:50000].std()
x_train.loc[value, 'STD_LAST_50000'] = x[-50000:].std()
x_train.loc[value, 'ABS_AVERAGE'] = np.abs(x ).mean()
x_train.loc[value, 'ABS_STD'] = np.abs(x ).std()
x_train.loc[value, 'ABS_MAX'] = np.abs(x ).max()
x_train.loc[value, 'ABS_MIN'] = np.abs(x ).min()
x_train.loc[value, '10Q'] = np.percentile(x, 0.10)
 x_train.loc[value, '25Q'] = np.percentile(x, 0.25)
 x_train.loc[value, '50Q'] = np.percentile(x, 0.50)
 x_train.loc[value, '75Q'] = np.percentile(x, 0.75)
 x_train.loc[value, '90Q'] = np.percentile(x, 0.90)
 x_train.loc[value, 'ABS_1Q'] = np.percentile(x, np.abs(0.01))
x_train.loc[value, 'ABS_5Q'] = np.percentile(x, np.abs(0.05))
x_train.loc[value, 'ABS_30Q'] = np.percentile(x, np.abs(0.30))
x_train.loc[value, 'ABS_60Q'] = np.percentile(x, np.abs(0.60))
x_train.loc[value, 'ABS_95Q'] = np.percentile(x, np.abs(0.95))
x_train.loc[value, 'ABS_99Q'] = np.percentile(x, np.abs(0.99))
x_train.loc[value, 'KURTOSIS'] = x.kurtosis()
x_train.loc[value, 'SKEW'] = x.skew()
x_train.loc[value, 'MEDIAN'] = x.median()
x_train.loc[value, 'HILBERT_MEAN'] = np.abs(hilbert(x)).mean()
x_train.loc[value, 'HANN_WINDOW_MEAN'] =(convolve(x, hann(150), mode = 'same')/ sum(hann(150)) ).mean()
SEED = 420
def decode_image(filename, label=None, image_size=None):
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
if image_size:
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=SEED):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label
def show_batch(ds, labels):
row = 6; col = 4;
row = min(row, BATCH_SIZE//col)
 for(img, label)in ds:
plt.figure(figsize=(15,int(15*row/col)))
 for j in range(row*col):
plt.subplot(row,col,j+1)
 plt.axis('off')
 plt.imshow(img[j,])
 plt.title(labels[label[j,].numpy().argmax() ])
 plt.show()
break<set_options>
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error
AUTO = tf.data.experimental.AUTOTUNE
def get_strategy() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 return strategy
strategy = get_strategy()<define_variables>
scaler = StandardScaler()
scaler.fit(x_train)
 x_train_scaled = scaler.transform(x_train)
 y_train_flatten = y_train.values.ravel()
LABELS = [
'healthy', 'Cedar_apple_rust', 'Black_rot', 'Apple_scab',
]
IMG_PATH = Path('.. /input/plantvillageapplecolor' )<categorify>
x_train_dataframe = pd.DataFrame(x_train_scaled)
 x_train_dataframe.head(10 )
all_img_ids = []
all_img_labels = []
for l in LABELS:
img_ids = [
'/'.join(l.parts[-2:])
 for l in(IMG_PATH /('Apple___' + l)).iterdir()
]
all_img_ids.extend(img_ids)
 all_img_labels.extend([l] * len(img_ids))
train_df = pd.DataFrame({'img_id': all_img_ids, 'img_label': all_img_labels})
 train_df.img_label = pd.Categorical(train_df.img_label, categories=LABELS, ordered=False)
 train_df = pd.concat([train_df, pd.get_dummies(train_df.img_label)], axis=1)
 train_df = train_df.sample(frac=1., random_state=SEED )<define_variables>
import keras
from keras.layers import Dense
from keras.layers import BatchNormalization
from keras.layers import Activation
from keras.layers import Dropout
from keras.models import Sequential
from keras import optimizers
NB_CLASSES = 4
IMG_SIZE = 256
EPOCHS = 40
BATCH_SIZE = 128 * strategy.num_replicas_in_sync
N_FOLDS = 5<define_variables>
kernel_init = 'he_normal'
def Model(input_dim, activation, classes):
model = Sequential()
model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(classes, kernel_initializer = kernel_init))
model.add(Activation('linear'))
return model
GCS_DS_PATH = KaggleDatasets().get_gcs_path('plantvillageapplecolor' )<categorify>
input_dim = x_train.shape[1]
activation = 'tanh'
classes = 1
model = Model(input_dim = input_dim, activation = activation, classes = classes)
train_labels = train_df[LABELS].values.astype(np.int64)
 train_paths = train_df.img_id.apply(lambda x: GCS_DS_PATH + '/' + x ).values
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(partial(decode_image, image_size=(IMG_SIZE, IMG_SIZE)) , num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<define_variables>
optim = optimizers.Adam(lr = 0.001)
 model.compile(loss = 'mean_absolute_error', optimizer = optim, metrics = ['mean_absolute_error'] )
show_batch(train_dataset, LABELS )<init_hyperparams>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col = 'seg_id')
 X_test = pd.DataFrame(columns = x_train.columns, dtype = np.float64, index = submission.index)
 for id in X_test.index:
seg = pd.read_csv('.. /input/test/' + id + '.csv')
 x = pd.Series(seg['acoustic_data'].values)
 X_test.loc[id, 'AVERAGE'] = x.mean()
X_test.loc[id, 'STD'] = x.std()
X_test.loc[id, 'MAX'] = x.max()
X_test.loc[id, 'MIN'] = x.min()
X_test.loc[id, 'SUM'] = x.sum()
X_test.loc[id, 'AVERAGE_FIRST_10000'] = x[:10000].mean()
X_test.loc[id, 'AVERAGE_LAST_10000'] = x[-10000:].mean()
X_test.loc[id, 'AVERAGE_FIRST_50000'] = x[:50000].mean()
X_test.loc[id, 'AVERAGE_LAST_50000'] = x[-50000:].mean()
X_test.loc[id, 'STD_FIRST_10000'] = x[:10000].std()
X_test.loc[id, 'STD_LAST_10000'] = x[-10000:].std()
X_test.loc[id, 'STD_FIRST_50000'] = x[:50000].std()
X_test.loc[id, 'STD_LAST_50000'] = x[-50000:].std()
X_test.loc[id, 'ABS_AVERAGE'] = np.abs(x ).mean()
X_test.loc[id, 'ABS_STD'] = np.abs(x ).std()
X_test.loc[id, 'ABS_MAX'] = np.abs(x ).max()
X_test.loc[id, 'ABS_MIN'] = np.abs(x ).min()
X_test.loc[id, '10Q'] = np.percentile(x, 0.10)
 X_test.loc[id, '25Q'] = np.percentile(x, 0.25)
 X_test.loc[id, '50Q'] = np.percentile(x, 0.50)
 X_test.loc[id, '75Q'] = np.percentile(x, 0.75)
 X_test.loc[id, '90Q'] = np.percentile(x, 0.90)
 X_test.loc[id, 'ABS_1Q'] = np.percentile(x, np.abs(0.01))
X_test.loc[id, 'ABS_5Q'] = np.percentile(x, np.abs(0.05))
X_test.loc[id, 'ABS_30Q'] = np.percentile(x, np.abs(0.30))
X_test.loc[id, 'ABS_60Q'] = np.percentile(x, np.abs(0.60))
X_test.loc[id, 'ABS_95Q'] = np.percentile(x, np.abs(0.95))
X_test.loc[id, 'ABS_99Q'] = np.percentile(x, np.abs(0.99))
X_test.loc[id, 'KURTOSIS'] = x.kurtosis()
X_test.loc[id, 'SKEW'] = x.skew()
X_test.loc[id, 'MEDIAN'] = x.median()
X_test.loc[id, 'HILBERT_MEAN'] = np.abs(hilbert(x)).mean()
X_test.loc[id, 'HANN_WINDOW_MEAN'] =(convolve(x, hann(150), mode = 'same')/ sum(hann(150)) ).mean()
LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 15
LR_SUSTAIN_EPOCHS = 3
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
def get_model() :
base_model = efn.EfficientNetB7(weights='imagenet', include_top=False, pooling='avg', input_shape=(None, None, 3))
x = base_model.output
predictions = Dense(NB_CLASSES, activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions )<train_on_grid>
X_test_scaled = scaler.transform(X_test)
 print(X_test_scaled.shape )
strategy = get_strategy()
with strategy.scope() :
model = get_model()
model.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=[tf.keras.metrics.AUC() ])
 model.fit(
 train_dataset,
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=[tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)],
epochs=EPOCHS
)<save_model>
input_dim = x_train.shape[1]
activation = 'tanh'
classes = 1
history = dict()
prediction_scores = dict()
for i in np.arange(0, 10):
optim = optimizers.Adam(lr = 0.001)
 ensemble_model = Model(input_dim = input_dim, activation = activation, classes = classes)
 ensemble_model.compile(loss = 'mean_absolute_error', optimizer = optim, metrics = ['mean_absolute_error'])
 print('TRAINING MODEL NO : {}'.format(i))
H = ensemble_model.fit(x_train_scaled, y_train_flatten,
batch_size = 64,
epochs = 200,
verbose = 1)
 history[i] = H
ensemble_model.save('MODEL_{}.model'.format(i))
predictions = ensemble_model.predict(X_test_scaled, verbose = 1, batch_size = 64)
 prediction_scores[i] = predictions
model.save("plant_village_pretrain_b7.h5" )<drop_column>
prediction1 = np.hstack([p.reshape(-1,1)for p in prediction_scores.values() ])
 prediction1 = np.mean(prediction1, axis = 1)
 print(prediction1.shape )
clear_session()
del model
del train_dataset
del train_labels
del strategy<set_options>
import lightgbm as lgb
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
gc.collect()<find_best_params>
params = {'boosting_type': 'gbdt',
'objective': 'regression_l1',
'nthread': 4,
'num_leaves': 64,
'learning_rate': 0.05,
'max_bin': 512,
'colsample_bytree': 1,
'reg_alpha': 0.1,
'reg_lambda': 0.1,
'min_split_gain': 0.1,
'min_child_weight': 0.1,
'min_child_samples': 20,
'metric' : 'mae'}
strategy = get_strategy()<define_variables>
gbm = lgb.LGBMRegressor(boosting_type= 'gbdt',
objective = 'regression_l1',
n_jobs = 4,
silent = True,
max_bin = params['max_bin'],
min_split_gain = params['min_split_gain'],
min_child_weight = params['min_child_weight'],
min_child_samples = params['min_child_samples'] )
NB_CLASSES = 4
IMG_SIZE = 768
EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
LABEL_COLS = ['healthy', 'multiple_diseases', 'rust', 'scab']<define_variables>
folds = KFold(n_splits = 12, shuffle = True, random_state = 101)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(x_train)) :
print('Fold', fold_n)
 X_train_f, X_valid = x_train.iloc[train_index], x_train.iloc[valid_index]
y_train_f, y_valid = y_train.iloc[train_index], y_train.iloc[valid_index]
gbm.fit(X_train_f, y_train_f.values.flatten() ,
eval_set=[(X_valid, y_valid.values.flatten())],
eval_metric = 'mae',
early_stopping_rounds = 200)
 print('Starting predicting...')
 y_pred = gbm.predict(X_valid, num_iteration = gbm.best_iteration_)
 print('The mae of prediction is:', mean_absolute_error(y_valid.values.flatten() , y_pred))
print('Feature importances:', list(gbm.feature_importances_))
GCS_DS_PATH = KaggleDatasets().get_gcs_path('plant-pathology-2020-fgvc7' )<load_from_csv>
submission['time_to_failure'] = prediction1
submission.to_csv('ensemble_submission.csv' )
path = '.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values<categorify>
lgbm_predictions = gbm.predict(X_test )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(partial(decode_image, image_size=(IMG_SIZE, IMG_SIZE)) , num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<categorify>
submission['time_to_failure'] = lgbm_predictions
print(submission.head())
 submission.to_csv('lgbm_submission.csv' )
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(partial(decode_image, image_size=(IMG_SIZE, IMG_SIZE)) , num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<define_variables>
submission['time_to_failure'] = 0.5 * lgbm_predictions + 0.5 * prediction1
submission.to_csv('lgbm_ensemble_submission.csv' )
show_batch(train_dataset, LABEL_COLS )<choose_model_class>
submission['time_to_failure'] = 0.2 * lgbm_predictions + 0.8 * prediction1
submission.to_csv('0.2lgbm_ensemble_0.8submission.csv' )
<choose_model_class><EOS> 
submission['time_to_failure'] = 0.8 * lgbm_predictions + 0.2 * prediction1
submission.to_csv('0.8lgbm_ensemble_0.2submission.csv' )