Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<install_modules>
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 print(os.listdir(".. /input"))
print(os.listdir(".. /input/gplearnmodels"))
!pip install -q efficientnet<import_modules>
df=pd.read_csv('.. /input/andrewstsfreshv19/ExtFeaturestrain.csv')
 test=pd.read_csv('.. /input/andrewstsfreshv19/ExtFeaturestest.csv')
 print(df.shape)
 X=df[df.columns.difference(['time_to_failure'])]
y=df['time_to_failure']
y_tr=y
scaler=MinMaxScaler(feature_range=(0, 1))
scaled=scaler.fit_transform(X.append(test))
train=pd.DataFrame(scaled[:X.shape[0]], columns=X.columns.values)
 test=pd.DataFrame(scaled[-test.shape[0]:], columns=X.columns.values)
 trainX, testX, trainy, testy = train_test_split(train, y, test_size=0.0,shuffle=True,random_state=42)
 X_tr_scaled = trainX
X_test_scaled = test
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from sklearn import metrics
from sklearn.model_selection import train_test_split<choose_model_class>
est_gp = SymbolicRegressor(population_size=10000,
tournament_size=50,
generations=1000,
stopping_criteria=0.0,
p_crossover=0.9, p_subtree_mutation=0.0001, p_hoist_mutation=0.0001, p_point_mutation=0.0001,
max_samples=1.0, verbose=1,
function_set =('tan', 'add', 'sub', 'mul', 'div','max', 'min'),
metric = 'mean absolute error', warm_start=True,
n_jobs = -1, parsimony_coefficient=0.00003, random_state=11 )
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync<define_variables>
with open('.. /input/gplearnmodels/gp_model.pkl', 'rb')as f:
est_gp = pickle.load(f )
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'<load_from_csv>
n=850
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
 signal.alarm(n)
 try:
est_gp.fit(train, y)
 except TimeoutException:
print('function terminated' )
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.15, random_state=2020 )<categorify>
print("This model was pretrained for {} generations".format(est_gp.run_details_['generation'][-1]))
est_gp.set_params(generations=est_gp.run_details_['generation'][-1]+1)
 est_gp.fit(train, y )
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def batchtest_to_numpy_images_and_labels(data):
images= data
numpy_images = images.numpy()
return numpy_images,None
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
tempo=""
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
if(label[0]==1):
tempo="healthy"
elif(label[1]==1):
tempo="multiple_diseases"
elif(label[2]==1):
tempo="rust"
else:
tempo="scab"
title = '' if label is None else tempo
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'])
 def displaytest_batch_of_images(databatch, predictions=None):

 images,labels = batchtest_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
tempo=""
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = ''
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()<normalization>
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
 submission.time_to_failure = est_gp.predict(X_test_scaled)
 submission.to_csv('submission.csv', index=True)
 submission.head()
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label<categorify>
params={'bagging_fraction': 0.63,
'boosting': 'gbdt',
'feature_fraction': 0.31,
'lambda_l1': 2.622427756417558,
'lambda_l2': 2.624427931714477,
'learning_rate': 0.041111529297691546,
'max_bin': 133,
'max_depth': 17,
'metric': 'RMSE',
'min_data_in_bin': 194,
'min_data_in_leaf': 11,
'num_leaves': 1058,
'objective': 'gamma',
'subsample': 0.9577983565538245}
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<define_variables>
prelgb1 = np.zeros(len(train))
mae = np.array([])
 predictions0 = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = train.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(trainX,trainy.values)) :
strLog = "fold {}".format(fold_)
 X_tr, X_val = trainX.iloc[trn_idx], trainX.iloc[val_idx]
y_tr, y_val = trainy.iloc[trn_idx], trainy.iloc[val_idx]
print(strLog)
 model0 = lgb.LGBMRegressor(**params, n_estimators = 10000, n_jobs = -1)
 model0.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)],
eval_metric='mae',
verbose=500,
early_stopping_rounds=100)
 mae = np.append(mae, mean_absolute_error(y_val, model0.predict(X_val)))
 print("Fold: {} Validation MAE: {}".format(strLog,mae[fold_]))
predictions0 += model0.predict(test, num_iteration=model0.best_iteration_)/ folds.n_splits
prelgb1[val_idx] = model0.predict(X_val ).reshape(-1,)
 print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(mae), np.std(mae)))
 submission.time_to_failure = predictions0
submission.to_csv('submissionlgb0.csv',index=True)
 submission.head()
prelgb1=pd.DataFrame({'id': trainy.index.values,'TTF': prelgb1} ).set_index('id' ).sort_index()
validating_dataset = valid_dataset.unbatch().batch(20)
 valid_batch = iter(validating_dataset )<define_variables>
blending1 = predictions0*1/2 + est_gp.predict(X_test_scaled)*1/2
submission['time_to_failure']=blending1
submission.to_csv('submissionblend1.csv',index=True)
 submission.head()
display_batch_of_images(next(valid_batch))<define_variables>
params ={'boosting': 'gblinear',
'colsample_bylevel': 0.87,
'colsample_bynode': 0.98,
'colsample_bytree': 0.63,
'eval_metric': 'mae',
'gamma': 2.3074997279871394,
'learning_rate': 0.19978743070807628,
'max_depth': 4,
'min_child_weight': 2.164915649342119,
'objective': 'reg:linear',
'reg_alpha': 2.5778910953585816,
'reg_lambda': 3.4511859649859136,
'subsample': 1.0,
'tree_method': 'exact'}
training_dataset = train_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset )<define_variables>
X=trainX
X_test=test
y=trainy
prexgb01 = np.zeros(len(train))
prediction01 = np.zeros(len(X_test))
n_fold = 5
folds = KFold(n_splits=n_fold, random_state=11)
 scores = []
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 prexgb01[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction01 += y_pred / folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 submission.time_to_failure = prediction01
submission.to_csv('submissionxgb0.csv',index=True)
 submission.head()
prexgb01=pd.DataFrame({'id': trainy.index.values,'TTF': prexgb01} ).set_index('id' ).sort_index()
<define_variables><EOS> 
blending =(predictions0 + prediction01 + est_gp.predict(X_test_scaled)) *1/3
submission['time_to_failure']=blending
submission.to_csv('submissionblend0.csv',index=True)
 submission.head()