Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<define_variables>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'<load_from_csv>
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change )
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train.head()<split>
class FeatureGenerator(object):
def __init__(self, dtype, n_jobs=1, chunk_size=None):
self.chunk_size = chunk_size
self.dtype = dtype
self.filename = None
self.n_jobs = n_jobs
self.test_files = []
if self.dtype == 'train':
self.filename = '.. /input/train.csv'
self.total_data = int(629145481 / self.chunk_size)
 else:
submission = pd.read_csv('.. /input/sample_submission.csv')
 for seg_id in submission.seg_id.values:
self.test_files.append(( seg_id, '.. /input/test/' + seg_id + '.csv'))
self.total_data = int(len(submission))
def read_chunks(self):
if self.dtype == 'train':
iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,
dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})
 for counter, df in enumerate(iter_df):
x = np.sign(df.acoustic_data.values)*np.log1p(np.abs(df.acoustic_data.values))
y = df.time_to_failure.values[-1]
seg_id = 'train_' + str(counter)
 del df
yield seg_id, x, y
else:
for seg_id, f in self.test_files:
df = pd.read_csv(f, dtype={'acoustic_data': np.float64})
 x = df.acoustic_data.values[-self.chunk_size:]
x = np.sign(x)*np.log1p(np.abs(x))
del df
yield seg_id, x, -999
def get_features(self, x, y, seg_id):

 x = pd.Series(x)
 zc = np.fft.fft(x)
 realFFT = pd.Series(np.real(zc))
imagFFT = pd.Series(np.imag(zc))
main_dict = self.features(x, y, seg_id)
 r_dict = self.features(realFFT, y, seg_id)
 i_dict = self.features(imagFFT, y, seg_id)
 for k, v in r_dict.items() :
if k in ['classic_sta_lta2_mean',
'exp_Moving_std_3000_mean',
'classic_sta_lta1_mean']:
main_dict[f'fftr_{k}'] = v
for k, v in i_dict.items() :
if k in ['classic_sta_lta2_mean']:
main_dict[f'ffti_{k}'] = v
return main_dict
def features(self, x, y, seg_id):
feature_dict = dict()
feature_dict['target'] = y
feature_dict['seg_id'] = seg_id
for p in [50]:
feature_dict[f'abs_percentile_{p}'] = np.percentile(np.abs(x), p)
 for autocorr_lag in [5]:
feature_dict[f'autocorrelation_{autocorr_lag}'] = feature_calculators.autocorrelation(x, autocorr_lag)
 for p in [95,99]:
feature_dict[f'binned_entropy_{p}'] = feature_calculators.binned_entropy(x, p)
 for slice_length, direction in product([50000], ['first']):
if direction == 'first':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[:slice_length])
 elif direction == 'last':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[-slice_length:])
 for peak in [10]:
feature_dict[f'num_peaks_{peak}'] = feature_calculators.number_peaks(x, peak)
 for p in [20]:
feature_dict[f'percentile_{p}'] = np.percentile(x, p)
 x_roll_std = x.rolling(1000 ).std().dropna().values
feature_dict[f'percentile_roll_std_30_window_1000'] = np.percentile(x_roll_std, 30)
 x_roll_std = x.rolling(500 ).std().dropna().values
feature_dict[f'percentile_roll_std_75_window_500'] = np.percentile(x_roll_std, 75)
 feature_dict[f'percentile_roll_std_80_window_500'] = np.percentile(x_roll_std, 80)
 ewma = pd.Series.ewm
feature_dict[f'exp_Moving_std_3000_mean'] =(ewma(x, span=3000 ).std(skipna=True)).mean(skipna=True)
 feature_dict['classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
feature_dict['classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
feature_dict['skew'] = x.skew()
return feature_dict
def generate(self):
feature_list = []
res = Parallel(n_jobs=self.n_jobs,
backend='threading' )(delayed(self.get_features )(x, y, s)
 for s, x, y in tqdm_notebook(self.read_chunks() , total=self.total_data))
for r in res:
feature_list.append(r)
 return pd.DataFrame(feature_list )
train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.1, random_state=2020 )<normalization>
training_fg = FeatureGenerator(dtype='train', n_jobs=4, chunk_size=150000)
 training_data = training_fg.generate()
training_data.columns
def decode_image(filename, label=None, image_size=(IM_Z, IM_Z)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label<categorify>
testing_fg = FeatureGenerator(dtype='test', n_jobs=4, chunk_size=150000)
 test_data = testing_fg.generate()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<choose_model_class>
X = training_data.drop(['target', 'seg_id'], axis=1)
 X_test = test_data.drop(['target', 'seg_id'], axis=1)
 test_segs = test_data.seg_id
y = training_data.target
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
ch_p = ModelCheckpoint(filepath="model_ir.h5", monitor='val_loss', save_weights_only=True,
verbose=1 )<choose_model_class>
ss = StandardScaler()
feats = X_test.columns
ss.fit(pd.concat([X[feats],X_test[feats]]))
X[feats] = ss.transform(X[feats])
 X_test[feats] = ss.transform(X_test[feats])
 X.insert(0,'seg_id',training_data.seg_id.values)
 X['time_to_failure'] = y.values
X_test.insert(0,'seg_id',test_segs.values)
with strategy.scope() :
model = tf.keras.Sequential([
tf.keras.applications.InceptionResNetV2(
 input_shape=(IM_Z, IM_Z, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
<choose_model_class>
def GPI(data):
return(5.683668 +
1.0*np.tanh(((((data["abs_percentile_50"])-(((data["percentile_roll_std_75_window_500"])* 2.0)))) -(((((((((((data["percentile_roll_std_75_window_500"])+(data["num_peaks_10"])))* 2.0)) +(data["abs_percentile_50"])))* 2.0)) * 2.0)))) +
1.0*np.tanh(((-1.0)-(((((data["percentile_roll_std_30_window_1000"])+(((( data["binned_entropy_95"])+(data["num_peaks_10"])) /2.0)))) +(data["percentile_roll_std_75_window_500"])))))+
1.0*np.tanh(((((((((data["abs_percentile_50"])+(data["binned_entropy_99"])))*(((( data["percentile_roll_std_75_window_500"])+(data["num_peaks_10"])) /2.0)))) *(( -1.0*(((( data["percentile_roll_std_75_window_500"])+(data["percentile_roll_std_30_window_1000"])))))))) * 2.0)) +
1.0*np.tanh(((( -1.0*(((((( data["fftr_exp_Moving_std_3000_mean"])*(((1.0)+(((data["percentile_roll_std_30_window_1000"])* 2.0)))))) / 2.0)))))*(((( data["num_peaks_10"])+(data["abs_percentile_50"])) /2.0)))) +
1.0*np.tanh(((((data["exp_Moving_std_3000_mean"])-(data["percentile_roll_std_75_window_500"])))+(((data["num_peaks_10"])*(((data["fftr_classic_sta_lta1_mean"])-(data["fftr_exp_Moving_std_3000_mean"]))))))))
def GPII(data):
return(5.683668 +
1.0*np.tanh(((((((((( -1.0*(((( data["num_peaks_10"])+(data["percentile_roll_std_75_window_500"])))))) * 2.0)) -(((data["num_peaks_10"])+(data["abs_percentile_50"])))))* 2.0)) * 2.0)) +
1.0*np.tanh(((((((( data["exp_Moving_std_3000_mean"])+(-2.0)))+(((((((data["percentile_roll_std_80_window_500"])+(data["autocorrelation_5"])))/ 2.0)) / 2.0)))/2.0)) -(((((data["percentile_roll_std_75_window_500"])* 2.0)) * 2.0)))) +
1.0*np.tanh(((( -1.0*(( data["fftr_exp_Moving_std_3000_mean"])))) *(((((data["num_peaks_10"])+(data["abs_percentile_50"])))*(((((data["percentile_roll_std_30_window_1000"])* 2.0)) +(((1.0)+(data["percentile_roll_std_30_window_1000"])))))))))+
1.0*np.tanh(((data["exp_Moving_std_3000_mean"])-(((data["percentile_roll_std_75_window_500"])+(((((((( data["abs_percentile_50"])+(data["percentile_roll_std_75_window_500"])) /2.0)) *(((( data["abs_percentile_50"])+(data["percentile_roll_std_75_window_500"])) /2.0)))) *(data["fftr_exp_Moving_std_3000_mean"])))))))+
1.0*np.tanh(((((((((( data["percentile_roll_std_80_window_500"])-(data["fftr_exp_Moving_std_3000_mean"])))+(((data["fftr_classic_sta_lta1_mean"])-(data["fftr_classic_sta_lta2_mean"])))) /2.0)) +(((data["fftr_classic_sta_lta1_mean"])-(data["fftr_exp_Moving_std_3000_mean"])))))*(data["num_peaks_10"]))))
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE<train_model>
mean_absolute_error(X.time_to_failure,GPI(X))
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule, ch_p],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)<predict_on_test>
mean_absolute_error(X.time_to_failure,GPII(X))
<save_to_csv><EOS> 
X_test['time_to_failure'] =(GPI(X_test)+GPII(X_test)) /2.
 X_test[['seg_id','time_to_failure']].to_csv('submission.csv',index=False )