Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<load_from_csv>
%%html
<h1><b>LIVE EARTHQUACK MAP</b></h1>
<iframe width="800" height="600" src="https://ds.iris.edu/seismon/" allowfullscreen style="align:center;"></iframe>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']<categorify>
%%html
<h1> <strong><span style="color:blue;">Can We Predict Earthquakes?</span></strong></h1>
<iframe width="800" height="400" src="https://www.youtube.com/embed/uUEzGcRJIZE" style="align:center;" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test<compute_test_metric>
%matplotlib inline
plt.style.use("fivethirtyeight")
n_labels = train_targets_scored.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))<choose_model_class>
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
 gc.collect()
def build_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model<split>
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def rmse(predictions, targets):
return np.sqrt(((predictions - targets)** 2 ).mean() )
%%time
n_seeds = 5
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 10
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
kf = KFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in kf.split(train_features):
print(f'FOLD: {fold}, SEED: {seed}')
 X_train, X_test = preprocessor(train_features.iloc[train].values,
train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values)
 X_train_2 = train_features.iloc[train][predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = build_model(n_features, n_features_2, n_labels)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=2, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=10, mode='min',restore_best_weights=True)
 hist = model.fit([X_train,X_train_2],y_train, batch_size=128, epochs=182,verbose=0,validation_data =([X_test,X_test_2],y_test),
callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('ResNet_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict([X_test,X_test_2])
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1
tf.keras.backend.clear_session()
del model, X_train, X_test, X_train_2, X_test_2<compute_test_metric>
gc.collect()
tf.print('
OOF score is ',oof )<load_from_csv>
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
X_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['ave', 'std', 'max', 'min','q95','q99', 'q05','q01',
'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend'])
 y_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x = seg['acoustic_data'].values
y = seg['time_to_failure'].values[-1]
y_train.loc[segment, 'time_to_failure'] = y
X_train.loc[segment, 'ave'] = x.mean()
X_train.loc[segment, 'std'] = x.std()
X_train.loc[segment, 'max'] = x.max()
X_train.loc[segment, 'min'] = x.min()
X_train.loc[segment, 'q95'] = np.quantile(x,0.95)
 X_train.loc[segment, 'q99'] = np.quantile(x,0.99)
 X_train.loc[segment, 'q05'] = np.quantile(x,0.05)
 X_train.loc[segment, 'q01'] = np.quantile(x,0.01)
 X_train.loc[segment, 'abs_max'] = np.abs(x ).max()
X_train.loc[segment, 'abs_mean'] = np.abs(x ).mean()
X_train.loc[segment, 'abs_std'] = np.abs(x ).std()
X_train.loc[segment, 'trend'] = add_trend_feature(x)
 X_train.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 display(X_train.head())
 gc.collect()
sub_resnet = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_resnet.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub_resnet.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_resnet<load_from_csv>
%%time
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train)
 gc.collect()
%%time
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop('sig_id',axis=1 )<categorify>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index )
cat = 'cp_dose'
le = preprocessing.LabelEncoder()
le.fit(train_features[cat])
 train_features[cat] = le.transform(train_features[cat])
 test_features[cat] = le.transform(test_features[cat] )<normalization>
for seg_id in tqdm(X_test.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values
X_test.loc[seg_id, 'ave'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
X_test.loc[seg_id, 'abs_mean'] = np.abs(x ).mean()
X_test.loc[seg_id, 'abs_std'] = np.abs(x ).std()
X_test.loc[seg_id, 'trend'] = add_trend_feature(x)
 X_test.loc[seg_id, 'abs_trend'] = add_trend_feature(x, abs_values=True )
scaler = preprocessing.MinMaxScaler()
scaler.fit(train_features.append(test_features.drop('cp_type',axis=1)))
 data_train = scaler.transform(train_features)
 data_test = scaler.transform(test_features.drop('cp_type',axis=1))<compute_train_metric>
f = [y_pred_nusvr,y_pred_SVR,y_pred_cat,y_pred_lgb,y_pred_Bayesian]
%%time
n_labels = train_targets_scored.shape[1]
n_features = data_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]
p_min = 0.001
p_max = 0.999
@tf.autograph.experimental.do_not_convert
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
n_seeds = 6
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
mskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in mskf.split(data_train,labels_train):
X_train = data_train[train]
X_test = data_train[test]
y_train = labels_train[train]
y_test = labels_train[test]
model = Sequential()
model.add(layers.Dense(2048, input_shape=(n_features,)))
 model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(1024))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(512))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(n_labels))
model.add(layers.Activation('sigmoid'))
model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.3, patience=5, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=15, mode='min',restore_best_weights=True)
 hist = model.fit(X_train,y_train, batch_size=128, epochs=182,verbose=0,validation_data =(X_test,y_test),callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('LabelSmooth_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict(X_test)
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1
tf.print('OOF score is ',oof )<load_from_csv>
f = np.transpose(f )
sub_smooth = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_smooth.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub_smooth.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_smooth<load_from_csv>
d = {'Model': ['nuSVR', 'SVR','Kernel Ridge','Lightgbm','Catboost','Stacking'], 'RMSE': [2.647,2.635,2.653,2.029,2.491,1.20],'F1_Score': [0.48,0.485,0.478,0.695,0.54,0.695]}
analysis_df = pd.DataFrame(d)
 analysis_df.index = analysis_df.Model
del analysis_df['Model']
display(analysis_df )
SEED = 34
NFOLDS = 5
DATA_DIR = '/kaggle/input/lish-moa/'
np.random.seed(SEED)
 train = pd.read_csv(DATA_DIR + 'train_features.csv')
 non_ctl_idx = train.loc[train['cp_type']!='ctl_vehicle'].index.to_list()
train_drug = pd.read_csv(DATA_DIR + 'train_drug.csv')
 targets = pd.read_csv(DATA_DIR + 'train_targets_scored.csv')
 test = pd.read_csv(DATA_DIR + 'test_features.csv')
 sub_xgb = pd.read_csv(DATA_DIR + 'sample_submission.csv')
 ctl_mask = train.cp_type=='ctl_vehicle'
train = train[~ctl_mask]
targets = targets[~ctl_mask]
X = train.iloc[:,1:].to_numpy()
X_test = test.iloc[:,1:].to_numpy()
y = targets.iloc[:,1:].to_numpy()
clf = Pipeline([('encode', CountEncoder(cols=[0, 2])) , 
('classify', MultiOutputClassifier(
 XGBClassifier(tree_method='gpu_hist')))
 ])
 params = {'classify__estimator__colsample_bytree': 0.6522,
'classify__estimator__gamma': 3.6975,
'classify__estimator__learning_rate': 0.0503,
'classify__estimator__max_delta_step': 2.0706,
'classify__estimator__max_depth': 10,
'classify__estimator__min_child_weight': 31.5800,
'classify__estimator__n_estimators': 166,
'classify__estimator__subsample': 0.8639
}
_ = clf.set_params(**params )<load_from_csv>
sclf = StackingRegressor(regressors=[svm1,svm2,svm3,svm4,svm5,svm6],
meta_regressor=SVR())
 sclf.fit(X_train_scaled, y_train.values.flatten() )
def assign_folds(seed,fold):
scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets = scored.columns[1:]
scored = scored.merge(train_drug, on='sig_id', how='left')
 scored = scored.iloc[non_ctl_idx]
vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored['fold'].values<define_variables>
X_test_scaled = scaler.transform(X_test)
 submission['time_to_failure'] = sclf.predict(X_test_scaled)
use_xgb = True<define_variables>
submission.to_csv("Advance_stack.csv" )
%%time
if use_xgb:
oof_preds = np.zeros(y.shape)
 test_preds = np.zeros(( test.shape[0], y.shape[1]))
oof_losses = []
x = assign_folds(SEED,NFOLDS)
 for n in range(NFOLDS):
print(f'Inferencing FOLD: {n}')
 trn_idx = np.where(x != n)[0]
val_idx = np.where(x == n)[0]
X_train, X_val = X[trn_idx], X[val_idx]
y_train, y_val = y[trn_idx], y[val_idx]
ctl_mask = X_train[:,0]=='ctl_vehicle'
X_train = X_train[~ctl_mask,:]
y_train = y_train[~ctl_mask]
clf = pickle.load(open(f'.. /input/moa-xgb-weights/xgb_cv_fold_{n}.dat', "rb"))
val_preds = clf.predict_proba(X_val)
 val_preds = np.array(val_preds)[:,:,1].T
oof_preds[val_idx] = val_preds
loss = log_loss(np.ravel(y_val), np.ravel(val_preds))
oof_losses.append(loss)
 preds = clf.predict_proba(X_test)
 preds = np.array(preds)[:,:,1].T
test_preds += preds / NFOLDS
print(oof_losses)
 print('Mean OOF loss across folds', np.mean(oof_losses))
print('STD OOF loss across folds', np.std(oof_losses))<compute_test_metric>
submission1 = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 submission1['time_to_failure'] = svm5.predict(X_test_scaled)
 submission1.to_csv("submission_lgbbestmodel.csv" )
control_mask = train['cp_type']=='ctl_vehicle'
oof_preds[control_mask] = 0
print('OOF log loss: ', log_loss(np.ravel(y), np.ravel(oof_preds)) )<prepare_output>
submission1 = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 submission1['time_to_failure'] = svm3.predict(X_test_scaled)
 submission1.to_csv("submission_svrbestmodel.csv" )
control_mask = test['cp_type']=='ctl_vehicle'
test_preds[control_mask] = 0
sub_xgb.iloc[:,1:] = test_preds
sub_xgb<drop_column>
submission1 = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 submission1['time_to_failure'] = svm1.predict(X_test_scaled)
 submission1.to_csv("submission_nusvrbestmodel.csv" )