Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<normalization>
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
df_train, df_test = scale_df(df_train, df_test, feature_cols )<normalization>
from keras.models import *
from keras.optimizers import *
from keras.regularizers import *
from keras.layers import *
from keras import backend as K
from keras.callbacks import *
df_train_sum, df_test_sum = scale_df(df_train_sum, df_test_sum, df_train_sum.columns )<save_to_csv>
print(os.listdir('.. /input/'))
train_X_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )
df_train.to_csv("features_train_scaled_raw.csv")
 df_test.to_csv("features_test_scaled_raw.csv")
 df_train_sum.to_csv("features_train_scaled_sum.csv")
 df_test_sum.to_csv("features_test_scaled_sum.csv" )<define_variables>
train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 print(train_X.shape)
 train_X.head()
df_features = df_train[feature_cols]<train_model>
y = pd.concat([y_0, y_1], axis=0)
 y = y.reset_index(drop=True)
 y[0].shape
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=69)
 sub_preds_rf = np.zeros(( df_test_sum.shape[0], 9))
oof_preds_rf = np.zeros(( df_train_sum.shape[0]))
score = 0
misclassified_indices = []
misclassified_tuples_all = []
for i,(train_index, test_index)in enumerate(folds.split(df_train_sum, y)) :
print('-'*20, i, '-'*20)
 clf = RandomForestClassifier(n_estimators = 200, n_jobs = -1)
 clf.fit(df_train_sum.iloc[train_index], y[train_index])
 oof_preds_rf[test_index] = clf.predict(df_train_sum.iloc[test_index])
 sub_preds_rf += clf.predict_proba(df_test_sum)/ folds.n_splits
score += clf.score(df_train_sum.iloc[test_index], y[test_index])
 print('score ', clf.score(df_train_sum.iloc[test_index], y[test_index]))
importances = clf.feature_importances_
features = df_train_sum.columns
feat_importances = pd.Series(importances, index=features)
 feat_importances.nlargest(30 ).sort_values().plot(kind='barh', color='
plt.show()
missed = y[test_index] != oof_preds_rf[test_index]
misclassified_indices.append(test_index[missed])
 misclassified_samples1 = y[test_index][missed]
misclassified_samples2 = oof_preds_rf[test_index][missed].astype("int")
 m1 = encoder.inverse_transform(misclassified_samples1)
 m2 = encoder.inverse_transform(misclassified_samples2)
 misclassified_tuples = [(a, b)for a, b in zip(m1, m2)]
misclassified_tuples_all.append(misclassified_tuples)
 print('Avg Accuracy', score / folds.n_splits)
<save_to_csv>
test_X = pd.read_csv(".. /input/lanl-master-s-features-creating-0/test_X_features_10.csv")
ss = pd.read_csv('.. /input/sample_submission.csv')
 ss['surface'] = encoder.inverse_transform(sub_preds_rf.argmax(axis=1))
ss.to_csv('rf.csv', index=False)
 ss.head(10 )<create_dataframe>
scaler = MinMaxScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )
df_missed_all = None
total_counter = collections.Counter()
def counter_to_df(df_missed_all, counter, new_col_name):
sorted_freqs = counter.most_common()
rows = [[row[0][0], row[0][1], row[1]] for row in sorted_freqs]
df_missed = pd.DataFrame(rows, columns=["expected", "actual", "count"])
 if df_missed_all is None:
df_missed_all = df_missed
else:
df_missed_all = df_missed_all.merge(df_missed, how='left', on=["expected", "actual"])
 created_col = df_missed_all.columns[-1]
print("renaming: "+new_col_name)
 df_missed_all.rename(columns={created_col: new_col_name}, inplace=True)
 df_missed_all[new_col_name].fillna(0, inplace=True)
 df_missed_all[new_col_name] = df_missed_all[new_col_name].astype(int)
 return df_missed_all
i = 1
for misclassified_tuples in misclassified_tuples_all:
counter = collections.Counter(misclassified_tuples)
 total_counter.update(misclassified_tuples)
 df_missed_all = counter_to_df(df_missed_all, counter, "count"+str(i))
i += 1
df_missed_all = counter_to_df(df_missed_all, total_counter, "all")
<sort_values>
train_columns = train_X.columns
n_fold = 5
df_missed_all.sort_values(by="all", ascending=False )<feature_engineering>
%%time
folds = KFold(n_splits=n_fold, shuffle = True, random_state=42)
 oof = np.zeros(len(train_X))
train_score = []
fold_idxs = []
predictions = np.zeros(len(test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 fold_idxs.append(val_idx)
 X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective="MAE", loss_function="MAE", boosting_type="Ordered", task_type="GPU")
 model.fit(X_tr,
y_tr,
eval_set=[(X_val, y_val)],
verbose=2500,
early_stopping_rounds=500)
 oof[val_idx] = model.predict(X_val)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(test_X[train_columns])/ folds.n_splits
train_score.append(model.best_score_['learn']["MAE"])
 cv_score = mean_absolute_error(train_y, oof)
 print(f"After {n_fold} test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}", end=" " )
miss_map = {}
hit_map = {}
ids, counts = np.unique(y, return_counts=True)
 totals = dict(zip(ids, counts))
total_misses = 0
total_hits = 0
for label_id in label_mapping.keys() :
label_name = label_mapping[label_id]
misses = df_missed_all[df_missed_all["expected"] == label_name]["all"].sum()
total_misses += misses
miss_map[label_name] = misses
total = totals[label_id]
hits = total - misses
total_hits += hits
hit_map[label_name] = hits
print(miss_map)
 print(hit_map)
 print(total_misses)
 print(total_hits)
 print(total_misses+total_hits )<merge>
today = str(datetime.date.today())
 submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission["time_to_failure"] = predictions
submission.to_csv(f'CatBoost_{today}_test_{cv_score:.3f}_train_{np.mean(train_score):.3f}.csv', index=False)
 submission.head()
df_hit_miss = pd.DataFrame.from_dict(hit_map, orient="index")
 df_miss = pd.DataFrame.from_dict(miss_map, orient="index")
 df_hit_miss = df_hit_miss.join(df_miss, how='outer', lsuffix='_left', rsuffix='_right')
 df_hit_miss.columns = ["hits", "misses"]
df_hit_miss.sort_values(by="hits", ascending=False )<count_values>
def create_model(input_dim=10):
model = Sequential()
model.add(Dense(256, activation="relu",input_dim=input_dim))
model.add(Dropout(0.5))
model.add(Dense(128, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(64, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation="linear"))
opt = adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
 model.compile(
 loss='mae',
optimizer=opt, 
)
 return model
patience = 50
call_ES = EarlyStopping(monitor='val_loss', min_delta=0, patience=patience, verbose=1, mode='auto', baseline=None, restore_best_weights=True )
idx_counter = collections.Counter()
for missed_idx in misclassified_indices:
idx_counter.update(missed_idx )<load_pretrained>
%%time
folds = KFold(n_splits=n_fold, shuffle = True, random_state=42)
 NN_oof = np.zeros(len(train_X))
train_score = []
fold_idxs = []
NN_predictions = np.zeros(len(test_X))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 fold_idxs.append(val_idx)
 X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = create_model(train_X.shape[-1])
 model.fit(X_tr, y_tr, epochs=500, batch_size=32, verbose=0, callbacks=[call_ES,], validation_data=[X_val, y_val])
 NN_oof[val_idx] = model.predict(X_val)[:,0]
NN_predictions += model.predict(test_X[train_columns])[:,0] / folds.n_splits
history = model.history.history
tr_loss = history["loss"]
val_loss = history["val_loss"]
print(f"loss: {tr_loss[-patience]:.3f} | val_loss: {val_loss[-patience]:.3f} | diff: {val_loss[-patience]-tr_loss[-patience]:.3f}")
 train_score.append(tr_loss[-patience])
 cv_score = mean_absolute_error(train_y, NN_oof)
 print(f"After {n_fold} test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}", end=" " )
<set_options>
today = str(datetime.date.today())
 submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission["time_to_failure"] = NN_predictions
submission.to_csv(f'NN_{today}_test_{cv_score:.3f}_train_{np.mean(train_score):.3f}.csv', index=False)
 submission.head()
warnings.filterwarnings('ignore')
 def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
with torch.no_grad() :
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
with torch.no_grad() :
for data in dataloader:
inputs = data['x'].to(device)
 outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
seed_everything(seed=42 )<set_options>
Scirpus_prediction = pd.read_csv(".. /input/andrews-new-script-plus-a-genetic-program-model/gpI.csv")
 Scirpus_prediction.head()
<load_from_csv><EOS> 
today = str(datetime.date.today())
 submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission["time_to_failure"] =(predictions+NN_predictions+Scirpus_prediction.time_to_failure.values)/3
submission.to_csv(f'FINAL_{today}_submission.csv', index=False)
 submission.head()