Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<merge>
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.simplefilter(action='ignore', category=UserWarning)
 sns.set()
init_notebook_mode(connected=True )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
data_type = {'acoustic_data': np.int16, 'time_to_failure': np.float32}
train = pd.read_csv('.. /input/train.csv', dtype=data_type)
 train.head()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
def add_trend_feature(arr, abs_values=False):

 idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def extract_features_from_segment(x):

 features = {}
features['ave'] = x.values.mean()
features['std'] = x.values.std()
features['max'] = x.values.max()
features['min'] = x.values.min()
features['q90'] = np.quantile(x.values, 0.90)
 features['q95'] = np.quantile(x.values, 0.95)
 features['q99'] = np.quantile(x.values, 0.99)
 features['q05'] = np.quantile(x.values, 0.05)
 features['q10'] = np.quantile(x.values, 0.10)
 features['q01'] = np.quantile(x.values, 0.01)
 features['std_to_mean'] = features['std'] / features['ave']
features['abs_max'] = np.abs(x.values ).max()
features['abs_mean'] = np.abs(x.values ).mean()
features['abs_std'] = np.abs(x.values ).std()
features['trend'] = add_trend_feature(x.values)
 features['abs_trend'] = add_trend_feature(x.values, abs_values=True)
 for w in [10, 50, 100, 1000]:
x_roll_abs_mean = x.abs().rolling(w ).mean().dropna().values
x_roll_mean = x.rolling(w ).mean().dropna().values
x_roll_std = x.rolling(w ).std().dropna().values
x_roll_min = x.rolling(w ).min().dropna().values
x_roll_max = x.rolling(w ).max().dropna().values
features['ave_roll_std_' + str(w)] = x_roll_std.mean()
features['std_roll_std_' + str(w)] = x_roll_std.std()
features['max_roll_std_' + str(w)] = x_roll_std.max()
features['min_roll_std_' + str(w)] = x_roll_std.min()
features['q01_roll_std_' + str(w)] = np.quantile(x_roll_std, 0.01)
 features['q05_roll_std_' + str(w)] = np.quantile(x_roll_std, 0.05)
 features['q10_roll_std_' + str(w)] = np.quantile(x_roll_std, 0.10)
 features['q95_roll_std_' + str(w)] = np.quantile(x_roll_std, 0.95)
 features['q99_roll_std_' + str(w)] = np.quantile(x_roll_std, 0.99)
 features['ave_roll_mean_' + str(w)] = x_roll_mean.mean()
features['std_roll_mean_' + str(w)] = x_roll_mean.std()
features['max_roll_mean_' + str(w)] = x_roll_mean.max()
features['min_roll_mean_' + str(w)] = x_roll_mean.min()
features['q05_roll_mean_' + str(w)] = np.quantile(x_roll_mean, 0.05)
 features['q95_roll_mean_' + str(w)] = np.quantile(x_roll_mean, 0.95)
 features['ave_roll_abs_mean_' + str(w)] = x_roll_abs_mean.mean()
features['std_roll_abs_mean_' + str(w)] = x_roll_abs_mean.std()
features['q05_roll_abs_mean_' + str(w)] = np.quantile(x_roll_abs_mean, 0.05)
 features['q95_roll_abs_mean_' + str(w)] = np.quantile(x_roll_abs_mean, 0.95)
 features['std_roll_min_' + str(w)] = x_roll_min.std()
features['max_roll_min_' + str(w)] = x_roll_min.max()
features['q05_roll_min_' + str(w)] = np.quantile(x_roll_min, 0.05)
 features['q95_roll_min_' + str(w)] = np.quantile(x_roll_min, 0.95)
 features['std_roll_max_' + str(w)] = x_roll_max.std()
features['min_roll_max_' + str(w)] = x_roll_max.min()
features['q05_roll_max_' + str(w)] = np.quantile(x_roll_max, 0.05)
 features['q95_roll_max_' + str(w)] = np.quantile(x_roll_max, 0.95)
 return features
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<load_from_csv>
X_train, target, quake = make_train(train, skip=150000)
 print("Train shape:", X_train.shape)
 X_train.head(3 )
def assign_folds(seed,fold):
scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets = scored.columns[1:]
scored = scored.merge(train_drug, on='sig_id', how='left')
 scored = scored.iloc[non_ctl_idx]
vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored['fold'].values<categorify>
def fold_generator(x, y, groups=None, num_folds=10, shuffle=True, seed=2019):
folds = KFold(num_folds, shuffle=shuffle, random_state=seed)
 for train_index, test_index in folds.split(x, y, groups):
yield train_index, test_index
folds = train.copy()
seed_folds = train.copy()
val_folds = assign_folds(34,5)
 folds['kfold'] = val_folds
folds<categorify>
rf = RandomForestRegressor(n_jobs=-1, max_depth=5)
 selector = BorutaPy(rf, n_estimators='auto', verbose=0,
random_state=1, max_iter=500)
 selector.fit(X_train.values, target.values)
 X_filtered = selector.transform(X_train.values)
 X_train = pd.DataFrame(X_filtered, columns=X_train.columns[selector.support_])
 X_train.head()
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
def make_pipeline(estimator):
pipeline = Pipeline([ 
('scaler', StandardScaler()), 
('model', estimator)
 ])
 return pipeline
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
def search_cv(x, y, pipeline, grid, max_iter=None, num_folds=10, shuffle=True):

 t0 = time.time()
cv = fold_generator(x, y, num_folds=num_folds)
 if max_iter is None:
search = GridSearchCV(pipeline, grid, cv=cv,
scoring='neg_mean_absolute_error')
 else:
search = RandomizedSearchCV(pipeline, grid, n_iter=max_iter, cv=cv,
scoring='neg_mean_absolute_error')
 search.fit(x, y)
 t0 = time.time() - t0
print("Best CV score: {:.4f}, time: {:.1f}s".format(-search.best_score_, t0))
print(search.best_params_)
 return search.best_estimator_
def make_predictions(x, y, pipeline, num_folds=10, shuffle=True, test=None, plot=True):

 if test is not None:
sub_prediction = np.zeros(test.shape[0])
 oof_prediction = np.zeros(x.shape[0])
 for tr_idx, val_idx in fold_generator(x, y, num_folds=num_folds):
pipeline.fit(x.iloc[tr_idx], y.iloc[tr_idx])
 oof_prediction[val_idx] = pipeline.predict(x.iloc[val_idx])
 if test is not None:
sub_prediction += pipeline.predict(test)/ num_folds
if plot:
plot_predictions(y, oof_prediction)
 if test is None:
return oof_prediction
else:
return oof_prediction, sub_prediction
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
grid = {'model__alpha': np.concatenate([np.linspace(0.001, 1, 200),
np.linspace(1, 100, 500)])}
ridge_pipe = make_pipeline(Ridge(random_state=2019))
ridge_pipe = search_cv(X_train, target, ridge_pipe, grid)
 ridge_oof = make_predictions(X_train, target, ridge_pipe )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.4)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<categorify>
ridge_oof[ridge_oof < 0] = 0
print("Mean error: {:.4f}".format(mean_absolute_error(target, ridge_oof)) )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
grid = {'model__gamma': np.linspace(1e-8, 1, 100),
'model__alpha': np.linspace(1e-6, 2, 200)}
kr_pipe = make_pipeline(KernelRidge(kernel='rbf'))
kr_pipe = search_cv(X_train, target, kr_pipe, grid, max_iter=60)
 kr_oof = make_predictions(X_train, target, kr_pipe )
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
grid = {'model__epsilon': np.linspace(0.001, 1, 100),
'model__C': np.linspace(0.01, 10, 100)}
svm_pipe = make_pipeline(SVR(kernel='rbf', gamma='scale'))
svm_pipe = search_cv(X_train, target, svm_pipe, grid, max_iter=60)
 svm_oof = make_predictions(X_train, target, svm_pipe )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<prepare_x_and_y>
grid = {
'model__max_depth': [4, 6, 8, 10, 12],
'model__max_features': ['auto', 'sqrt', 'log2'],
'model__min_samples_leaf': [2, 4, 8, 12, 14, 16, 20],
'model__min_samples_split': [2, 4, 6, 8, 12, 16, 20],
}
rf_pipe = make_pipeline(RandomForestRegressor(criterion='mae', n_estimators=40))
rf_pipe = search_cv(X_train, target, rf_pipe, grid, max_iter=10)
 rf_oof = make_predictions(X_train, target, rf_pipe )
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 asn_folds = assign_folds(seed,5)
 seed_folds['kfold'] = asn_folds
trn_idx = train[seed_folds['kfold'] != fold].index
val_idx = train[seed_folds['kfold'] == fold].index
train_df = train[seed_folds['kfold'] != fold].reset_index(drop=True)
 valid_df = train[seed_folds['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"RankGauss_{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"RankGauss_{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
grid = {'model__learning_rate': np.linspace(1e-5, 0.9, 100)}
base = Ridge(alpha=2)
 ada_pipe = make_pipeline(AdaBoostRegressor(base_estimator=base, n_estimators=200))
ada_pipe = search_cv(X_train, target, ada_pipe, grid)
 ada_oof = make_predictions(X_train, target, ada_pipe )
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
fixed_params = {
'objective': 'regression_l1',
'boosting': 'gbdt',
'verbosity': -1,
'random_seed': 19,
'n_estimators': 80000,
'max_depth': -1
}
param_grid = {
'learning_rate': [0.1, 0.05, 0.01, 0.005],
'num_leaves': list(range(2, 60, 2)) ,
'feature_fraction': [0.8, 0.85, 0.9, 0.95, 1],
'subsample': [0.8, 0.9, 0.95, 1],
'lambda_l1': [0, 0.2, 0.4, 0.6, 0.8, 2, 5],
'lambda_l2': [0, 0.2, 0.4, 0.6, 0.8, 2, 5],
'min_data_in_leaf': [10, 20, 40, 60, 100],
'min_gain_to_split': [0, 0.001, 0.01, 0.1],
}
best_score = 999
dataset = lgb.Dataset(X_train, label=target)
 for i in range(600):
params = {k: random.choice(v)for k, v in param_grid.items() }
params.update(fixed_params)
 result = lgb.cv(params, dataset, nfold=10, early_stopping_rounds=100,
stratified=False)
 if result['l1-mean'][-1] < best_score:
best_score = result['l1-mean'][-1]
best_params = params
best_nrounds = len(result['l1-mean'] )
%%time
SEED = [34, 14, 75]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<set_options>
print("Best mean score: {:.4f}, num rounds: {}".format(best_score, best_nrounds))
print(best_params)
 gb_pipe = make_pipeline(lgb.LGBMRegressor(**best_params))
gb_oof = make_predictions(X_train, target, gb_pipe )
<merge><EOS> 
X_test = make_test()
X_test = X_test[X_train.columns]
gb_oof, gb_sub = make_predictions(X_train, target, gb_pipe,
test=X_test, plot=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['time_to_failure'] = gb_sub
submission.to_csv('submission_gb.csv', index=False)
 submission.time_to_failure.describe()