Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<categorify>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<categorify>
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )
class DenoisingAutoEncoder(nn.Module):
def __init__(self, in_out_channels):
super(DenoisingAutoEncoder, self ).__init__()
self.encoder=nn.Sequential(
 nn.Linear(in_out_channels,256),
nn.ReLU(True),
nn.Linear(256,128),
nn.ReLU(True),
nn.Linear(128,64),
nn.ReLU(True)
)
 self.decoder=nn.Sequential(
 nn.Linear(64,128),
nn.ReLU(True),
nn.Linear(128,256),
nn.ReLU(True),
nn.Linear(256,in_out_channels), 
)
 def forward(self,x):
x=self.encoder(x)
 x=self.decoder(x)
 return x
class DAEDataset:
def __init__(self, ids, ys=None):
self.ids = ids
self.ys = ys
def __len__(self):
return len(self.ids)
 def __getitem__(self, item):
if self.ys is not None:
return {
'ids': torch.tensor(self.ids[item], dtype=torch.float32),
'targets': torch.tensor(self.ys[item], dtype=torch.float32),
}
else:
return {
'ids': torch.tensor(self.ids[item], dtype=torch.float32),
}
def dae_test_fn(data_loader, model, device):
model.eval()
preds = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"].to(device, dtype=torch.float32)
 outputs = model.encoder(ids)
 outputs = outputs.cpu().detach().numpy()
preds.append(outputs)
 preds = np.concatenate(preds, 0)
 return preds<categorify>
etq_meta = [
{"start":0, "end":5656574},
{"start":5656574, "end":50085878},
{"start":50085878, "end":104677356},
{"start":104677356, "end":138772453},
{"start":138772453, "end":187641820},
{"start":187641820, "end":218652630},
{"start":218652630, "end":245829585},
{"start":245829585, "end":307838917},
{"start":307838917, "end":338276287},
{"start":338276287, "end":375377848},
{"start":375377848, "end":419368880},
{"start":419368880, "end":461811623},
{"start":461811623, "end":495800225},
{"start":495800225, "end":528777115},
{"start":528777115, "end":585568144},
{"start":585568144, "end":621985673},
{"start":621985673, "end":629145480},
]
df = []
for i in [2, 7, 0, 4, 11, 13, 9, 1, 14, 10]:
df.append(train[etq_meta[i]['start']:etq_meta[i]['start']+150000*(( etq_meta[i]['end'] - etq_meta[i]['start'])//150000)] )
data_train = train_features[GENES+CELLS]
data_test = test_features[GENES+CELLS]
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = DenoisingAutoEncoder(len(GENES))
model.to(device)
 ae_genes_train_dataset = DAEDataset(
 ids=data_train[GENES].values, 
)
 ae_genes_test_dataset = DAEDataset(
 ids=data_test[GENES].values, 
)
 ae_genes_train_loader = torch.utils.data.DataLoader(
 ae_genes_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_genes_test_loader = torch.utils.data.DataLoader(
 ae_genes_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_genes_train_dataset, ae_genes_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v3/GENES_Reconstruction_model.pth'))
ae_genes_train_reconstruction = dae_test_fn(ae_genes_train_loader, model, device)
 ae_genes_test_reconstruction = dae_test_fn(ae_genes_test_loader, model, device)
 model = DenoisingAutoEncoder(len(CELLS))
model.to(device)
 ae_cells_train_dataset = DAEDataset(
 ids=data_train[CELLS].values, 
)
 ae_cells_test_dataset = DAEDataset(
 ids=data_test[CELLS].values, 
)
 ae_cells_train_loader = torch.utils.data.DataLoader(
 ae_cells_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_cells_test_loader = torch.utils.data.DataLoader(
 ae_cells_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_cells_train_dataset, ae_cells_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v3/CELLS_Reconstruction_model.pth'))
ae_cells_train_reconstruction = dae_test_fn(ae_cells_train_loader, model, device)
 ae_cells_test_reconstruction = dae_test_fn(ae_cells_test_loader, model, device )<create_dataframe>
train = pd.concat(df)
 del df
gc.collect()
ae_cells_train = pd.DataFrame(ae_cells_train_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_train = pd.DataFrame(ae_genes_train_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_test = pd.DataFrame(ae_cells_test_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_test = pd.DataFrame(ae_genes_test_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_train.shape, ae_cells_test.shape, ae_genes_train.shape, ae_genes_test.shape<create_dataframe>
num_seg = len(train)//150000
train_X = []
train_y = []
for i in tqdm_notebook(range(num_seg)) :
if 100000 * i + 150000 < len(train):
train_X.append(train['acoustic_data'].iloc[150000 * i:150000 * i + 150000])
 train_y.append(train['time_to_failure'].iloc[150000 * i + 149999])
 del train
gc.collect()
train_X = np.array(train_X,dtype = np.float32)
 train_y = np.array(train_y,dtype = np.float32 )
n_comp_GENES = 450
n_comp_CELLS = 2
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 train_features = pd.concat(( train_features, ae_genes_train, ae_cells_train), axis=1)
 test_features = pd.concat(( test_features, ae_genes_test, ae_cells_test), axis=1)
 train_features.shape, test_features.shape<create_dataframe>
X_mean = train_X.mean(0)
 X_std = train_X.std(0)
 train_X -= X_mean
train_X /= X_std
y_mean = train_y.mean()
y_std = train_y.std()
train_y -= y_mean
train_y /= y_std
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = pd.merge(train, train_drug, on='sig_id')
 groups = np.array(train.drug_id.values)
 folds = train.copy()
n_folds = 7
mskf = MultilabelStratifiedKFold(n_splits=n_folds)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target, groups=groups)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
len(feature_cols )<define_variables>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = []
for i, seg_id in enumerate(tqdm_notebook(submission.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 test_X.append(seg['acoustic_data'])
 test_X = np.array(test_X,dtype = np.float32)
 test_X -= X_mean
test_X /= X_std
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = n_folds
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024<prepare_x_and_y>
train_X = np.expand_dims(train_X,-1)
 test_X = np.expand_dims(test_X,-1 )
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.05, div_factor=1.5e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<normalization>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<normalization>
def keras_model() :
def wave_block(x,filters,kernel_size,n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters=filters,
kernel_size=1,
padding='same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters=filters,
kernel_size=kernel_size,
padding='same',
activation = 'tanh',
dilation_rate=dilation_rate )(x)
 sigm_out = Conv1D(filters=filters,
kernel_size=kernel_size,
padding='same',
activation = 'sigmoid',
dilation_rate=dilation_rate )(x)
 x = Multiply()([tanh_out,sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding='same' )(x)
 res_x = Add()([res_x,x])
 return res_x
clear_session()
inp = Input(shape=(150000,1))
Collect = []
x = wave_block(inp,16,3,8)
 x = AveragePooling1D(10 )(x)
 x = wave_block(x,32,3,5)
 x = AveragePooling1D(10 )(x)
 x = wave_block(x,64,3,3)
 x = AveragePooling1D(10 )(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(150 )(x)
 x = Dropout(0.2 )(x)
 x = Dense(128, activation="relu" )(x)
 x = Dense(1 )(x)
 model = Model(inputs = inp, outputs=x)
 return model
model = keras_model()
model.summary()
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.25)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<define_variables>
def step_decay(epoch):
x = 0.0001
if epoch >= 40: x = 0.00001
return x
lr_decay = LearningRateScheduler(step_decay )
SEED = [6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED )<save_to_csv>
prediction = np.zeros(len(test_X))
model = keras_model()
model.compile(Adam(lr=0.00001), loss='mse', metrics=['mae'])
 model.fit(train_X,train_y,
batch_size = 16,
epochs=100,
verbose=2 
)
 prediction = model.predict(test_X)[:,0]
submission['time_to_failure'] = prediction * y_std + y_mean
print(submission.head())
 submission.to_csv('submission.csv' )
<save_to_csv><EOS>