Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<load_pretrained>
warnings.filterwarnings("ignore")
 PandasDataFrame = TypeVar('pandas.core.frame.DataFrame')
train_images_dir = PATH + 'train_images/{}.jpg'
test_images_dir = PATH + 'test_images/{}.jpg'
df_train=pd.read_pickle('.. /input/carpose/' + 'df_train.pkl')
 df_dev=pd.read_pickle('.. /input/carpose/' + 'df_dev.pkl')
 df_test=pd.read_pickle('.. /input/carpose/' + 'df_test.pkl')
 train_dataset = CarDataset(df_train, train_images_dir)
 dev_dataset = CarDataset(df_dev, train_images_dir)
 test_dataset = CarDataset(df_test, test_images_dir )<load_pretrained>
IS_LOCAL = False
if(IS_LOCAL):
PATH=".. /input/LANL/"
else:
PATH=".. /input/"
os.listdir(PATH)
 train_X = pd.read_csv(os.path.join(PATH,'.. /input/all-features/train_X.csv'))
train_y = pd.read_csv(os.path.join(PATH,'.. /input/all-features/train_y.csv'))
train_X.shape, train_y.shape
BATCH_SIZE = 2
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)
 dev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2 )<choose_model_class>
submission = pd.read_csv(os.path.join(PATH,'LANL-Earthquake-Prediction/sample_submission.csv'), index_col='seg_id')
 test_X = pd.read_csv(os.path.join(PATH,'.. /input/all-features/test_X.csv'))
submission.shape, test_X.shape
class double_conv(nn.Module):

 def __init__(self, in_ch, out_ch):
super(double_conv, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True),
nn.Conv2d(out_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True)
)
 def forward(self, x):
x = self.conv(x)
 return x
class up(nn.Module):
def __init__(self, in_ch, out_ch, bilinear=True):
super(up, self ).__init__()
if bilinear:
self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
 else:
self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)
 self.conv = double_conv(in_ch, out_ch)
 def forward(self, x1, x2=None):
x1 = self.up(x1)
 diffY = x2.size() [2] - x1.size() [2]
diffX = x2.size() [3] - x1.size() [3]
x1 = F.pad(x1,(diffX // 2, diffX - diffX//2,
diffY // 2, diffY - diffY//2))
if x2 is not None:
x = torch.cat([x2, x1], dim=1)
 else:
x = x1
x = self.conv(x)
 return x
def get_mesh(batch_size, shape_x, shape_y):
mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))
mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mesh = torch.cat([torch.tensor(mg_x ).to(device), torch.tensor(mg_y ).to(device)], 1)
 return mesh<define_variables>

 def fold_generator(x, y, groups=None, num_folds=10, shuffle=True, seed=2019):
folds = KFold(num_folds, shuffle=shuffle, random_state=seed)
 for train_index, test_index in folds.split(x, y, groups):
yield train_index, test_index
__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
'resnet152', 'resnext50_32x4d', 'resnext101_32x8d',
'wide_resnet50_2', 'wide_resnet101_2']
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',
'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth',
'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth',
'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth',
}
def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=dilation, groups=groups, bias=False, dilation=dilation)
 def conv1x1(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
 class BasicBlock(nn.Module):
expansion = 1
__constants__ = ['downsample']
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
base_width=64, dilation=1, norm_layer=None):
super(BasicBlock, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
if groups != 1 or base_width != 64:
raise ValueError('BasicBlock only supports groups=1 and base_width=64')
 if dilation > 1:
raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
 self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = norm_layer(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = norm_layer(planes)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
identity = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
__constants__ = ['downsample']
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
base_width=64, dilation=1, norm_layer=None):
super(Bottleneck, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
width = int(planes *(base_width / 64.))* groups
self.conv1 = conv1x1(inplanes, width)
 self.bn1 = norm_layer(width)
 self.conv2 = conv3x3(width, width, stride, groups, dilation)
 self.bn2 = norm_layer(width)
 self.conv3 = conv1x1(width, planes * self.expansion)
 self.bn3 = norm_layer(planes * self.expansion)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
identity = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,
groups=1, width_per_group=64, replace_stride_with_dilation=None,
norm_layer=None):
super(ResNet, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self._norm_layer = norm_layer
self.inplanes = 64
self.dilation = 1
if replace_stride_with_dilation is None:
replace_stride_with_dilation = [False, False, False]
if len(replace_stride_with_dilation)!= 3:
raise ValueError("replace_stride_with_dilation should be None "
"or a 3-element tuple, got {}".format(replace_stride_with_dilation))
self.groups = groups
self.base_width = width_per_group
self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,
bias=False)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
dilate=replace_stride_with_dilation[0])
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilate=replace_stride_with_dilation[1])
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
dilate=replace_stride_with_dilation[2])
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m,(nn.BatchNorm2d, nn.GroupNorm)) :
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 if zero_init_residual:
for m in self.modules() :
if isinstance(m, Bottleneck):
nn.init.constant_(m.bn3.weight, 0)
 elif isinstance(m, BasicBlock):
nn.init.constant_(m.bn2.weight, 0)
 def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
norm_layer = self._norm_layer
downsample = None
previous_dilation = self.dilation
if dilate:
self.dilation *= stride
stride = 1
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 conv1x1(self.inplanes, planes * block.expansion, stride),
norm_layer(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
self.base_width, previous_dilation, norm_layer))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, groups=self.groups,
base_width=self.base_width, dilation=self.dilation,
norm_layer=norm_layer))
return nn.Sequential(*layers)

def forward(self, x):
conv1 = F.relu(self.bn1(self.conv1(x)) , inplace=True)
 conv1 = F.max_pool2d(conv1, 3, stride=2, padding=1)
 feats4 = self.layer1(conv1)
 feats8 = self.layer2(feats4)
 feats16 = self.layer3(feats8)
 feats32 = self.layer4(feats16)
 return feats8, feats16, feats32
def _resnet(arch, block, layers, pretrained, progress, **kwargs):
model = ResNet(block, layers, **kwargs)
 if pretrained:
state_dict = load_state_dict_from_url(model_urls[arch],
progress=progress)
 model.load_state_dict(state_dict)
 return model
def resnet18(pretrained=False, progress=True, **kwargs):
r
return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress,
**kwargs)
 def resnet34(pretrained=False, progress=True, **kwargs):
r
return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress,
**kwargs)
 def resnet50(pretrained=False, progress=True, **kwargs):
r
return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress,
**kwargs)
 def resnet101(pretrained=False, progress=True, **kwargs):
r
return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress,
**kwargs)
 def resnet152(pretrained=False, progress=True, **kwargs):
r
return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress,
**kwargs)
 def resnext50_32x4d(pretrained=False, progress=True, **kwargs):
r
kwargs['groups'] = 32
kwargs['width_per_group'] = 4
return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3],
pretrained, progress, **kwargs)
 def resnext101_32x8d(pretrained=False, progress=True, **kwargs):
r
kwargs['groups'] = 32
kwargs['width_per_group'] = 8
return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3],
pretrained, progress, **kwargs)
 def wide_resnet50_2(pretrained=False, progress=True, **kwargs):
r
kwargs['width_per_group'] = 64 * 2
return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3],
pretrained, progress, **kwargs)
 def wide_resnet101_2(pretrained=False, progress=True, **kwargs):
r
kwargs['width_per_group'] = 64 * 2
return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3],
pretrained, progress, **kwargs )<train_on_grid>

 def search_cv(x, y, model, grid, max_iter=None, num_folds=10, shuffle=True):
t0 = time.time()
cv = fold_generator(x, y, num_folds=num_folds)
 if max_iter is None:
search = GridSearchCV(model, grid, cv=cv,
scoring='neg_mean_absolute_error')
 else:
search = RandomizedSearchCV(model, grid, n_iter=max_iter, cv=cv,
scoring='neg_mean_absolute_error')
 search.fit(x, y, silent=True)
 t0 = time.time() - t0
print("Best CV score: {:.4f}, time: {:.1f}s".format(-search.best_score_, t0))
print(search.best_params_)
 return search.best_estimator_
class CentResnet(nn.Module):

 def __init__(self, n_classes):
super(CentResnet, self ).__init__()
self.base_model = base_model
self.lat8 = nn.Conv2d(512, 256, 1)
 self.lat16 = nn.Conv2d(1024, 256, 1)
 self.lat32 = nn.Conv2d(2048, 256, 1)
 self.bn8 = nn.GroupNorm(16, 256)
 self.bn16 = nn.GroupNorm(16, 256)
 self.bn32 = nn.GroupNorm(16, 256)
 self.conv0 = double_conv(5, 64)
 self.conv1 = double_conv(64, 128)
 self.conv2 = double_conv(128, 512)
 self.conv3 = double_conv(512, 1024)
 self.mp = nn.MaxPool2d(2)
 self.up1 = up(1282 , 512)
 self.up2 = up(512 + 512, 256)
 self.outc = nn.Conv2d(256, n_classes, 1)
 def forward(self, x):
batch_size = x.shape[0]
mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])
 x0 = torch.cat([x, mesh1], 1)
 x1 = self.mp(self.conv0(x0))
x2 = self.mp(self.conv1(x1))
x3 = self.mp(self.conv2(x2))
x4 = self.mp(self.conv3(x3))
feats8, feats16, feats32 = self.base_model(x)
 lat8 = F.relu(self.bn8(self.lat8(feats8)))
 lat16 = F.relu(self.bn16(self.lat16(feats16)))
 lat32 = F.relu(self.bn32(self.lat32(feats32)))
 mesh2 = get_mesh(batch_size, lat32.shape[2], lat32.shape[3])
 feats = torch.cat([lat32, mesh2], 1)
 x = self.up1(feats, x4)
 x = self.up2(x, x3)
 x = self.outc(x)
 return x<choose_model_class>

 def make_predictions(x, y, model, num_folds=10, shuffle=True, test=None, plot=True):
if test is not None:
sub_prediction = np.zeros(test.shape[0])
 oof_prediction = np.zeros(x.shape[0])
 for tr_idx, val_idx in fold_generator(x, y, num_folds=num_folds):
model.fit(x.iloc[tr_idx], y.iloc[tr_idx])
 oof_prediction[val_idx] = model.predict(x.iloc[val_idx])
 if test is not None:
sub_prediction += model.predict(test)/ num_folds
if plot:
plot_predictions(y, oof_prediction)
 if test is None:
return oof_prediction
else:
return oof_prediction, sub_prediction
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device)
 n_epochs = 10
model = CentResnet(8 ).to(device)
 optimizer = optim.AdamW(model.parameters() , lr=0.001)
 exp_lr_scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=0.001,cycle_momentum=False,max_lr=0.0013,step_size_up=2000 )<compute_test_metric>
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)
 scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)
 scaled_test_X.shape
def criterion(prediction, mask, regr,weight=0.4, size_average=True):
pred_mask = torch.sigmoid(prediction[:, 0])
 mask_loss = mask * torch.log(pred_mask + 1e-12)+(1 - mask)* torch.log(1 - pred_mask + 1e-12)
 mask_loss = -mask_loss.mean(0 ).sum()
pred_regr = prediction[:, 1:]
regr_loss =(torch.abs(pred_regr - regr ).sum(1)* mask ).sum(1 ).sum(1)/ mask.sum(1 ).sum(1)
 regr_loss = regr_loss.mean(0)
 loss = mask_loss +regr_loss
if not size_average:
loss *= prediction.shape[0]
return loss ,mask_loss , regr_loss<train_model>
grid = {
'depth': [4, 6, 8],
'random_seed' : [400, 100, 200],
'learning_rate' : [0.01, 0.03, 0.1]
}
cat_model = CatBoostRegressor(loss_function='MAE', task_type = "GPU")
 cat_model = search_cv(scaled_train_X, train_y, cat_model, grid, max_iter=3 )
def train(epoch, history=None):
model.train()
t = tqdm(train_loader)
 for batch_idx,(img_batch, mask_batch, regr_batch)in enumerate(t):
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 optimizer.zero_grad()
output = model(img_batch)
 if epoch < SWITCH_LOSS_EPOCH :
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,1)
 else:
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5)
 t.set_description(f'train_loss(l={loss:.3f} )(m={mask_loss:.2f} )(r={regr_loss:.4f}')
 if history is not None:
history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()
loss.backward()
optimizer.step()
exp_lr_scheduler.step()
print('Train Epoch: {} \tLR: {:.6f}\tLoss: {:.6f}\tMaskLoss: {:.6f}\tRegLoss: {:.6f}'.format(
 epoch,
optimizer.state_dict() ['param_groups'][0]['lr'],
loss.data,
mask_loss.data,
regr_loss.data))
def evaluate(epoch, history=None):
model.eval()
loss = 0
valid_loss = 0
valid_mask_loss = 0
valid_regr_loss = 0
with torch.no_grad() :
for img_batch, mask_batch, regr_batch in dev_loader:
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 output = model(img_batch)
 if epoch < SWITCH_LOSS_EPOCH :
loss,mask_loss, regr_loss= criterion(output, mask_batch, regr_batch,1, size_average=False)
 valid_loss += loss.data
valid_mask_loss += mask_loss.data
valid_regr_loss += regr_loss.data
else :
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5, size_average=False)
 valid_loss += loss.data
valid_mask_loss += mask_loss.data
valid_regr_loss += regr_loss.data
valid_loss /= len(dev_loader.dataset)
 valid_mask_loss /= len(dev_loader.dataset)
 valid_regr_loss /= len(dev_loader.dataset)
 if history is not None:
history.loc[epoch, 'dev_loss'] = valid_loss.cpu().numpy()
history.loc[epoch, 'mask_loss'] = valid_mask_loss.cpu().numpy()
history.loc[epoch, 'regr_loss'] = valid_regr_loss.cpu().numpy()
print('Dev loss: {:.4f}'.format(valid_loss)) 
<save_model>
oof = make_predictions(scaled_train_X, train_y, cat_model )
torch.save(model.state_dict() , './resnext50.pth' )<statistical_test>
oof, sub = make_predictions(scaled_train_X, train_y, cat_model,
test=scaled_test_X, plot=False )
act = torch.nn.Sigmoid()
logtens = torch.from_numpy(logits)
 probs = act(logtens)
 probs = probs[probs>0]
print(probs )<train_model>
submission.time_to_failure = sub
submission.to_csv('submission.csv',index=True )