Kaggle competition: LANL Earthquake Prediction
DISTANCE_THRESH_CLEAR = 2
def convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):
return x * fx / z + cx, y * fy / z + cy
def optimize_xy(r, c, x0, y0, z0):
def distance_fn(xyz):
x, y, z = xyz
x, y = convert_3d_to_2d(x, y, z0)
 y, x = x, y
x =(x - IMG_SHAPE[0] // 2)* IMG_HEIGHT /(IMG_SHAPE[0] // 2)/ MODEL_SCALE
x = np.round(x ).astype('int')
 y =(y + IMG_SHAPE[1] // 4)* IMG_WIDTH /(IMG_SHAPE[1] * 1.5)/ MODEL_SCALE
y = np.round(y ).astype('int')
 return(x-r)**2 +(y-c)**2
res = minimize(distance_fn, [x0, y0, z0], method='Powell')
 x_new, y_new, z_new = res.x
return x_new, y_new, z0
def clear_duplicates(coords):
for c1 in coords:
xyz1 = np.array([c1['x'], c1['y'], c1['z']])
 for c2 in coords:
xyz2 = np.array([c2['x'], c2['y'], c2['z']])
 distance = np.sqrt(((xyz1 - xyz2)**2 ).sum())
 if distance < DISTANCE_THRESH_CLEAR:
if c1['confidence'] < c2['confidence']:
c1['confidence'] = -1
return [c for c in coords if c['confidence'] > 0]
def extract_coords(prediction):
logits = prediction[0]
regr_output = prediction[1:]
points = np.argwhere(logits > -0.5)
 col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])
 coords = []
for r, c in points:
regr_dict = dict(zip(col_names, regr_output[:, r, c]))
coords.append(_regr_back(regr_dict))
coords[-1]['confidence'] = 1 /(1 + np.exp(-logits[r, c]))
coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])
 coords = clear_duplicates(coords)
 return coords
def coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):
s = []
for c in coords:
for n in names:
s.append(str(c.get(n, 0)))
 return ' '.join(s )<load_pretrained>
%matplotlib inline
sns.set()
warnings.filterwarnings("ignore" )
val_preds = []
dev_loader = DataLoader(dataset=dev_dataset, batch_size=2, shuffle=False, num_workers=2)
 model.eval()
for img, _, _ in tqdm(dev_loader):
with torch.no_grad() :
output = model(img.to(device))
output = output.data.cpu().numpy()
for out in output:
coords = extract_coords(out)
 s = coords2str(coords)
 val_preds.append(s )<save_to_csv>
train_features_ = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/train_X.csv')
 train_y_ = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/train_y.csv')
 train_y_extra_ = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/train_y_extra.csv')
 test_features_ = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/test_X.csv' )
df_dev['PredictionString'] = val_preds
df_dev.head()
df_dev.to_csv('val_predictions.csv', index=False )<load_pretrained>
X_train = train_features_.copy()
X_test = test_features_.copy()
y_train = train_y_
predictions = []
test_loader = DataLoader(dataset=test_dataset, batch_size=2, shuffle=False, num_workers=2)
 model.eval()
for img, _, _ in tqdm(test_loader):
with torch.no_grad() :
output = model(img.to(device))
output = output.data.cpu().numpy()
for out in output:
coords = extract_coords(out)
 s = coords2str(coords)
 predictions.append(s )<save_to_csv>
rows = 150_000
y_train_new = pd.DataFrame(columns=['taf'], dtype=np.float64, index = y_train.index)
 y_train_new['taf'] = train_y_extra_['time_after_failure']
test = pd.read_csv(PATH + 'sample_submission.csv')
 test['PredictionString'] = predictions
test.to_csv('predictions.csv', index=False)
 test.head()<data_type_conversions>
cols = list(np.abs(X_train.corrwith(y_train_new['taf'])).sort_values(ascending=False ).head(50 ).index)
 cols[::5]
def expand_df(df, PredictionStringCols):
df = df.dropna().copy()
df['NumCars'] = [int(( x.count(' ')+1)/7)for x in df['PredictionString']]
image_id_expanded = [item for item, count in zip(df['ImageId'], df['NumCars'])for i in range(count)]
prediction_strings_expanded = df['PredictionString'].str.split(' ',expand = True ).values.reshape(-1,7 ).astype(float)
 prediction_strings_expanded = prediction_strings_expanded[~np.isnan(prediction_strings_expanded ).all(axis=1)]
df = pd.DataFrame(
 {
'ImageId': image_id_expanded,
PredictionStringCols[0]:prediction_strings_expanded[:,0],
PredictionStringCols[1]:prediction_strings_expanded[:,1],
PredictionStringCols[2]:prediction_strings_expanded[:,2],
PredictionStringCols[3]:prediction_strings_expanded[:,3],
PredictionStringCols[4]:prediction_strings_expanded[:,4],
PredictionStringCols[5]:prediction_strings_expanded[:,5],
PredictionStringCols[6]:prediction_strings_expanded[:,6]
})
 return df
def str2coords(s, names):
coords = []
for l in np.array(s.split() ).reshape([-1, 7]):
coords.append(dict(zip(names, l.astype('float'))))
return coords
def TranslationDistance(p,g, abs_dist = False):
dx = p['x'] - g['x']
dy = p['y'] - g['y']
dz = p['z'] - g['z']
diff0 =(g['x']**2 + g['y']**2 + g['z']**2)**0.5
diff1 =(dx**2 + dy**2 + dz**2)**0.5
if abs_dist:
diff = diff1
else:
diff = diff1/diff0
return diff
def RotationDistance(p, g):
true=[ g['pitch'] ,g['yaw'] ,g['roll'] ]
pred=[ p['pitch'] ,p['yaw'] ,p['roll'] ]
q1 = R.from_euler('xyz', true)
 q2 = R.from_euler('xyz', pred)
 diff = R.inv(q2)* q1
W = np.clip(diff.as_quat() [-1], -1., 1.)
W =(acos(W)*360)/pi
if W > 180:
W = 360 - W
return W<define_search_space>
coef = [0.35, 0.5, 0.05, 0.1]
def custom_objective(y_true, y_pred):
c = 0.5
residual = y_pred - y_true
grad = c * residual /(np.abs(residual)+ c)
 hess = c ** 2 /(np.abs(residual)+ c)** 2
h = 1.2
scale = 1 +(residual / h)** 2
scale_sqrt = np.sqrt(scale)
 grad_huber = residual / scale_sqrt
hess_huber = 1 / scale / scale_sqrt
grad_rmse = residual
hess_rmse = 1.0
grad_mae = np.array(residual)
 grad_mae[grad_mae > 0] = 1.
 grad_mae[grad_mae <= 0] = -1.
 hess_mae = 1.0
return coef[0] * grad + coef[1] * grad_huber + coef[2] * grad_rmse + coef[3] * grad_mae, \
coef[0] * hess + coef[1] * hess_huber + coef[2] * hess_rmse + coef[3] * hess_mae
def logcosh_objective(y_true, y_pred):
d = y_pred - y_true
grad = np.tanh(d)/y_true
hess =(1.0 - grad*grad)/y_true
return grad, hess
def huber_objective(y_true, y_pred):
d = y_pred - y_true
h = 1.2
scale = 1 +(d / h)** 2
scale_sqrt = np.sqrt(scale)
 grad = d / scale_sqrt
hess = 1 / scale / scale_sqrt
return grad, hess
thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]
thres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]
def check_match(idx):
keep_gt=False
thre_tr_dist = thres_tr_list[idx]
thre_ro_dist = thres_ro_list[idx]
train_dict = {imgID:str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z'])for imgID,s in zip(train_df['ImageId'],train_df['PredictionString'])}
valid_dict = {imgID:str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score'])for imgID,s in zip(valid_df['ImageId'],valid_df['PredictionString'])}
result_flg = []
scores = []
MAX_VAL = 10**10
for img_id in valid_dict:
for pcar in sorted(valid_dict[img_id], key=lambda x: -x['carid_or_score']):
min_tr_dist = MAX_VAL
min_idx = -1
for idx, gcar in enumerate(train_dict[img_id]):
tr_dist = TranslationDistance(pcar,gcar)
 if tr_dist < min_tr_dist:
min_tr_dist = tr_dist
min_ro_dist = RotationDistance(pcar,gcar)
 min_idx = idx
if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:
if not keep_gt:
train_dict[img_id].pop(min_idx)
 result_flg.append(1)
 else:
result_flg.append(0)
 scores.append(pcar['carid_or_score'])
 return result_flg, scores<load_from_csv>
params = {'bagging_fraction': 0.71,
'boosting': 'gbdt',
'feature_fraction': 0.94,
'lambda_l1': 3.131216244016188,
'lambda_l2': 2.4124061313905836,
'learning_rate': 0.049886848207269734,
'max_bin': 193,
'max_depth': 15,
'metric': 'MAE',
'min_data_in_bin': 167,
'min_data_in_leaf': 62,
'min_gain_to_split': 2.07,
'num_leaves': 38,
'objective': custom_objective,
'subsample': 0.9133120405819966}
valid_df = pd.read_csv('val_predictions.csv')
 expanded_valid_df = expand_df(valid_df, ['pitch','yaw','roll','x','y','z','Score'])
 valid_df = valid_df.fillna('')
 train_df = pd.read_csv('.. /input/pku-autonomous-driving/train.csv')
 train_df = train_df[train_df.ImageId.isin(valid_df.ImageId.unique())]
expanded_train_df = expand_df(train_df, ['model_type','pitch','yaw','roll','x','y','z'])
 max_workers = 10
n_gt = len(expanded_train_df)
 ap_list = []
p = Pool(processes=max_workers)
 for result_flg, scores in p.imap(check_match, range(10)) :
if np.sum(result_flg)> 0:
n_tp = np.sum(result_flg)
 recall = n_tp/n_gt
ap = average_precision_score(result_flg, scores)*recall
else:
ap = 0
ap_list.append(ap)
 map = np.mean(ap_list)
 print('map:', map )<import_modules>
features_select = ['abs_min',
'abs_q01',
'abs_q05',
'abs_trend',
'autocorrelation_10',
'autocorrelation_1000',
'autocorrelation_500',
'av_change_abs_roll_mean_10',
'av_change_abs_roll_std_10',
'av_change_abs_roll_std_100',
'av_change_abs_roll_std_1000',
'avg_first_10000',
'avg_last_10000',
'avg_last_5000',
'c3_5',
'c3_500',
'classic_sta_lta1_mean',
'classic_sta_lta4_mean',
'classic_sta_lta5_mean',
'count_big_50000_less_threshold_5',
'energy_spectra_2640hz',
'energy_spectra_9306hz',
'energy_spectra_norm_109306hz_denoised',
'energy_spectra_norm_129306hz',
'energy_spectra_norm_135973hz_denoised',
'energy_spectra_norm_142640hz_denoised',
'energy_spectra_norm_149306hz_denoised',
'energy_spectra_norm_15973hz_denoised',
'energy_spectra_norm_169306hz_denoised',
'energy_spectra_norm_22640hz_denoised',
'energy_spectra_norm_29306hz_denoised',
'energy_spectra_norm_35973hz_denoised',
'energy_spectra_norm_42640hz_denoised',
'energy_spectra_norm_49306hz_denoised',
'energy_spectra_norm_55973hz_denoised',
'energy_spectra_norm_62640hz_denoised',
'energy_spectra_norm_89306hz_denoised',
'energy_spectra_norm_9306hz_denoised',
'energy_spectra_norm_95973hz_denoised',
'fft_1000_roll_std_20',
'fft_1000_roll_std_25',
'fft_1000_roll_std_70',
'fft_100_roll_std_1',
'fft_100_roll_std_20',
'fft_100_roll_std_70',
'fft_100_roll_std_75',
'fft_10_roll_std_75',
'fft_mean_change_rate',
'fft_min',
'fft_min_roll_mean_100',
'fft_min_roll_std_100',
'fft_skew',
'fft_skew_first_50000',
'fft_spkt_welch_density_100',
'fft_spkt_welch_density_5',
'fft_spkt_welch_density_50',
'fft_time_rev_asym_stat_10',
'fft_time_rev_asym_stat_100',
'iqr',
'kstat_3',
'mad',
'max_first_5000',
'max_last_10000',
'max_roll_mean_1000',
'max_to_min',
'mean_change_abs',
'med',
'min_last_10000',
'min_roll_std_100',
'num_crossings',
'num_peaks_10',
'num_peaks_5',
'q01_roll_mean_100',
'q01_roll_std_10',
'q01_roll_std_1000',
'q05_roll_mean_100',
'q05_roll_std_1000',
'q95',
'q95_roll_mean_10',
'q95_roll_mean_100',
'q95_roll_std_1000',
'q99_roll_mean_1000',
'skew',
'std_0_to_10',
'std_first_5000',
'std_neg_10_to_0',
'std_neg_2_to_2',
'std_neg_5_to_5',
'std_roll_mean_1000']
PATH = '.. /input/pku-autonomous-driving/'
os.listdir(PATH )<define_variables>
X_train_select = X_train[features_select].copy()
X_test_select = X_test[features_select].copy()
SWITCH_LOSS_EPOCH = 5
print(torch.__version__ )<load_from_csv>
ttf_pred_orig, ttf_test_orig, ttf_pred_oof, model1= get_prediction(X_train_select, y_train,
X_test = X_test_select)
 print("Training MAE for time_to_failure is {:.7f}"\ 
.format(np.abs(y_train['time_to_failure'] - ttf_pred_orig ).mean()))
print("OOF MAE for time_to_failure is {:.7f}"\ 
.format(np.abs(y_train['time_to_failure'] - ttf_pred_oof ).mean()))
train = pd.read_csv(PATH + 'train.csv')
 test = pd.read_csv(PATH + 'sample_submission.csv')
 bad_list = ['ID_1a5a10365',
'ID_1db0533c7',
'ID_53c3fe91a',
'ID_408f58e9f',
'ID_4445ae041',
'ID_bb1d991f6',
'ID_c44983aeb',
'ID_f30ebe4d4']
train = train.loc[~train['ImageId'].isin(bad_list)]
camera_matrix = np.array([[2304.5479, 0, 1686.2379],
[0, 2305.8757, 1354.9849],
[0, 0, 1]], dtype=np.float32)
 camera_matrix_inv = np.linalg.inv(camera_matrix)
 train.head()<categorify>
taf_pred, _, taf_pred_oof, model2 = get_prediction(X_train_select, y_train_new)
 print("Training MAE for time_after_failure is {:.7f}"\ 
.format(( y_train_new['taf'] - taf_pred ).abs().mean()))
print("OOF MAE for time_after_failure is {:.7f}"\ 
.format(np.abs(y_train_new['taf'] - taf_pred_oof ).mean()))
def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):

 coords = []
for l in np.array(s.split() ).reshape([-1, 7]):
coords.append(dict(zip(names, l.astype('float'))))
if 'id' in coords[-1]:
coords[-1]['id'] = int(coords[-1]['id'])
 return coords<train_model>
X_train['taf_estimate'] = taf_pred_oof
ttf_pred,_,ttf_pred_oof, model3 = get_prediction(X_train, y_train)
 print("
Training MAE for time_to_failure is {:.7f}"\ 
.format(np.abs(y_train['time_to_failure'] - ttf_pred ).mean()))
print("OOF MAE for time_to_failure is {:.7f}"\ 
.format(np.abs(y_train['time_to_failure'] - ttf_pred_oof ).mean()))
inp = train['PredictionString'][0]
print('Example input:
', inp)
 print()
print('Output:
', str2coords(inp))<categorify>
important_feature_index = np.argsort(model3.feature_importances_)[::-1]
X_train.columns[important_feature_index[:10]]
def get_img_coords(s):

 coords = str2coords(s)
 xs = [c['x'] for c in coords]
ys = [c['y'] for c in coords]
zs = [c['z'] for c in coords]
P = np.array(list(zip(xs, ys, zs)) ).T
img_p = np.dot(camera_matrix, P ).T
img_p[:, 0] /= img_p[:, 2]
img_p[:, 1] /= img_p[:, 2]
img_xs = img_p[:, 0]
img_ys = img_p[:, 1]
img_zs = img_p[:, 2]
return img_xs, img_ys
plt.figure(figsize=(14,14))
plt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'))
plt.scatter(*get_img_coords(train['PredictionString'][2217]), color='red', s=100);<prepare_x_and_y>
X_train = X_train_select.copy()
X_test = X_test_select.copy()
def euler_to_Rot(yaw, pitch, roll):
Y = np.array([[cos(yaw), 0, sin(yaw)],
[0, 1, 0],
[-sin(yaw), 0, cos(yaw)]])
 P = np.array([[1, 0, 0],
[0, cos(pitch), -sin(pitch)],
[0, sin(pitch), cos(pitch)]])
 R = np.array([[cos(roll), -sin(roll), 0],
[sin(roll), cos(roll), 0],
[0, 0, 1]])
 return np.dot(Y, np.dot(P, R))<feature_engineering>
X_bsp_train = X_train[1200:]
X_bsp_cv = X_train[:1200]
y_bsp_train = y_train[1200:]
y_bsp_cv = y_train[:1200]
y_bsp_train_new = y_train_new[1200:]
y_bsp_cv_new = y_train_new[:1200]
IMG_WIDTH = 1600
IMG_HEIGHT = 700
MODEL_SCALE = 8
def _regr_preprocess(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] / 100
regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)
 regr_dict['pitch_sin'] = sin(regr_dict['pitch'])
 regr_dict['pitch_cos'] = cos(regr_dict['pitch'])
 regr_dict.pop('pitch')
 regr_dict.pop('id')
 return regr_dict
def _regr_back(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] * 100
regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)
 pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 regr_dict['pitch'] = np.arccos(pitch_cos)* np.sign(pitch_sin)
 return regr_dict
def preprocess_image(img):
img = img[img.shape[0] // 2:]
bg = np.ones_like(img)* img.mean(1, keepdims=True ).astype(img.dtype)
 bg = bg[:, :img.shape[1] // 4]
img = np.concatenate([bg, img, bg], 1)
 img = cv2.resize(img,(IMG_WIDTH, IMG_HEIGHT))
return(img / 255 ).astype('float32')
 def get_mask_and_regr(img, labels):
mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')
 regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']
regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')
 coords = str2coords(labels)
 xs, ys = get_img_coords(labels)
 for x, y, regr_dict in zip(xs, ys, coords):
x, y = y, x
x =(x - img.shape[0] // 2)* IMG_HEIGHT /(img.shape[0] // 2)/ MODEL_SCALE
x = np.round(x ).astype('int')
 y =(y + img.shape[1] // 4)* IMG_WIDTH /(img.shape[1] * 1.5)/ MODEL_SCALE
y = np.round(y ).astype('int')
 if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:
mask[x, y] = 1
regr_dict = _regr_preprocess(regr_dict)
 regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]
return mask, regr<categorify>
n_fold = 6
random_state = 802
class CarDataset(Dataset):

 def __init__(self, dataframe, root_dir, training=True, transform=None):
self.df = dataframe
self.root_dir = root_dir
self.transform = transform
self.training = training
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
idx, labels = self.df.values[idx]
img_name = self.root_dir.format(idx)
 img0 = imread(img_name, True)
 img = preprocess_image(img0)
 img = np.rollaxis(img, 2, 0)
 if self.training:
mask, regr = get_mask_and_regr(img0, labels)
 regr = np.rollaxis(regr, 2, 0)
 else:
mask, regr = 0, 0
return [img, mask, regr]<split>
ttf_est_train, ttf_est_cv, ttf_est_oof, _ = get_prediction(X_bsp_train,
y_bsp_train,
X_test=X_bsp_cv,
n_fold=n_fold,
random_state=random_state)
 print("TTF CV MAE: {:.7f}"\ 
.format(np.abs(y_bsp_cv['time_to_failure'] - ttf_est_cv ).mean()))
print("TTF OOF MAE: {:.7f}"\ 
.format(np.abs(y_bsp_train['time_to_failure'] - ttf_est_oof ).mean()))
train_images_dir = PATH + 'train_images/{}.jpg'
test_images_dir = PATH + 'test_images/{}.jpg'
df_train, df_dev = train_test_split(train, test_size=0.1, random_state=63)
 df_test = test
train_dataset = CarDataset(df_train, train_images_dir)
 dev_dataset = CarDataset(df_dev, train_images_dir)
 test_dataset = CarDataset(df_test, test_images_dir )<load_pretrained>
taf_est_train, taf_est_cv, taf_est_oof, _ = get_prediction(X_bsp_train,
y_bsp_train_new,
X_test=X_bsp_cv,
n_fold=n_fold,
random_state=random_state )
BATCH_SIZE = 1
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 dev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)
 test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4 )<choose_model_class>
X_bsp_train = X_bsp_train.assign(taf_estimate = taf_est_oof);
X_bsp_cv = X_bsp_cv.assign(taf_estimate = taf_est_cv);
class double_conv(nn.Module):

 def __init__(self, in_ch, out_ch):
super(double_conv, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True),
nn.Conv2d(out_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True)
)
 def forward(self, x):
x = self.conv(x)
 return x
class up(nn.Module):
def __init__(self, in_ch, out_ch, bilinear=True):
super(up, self ).__init__()
if bilinear:
self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
 else:
self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)
 self.conv = double_conv(in_ch, out_ch)
 def forward(self, x1, x2=None):
x1 = self.up(x1)
 diffY = x2.size() [2] - x1.size() [2]
diffX = x2.size() [3] - x1.size() [3]
x1 = F.pad(x1,(diffX // 2, diffX - diffX//2,
diffY // 2, diffY - diffY//2))
if x2 is not None:
x = torch.cat([x2, x1], dim=1)
 else:
x = x1
x = self.conv(x)
 return x
def get_mesh(batch_size, shape_x, shape_y):
mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))
mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mesh = torch.cat([torch.tensor(mg_x ).to(device), torch.tensor(mg_y ).to(device)], 1)
 return mesh<load_pretrained>
ttf_est_train, ttf_est_cv, ttf_est_oof, model_trial = get_prediction(X_bsp_train, y_bsp_train,
X_test=X_bsp_cv,
n_fold=n_fold,random_state=random_state)
 print("TTF CV MAE after adding TAF to training features: {:.7f}"\ 
.format(np.abs(y_bsp_cv['time_to_failure'] - ttf_est_cv ).mean()))
print("TTF OOF MAE after adding TAF to training features: {:.7f}"\ 
.format(np.abs(y_bsp_train['time_to_failure'] - ttf_est_oof ).mean()))
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth',
'inceptionresnetv2': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',
}
def conv3x3(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 def conv1x1(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
 class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = nn.GroupNorm(16, planes)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = nn.GroupNorm(16, planes)
 if stride != 1 or inplanes != planes:
self.downsample = nn.Sequential(
 conv1x1(inplanes, planes, stride), nn.GroupNorm(16, planes))
else:
self.downsample = None
def forward(self, x):
identity = x
out = F.relu(self.bn1(self.conv1(x)) , inplace=True)
 out = self.bn2(self.conv2(out))
if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = F.relu(out, inplace=True)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1):
super(Bottleneck, self ).__init__()
self.conv1 = conv1x1(inplanes, planes)
 self.bn1 = nn.GroupNorm(16, planes)
 self.conv2 = conv3x3(planes, planes, stride)
 self.bn2 = nn.GroupNorm(16, planes)
 self.conv3 = conv1x1(planes, planes * self.expansion)
 self.bn3 = nn.GroupNorm(16, planes * self.expansion)
 if stride != 1 or inplanes != planes * self.expansion:
self.downsample = nn.Sequential(
 conv1x1(inplanes, planes * self.expansion, stride),
nn.GroupNorm(16, planes * self.expansion))
else:
self.downsample = None
def forward(self, x):
identity = x
out = F.relu(self.bn1(self.conv1(x)) , inplace=True)
 out = F.relu(self.bn2(self.conv2(out)) , inplace=True)
 out = self.bn3(self.conv3(out))
if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = F.relu(out)
 return out
class ResNetFeatures(nn.Module):
def __init__(self, block, layers, num_classes=1000, zero_init_residual=False):
super(ResNetFeatures, self ).__init__()
self.inplanes = 64
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
bias=False)
 self.bn1 = nn.GroupNorm(16, 64)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 if zero_init_residual:
for m in self.modules() :
if isinstance(m, Bottleneck):
nn.init.constant_(m.bn3.weight, 0)
 elif isinstance(m, BasicBlock):
nn.init.constant_(m.bn2.weight, 0)
 def _make_layer(self, block, planes, blocks, stride=1):
layers = []
layers.append(block(self.inplanes, planes, stride))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
 def forward(self, x):
conv1 = F.relu(self.bn1(self.conv1(x)) , inplace=True)
 conv1 = F.max_pool2d(conv1, 3, stride=2, padding=1)
 feats4 = self.layer1(conv1)
 feats8 = self.layer2(feats4)
 feats16 = self.layer3(feats8)
 feats32 = self.layer4(feats16)
 return feats8, feats16, feats32
def resnet18(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['resnet18']))
return model
def resnext50(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['resnext50_32x4d']))
return model
def resnet34(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['resnet34']))
return model
def inceptionresnetv2(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['inceptionresnetv2']))
return model
def densenet201(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['densenet201']))
return model
def _load_pretrained(model, pretrained):
model_dict = model.state_dict()
pretrained = {k : v for k, v in pretrained.items() if k in model_dict}
model_dict.update(pretrained)
 model.load_state_dict(model_dict )<choose_model_class>
from eli5 import show_weights
from eli5.sklearn import PermutationImportance
from IPython.display import display
base_model = resnext50(pretrained=False)
 base_model<define_search_model>
perm = PermutationImportance(model_trial, random_state=1 ).fit(X_bsp_cv,y_bsp_cv)
 display(show_weights(perm, feature_names = X_bsp_cv.columns.tolist()))
class CentResnet(nn.Module):

 def __init__(self, n_classes):
super(CentResnet, self ).__init__()
self.base_model = base_model
self.lat8 = nn.Conv2d(128, 256, 1)
 self.lat16 = nn.Conv2d(256, 256, 1)
 self.lat32 = nn.Conv2d(512, 256, 1)
 self.bn8 = nn.GroupNorm(16, 256)
 self.bn16 = nn.GroupNorm(16, 256)
 self.bn32 = nn.GroupNorm(16, 256)
 self.conv0 = double_conv(5, 64)
 self.conv1 = double_conv(64, 128)
 self.conv2 = double_conv(128, 512)
 self.conv3 = double_conv(512, 1024)
 self.mp = nn.MaxPool2d(2)
 self.up1 = up(1282 , 512)
 self.up2 = up(512 + 512, 256)
 self.outc = nn.Conv2d(256, n_classes, 1)
 def forward(self, x):
batch_size = x.shape[0]
mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])
 x0 = torch.cat([x, mesh1], 1)
 x1 = self.mp(self.conv0(x0))
x2 = self.mp(self.conv1(x1))
x3 = self.mp(self.conv2(x2))
x4 = self.mp(self.conv3(x3))
feats8, feats16, feats32 = self.base_model(x)
 lat8 = F.relu(self.bn8(self.lat8(feats8)))
 lat16 = F.relu(self.bn16(self.lat16(feats16)))
 lat32 = F.relu(self.bn32(self.lat32(feats32)))
 mesh2 = get_mesh(batch_size, lat32.shape[2], lat32.shape[3])
 feats = torch.cat([lat32, mesh2], 1)
 x = self.up1(feats, x4)
 x = self.up2(x, x3)
 x = self.outc(x)
 return x<set_options>
X_bsp_train['ttf_estimate'] = ttf_est_oof
X_bsp_cv['ttf_estimate'] = ttf_est_cv
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss<choose_model_class>
num_bootstrap = 2
coeff_bsp = [0.4, 0.6]
for b in tqdm_notebook(range(num_bootstrap)) :
print("
Bootstrapping iteration {0}.".format(b+1))
random_state = b
taf_est_train, taf_est_cv, taf_est_oof, model_taf = \
get_prediction(X_bsp_train.drop(columns=['taf_estimate']),
y_bsp_train_new,
X_test=X_bsp_cv.drop(columns=['taf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TAF training OOF MAE: {:.7f}"\ 
.format(np.abs(y_bsp_train_new['taf'] - taf_est_oof ).mean()))
print("TAF CV MAE: {:.7f}"\ 
.format(np.abs(y_bsp_cv_new['taf'] - taf_est_cv ).mean()))
print("Mean abs change in TAF feature for CV: {:.7f}"\ 
.format(np.abs(X_bsp_cv['taf_estimate'] - taf_est_cv ).mean()))
X_bsp_train['taf_estimate'] = coeff_bsp[0]*X_bsp_train['taf_estimate'] + coeff_bsp[1]*taf_est_oof
X_bsp_cv['taf_estimate'] = coeff_bsp[0]*X_bsp_cv['taf_estimate'] + coeff_bsp[1]*taf_est_cv
ttf_est_train, ttf_est_cv, ttf_est_oof, model = \
get_prediction(X_bsp_train.drop(columns=['ttf_estimate']),
y_bsp_train,
X_test=X_bsp_cv.drop(columns=['ttf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TTF training OOF MAE: {:.7f}"\ 
.format(np.abs(y_bsp_train['time_to_failure'] - ttf_est_oof ).mean()))
print("TTF CV MAE: {:.7f}"\ 
.format(np.abs(y_bsp_cv['time_to_failure'] - ttf_est_cv ).mean()))
print("Mean abs change in TTF feature for CV: {:.7f}"\ 
.format(np.abs(X_bsp_cv['ttf_estimate'] - ttf_est_cv ).mean()))
X_bsp_train['ttf_estimate'] = coeff_bsp[0]*X_bsp_train['ttf_estimate'] + coeff_bsp[1]*ttf_est_oof
X_bsp_cv['ttf_estimate'] = coeff_bsp[0]*X_bsp_cv['ttf_estimate'] + coeff_bsp[1]*ttf_est_cv
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device)
 n_epochs = 2
model = CentResnet(8 ).to(device)
 optimizer = optim.AdamW(model.parameters() , lr=0.001)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=max(n_epochs, 10)* len(train_loader)// 3, gamma=0.1 )<define_variables>
important_feature_index = np.argsort(model.feature_importances_)[::-1]
X_bsp_train.columns[important_feature_index[:20]]
img_batch = torch.randn(( 1,3,700,1600))<train_model>
X_train = train_features_[features_select]
X_test = test_features_[features_select]
test = model(img_batch.to(device))<compute_test_metric>
print("Final bootstrapping is performed based on {} features.".format(X_train.shape[-1]))
def criterion(prediction, mask, regr,weight=0.4, size_average=True):
pred_mask = torch.sigmoid(prediction[:, 0])
 mask_loss = mask * torch.log(pred_mask + 1e-12)+(1 - mask)* torch.log(1 - pred_mask + 1e-12)
 mask_loss = -mask_loss.mean(0 ).sum()
pred_regr = prediction[:, 1:]
regr_loss =(torch.abs(pred_regr - regr ).sum(1)* mask ).sum(1 ).sum(1)/ mask.sum(1 ).sum(1)
 regr_loss = regr_loss.mean(0)
 loss = mask_loss +regr_loss
if not size_average:
loss *= prediction.shape[0]
return loss ,mask_loss , regr_loss<train_model>
n_fold = 6
random_state = 1127
params = {'bagging_fraction': 0.51,
'boosting': 'gbdt',
'feature_fraction': 0.9,
'lambda_l1': 2,
'lambda_l2': 0.03,
'learning_rate': 0.1,
'max_bin': 48,
'max_depth': 8,
'metric': 'MAE',
'min_data_in_bin': 57,
'min_data_in_leaf': 11,
'min_gain_to_split': 0.53,
'num_leaves': 83,
'objective': custom_objective,
'subsample': 0.55}
def train(epoch, history=None):
model.train()
t = tqdm(train_loader)
 for batch_idx,(img_batch, mask_batch, regr_batch)in enumerate(t):
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 optimizer.zero_grad()
output = model(img_batch)
 if epoch < SWITCH_LOSS_EPOCH :
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,1)
 else:
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5)
 t.set_description(f'train_loss(l={loss:.3f} )(m={mask_loss:.2f} )(r={regr_loss:.4f}')
 if history is not None:
history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()
loss.backward()
optimizer.step()
exp_lr_scheduler.step()
print('Train Epoch: {} \tLR: {:.6f}\tLoss: {:.6f}\tMaskLoss: {:.6f}\tRegLoss: {:.6f}'.format(
 epoch,
optimizer.state_dict() ['param_groups'][0]['lr'],
loss.data,
mask_loss.data,
regr_loss.data))
def evaluate(epoch, history=None):
model.eval()
loss = 0
valid_loss = 0
valid_mask_loss = 0
valid_regr_loss = 0
with torch.no_grad() :
for img_batch, mask_batch, regr_batch in dev_loader:
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 output = model(img_batch)
 if epoch < SWITCH_LOSS_EPOCH :
loss,mask_loss, regr_loss= criterion(output, mask_batch, regr_batch,1, size_average=False)
 valid_loss += loss.data
valid_mask_loss += mask_loss.data
valid_regr_loss += regr_loss.data
else :
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5, size_average=False)
 valid_loss += loss.data
valid_mask_loss += mask_loss.data
valid_regr_loss += regr_loss.data
valid_loss /= len(dev_loader.dataset)
 valid_mask_loss /= len(dev_loader.dataset)
 valid_regr_loss /= len(dev_loader.dataset)
 if history is not None:
history.loc[epoch, 'dev_loss'] = valid_loss.cpu().numpy()
history.loc[epoch, 'mask_loss'] = valid_mask_loss.cpu().numpy()
history.loc[epoch, 'regr_loss'] = valid_regr_loss.cpu().numpy()
print('Dev loss: {:.4f}'.format(valid_loss))<save_model>
taf_est_train, taf_est_test, taf_est_oof, _ = get_prediction(X_train, y_train_new,
X_test=X_test,
n_fold=n_fold,random_state=random_state )
torch.save(model.state_dict() , './resnext50.pth' )<statistical_test>
X_train['taf_estimate'] = taf_est_oof
X_test['taf_estimate'] = taf_est_test
ttf_est_train, ttf_est_test, ttf_est_oof, _ = get_prediction(X_train, y_train,
X_test=X_test,
n_fold=n_fold,random_state=random_state)
 ttf_est_test_bsp1 = ttf_est_test
print("TTF OOF MAE after adding TAF to training features: {:.7f}"\ 
.format(( np.abs(y_train['time_to_failure'] - ttf_est_oof)).mean()))
act = torch.nn.Sigmoid()
logtens = torch.from_numpy(logits)
 probs = act(logtens)
 probs = probs[probs>0]
print(probs )<train_model>
X_train['ttf_estimate'] = ttf_est_oof
X_test['ttf_estimate'] = ttf_est_test
DISTANCE_THRESH_CLEAR = 2
def convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):
return x * fx / z + cx, y * fy / z + cy
def optimize_xy(r, c, x0, y0, z0):
def distance_fn(xyz):
x, y, z = xyz
x, y = convert_3d_to_2d(x, y, z0)
 y, x = x, y
x =(x - IMG_SHAPE[0] // 2)* IMG_HEIGHT /(IMG_SHAPE[0] // 2)/ MODEL_SCALE
x = np.round(x ).astype('int')
 y =(y + IMG_SHAPE[1] // 4)* IMG_WIDTH /(IMG_SHAPE[1] * 1.5)/ MODEL_SCALE
y = np.round(y ).astype('int')
 return(x-r)**2 +(y-c)**2
res = minimize(distance_fn, [x0, y0, z0], method='Powell')
 x_new, y_new, z_new = res.x
return x_new, y_new, z0
def clear_duplicates(coords):
for c1 in coords:
xyz1 = np.array([c1['x'], c1['y'], c1['z']])
 for c2 in coords:
xyz2 = np.array([c2['x'], c2['y'], c2['z']])
 distance = np.sqrt(((xyz1 - xyz2)**2 ).sum())
 if distance < DISTANCE_THRESH_CLEAR:
if c1['confidence'] < c2['confidence']:
c1['confidence'] = -1
return [c for c in coords if c['confidence'] > 0]
def extract_coords(prediction):
logits = prediction[0]
regr_output = prediction[1:]
points = np.argwhere(logits > 0)
 col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])
 coords = []
for r, c in points:
regr_dict = dict(zip(col_names, regr_output[:, r, c]))
coords.append(_regr_back(regr_dict))
coords[-1]['confidence'] = 1 /(1 + np.exp(-logits[r, c]))
coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])
 coords = clear_duplicates(coords)
 return coords
def coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):
s = []
for c in coords:
for n in names:
s.append(str(c.get(n, 0)))
 return ' '.join(s )<load_pretrained>
num_bootstrap = 20
coeff_bsp = [0.4, 0.6]
for b in tqdm_notebook(range(num_bootstrap)) :
print("
Bootstrapping iteration {0}.".format(b+1))
random_state = b
taf_est_train, taf_est_test, taf_est_oof, _ = \
get_prediction(X_train.drop(columns=['taf_estimate']),
y_train_new,
X_test=X_test.drop(columns=['taf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TAF training OOF MAE: {:.7f}"\ 
.format(np.abs(y_train_new['taf'] - taf_est_oof ).mean()))
X_train['taf_estimate'] = coeff_bsp[0]*X_train['taf_estimate'] + coeff_bsp[1]*taf_est_oof
X_test['taf_estimate'] = coeff_bsp[0]*X_test['taf_estimate'] + coeff_bsp[1]*taf_est_test
ttf_est_train, ttf_est_test, ttf_est_oof, model = \
get_prediction(X_train.drop(columns=['ttf_estimate']),
y_train,
X_test=X_test.drop(columns=['ttf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TTF training OOF MAE: {:.7f}"\ 
.format(( np.abs(y_train['time_to_failure'] - ttf_est_oof)).mean()))
X_train['ttf_estimate'] = coeff_bsp[0]*X_train['ttf_estimate'] + coeff_bsp[1]*ttf_est_oof
X_test['ttf_estimate'] = coeff_bsp[0]*X_test['ttf_estimate'] + coeff_bsp[1]*ttf_est_test
<save_to_csv><EOS> 
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv',
index_col='seg_id')
 submission_nonbsp = submission.copy()
submission_bsp1 = submission.copy()
submission['time_to_failure'] = ttf_est_test
submission_bsp1['time_to_failure'] = ttf_est_test_bsp1
submission_nonbsp['time_to_failure'] = ttf_test_orig
submission.to_csv('submission.csv')
 submission_nonbsp.to_csv('submission_nonbsp.csv')
 submission_bsp1.to_csv('submission_bsp1.csv' )