Kaggle competition: Peking University/Baidu - Autonomous Driving
<SOS> metric: PKUAutoDrivingAP Kaggle data source: peking-university/baidu-autonomous-driving<choose_model_class>
!pip -q install efficientnet-pytorch
estimator = SVR(gamma='scale', epsilon=0.2, tol=1e-3, max_iter=5000)
 Cs = np.logspace(-1, 1, 11)
 cv = 5<predict_on_test>
RandomGamma, HueSaturationValue, RGBShift, MotionBlur, Blur,
GaussNoise, ChannelShuffle 
)
 osj = os.path.join
osdir = os.listdir
base_seed = 1289
def torch_seed_everything(seed=base_seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch_seed_everything()
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=Cs, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )<prepare_output>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device)
 if device == 'cuda':
torch.cuda.empty_cache()
model, optimizer, loss =0,0,0
gc.collect()
predictions_svr[target] = pred_te<load_from_csv>
platform = 'Kaggle'
if platform == 'Colab':
drive.mount("/content/drive", force_remount=False)
submission = pd.read_csv(opj('/kaggle', 'input', 'trends-assessment-prediction', 'sample_submission.csv')).set_index('Id')
 submission.head()<normalization>
debug = False
effnet_ver = 'b0'
dropout_rate = 0.02
is_resume_train = True
n_samples = 100_000 if not debug else 20
img_h, img_w = 512, 2048
img_orig_h, img_orig_w = 2710, 3384
model_scale = 8
batch_size = 2
num_epochs = 6 if not debug else 2
num_epochs_resume = 3 if not debug else 2
start_epoch = 0
grad_accu_steps = 1
if debug:
grad_accu_steps = 1
num_splits = 8
fold_to_train = 3
date_time_str = time.strftime("%d_%m_time_%H_%M", time.localtime())
 path_dict = {'Kaggle': '.. /input/pku-autonomous-driving/',
'Colab': './drive/My Drive/Colab/Peking/Data/',
'gcp': 'gs://peking_12/data/raw_data',
'local': '/home/polk/Kaggle/Peking/data/raw_data'}
train_test_path_dict = {'Kaggle': '.. /input/pku-autonomous-driving/',
'Colab': './drive/My Drive/Colab/Peking/Data/',
'gcp': 'gs://peking_12/data/raw_data',
'local': '/home/polk/Kaggle/Peking/data'}
path_out_dict = {'Kaggle': './',
'Colab': f"./drive/My Drive/Colab/Peking/Output/Output_kaggle_eff_B0_{date_time_str}",
'gcp': f"/home/ik/peking/logs/output_{date_time_str}",
'local': f"/home/polk/Kaggle/Peking/Output_eff_{date_time_str}"}
path = path_dict[platform]
train_test_path = train_test_path_dict[platform]
out_path = path_out_dict[platform]
prev_model_fn_path = '.. /input/dropout-change-eff-b0-2-01-no-aug-epochs-6/model_fld_3_epoch_end_date_02_01_time_06_25.pth'
if platform != 'Kaggle':
if not os.path.exists(out_path): os.mkdir(out_path)
 print("
train and test images dir:", train_test_path)
 print("
train.csv, samples_submission.csv,...directory:", path)
 print("
logs, output dir:", out_path)
 print("
path directory list:", os.listdir(path))
def back_transform(y_test, unique_values, scaler=None, tidx=0):
pow_age = 1.0
pow_d1v1 = 1.5
pow_d1v2 = 1.5
pow_d2v1 = 1.5
pow_d2v2 = 1.5
pow_d21 = 1.5
pow_d22 = 1.0
powers = [pow_age, pow_d1v1, pow_d1v2, pow_d2v1, pow_d2v2, pow_d21, pow_d22]
for i, a in enumerate(y_test):
y_test[i] = unique_values[np.argmin(np.abs(a-unique_values)) ]
y_test = scaler.inverse_transform(np.transpose([y_test] * 7)) [:, tidx]
y_test = np.power(y_test, 1./powers[tidx])
 return y_test<save_to_csv>
train = pd.read_csv(osj(path, 'train.csv'), nrows = n_samples)
 drop_images = ['ID_1a5a10365', 'ID_4d238ae90.jpg', 'ID_408f58e9f', 'ID_bb1d991f6', 'ID_c44983aeb']
train = train[~train['ImageId'].isin(drop_images)]
test = pd.read_csv(osj(path, 'sample_submission.csv'), nrows = n_samples)
 camera_matrix = np.array([[2304.5479, 0, 1686.2379],
[0, 2305.8757, 1354.9849],
[0, 0, 1]], dtype=np.float32)
 camera_matrix_inv = np.linalg.inv(camera_matrix)
 print("train.shape:", train.shape, "\ttest.shape:", test.shape)
 train.head(2 )
prediction_dict = predictions_ridge
for i, t in enumerate(['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']):
unique_values = sorted(targets[t].dropna().unique())
 pred_values = back_transform(prediction_dict[t], unique_values, scaler=scaler_targets, tidx=i)
 submission.iloc[submission.index.str.contains(t), 0] = pred_values
display(submission.head(10))
submission.to_csv('submission_ridge.csv' )<save_to_csv>
def imread(path, fast_mode=False):
img = cv2.imread(path)
 if not fast_mode and img is not None and len(img.shape)== 3:
img = np.array(img[:, :, ::-1])
 return img
def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):

 coords = []
for l in np.array(s.split() ).reshape([-1, 7]):
coords.append(dict(zip(names, l.astype('float'))))
if 'id' in coords[-1]:
coords[-1]['id'] = int(coords[-1]['id'])
 return coords
def rotate(x, angle):
x = x + angle
x = x -(x + np.pi)//(2 * np.pi)* 2 * np.pi
return x
def add_number_of_cars(df):

 df['numcars'] = [int(( x.count(' ')+1)/7)for x in df['PredictionString']]
return df
prediction_dict = predictions_svr
for i, t in enumerate(['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']):
unique_values = sorted(targets[t].dropna().unique())
 pred_values = back_transform(prediction_dict[t], unique_values, scaler=scaler_targets, tidx=i)
 submission.iloc[submission.index.str.contains(t), 0] = pred_values
display(submission.head(10))
submission.to_csv('submission_svr.csv' )<define_variables>
DISTANCE_THRESH_CLEAR = 2
def convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):
return x * fx / z + cx, y * fy / z + cy
def optimize_xy(r, c, x0, y0, z0):
def distance_fn(xyz):
x, y, z = xyz
x, y = convert_3d_to_2d(x, y, z0)
 y, x = x, y
x =(x - img_orig_h // 2)* img_h /(img_orig_h // 2)/ model_scale
x = np.round(x ).astype('int')
 y =(y + img_orig_w // 4)* img_w /(img_orig_w * 1.5)/ model_scale
y = np.round(y ).astype('int')
 return(x-r)**2 +(y-c)**2
res = minimize(distance_fn, [x0, y0, z0], method='Powell')
 x_new, y_new, z_new = res.x
return x_new, y_new, z0
def clear_duplicates(coords):
for c1 in coords:
xyz1 = np.array([c1['x'], c1['y'], c1['z']])
 for c2 in coords:
xyz2 = np.array([c2['x'], c2['y'], c2['z']])
 distance = np.sqrt(((xyz1 - xyz2)**2 ).sum())
 if distance < DISTANCE_THRESH_CLEAR:
if c1['confidence'] < c2['confidence']:
c1['confidence'] = -1
return [c for c in coords if c['confidence'] > 0]
def extract_coords(prediction, threshold=0):
logits = prediction[0]
regr_output = prediction[1:]
points = np.argwhere(logits > threshold)
 col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])
 coords = []
for r, c in points:
regr_dict = dict(zip(col_names, regr_output[:, r, c]))
coords.append(_regr_back(regr_dict))
coords[-1]['confidence'] = 1 /(1 + np.exp(-logits[r, c]))
coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])
 coords = clear_duplicates(coords)
 return coords
def coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):
s = []
for c in coords:
for n in names:
s.append(str(c.get(n, 0)))
 return ' '.join(s)
 def get_img_coords(s):

 coords = str2coords(s)
 xs = [c['x'] for c in coords]
ys = [c['y'] for c in coords]
zs = [c['z'] for c in coords]
P = np.array(list(zip(xs, ys, zs)) ).T
img_p = np.dot(camera_matrix, P ).T
img_p[:, 0] /= img_p[:, 2]
img_p[:, 1] /= img_p[:, 2]
img_xs = img_p[:, 0]
img_ys = img_p[:, 1]
img_zs = img_p[:, 2]
return img_xs, img_ys
!cp.. /input/rapids/rapids.0.13.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null
sys.path = ['/opt/conda/envs/rapids/lib/python3.6/site-packages'] + sys.path
sys.path = ['/opt/conda/envs/rapids/lib/python3.6'] + sys.path
sys.path = ['/opt/conda/envs/rapids/lib'] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/
print(f'cuDF Version: {cudf.__version__}
cuPY Version: {cp.__version__}
cuML Version: {cuml.__version__}' )<set_options>
def _regr_preprocess(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] / 100
regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)
 regr_dict['pitch_sin'] = sin(regr_dict['pitch'])
 regr_dict['pitch_cos'] = cos(regr_dict['pitch'])
 regr_dict.pop('pitch')
 regr_dict.pop('id')
 return regr_dict
def _regr_back(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] * 100
regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)
 pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 regr_dict['pitch'] = np.arccos(pitch_cos)* np.sign(pitch_sin)
 return regr_dict
def preprocess_image(img):
img = img[img.shape[0] // 2:]
bg = np.ones_like(img)* img.mean(1, keepdims=True ).astype(img.dtype)
 bg = bg[:, :img.shape[1] // 4]
img = np.concatenate([bg, img, bg], 1)
 img = cv2.resize(img,(img_w, img_h))
return img
def get_mask_and_regr(img_orig_height, img_orig_width, labels):
mask = np.zeros([img_h // model_scale, img_w // model_scale], dtype='float32')
 regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']
regr = np.zeros([img_h // model_scale, img_w // model_scale, 7], dtype='float32')
 coords = str2coords(labels)
 xs, ys = get_img_coords(labels)
 for x, y, regr_dict in zip(xs, ys, coords):
x, y = y, x
x =(x - img_orig_height // 2)* img_h /(img_orig_height // 2)/ model_scale
x = np.round(x ).astype('int')
 y =(y + img_orig_width // 4)* img_w /(img_orig_width * 1.5)/ model_scale
y = np.round(y ).astype('int')
 if x >= 0 and x < img_h // model_scale and y >= 0 and y < img_w // model_scale:
mask[x, y] = 1
regr_dict = _regr_preprocess(regr_dict)
 regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]
return mask, regr
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 50)
 pd.set_option('display.max_columns', 50)
 SEED = 7021991<load_from_csv>
def remove_out_image_cars(df):
def isnot_out(x,y):
return(x>=0)&(x<=img_orig_w)&(y>=0)&(y<=img_orig_h)
 df = add_number_of_cars(df)
 new_str_coords = []
counter_all_ls = []
for idx,str_coords in enumerate(df['PredictionString']):
coords = str2coords(str_coords, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z'])
 xs, ys = get_img_coords(str_coords)
 counter = 0
coords_new = []
for(item,x,y)in zip(coords, xs, ys):
if isnot_out(x, y):
coords_new.append(item)
 counter += 1
new_str_coords.append(coords2str(coords_new, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']))
counter_all_ls.append(counter)
 df['new_pred_string'] = new_str_coords
df['new_numcars'] = counter_all_ls
print("num of cars outside image bounds:", df['numcars'].sum() -df['new_numcars'].sum() ,
"out of all", df['numcars'].sum() , " cars in train")
 del df['PredictionString'], train['numcars']
df.rename(columns={'new_pred_string': 'PredictionString'}, inplace=True)
 return df
train = remove_out_image_cars(train )
df = pd.read_pickle('.. /input/trends-neuroimaging-data-analysis-3d-features/trends_tabular_data.pkl')
 df_fnc = pd.read_csv('.. /input/trends-assessment-prediction/fnc.csv')
 df_loading = pd.read_csv('.. /input/trends-assessment-prediction/loading.csv')
 fnc_features = [feature for feature in df_fnc.columns.tolist() [1:]]
shifted_loading_features = ['IC_18', 'IC_20']
loading_features = [feature for feature in df_loading.columns.tolist() [1:] if feature not in shifted_loading_features]
target_features = ['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']
utility_features = ['is_train', 'site', 'site_predicted']
spatial_features = [feature for feature in df.columns.tolist() [1:] if feature not in fnc_features and
feature not in loading_features and
feature not in shifted_loading_features and
feature not in target_features and
feature not in utility_features]
df.loc[:, spatial_features] = StandardScaler().fit_transform(df[spatial_features])
 boundingbox_features = [feature for feature in spatial_features if 'box' in feature]
statistical_features = [feature for feature in spatial_features if 'box' not in feature]
print(f'TReNDS Neuroimaging - Tabular Data Shape = {df.shape}')
 print(f'TReNDS Neuroimaging - Tabular Data Memory Usage = {df.memory_usage().sum() / 1024 ** 2:.2f} MB')
 del df_fnc, df_loading, shifted_loading_features<feature_engineering>
def euler_to_Rot(yaw, pitch, roll):
Y = np.array([[cos(yaw), 0, sin(yaw)],
[0, 1, 0],
[-sin(yaw), 0, cos(yaw)]])
 P = np.array([[1, 0, 0],
[0, cos(pitch), -sin(pitch)],
[0, sin(pitch), cos(pitch)]])
 R = np.array([[cos(roll), -sin(roll), 0],
[sin(roll), cos(roll), 0],
[0, 0, 1]])
 return np.dot(Y, np.dot(P, R))
def draw_line(image, points):
color =(255, 0, 0)
 cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)
 cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)
 cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)
 cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)
 return image
def draw_points(image, points):
for(p_x, p_y, p_z)in points:
cv2.circle(image,(p_x, p_y), int(1000 / p_z),(0, 255, 0), -1)
 return image
def visualize(img, coords):
x_l = 1.02
y_l = 0.80
z_l = 2.31
img = img.copy()
for point in coords:
x, y, z = point['x'], point['y'], point['z']
yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']
Rt = np.eye(4)
 t = np.array([x, y, z])
 Rt[:3, 3] = t
Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll ).T
Rt = Rt[:3, :]
P = np.array([[x_l, -y_l, -z_l, 1],
[x_l, -y_l, z_l, 1],
[-x_l, -y_l, z_l, 1],
[-x_l, -y_l, -z_l, 1],
[0, 0, 0, 1]] ).T
img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))
img_cor_points = img_cor_points.T
img_cor_points[:, 0] /= img_cor_points[:, 2]
img_cor_points[:, 1] /= img_cor_points[:, 2]
img_cor_points = img_cor_points.astype(int)
 img = draw_line(img, img_cor_points)
 img = draw_points(img, img_cor_points[-1:])
 return img
df_train = df[df['is_train'] == 1].copy(deep=True ).drop(columns=utility_features ).reset_index(drop=True)
 df_test = df[df['is_train'] == 0].copy(deep=True ).drop(columns=(target_features + utility_features)).reset_index(drop=True)
 df_train['is_filtered'] = 0
df_train.loc[df_train['IC_02'] < -0.02, 'is_filtered'] = 1
filtered_idx = df_train[df_train['is_filtered'] == 1].index
df_train.drop(filtered_idx, inplace=True)
 df_train.drop(columns=['is_filtered'], inplace=True)
 print(f'Training Data Shape = {df_train.shape}')
 print(f'Training Data Memory Usage = {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')
 print(f'Test Data Shape = {df_test.shape}')
 print(f'Test Data Memory Usage = {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB')
 del df, filtered_idx, utility_features<define_search_space>
means = np.array([80.302637, 73.818344, 69.698726] ).reshape(1,1,3)
 stds = np.array([52.455102, 52.848600, 50.285960] ).reshape(1,1,3)
 def normalize(img, means, stds, tensor=False):
return(img - means)/stds
def denormalize(img, means, stds, resize_to_original=False):

 img = np.moveaxis(img, 0, 2)
 img = img*stds + means
img = np.clip(img, 0, 255 ).astype('uint8')
 if resize_to_original:
img = img[:,(img_w//4):(img_w - img_w//4),:]
img = cv2.copyMakeBorder(img, img.shape[0], 0,0,0, cv2.BORDER_CONSTANT)
 img = cv2.resize(img,(img_orig_w, img_orig_h))
return img
class CarDataset(Dataset):

 def __init__(self, dataframe, root_dir, root_dir_dropmasks=osj(path,'test_masks/{}.jpg'), training=True, transform=None):
self.df = dataframe
self.root_dir = root_dir
self.root_dir_dropmasks = root_dir_dropmasks
self.transform = transform
self.training = training
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
idx, labels = self.df.values[idx]
img_name = self.root_dir.format(idx)
 img = imread(img_name, True)
 img = preprocess_image(img)
 if self.transform:
img = self.transform(image=img)['image']
img = normalize(img, means, stds)
 img = np.rollaxis(img, 2, 0)
 if self.training:
mask, regr = get_mask_and_regr(img_orig_h, img_orig_w, labels)
 regr = np.rollaxis(regr, 2, 0)
 dropmask = 0
else:
mask, regr = 0, 0
dropmask_name = self.root_dir_dropmasks.format(idx)
 if os.path.isfile(dropmask_name):
dropmask = imread(dropmask_name, True)
 dropmask = preprocess_image(dropmask)
 else:
dropmask = np.zeros(( img_h, img_w, 3))
img = torch.as_tensor(img, dtype=torch.float32)
 mask = torch.as_tensor(mask, dtype=torch.float32)
 regr = torch.as_tensor(regr, dtype=torch.float32)
 dropmask = torch.as_tensor(dropmask, dtype=torch.float32)
 return [img, mask, regr, dropmask]
class backup_CarDataset(Dataset):

 def __init__(self, dataframe, root_dir, training=True, transform=None, means = means, stds = stds):
self.df = dataframe
self.root_dir = root_dir
self.transform = transform
self.training = training
self.means = means
self.stds = stds
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
idx, labels = self.df.values[idx]
img_name = self.root_dir.format(idx)
 img = imread(img_name, True)
 img = preprocess_image(img)
 if self.transform:
img = self.transform(image=img)['image']
img = normalize(img, self.means, self.stds)
 img = np.rollaxis(img, 2, 0)
 if self.training:
mask, regr = get_mask_and_regr(labels)
 regr = np.rollaxis(regr, 2, 0)
 else:
mask, regr = 0, 0
img = torch.Tensor(img); mask = torch.Tensor(mask); regr = torch.Tensor(regr)
 return [img, mask, regr]
class LinearModelEnsemble:
def __init__(self, model, model_name, model_features, model_parameters, cv, n_rounds, loading_fraction, fnc_fraction, statistical_fraction, boundingbox_fraction, verbose):
self.target_weights = np.array([0.3, 0.175, 0.175, 0.175, 0.175])
 self.target_idx = {
'age': df_train.loc[~df_train['age'].isnull() ].index,
'domain1_var1': df_train.loc[~df_train['domain1_var1'].isnull() ].index,
'domain1_var2': df_train.loc[~df_train['domain1_var2'].isnull() ].index,
'domain2_var1': df_train.loc[~df_train['domain2_var1'].isnull() ].index,
'domain2_var2': df_train.loc[~df_train['domain2_var2'].isnull() ].index
}
self.random_states = [1337, 5041992, 7021991, 42, 0]
self.model = model
self.model_name = model_name
self.model_features = model_features
self.model_parameters = model_parameters
self.cv = cv
self.n_rounds = n_rounds
self.loading_fraction = loading_fraction
self.fnc_fraction = fnc_fraction
self.statistical_fraction = statistical_fraction
self.boundingbox_fraction = boundingbox_fraction
self.verbose = verbose
def normalized_absolute_error(self, y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred)) / np.sum(y_true))
def _ridge_train_and_predict(self, X_train, y_train, X_test, cv, r):
self.ridge_scores = {
'age': [],
'domain1_var1': [],
'domain1_var2': [],
'domain2_var1': [],
'domain2_var2': [],
}
self.ridge_oof_scores = []
for i, target in enumerate(target_features):
if self.verbose:
print(f'
Fitting {target}
{(len(target)+ 8)* "-"}')
 X_trn = X_train.loc[self.target_idx[target], self.model_features[target]].copy(deep=True)
 y_trn = y_train.loc[self.target_idx[target], target].copy(deep=True)
 train_predictions = pd.DataFrame(y_trn.copy(deep=True))
if r == 0:
df_train[f'{self.model_name}_{target}'] = 0
df_test[f'{self.model_name}_{target}'] = 0
for fold,(trn_idx, val_idx)in enumerate(cv.split(X_trn), 1):
trn_data = X_trn.iloc[trn_idx, :]
trn_labels = y_trn.iloc[trn_idx]
val_data = X_trn.iloc[val_idx, :]
val_labels = y_trn.iloc[val_idx]
ridge_regression = Ridge(**self.model_parameters[target])
 ridge_regression.fit(trn_data, trn_labels)
 fold_train_predictions = ridge_regression.predict(val_data)
 df_train.loc[X_trn.iloc[val_idx, :].index, f'{self.model_name}_{target}'] +=(fold_train_predictions / self.n_rounds)
 train_predictions.loc[X_trn.iloc[val_idx, :].index, target] = fold_train_predictions
fold_test_predictions = ridge_regression.predict(X_test.loc[:, self.model_features[target]])
 df_test.loc[X_test.index, f'{self.model_name}_{target}'] +=(( fold_test_predictions / cv.n_splits)/ self.n_rounds)
 ridge_score = self.normalized_absolute_error(val_labels.values, fold_train_predictions)
 self.ridge_scores[target].append(ridge_score)
 if self.verbose:
print(f'Fold {fold} - Ridge Regression NAE {ridge_score:.6}')
 oof_nae = self.normalized_absolute_error(df_train.loc[self.target_idx[target], target], train_predictions[target])
 self.ridge_oof_scores.append(oof_nae)
 if self.verbose:
print(f'{52 * "-"}
Ridge Regression({target})Mean NAE {np.mean(self.ridge_scores[target]):.6} [STD:{np.std(self.ridge_scores[target]):.6}]')
 print(f'Ridge Regression({target})OOF NAE {oof_nae:.6}')
 weighted_nae = np.sum(np.array(self.ridge_oof_scores)* self.target_weights)
 print(f'{52 * "-"}
Ridge Regression Global Weighted NAE {weighted_nae:.6}')
 def _svr_train_and_predict(self, X_train, y_train, X_test, cv, r):
self.svr_scores = {
'age': [],
'domain1_var1': [],
'domain1_var2': [],
'domain2_var1': [],
'domain2_var2': [],
}
self.svr_oof_scores = []
for i, target in enumerate(target_features):
if self.verbose:
print(f'
Fitting {target}
{(len(target)+ 8)* "-"}')
 X_trn = X_train.loc[self.target_idx[target], self.model_features[target]].copy(deep=True)
 y_trn = y_train.loc[self.target_idx[target], target].copy(deep=True)
 train_predictions = pd.DataFrame(y_trn.copy(deep=True))
if r == 0:
df_train[f'{self.model_name}_{target}'] = 0
df_test[f'{self.model_name}_{target}'] = 0
for fold,(trn_idx, val_idx)in enumerate(cv.split(X_trn), 1):
trn_data = X_trn.iloc[trn_idx, :]
trn_labels = y_trn.iloc[trn_idx]
val_data = X_trn.iloc[val_idx, :]
val_labels = y_trn.iloc[val_idx]
svr_model = cuml.SVR(**self.model_parameters[target])
 svr_model.fit(trn_data.values, trn_labels.values)
 fold_train_predictions = svr_model.predict(val_data.values)
 df_train.loc[X_trn.iloc[val_idx, :].index, f'{self.model_name}_{target}'] +=(cp.asnumpy(fold_train_predictions.values)/ self.n_rounds)
 train_predictions.loc[X_trn.iloc[val_idx, :].index, target] = fold_train_predictions
fold_test_predictions = svr_model.predict(X_test.loc[:, self.model_features[target]].values)
 df_test.loc[X_test.index, f'{self.model_name}_{target}'] +=(( cp.asnumpy(fold_test_predictions.values)/ cv.n_splits)/ self.n_rounds)
 svr_score = self.normalized_absolute_error(val_labels.values, cp.asnumpy(fold_train_predictions.values))
self.svr_scores[target].append(svr_score)
 if self.verbose:
print(f'Fold {fold} - SVR NAE {svr_score:.6}')
 oof_nae = self.normalized_absolute_error(df_train.loc[self.target_idx[target], target], train_predictions[target])
 self.svr_oof_scores.append(oof_nae)
 if self.verbose:
print(f'{52 * "-"}
SVR({target})Mean NAE {np.mean(self.svr_scores[target]):.6} [STD:{np.std(self.svr_scores[target]):.6}]')
 print(f'SVR({target})OOF NAE {oof_nae:.6}')
 weighted_nae = np.sum(np.array(self.svr_oof_scores)* self.target_weights)
 print(f'{52 * "-"}
SVR Global Weighted NAE {weighted_nae:.6}')
 def plot_regression(self):
fig, axes = plt.subplots(nrows=5, ncols=2, figsize=(25, 50))
for i, target in enumerate(target_features):
target_oof_predictions = df_train.loc[self.target_idx[target], f'{self.model_name}_{target}']
target_y_true = df_train.loc[self.target_idx[target], target]
target_test_predictions = df_test.loc[:, f'{self.model_name}_{target}']
sns.scatterplot(target_y_true, target_oof_predictions, ax=axes[i][0])
 sns.distplot(target_oof_predictions, label='Train Predictions', ax=axes[i][1])
 sns.distplot(target_test_predictions, label='Test Predictions', ax=axes[i][1])
 axes[i][0].set_xlabel(f'Labels', size=18)
 axes[i][0].set_ylabel(f'OOF Predictions', size=18)
 axes[i][1].set_xlabel('')
 axes[i][1].legend(prop={'size': 18})
 for j in range(2):
axes[i][j].tick_params(axis='x', labelsize=15)
 axes[i][j].tick_params(axis='y', labelsize=15)
 axes[i][0].set_title(f'{target} Labels vs OOF Predictions', size=20, pad=20)
 axes[i][1].set_title(f'{target} Predictions Distributions', size=20, pad=20)
 plt.show()
def run(self, X_train, y_train, X_test):
if self.model == 'RidgeRegression':
print(f'
{52 * "-"}')
 for r in range(self.n_rounds):
self.cv.random_state = self.random_states[r]
print(f'
Round {r + 1} - Seed {self.cv.random_state}
{(14 + len(str(self.cv.random_state)) + len(str(r)))* "-"}')
 if self.loading_fraction < 1.0:
np.random.seed(self.random_states[r])
 loading_feature_count = int(np.ceil(len(loading_features)* self.loading_fraction))
loading_drop_features = list(set(loading_features)- set(np.random.choice(loading_features, loading_feature_count, replace=False)))
 else:
loading_drop_features = []
print(f'Dropped {len(loading_drop_features)}/{len(loading_features)} Random Loading Features')
 if self.fnc_fraction < 1.0:
np.random.seed(self.random_states[r])
 fnc_feature_count = int(np.ceil(len(fnc_features)* self.fnc_fraction))
fnc_drop_features = list(set(fnc_features)- set(np.random.choice(fnc_features, fnc_feature_count, replace=False)))
 else:
fnc_drop_features = []
print(f'Dropped {len(fnc_drop_features)}/{len(fnc_features)} Random FNC Features')
 if self.statistical_fraction < 1.0:
np.random.seed(self.random_states[r])
 statistical_feature_count = int(np.ceil(len(statistical_features)* self.statistical_fraction))
statistical_drop_features = list(set(statistical_features)- set(np.random.choice(statistical_features, statistical_feature_count, replace=False)))
 else:
statistical_drop_features = []
print(f'Dropped {len(statistical_drop_features)}/{len(statistical_features)} Random Statistical Features')
 if self.boundingbox_fraction < 1.0:
np.random.seed(self.random_states[r])
 boundingbox_feature_count = int(np.ceil(len(boundingbox_features)* self.boundingbox_fraction))
boundingbox_drop_features = list(set(boundingbox_features)- set(np.random.choice(boundingbox_features, boundingbox_feature_count, replace=False)))
 else:
boundingbox_drop_features = []
print(f'Dropped {len(boundingbox_drop_features)}/{len(boundingbox_features)} Random Bounding Box Features')
 drop_features = loading_drop_features + fnc_drop_features + statistical_drop_features + boundingbox_drop_features
self.model_features = {
'age': [feature for feature in self.model_features['age'] if feature not in drop_features],
'domain1_var1': [feature for feature in self.model_features['domain1_var1'] if feature not in drop_features],
'domain1_var2': [feature for feature in self.model_features['domain1_var2'] if feature not in drop_features],
'domain2_var1': [feature for feature in self.model_features['domain2_var1'] if feature not in drop_features],
'domain2_var2': [feature for feature in self.model_features['domain2_var2'] if feature not in drop_features]
}
self._ridge_train_and_predict(X_train, y_train, X_test, cv=self.cv, r=r)
 print(f'
Final Results(Blend of {self.n_rounds} Rounds)
 {(32 + len(str(self.n_rounds)))* "-"}')
 self.blend_oof_scores = []
for target in target_features:
blend_oof_nae = self.normalized_absolute_error(df_train.loc[self.target_idx[target], target], df_train.loc[self.target_idx[target], f'{self.model_name}_{target}'])
 self.blend_oof_scores.append(blend_oof_nae)
 print(f'Ridge Regression({target})Rounds OOF NAE {blend_oof_nae:.6}')
 weighted_nae = np.sum(np.array(self.blend_oof_scores)* self.target_weights)
 print(f'{52 * "-"}
Ridge Regression Global Weighted NAE {weighted_nae:.6}')
 elif self.model == 'SVR':
print(f'
{39 * "-"}')
 for r in range(self.n_rounds):
self.cv.random_state = self.random_states[r]
print(f'
Round {r + 1} - Seed {self.cv.random_state}
{(14 + len(str(self.cv.random_state)) + len(str(r)))* "-"}')
 if self.loading_fraction < 1.0:
np.random.seed(self.random_states[r])
 loading_feature_count = int(np.ceil(len(loading_features)* self.loading_fraction))
loading_drop_features = list(set(loading_features)- set(np.random.choice(loading_features, loading_feature_count, replace=False)))
 else:
loading_drop_features = []
print(f'Dropped {len(loading_drop_features)}/{len(loading_features)} Random Loading Features')
 if self.fnc_fraction < 1.0:
np.random.seed(self.random_states[r])
 fnc_feature_count = int(np.ceil(len(fnc_features)* self.fnc_fraction))
fnc_drop_features = list(set(fnc_features)- set(np.random.choice(fnc_features, fnc_feature_count, replace=False)))
 else:
fnc_drop_features = []
print(f'Dropped {len(fnc_drop_features)}/{len(fnc_features)} Random FNC Features')
 if self.statistical_fraction < 1.0:
np.random.seed(self.random_states[r])
 statistical_feature_count = int(np.ceil(len(statistical_features)* self.statistical_fraction))
statistical_drop_features = list(set(statistical_features)- set(np.random.choice(statistical_features, statistical_feature_count, replace=False)))
 else:
statistical_drop_features = []
print(f'Dropped {len(statistical_drop_features)}/{len(statistical_features)} Random Statistical Features')
 if self.boundingbox_fraction < 1.0:
np.random.seed(self.random_states[r])
 boundingbox_feature_count = int(np.ceil(len(boundingbox_features)* self.boundingbox_fraction))
boundingbox_drop_features = list(set(boundingbox_features)- set(np.random.choice(boundingbox_features, boundingbox_feature_count, replace=False)))
 else:
boundingbox_drop_features = []
print(f'Dropped {len(boundingbox_drop_features)}/{len(boundingbox_features)} Random Bounding Box Features')
 drop_features = loading_drop_features + fnc_drop_features + statistical_drop_features + boundingbox_drop_features
self.model_features = {
'age': [feature for feature in self.model_features['age'] if feature not in drop_features],
'domain1_var1': [feature for feature in self.model_features['domain1_var1'] if feature not in drop_features],
'domain1_var2': [feature for feature in self.model_features['domain1_var2'] if feature not in drop_features],
'domain2_var1': [feature for feature in self.model_features['domain2_var1'] if feature not in drop_features],
'domain2_var2': [feature for feature in self.model_features['domain2_var2'] if feature not in drop_features]
}
self._svr_train_and_predict(X_train, y_train, X_test, cv=self.cv, r=r)
 print(f'
Final Results(Blend of {self.n_rounds} Rounds)
 {(32 + len(str(self.n_rounds)))* "-"}')
 self.blend_oof_scores = []
for target in target_features:
blend_oof_nae = self.normalized_absolute_error(df_train.loc[self.target_idx[target], target], df_train.loc[self.target_idx[target], f'{self.model_name}_{target}'])
 self.blend_oof_scores.append(blend_oof_nae)
 print(f'SVR({target})Rounds OOF NAE {blend_oof_nae:.6}')
 weighted_nae = np.sum(np.array(self.blend_oof_scores)* self.target_weights)
 print(f'{39 * "-"}
SVR Global Weighted NAE {weighted_nae:.6}')
 self.plot_regression() 
<prepare_x_and_y>
class double_conv(nn.Module):

 def __init__(self, in_ch, out_ch):
super(double_conv, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True),
nn.Conv2d(out_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True)
)
 def forward(self, x):
x = self.conv(x)
 return x
class up(nn.Module):
def __init__(self, in_ch, out_ch, bilinear=True):
super(up, self ).__init__()
if bilinear:
self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
 else:
self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)
 self.conv = double_conv(in_ch, out_ch)
 def forward(self, x1, x2=None):
x1 = self.up(x1)
 diffY = x2.size() [2] - x1.size() [2]
diffX = x2.size() [3] - x1.size() [3]
x1 = F.pad(x1,(diffX // 2, diffX - diffX//2,
diffY // 2, diffY - diffY//2))
if x2 is not None:
x = torch.cat([x2, x1], dim=1)
 else:
x = x1
x = self.conv(x)
 return x
def get_mesh(batch_size, shape_x, shape_y):
mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))
mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mesh = torch.cat([torch.tensor(mg_x ).to(device), torch.tensor(mg_y ).to(device)], 1)
 return mesh
X_train = df_train[(loading_features + spatial_features)].copy(deep=True)
 y_train = df_train[target_features].copy(deep=True)
 X_test = df_test[(loading_features + spatial_features)].copy(deep=True)
 for df in [X_train, X_test]:
df[statistical_features] /= 5000
df[boundingbox_features] /= 5000
np.random.seed(SEED)
 y_train['age'] += np.random.normal(loc=0, scale=0.1, size=len(y_train))
model_features = {
'age': loading_features + statistical_features + boundingbox_features,
'domain1_var1': loading_features + statistical_features,
'domain1_var2': loading_features + statistical_features,
'domain2_var1': loading_features + statistical_features,
'domain2_var2': loading_features + statistical_features
}
model_parameters = {
'age': {
'alpha': 0.0002
},
'domain1_var1': {
'alpha': 0.0004
},
'domain1_var2': {
'alpha': 0.05
},
'domain2_var1': {
'alpha': 0.00075
},
'domain2_var2': {
'alpha': 0.001
}
}
linear_model_ensemble_parameters = {
'model': 'RidgeRegression',
'model_name': 'ridge_regression_loading',
'model_features': model_features,
'model_parameters': model_parameters,
'cv': KFold(n_splits=10, shuffle=True, random_state=SEED),
'n_rounds': 5,
'loading_fraction': 1,
'fnc_fraction': 1,
'statistical_fraction': 1,
'boundingbox_fraction': 0.95,
'verbose': False
}
linear_model_ensemble = LinearModelEnsemble(**linear_model_ensemble_parameters)
 linear_model_ensemble.run(X_train, y_train, X_test)
 del X_train, y_train, X_test, model_features, model_parameters, linear_model_ensemble_parameters, linear_model_ensemble<prepare_x_and_y>
def set_dropout(model, drop_rate):
for name, child in model.named_children() :
if isinstance(child, torch.nn.Dropout):
child.p = drop_rate
print("name:", name)
 print("children:
", child)
 def effnet_dropout(drop_rate):
base_model0 = EfficientNet.from_pretrained(f"efficientnet-{effnet_ver}")
 set_dropout(base_model0, drop_rate)
 return base_model0
class MyUNet(nn.Module):

 def __init__(self, n_classes, drop_rate):
super(MyUNet, self ).__init__()
self.drop_rate = dropout_rate
self.base_model = effnet_dropout(drop_rate = self.drop_rate)
 self.conv0 = double_conv(5, 64)
 self.conv1 = double_conv(64, 128)
 self.conv2 = double_conv(128, 512)
 self.conv3 = double_conv(512, 1024)
 self.mp = nn.MaxPool2d(2)
 if effnet_ver == 'b0': self.up1 = up(1282 + 1024, 512)
 elif effnet_ver == 'b2': self.up1 = up(1410 + 1024, 512)
 elif effnet_ver == 'b4': self.up1 = up(1794 + 1024, 512)
 elif effnet_ver == 'b5': self.up1 = up(2050 + 1024, 512)
 self.up2 = up(512 + 512, 256)
 self.outc = nn.Conv2d(256, n_classes, 1)
 def forward(self, x):
batch_size = x.shape[0]
mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])
 x0 = torch.cat([x, mesh1], 1)
 x1 = self.mp(self.conv0(x0))
x2 = self.mp(self.conv1(x1))
x3 = self.mp(self.conv2(x2))
x4 = self.mp(self.conv3(x3))
feats = self.base_model.extract_features(x)
 mesh2 = get_mesh(batch_size, feats.shape[2], feats.shape[3])
 feats = torch.cat([feats, mesh2], 1)
 x = self.up1(feats, x4)
 x = self.up2(x, x3)
 x = self.outc(x)
 return x
X_train = df_train[(loading_features + spatial_features)].copy(deep=True)
 y_train = df_train[target_features].copy(deep=True)
 X_test = df_test[(loading_features + spatial_features)].copy(deep=True)
 for df in [X_train, X_test]:
df[statistical_features] /= 5000
df[boundingbox_features] /= 5000
np.random.seed(SEED)
 y_train['age'] += np.random.normal(loc=0, scale=0.1, size=len(y_train))
model_features = {
'age': loading_features + statistical_features + boundingbox_features,
'domain1_var1': loading_features + statistical_features,
'domain1_var2': loading_features + statistical_features,
'domain2_var1': loading_features + statistical_features,
'domain2_var2': loading_features + statistical_features
}
model_parameters = {
'age': {
'C': 100,
'kernel': 'rbf',
'cache_size': 3000
},
'domain1_var1': {
'C': 2,
'kernel': 'rbf',
'cache_size': 3000
},
'domain1_var2': {
'C': 1,
'kernel': 'rbf',
'cache_size': 3000
},
'domain2_var1': {
'C': 3,
'kernel': 'rbf',
'cache_size': 3000
},
'domain2_var2': {
'C': 9,
'kernel': 'rbf',
'cache_size': 3000
}
}
linear_model_ensemble_parameters = {
'model': 'SVR',
'model_name': 'svr_loading',
'model_features': model_features,
'model_parameters': model_parameters,
'cv': KFold(n_splits=10, shuffle=True, random_state=None),
'n_rounds': 5,
'loading_fraction': 1,
'fnc_fraction': 1,
'statistical_fraction': 1,
'boundingbox_fraction': 0.95,
'verbose': False
}
linear_model_ensemble = LinearModelEnsemble(**linear_model_ensemble_parameters)
 linear_model_ensemble.run(X_train, y_train, X_test)
 del X_train, y_train, X_test, model_features, model_parameters, linear_model_ensemble_parameters, linear_model_ensemble<prepare_x_and_y>
def build_model(drop_rate):
print("device:", device)
 model = MyUNet(8, drop_rate ).to(device)
 return model
X_train = df_train[(loading_features + fnc_features + spatial_features)].copy(deep=True)
 y_train = df_train[target_features].copy(deep=True)
 X_test = df_test[(loading_features + fnc_features + spatial_features)].copy(deep=True)
 for df in [X_train, X_test]:
df[fnc_features] /= 500
df[statistical_features] /= 5000
df[boundingbox_features] /= 5000
np.random.seed(SEED)
 y_train['age'] += np.random.normal(loc=0, scale=0.1, size=len(y_train))
model_features = {
'age': loading_features + fnc_features + statistical_features + boundingbox_features,
'domain1_var1': loading_features + fnc_features + statistical_features,
'domain1_var2': loading_features + fnc_features + statistical_features,
'domain2_var1': loading_features + fnc_features + statistical_features,
'domain2_var2': loading_features + fnc_features + statistical_features
}
model_parameters = {
'age': {
'alpha': 0.0004
},
'domain1_var1': {
'alpha': 0.004
},
'domain1_var2': {
'alpha': 0.02
},
'domain2_var1': {
'alpha': 0.002
},
'domain2_var2': {
'alpha': 0.003
}
}
linear_model_ensemble_parameters = {
'model': 'RidgeRegression',
'model_name': 'ridge_regression_all',
'model_features': model_features,
'model_parameters': model_parameters,
'cv': KFold(n_splits=10, shuffle=True, random_state=SEED),
'n_rounds': 5,
'loading_fraction': 1,
'fnc_fraction': 0.95,
'statistical_fraction': 1,
'boundingbox_fraction': 1,
'verbose': False
}
linear_model_ensemble = LinearModelEnsemble(**linear_model_ensemble_parameters)
 linear_model_ensemble.run(X_train, y_train, X_test)
 del X_train, y_train, X_test, model_features, model_parameters, linear_model_ensemble_parameters, linear_model_ensemble<prepare_x_and_y>
def criter_metric(prediction, mask, regr, size_average=True):
pred_mask = torch.sigmoid(prediction[:, 0])
 mask_loss = mask * torch.log(pred_mask + 1e-12)+(1 - mask)* torch.log(1 - pred_mask + 1e-12)
 mask_loss = -mask_loss.mean(0 ).sum()
pred_regr = prediction[:, 1:]
regr_loss =(torch.abs(pred_regr - regr ).sum(1)* mask ).sum(1 ).sum(1)/ mask.sum(1 ).sum(1)
 regr_loss = regr_loss.mean(0)
 loss = mask_loss + regr_loss
if not size_average:
loss *= prediction.shape[0]
return loss, mask_loss, regr_loss
w_mask = 0.1; w_regr = 0.9
w_mask = torch.tensor(w_mask, device=device); w_regr = torch.tensor(w_regr, device=device)
 gamma=torch.tensor(5, device=device)
 def criter_objective(prediction, mask, regr, size_average=True):
pred_mask = torch.sigmoid(prediction[:, 0])
 mask_loss = mask *(( 1-pred_mask)**gamma)* torch.log(pred_mask+1e-12)\
+(1 - mask)*torch.log(1-pred_mask+1e-12)
 mask_loss = -mask_loss.mean(0 ).sum()
pred_regr = prediction[:, 1:]
regr_loss =(torch.abs(pred_regr - regr ).sum(1)* mask ).sum(1 ).sum(1)/ mask.sum(1 ).sum(1)
 regr_loss = regr_loss.mean(0)
 loss = w_mask*mask_loss + w_regr*regr_loss
if not size_average:
loss *= prediction.shape[0]
return loss
X_train = df_train[(loading_features + fnc_features + spatial_features)].copy(deep=True)
 y_train = df_train[target_features].copy(deep=True)
 X_test = df_test[(loading_features + fnc_features + spatial_features)].copy(deep=True)
 for df in [X_train, X_test]:
df[fnc_features] /= 500
df[statistical_features] /= 5000
df[boundingbox_features] /= 5000
np.random.seed(SEED)
 y_train['age'] += np.random.normal(loc=0, scale=0.1, size=len(y_train))
model_features = {
'age': loading_features + fnc_features + statistical_features + boundingbox_features,
'domain1_var1': loading_features + fnc_features + statistical_features,
'domain1_var2': loading_features + fnc_features + statistical_features,
'domain2_var1': loading_features + fnc_features + statistical_features,
'domain2_var2': loading_features + fnc_features + statistical_features
}
model_parameters = {
'age': {
'C': 72,
'kernel': 'rbf',
'cache_size': 3000
},
'domain1_var1': {
'C': 7,
'kernel': 'rbf',
'cache_size': 3000
},
'domain1_var2': {
'C': 1,
'kernel': 'rbf',
'cache_size': 3000
},
'domain2_var1': {
'C': 10,
'kernel': 'rbf',
'cache_size': 3000
},
'domain2_var2': {
'C': 10,
'kernel': 'rbf',
'cache_size': 3000
}
}
linear_model_ensemble_parameters = {
'model': 'SVR',
'model_name': 'svr_all',
'model_features': model_features,
'model_parameters': model_parameters,
'cv': KFold(n_splits=10, shuffle=True, random_state=SEED),
'n_rounds': 5,
'loading_fraction': 1,
'fnc_fraction': 1,
'statistical_fraction': 1,
'boundingbox_fraction': 1,
'verbose': False
}
linear_model_ensemble = LinearModelEnsemble(**linear_model_ensemble_parameters)
 linear_model_ensemble.run(X_train, y_train, X_test)
 del X_train, y_train, X_test, model_features, model_parameters, linear_model_ensemble_parameters<compute_train_metric>
def evaluate(model, epoch, history=None):
model.eval()
val_loss, mask_loss, regr_loss = torch.tensor(0., requires_grad=False), torch.tensor(0., requires_grad=False), torch.tensor(0., requires_grad=False)
 with torch.no_grad() :
for img_batch, mask_batch, regr_batch, _ in dev_loader:
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 output = model(img_batch)
 val_loss_batch, mask_loss_batch, regr_loss_batch = criter_metric(output, mask_batch, regr_batch, size_average=False)
 val_loss += val_loss_batch
mask_loss += mask_loss_batch
regr_loss += regr_loss_batch
val_loss /= len(dev_loader.dataset)
 mask_loss /= len(dev_loader.dataset)
 regr_loss /= len(dev_loader.dataset)
 return val_loss.cpu().numpy() , mask_loss.cpu().numpy() , regr_loss.cpu().numpy()
def save_checkpoint(state, model_name=None):

 if not model_name: model_name = f"model_date_{date_time_str}.pth"
torch.save(state, osj(out_path, model_name))
def train_eval_model(model, epoch, best_val_loss, history=None):
model.train()
freq_eval = len(train_loader.dataset)//4
for batch_idx,(img_batch, mask_batch, regr_batch, _)in enumerate(tqdm(train_loader)) :
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 output = model(img_batch)
 loss = criter_objective(output, mask_batch, regr_batch)
 train_loss = loss.data.cpu().numpy()
loss = loss / grad_accu_steps
loss.backward()
if(batch_idx+1)% grad_accu_steps == 0:
optimizer.step()
optimizer.zero_grad()
lr_value = optimizer.state_dict() ['param_groups'][0]['lr']
cond1 =(( batch_idx+1)% freq_eval)==0
cond2 = abs(batch_idx-len(train_loader)) *10 > len(train_loader)
 is_end_epoch =(( batch_idx+1)% len(train_loader)) == 0
if is_end_epoch |(cond1 & cond2):
t_eval0 = time.time()
val_loss, mask_val_loss, regr_val_loss = evaluate(model, dev_loader)
 print("
=========================")
 print(f"epoch={epoch}, batch={batch_idx} => lr={lr_value:.7f}, trn loss={loss:4f}")
 print(f"VAL LOSS: {val_loss:.2f}, \tBest Val Loss: {best_val_loss:.2f} \teval time={(time.time() -t_eval0)/60:.2f} min.")
 print("mask_val_loss:", mask_val_loss, "\tregr_val_loss", regr_val_loss)
 if is_end_epoch: print(f"
===END of EPOCH {epoch}================")
 history.loc[epoch*len(train_loader)+batch_idx, ['train_loss','val_loss', 'lr','mask_val_loss','regr_val_loss']] =\
[train_loss, val_loss, lr_value, mask_val_loss, regr_val_loss ]
history.to_csv(osj(out_path, f'history_{fld}.csv'))
is_best = val_loss < best_val_loss
best_val_loss = min(val_loss, best_val_loss)
 history.loc[epoch*len(train_loader)+batch_idx, ['train_loss', 'lr']] = [train_loss, lr_value ]
return model, optimizer, history, best_val_loss, val_loss
def normalized_absolute_error(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred)) / np.sum(y_true))
target_weights = np.array([0.3, 0.175, 0.175, 0.175, 0.175])
 ridge_loading_scores = []
svr_loading_scores = []
ridge_all_scores = []
svr_all_scores = []
blend_scores = []
print('---------- age ----------
')
 age_labels = df_train[~df_train['age'].isnull() ]['age']
ridge_loading_age_oof_score = normalized_absolute_error(age_labels, df_train.loc[~df_train['age'].isnull() ]['ridge_regression_loading_age'])
 svr_loading_age_oof_score = normalized_absolute_error(age_labels, df_train.loc[~df_train['age'].isnull() ]['svr_loading_age'])
 ridge_all_age_oof_score = normalized_absolute_error(age_labels, df_train.loc[~df_train['age'].isnull() ]['ridge_regression_all_age'])
 svr_all_age_oof_score = normalized_absolute_error(age_labels, df_train.loc[~df_train['age'].isnull() ]['svr_all_age'])
 df_train.loc[~df_train['age'].isnull() , 'blend_age'] =(df_train.loc[~df_train['age'].isnull() , 'ridge_regression_loading_age'] * 0)+\ 
(df_train.loc[~df_train['age'].isnull() , 'svr_loading_age'] * 0)+\ 
(df_train.loc[~df_train['age'].isnull() , 'ridge_regression_all_age'] * 0.5)+\ 
(df_train.loc[~df_train['age'].isnull() , 'svr_all_age'] * 0.5)
 df_test['blend_age'] =(df_test['ridge_regression_loading_age'] * 0)+\ 
(df_test['svr_loading_age'] * 0)+\ 
(df_test['ridge_regression_all_age'] * 0.5)+\ 
(df_test['svr_all_age'] * 0.5)
 blend_age_oof_score = normalized_absolute_error(age_labels, df_train[~df_train['age'].isnull() ]['blend_age'])
 ridge_loading_scores.append(ridge_loading_age_oof_score)
 svr_loading_scores.append(svr_loading_age_oof_score)
 ridge_all_scores.append(ridge_all_age_oof_score)
 svr_all_scores.append(svr_all_age_oof_score)
 blend_scores.append(blend_age_oof_score)
 print(f'Ridge Regression Loading(age)OOF NAE {ridge_loading_age_oof_score:.6}')
 print(f'SVR Loading(age)OOF NAE {svr_loading_age_oof_score:.6}')
 print(f'Ridge Regression All(age)OOF NAE {ridge_all_age_oof_score:.6}')
 print(f'SVR(age)All OOF NAE {svr_all_age_oof_score:.6}')
 print(f'Blend(age)OOF NAE {blend_age_oof_score:.6}')
 print('
---------- domain1_var1 ----------
')
 domain1_var1_labels = df_train[~df_train['domain1_var1'].isnull() ]['domain1_var1']
ridge_loading_domain1_var1_oof_score = normalized_absolute_error(domain1_var1_labels, df_train.loc[~df_train['domain1_var1'].isnull() ]['ridge_regression_loading_domain1_var1'])
 svr_loading_domain1_var1_oof_score = normalized_absolute_error(domain1_var1_labels, df_train.loc[~df_train['domain1_var1'].isnull() ]['svr_loading_domain1_var1'])
 ridge_all_domain1_var1_oof_score = normalized_absolute_error(domain1_var1_labels, df_train.loc[~df_train['domain1_var1'].isnull() ]['ridge_regression_all_domain1_var1'])
 svr_all_domain1_var1_oof_score = normalized_absolute_error(domain1_var1_labels, df_train.loc[~df_train['domain1_var1'].isnull() ]['svr_all_domain1_var1'])
 df_train.loc[~df_train['domain1_var1'].isnull() , 'blend_domain1_var1'] =(df_train.loc[~df_train['domain1_var1'].isnull() , 'ridge_regression_loading_domain1_var1'] * 0.005)+\ 
(df_train.loc[~df_train['domain1_var1'].isnull() , 'svr_loading_domain1_var1'] * 0.005)+\ 
(df_train.loc[~df_train['domain1_var1'].isnull() , 'ridge_regression_all_domain1_var1'] * 0.495)+\ 
(df_train.loc[~df_train['domain1_var1'].isnull() , 'svr_all_domain1_var1'] * 0.495)
 df_test['blend_domain1_var1'] =(df_test['ridge_regression_loading_domain1_var1'] * 0.005)+\ 
(df_test['svr_loading_domain1_var1'] * 0.005)+\ 
(df_test['ridge_regression_all_domain1_var1'] * 0.495)+\ 
(df_test['svr_all_domain1_var1'] * 0.495)
 blend_domain1_var1_oof_score = normalized_absolute_error(domain1_var1_labels, df_train[~df_train['domain1_var1'].isnull() ]['blend_domain1_var1'])
 ridge_loading_scores.append(ridge_loading_domain1_var1_oof_score)
 svr_loading_scores.append(svr_loading_domain1_var1_oof_score)
 ridge_all_scores.append(ridge_all_domain1_var1_oof_score)
 svr_all_scores.append(svr_all_domain1_var1_oof_score)
 blend_scores.append(blend_domain1_var1_oof_score)
 print(f'Ridge Regression Loading(domain1_var1)OOF NAE {ridge_loading_domain1_var1_oof_score:.6}')
 print(f'SVR Loading(domain1_var1)OOF NAE {svr_loading_domain1_var1_oof_score:.6}')
 print(f'Ridge Regression All(domain1_var1)OOF NAE {ridge_all_domain1_var1_oof_score:.6}')
 print(f'SVR(domain1_var1)All OOF NAE {svr_all_domain1_var1_oof_score:.6}')
 print(f'Blend(domain1_var1)OOF NAE {blend_domain1_var1_oof_score:.6}')
 print('
---------- domain1_var2 ----------
')
 domain1_var2_labels = df_train[~df_train['domain1_var2'].isnull() ]['domain1_var2']
ridge_loading_domain1_var2_oof_score = normalized_absolute_error(domain1_var2_labels, df_train.loc[~df_train['domain1_var2'].isnull() ]['ridge_regression_loading_domain1_var2'])
 svr_loading_domain1_var2_oof_score = normalized_absolute_error(domain1_var2_labels, df_train.loc[~df_train['domain1_var2'].isnull() ]['svr_loading_domain1_var2'])
 ridge_all_domain1_var2_oof_score = normalized_absolute_error(domain1_var2_labels, df_train.loc[~df_train['domain1_var2'].isnull() ]['ridge_regression_all_domain1_var2'])
 svr_all_domain1_var2_oof_score = normalized_absolute_error(domain1_var2_labels, df_train.loc[~df_train['domain1_var2'].isnull() ]['svr_all_domain1_var2'])
 df_train.loc[~df_train['domain1_var2'].isnull() , 'blend_domain1_var2'] =(df_train.loc[~df_train['domain1_var2'].isnull() , 'ridge_regression_loading_domain1_var2'] * 0.005)+\ 
(df_train.loc[~df_train['domain1_var2'].isnull() , 'svr_loading_domain1_var2'] * 0.005)+\ 
(df_train.loc[~df_train['domain1_var2'].isnull() , 'ridge_regression_all_domain1_var2'] * 0.495)+\ 
(df_train.loc[~df_train['domain1_var2'].isnull() , 'svr_all_domain1_var2'] * 0.495)
 df_test['blend_domain1_var2'] =(df_test['ridge_regression_loading_domain1_var2'] * 0.005)+\ 
(df_test['svr_loading_domain1_var2'] * 0.005)+\ 
(df_test['ridge_regression_all_domain1_var2'] * 0.495)+\ 
(df_test['svr_all_domain1_var2'] * 0.495)
 blend_domain1_var2_oof_score = normalized_absolute_error(domain1_var2_labels, df_train[~df_train['domain1_var2'].isnull() ]['blend_domain1_var2'])
 ridge_loading_scores.append(ridge_loading_domain1_var2_oof_score)
 svr_loading_scores.append(svr_loading_domain1_var2_oof_score)
 ridge_all_scores.append(ridge_all_domain1_var2_oof_score)
 svr_all_scores.append(svr_all_domain1_var2_oof_score)
 blend_scores.append(blend_domain1_var2_oof_score)
 print(f'Ridge Regression Loading(domain1_var2)OOF NAE {ridge_loading_domain1_var2_oof_score:.6}')
 print(f'SVR Loading(domain1_var2)OOF NAE {svr_loading_domain1_var2_oof_score:.6}')
 print(f'Ridge Regression All(domain1_var2)OOF NAE {ridge_all_domain1_var2_oof_score:.6}')
 print(f'SVR(domain1_var2)All OOF NAE {svr_all_domain1_var2_oof_score:.6}')
 print(f'Blend(domain1_var2)OOF NAE {blend_domain1_var2_oof_score:.6}')
 print('
---------- domain2_var1 ----------
')
 domain2_var1_labels = df_train[~df_train['domain2_var1'].isnull() ]['domain2_var1']
ridge_loading_domain2_var1_oof_score = normalized_absolute_error(domain2_var1_labels, df_train.loc[~df_train['domain2_var1'].isnull() ]['ridge_regression_loading_domain2_var1'])
 svr_loading_domain2_var1_oof_score = normalized_absolute_error(domain2_var1_labels, df_train.loc[~df_train['domain2_var1'].isnull() ]['svr_loading_domain2_var1'])
 ridge_all_domain2_var1_oof_score = normalized_absolute_error(domain2_var1_labels, df_train.loc[~df_train['domain2_var1'].isnull() ]['ridge_regression_all_domain2_var1'])
 svr_all_domain2_var1_oof_score = normalized_absolute_error(domain2_var1_labels, df_train.loc[~df_train['domain2_var1'].isnull() ]['svr_all_domain2_var1'])
 df_train.loc[~df_train['domain2_var1'].isnull() , 'blend_domain2_var1'] =(df_train.loc[~df_train['domain2_var1'].isnull() , 'ridge_regression_loading_domain2_var1'] * 0.005)+\ 
(df_train.loc[~df_train['domain2_var1'].isnull() , 'svr_loading_domain2_var1'] * 0.005)+\ 
(df_train.loc[~df_train['domain2_var1'].isnull() , 'ridge_regression_all_domain2_var1'] * 0.495)+\ 
(df_train.loc[~df_train['domain2_var1'].isnull() , 'svr_all_domain2_var1'] * 0.495)
 df_test['blend_domain2_var1'] =(df_test['ridge_regression_loading_domain2_var1'] * 0.005)+\ 
(df_test['svr_loading_domain2_var1'] * 0.005)+\ 
(df_test['ridge_regression_all_domain2_var1'] * 0.495)+\ 
(df_test['svr_all_domain2_var1'] * 0.495)
 blend_domain2_var1_oof_score = normalized_absolute_error(domain2_var1_labels, df_train[~df_train['domain2_var1'].isnull() ]['blend_domain2_var1'])
 ridge_loading_scores.append(ridge_loading_domain2_var1_oof_score)
 svr_loading_scores.append(svr_loading_domain2_var1_oof_score)
 ridge_all_scores.append(ridge_all_domain2_var1_oof_score)
 svr_all_scores.append(svr_all_domain2_var1_oof_score)
 blend_scores.append(blend_domain2_var1_oof_score)
 print(f'Ridge Regression Loading(domain2_var1)OOF NAE {ridge_loading_domain2_var1_oof_score:.6}')
 print(f'SVR Loading(domain2_var1)OOF NAE {svr_loading_domain2_var1_oof_score:.6}')
 print(f'Ridge Regression All(domain2_var1)OOF NAE {ridge_all_domain2_var1_oof_score:.6}')
 print(f'SVR(domain2_var1)All OOF NAE {svr_all_domain2_var1_oof_score:.6}')
 print(f'Blend(domain2_var1)OOF NAE {blend_domain2_var1_oof_score:.6}')
 print('
---------- domain2_var2 ----------
')
 domain2_var2_labels = df_train[~df_train['domain2_var2'].isnull() ]['domain2_var2']
ridge_loading_domain2_var2_oof_score = normalized_absolute_error(domain2_var2_labels, df_train.loc[~df_train['domain2_var2'].isnull() ]['ridge_regression_loading_domain2_var2'])
 svr_loading_domain2_var2_oof_score = normalized_absolute_error(domain2_var2_labels, df_train.loc[~df_train['domain2_var2'].isnull() ]['svr_loading_domain2_var2'])
 ridge_all_domain2_var2_oof_score = normalized_absolute_error(domain2_var2_labels, df_train.loc[~df_train['domain2_var2'].isnull() ]['ridge_regression_all_domain2_var2'])
 svr_all_domain2_var2_oof_score = normalized_absolute_error(domain2_var2_labels, df_train.loc[~df_train['domain2_var2'].isnull() ]['svr_all_domain2_var2'])
 df_train.loc[~df_train['domain2_var2'].isnull() , 'blend_domain2_var2'] =(df_train.loc[~df_train['domain2_var2'].isnull() , 'ridge_regression_loading_domain2_var2'] * 0.005)+\ 
(df_train.loc[~df_train['domain2_var2'].isnull() , 'svr_loading_domain2_var2'] * 0.005)+\ 
(df_train.loc[~df_train['domain2_var2'].isnull() , 'ridge_regression_all_domain2_var2'] * 0.495)+\ 
(df_train.loc[~df_train['domain2_var2'].isnull() , 'svr_all_domain2_var2'] * 0.495)
 df_test['blend_domain2_var2'] =(df_test['ridge_regression_loading_domain2_var2'] * 0.005)+\ 
(df_test['svr_loading_domain2_var2'] * 0.005)+\ 
(df_test['ridge_regression_all_domain2_var2'] * 0.495)+\ 
(df_test['svr_all_domain2_var2'] * 0.495)
 blend_domain2_var2_oof_score = normalized_absolute_error(domain2_var2_labels, df_train[~df_train['domain2_var2'].isnull() ]['blend_domain2_var2'])
 ridge_loading_scores.append(ridge_loading_domain2_var2_oof_score)
 svr_loading_scores.append(svr_loading_domain2_var2_oof_score)
 ridge_all_scores.append(ridge_all_domain2_var2_oof_score)
 svr_all_scores.append(svr_all_domain2_var2_oof_score)
 blend_scores.append(blend_domain2_var2_oof_score)
 print(f'Ridge Regression Loading(domain2_var2)OOF NAE {ridge_loading_domain2_var2_oof_score:.6}')
 print(f'SVR Loading(domain2_var2)OOF NAE {svr_loading_domain2_var2_oof_score:.6}')
 print(f'Ridge Regression All(domain2_var2)OOF NAE {ridge_all_domain2_var2_oof_score:.6}')
 print(f'SVR(domain2_var2)All OOF NAE {svr_all_domain2_var2_oof_score:.6}')
 print(f'Blend(domain2_var2)OOF NAE {blend_domain2_var2_oof_score:.6}')
 print('
---------- Final Score ----------
')
 ridge_loading_weighted_nae = np.sum(np.array(ridge_loading_scores)* target_weights)
 svr_loading_weighted_nae = np.sum(np.array(svr_loading_scores)* target_weights)
 ridge_all_weighted_nae = np.sum(np.array(ridge_all_scores)* target_weights)
 svr_all_weighted_nae = np.sum(np.array(svr_all_scores)* target_weights)
 blend_weighted_nae = np.sum(np.array(blend_scores)* target_weights)
 print(f'Ridge Regression Loading Global Weighted NAE {ridge_loading_weighted_nae:.6}')
 print(f'SVR Loading Global Weighted NAE {svr_loading_weighted_nae:.6}')
 print(f'Ridge Regression All Global Weighted NAE {ridge_all_weighted_nae:.6}')
 print(f'SVR All Weighted NAE {svr_all_weighted_nae:.6}')
 print(f'Blend Global Weighted NAE {blend_weighted_nae:.6}' )<compute_test_metric>
train_images_dir = osj(train_test_path,'train_images/{}.jpg')
 test_images_dir = osj(train_test_path,'test_images/{}.jpg')
 df_test = test
kfold = KFold(n_splits=num_splits, shuffle=True, random_state= base_seed)
 fld = fold_to_train
trn_idx, val_idx = list(kfold.split(train)) [fld]
train_dataset = CarDataset(train[['ImageId', 'PredictionString']].iloc[trn_idx], train_images_dir,
transform = albu_transform_train)
 dev_dataset = CarDataset(train[['ImageId', 'PredictionString']].iloc[val_idx], train_images_dir,
transform = albu_transform_valid)
 train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)
 dev_loader = DataLoader(dataset=dev_dataset, batch_size=batch_size, shuffle=False, num_workers=4)
 model = build_model(drop_rate=dropout_rate)
 optimizer = optim.Adam(model.parameters() , lr=0.001)
 scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[7,10,14,17,20], gamma=0.5)
 if is_resume_train:
checkpoint = torch.load(prev_model_fn_path, map_location=device)
 model.load_state_dict(checkpoint['state_dict'])
 optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 epoch_start = checkpoint['epoch']
n_epochs = num_epochs_resume
best_val_loss = prev_epoch_val_loss = checkpoint['best_val_loss']
prev_epoch_val_loss = copy.copy(best_val_loss)
 scheduler.load_state_dict(checkpoint['scheduler'])
 del checkpoint; _ = gc.collect()
else:
epoch_start = 0
n_epochs = num_epochs
best_val_loss, prev_epoch_val_loss = 10e+6, 10e+6
history = pd.DataFrame(np.full(( len(train_loader)*n_epochs, 5), fill_value=-1, dtype=np.float32),
columns=['train_loss','val_loss','lr','mask_val_loss','regr_val_loss'])
 print(f"
======================
Train start.Output directory {out_path}")
 print("efficientnet-", effnet_ver, "n_epochs:", n_epochs, "len(train_loader):", len(train_loader))
for epoch in range(epoch_start, n_epochs+epoch_start):
torch.cuda.empty_cache()
gc.collect()
model, optimizer, history, best_val_loss, val_loss = train_eval_model(model, epoch, best_val_loss, history)
 scheduler.step()
is_last_epoch =(epoch==(n_epochs+epoch_start-1))
if(val_loss < prev_epoch_val_loss)|(is_last_epoch):
params_save_model = { 'epoch': start_epoch + epoch, 'loss': history['train_loss'].iat[-1],
'best_val_loss': best_val_loss,
'state_dict': model.state_dict() ,
'optimizer_state_dict': optimizer.state_dict() ,
'scheduler': scheduler.state_dict() }
print(f"Val_loss improved or end training => Saving at epoch {epoch} end w val_loss={val_loss:.2f}, best_val_loss = {best_val_loss:.2f}")
 save_checkpoint(params_save_model, model_name = f"model_fld_{fld}_islastepoch_{is_last_epoch:d}_date_{date_time_str}.pth")
 prev_epoch_val_loss = val_loss
history.to_csv(osj(out_path, f"history_final_fold_{fld}.csv"), index=False )
def get_score_by_deciles(target, predictions):
y_true = df_train[target]
y_pred = df_train[predictions]
print(f'{target} NAE in Deciles
{("-" *(15 + len(target)))}')
 deciles = np.arange(0, 1, 0.1)
 for decile in deciles:
decile_lower_bound = y_true.quantile(decile)
 decile_upper_bound = y_true.quantile(decile + 0.1)
 decile_y_true = y_true.loc[(y_true >= decile_lower_bound)&(y_true < decile_upper_bound)]
decile_y_pred = y_pred.loc[(y_true >= decile_lower_bound)&(y_true < decile_upper_bound)]
decile_nae = normalized_absolute_error(decile_y_true, decile_y_pred)
 print(f'Decile: {decile:.6}-{decile + 0.1:.6}({decile_lower_bound:.6}-{decile_upper_bound:.6})NAE {decile_nae:.6}')
 global_nae = normalized_absolute_error(y_true, y_pred)
 print(f'{("-" *(15 + len(target)))}
{target} Global NAE {global_nae:.6}
')
 for target in target_features:
get_score_by_deciles(target, f'blend_{target}' )<categorify>
history_old = pd.read_csv('.. /input/dropout-change-eff-b0-2-01-no-aug-epochs-6/history_final_fold_3.csv')
 history_old = history_old[history_old['val_loss'] > 0]
history_new = history[history['val_loss']>0]
history_new.index = history_new.index + history_old.index.max() +1
history_new = pd.concat([history_old, history_new], axis=0)
 history_new.tail()
column_mapping = {'blend_age': 'age',
'blend_domain1_var1': 'domain1_var1',
'blend_domain1_var2': 'domain1_var2',
'blend_domain2_var1': 'domain2_var1',
'blend_domain2_var2': 'domain2_var2'}
df_submission = pd.melt(df_test.rename(columns=column_mapping)[['Id', 'age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']], id_vars=['Id'], value_name='Predicted')
 df_submission['Id'] = df_submission['Id'].astype('str')+ '_' + df_submission['variable'].astype('str')
 df_submission = df_submission.drop(columns=['variable'] ).sort_values('Id')
 df_submission.to_csv('linear_model_blend_submission.csv', index=False)
 df_submission.head(10 )<import_modules>
torch.cuda.empty_cache()
gc.collect()
threshold0 = -1.6
n_imgs = 6 if not debug else 2
for idx in random.sample(range(len(dev_dataset)) , n_imgs):
img, mask, regr, _ = dev_dataset[idx]
output = model(img[None].to(device)).data.cpu().numpy()
img = img.data.cpu().numpy()
mask = mask.data.cpu().numpy()
regr = regr.data.cpu().numpy()
coords_pred = extract_coords(output[0], threshold=threshold0)
 coords_true = extract_coords(np.concatenate([mask[None], regr], 0))
img = denormalize(img, means, stds, resize_to_original=True)
 fig, axes = plt.subplots(1, 2, figsize=(16,12))
axes[0].set_title('Ground truth')
 axes[0].imshow(visualize(img, coords_true)[(img.shape[0]//2):])
 axes[1].set_title('Prediction')
 axes[1].imshow(visualize(img, coords_pred)[(img.shape[0]//2):])
 plt.show()
import numpy as np
import pandas as pd
import random
import gc
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.metrics import mean_absolute_error, mean_squared_error
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.linear_model import ElasticNet, Ridge, Lasso, BayesianRidge, ARDRegression
from sklearn.svm import LinearSVR, NuSVR, SVR
from sklearn.ensemble import BaggingRegressor<set_options>
is_last_epoch = False
model_name = f"model_fld_{fld}_islastepoch_{is_last_epoch:d}_date_{date_time_str}.pth"
try:
checkpoint = torch.load(osj(out_path, model_name), map_location=device)
 print("best model of the current session loaded.")
 except:
checkpoint = torch.load(prev_model_fn_path, map_location=device)
 print("NO best model in the current session.Previously trained model loaded")
 model.load_state_dict(checkpoint['state_dict'] )
warnings.filterwarnings("ignore" )<compute_test_metric>
def predict_test(model, threshold = 0):
preds, preds_before_drop_mask = [], []
test_loader = DataLoader(dataset=test_dataset, batch_size=4, shuffle=False, num_workers=4)
 model.eval()
for img_batch, _, _, test_mask_batch in tqdm(test_loader):
with torch.no_grad() :
output = model(img_batch.to(device)).data.cpu().numpy()
test_mask_batch = test_mask_batch.data.cpu().numpy()
for out, test_mask in zip(output, test_mask_batch):
coords = extract_coords(out, threshold= threshold)
 s = coords2str(coords)
 preds_before_drop_mask.append(s)
 if isinstance(test_mask, np.ndarray):
test_mask = cv2.resize(test_mask,(256,64)) [...,0]
test_mask = np.where(test_mask>255//2, 100, 0)
 out[0,...] = out[0,...] - test_mask
coords = extract_coords(out, threshold= threshold)
 s = coords2str(coords)
 preds.append(s)
 return preds, preds_before_drop_mask
def metric(y_true, y_pred):
m1 = np.sum(np.abs(y_true - y_pred), axis=0)
 mae = m1/len(y_true)
 return mae, np.mean(m1/np.sum(y_true, axis=0))<load_from_csv>
threshold0 = -1.5
test_thresh, test_thresh_before = {}, {}
test_batch_size = 4
test_images_dir = osj(train_test_path,'test_images/{}.jpg')
 test_masks_dir = osj(path,'test_masks/{}.jpg')
 test_dataset = CarDataset(test, test_images_dir, test_masks_dir, training=False)
 test_loader = DataLoader(dataset=test_dataset, batch_size=test_batch_size, shuffle=False, num_workers=4)
 for thresh in tqdm([threshold0]):
preds_test = pd.read_csv(osj(path,'sample_submission.csv'), nrows = n_samples)
 preds_test_before = preds_test.copy()
temp_preds, temp_preds_before_drop_mask = predict_test(model, thresh)
 preds_test['PredictionString'] = temp_preds[:n_samples]
preds_test_before['PredictionString'] = temp_preds_before_drop_mask[:n_samples]
test_thresh['thresh_'+str(thresh)] = preds_test
test_thresh_before['thresh_'+str(thresh)] = preds_test_before
fnc_df = pd.read_csv(".. /input/trends-assessment-prediction/fnc.csv")
 loading_df = pd.read_csv(".. /input/trends-assessment-prediction/loading.csv")
 fnc_features, loading_features = list(fnc_df.columns[1:]), list(loading_df.columns[1:])
 df = fnc_df.merge(loading_df, on="Id")
 labels_df = pd.read_csv(".. /input/trends-assessment-prediction/train_scores.csv")
 labels_df["is_train"] = True
df = df.merge(labels_df, on="Id", how="left")
 test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()
df.shape, test_df.shape<load_from_csv>
sub = test_thresh['thresh_'+str(thresh)]
sub_before = test_thresh_before['thresh_'+str(thresh)]
numcars = pd.DataFrame({'ImageId': sub['ImageId'].values, 'numcars': [int(( str(x ).count(' ')+1)/7)for x in sub['PredictionString']]})
 numcars_before = pd.DataFrame({'ImageId': sub_before['ImageId'].values, 'numcars': [int(( str(x ).count(' ')+1)/7)for x in sub_before['PredictionString']]})
 numcars_compare = pd.concat([numcars[numcars_before['numcars']!=numcars['numcars']],
numcars_before[numcars_before['numcars']!=numcars['numcars']]], axis=1)
 numcars_compare.columns = ['ImageId_new','numcars_new', 'ImageId_old','numcars_old']
numcars_compare = numcars_compare[['ImageId_old','ImageId_new','numcars_old','numcars_new']]
print("
Number of images affected by test_masks reduction", numcars_compare.shape[0])
 print()
numcars_compare.head()
<define_search_space><EOS> 
sub.to_csv(osj(out_path, f'sub_thresh_{threshold0:.2f}_cv_3045_eff_b0_dropout_change_resume_KGL.csv'), index=False)
 sub.head()