Kaggle competition: Facial Keypoints Detection
<SOS> metric: RMSE Kaggle data source: facial-keypoints-detection<merge>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import albumentations as A
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Input, Dense, Flatten
from tensorflow.keras.layers import Convolution2D, MaxPool2D, Dropout, BatchNormalization, Dropout, LeakyReLU
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications import ResNet50
from tensorflow.python.keras.utils.data_utils import Sequence
cols_to_use = gameCities.columns.difference(test.columns ).tolist() + ["Season", "WTeamID", "LTeamID"]
test = test.merge(gameCities[cols_to_use].drop_duplicates(subset=["Season", "WTeamID", "LTeamID"]),
how="left", on=["Season", "WTeamID", "LTeamID"])
 del gameCities
gc.collect()
test.head()
cols_to_use = data_dict["WSeasons"].columns.difference(test.columns ).tolist() + ["Season"]
test = test.merge(data_dict["WSeasons"][cols_to_use].drop_duplicates(subset=["Season"]),
how="left", on=["Season"])
 test.head()
cols_to_use = data_dict["WTeams"].columns.difference(test.columns ).tolist()
test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["WTeamID"], right_on=["TeamID"])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["LTeamID"], right_on=["TeamID"], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 test.head()
cols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(test.columns ).tolist() + ['Season']
test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 print(test.shape)
 test.head()<drop_column>
SEED = 1
np.random.seed(SEED)
 tf.random.set_seed(SEED)
 random.seed(SEED)
 if SEED is not None:
os.environ['PYTHONHASHSEED'] = str(SEED )
not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist() ]
print(not_exist_in_test)
 train = train.drop(not_exist_in_test, axis=1)
 train.head()<groupby>
!mkdir data/
!unzip -q /kaggle/input/facial-keypoints-detection/training.zip -d data/train
!unzip -q /kaggle/input/facial-keypoints-detection/test.zip -d data/test
team_win_score = regularSeason.groupby(['Season', 'WTeamID'] ).agg({'WScore':['sum', 'count', 'var']} ).reset_index()
team_win_score.columns = [' '.join(col ).strip() for col in team_win_score.columns.values]
team_loss_score = regularSeason.groupby(['Season', 'LTeamID'] ).agg({'LScore':['sum', 'count', 'var']} ).reset_index()
team_loss_score.columns = [' '.join(col ).strip() for col in team_loss_score.columns.values]
del regularSeason
gc.collect()<merge>
def format_dataset(dataframe):
X = np.array([ format_image(x)for x in dataframe['Image'] ])
 if len(dataframe.columns)> 2:
y = dataframe.drop('Image', axis=1 ).values
return X, y
return X
def format_image(img_row):

 img = img_row.split(' ')
 img = np.array(img, dtype=np.float32)
 img = img.reshape(( 96,96,1))
img = img / 255.
 return img
def format_keypoints(keypoint):

 return(keypoint - 48.) / 48.
 def unformat_keypoints(keypoint):

 return keypoint*48 + 48
def show_sample(img, keypoints, axis=None, color='b'):

 if axis is None:
fig, axis = plt.subplots()
axis.scatter(keypoints[0::2], keypoints[1::2], s=10, c=color)
 axis.imshow(img.squeeze() , cmap='gray')
 def show_random_samples(X, y, n_rows=2, n_cols=5):

 fig = plt.figure(figsize=(2*n_cols, 2*n_rows), dpi=100)
 for i, idx in enumerate(np.random.randint(0, len(y), n_rows*n_cols)) :
axis = fig.add_subplot(n_rows, n_cols, i+1, xticks=[], yticks=[])
 show_sample(X[idx], y[idx], axis=axis)
 axis.set_title(f'Sample
train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 train.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 train.head()<merge>
train_dir = 'data/train/training.csv'
test_dir = 'data/test/test.csv'
train_data = pd.read_csv(train_dir)
 test_data = pd.read_csv(test_dir )
test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 test.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 test.head()<feature_engineering>
print(f'Train sample: {len(train_data)}')
 print('Pourcentage of missing values:')
 train_data.isna().mean().round(4)* 100
def preprocess(df):
df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']
df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']
df['x_count'] = df['WScore count_x'] + df['LScore count_y']
df['y_count'] = df['WScore count_y'] + df['WScore count_x']
df['x_var'] = df['WScore var_x'] + df['LScore count_y']
df['y_var'] = df['WScore var_y'] + df['WScore var_x']
return df
train = preprocess(train)
 test = preprocess(test )<drop_column>
X_train, y_train = format_dataset(train_data)
 X_test = format_dataset(test_data)
 print(X_train.shape, y_train.shape, X_test.shape )
train_win = train.copy()
train_los = train.copy()
train_win = train_win[['Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_los = train_los[['Seed_L', 'Seed_W', 'TeamName_L', 'TeamName_W',
'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_win.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
train_los.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
test = test[['ID', 'Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
test.columns = ['ID', 'Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']<feature_engineering>
show_random_samples(X_train, y_train )
def feature_engineering(df):
df['Seed_diff'] = df['Seed_1'] - df['Seed_2']
df['Score_diff'] = df['Score_1'] - df['Score_2']
df['Count_diff'] = df['Count_1'] - df['Count_2']
df['Var_diff'] = df['Var_1'] - df['Var_2']
df['Mean_score1'] = df['Score_1'] / df['Count_1']
df['Mean_score2'] = df['Score_2'] / df['Count_2']
df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']
df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']
df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']
return df
train_win = feature_engineering(train_win)
 train_los = feature_engineering(train_los)
 test = feature_engineering(test )<concatenate>
def create_small_dense_network() :
model = Sequential()
model.add(Input(shape=(96, 96, 1)))
 model.add(Flatten())
 model.add(Dense(units=100, activation='relu'))
model.add(Dense(units=30, activation=None))
return model
data = pd.concat(( train_win, train_los)).reset_index(drop=True)
 print(data.shape)
 data.head()<categorify>
model1 = create_small_dense_network()
es = EarlyStopping(monitor='val_loss', patience=10)
 mc = ModelCheckpoint('best_model1.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True)
 model1.compile(optimizer='adam', loss='mean_squared_error', metrics=['mae'])
 hist1 = model1.fit(X_train, y_train, epochs=500, batch_size=256, verbose=0, validation_split=0.2, callbacks=[es, mc] )
categoricals = ["TeamName_1", "TeamName_2"]
for c in categoricals:
le = LabelEncoder()
data[c] = data[c].fillna("NaN")
 data[c] = le.fit_transform(data[c])
 test[c] = le.transform(test[c])
 data.head()<drop_column>
model1.load_weights('best_model1.h5')
 show_random_preds(model1, X_test )
target = 'result'
features = data.columns.values.tolist()
features.remove(target )<train_on_grid>
def create_convnet(n_outputs=30):
model = Sequential()
model.add(Convolution2D(32,(5,5), padding='same', input_shape=(96,96,1)))
 model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(32,(5,5), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Convolution2D(64,(3,3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(64,(3,3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Convolution2D(96,(3,3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(96,(3,3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Convolution2D(128,(3,3),padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(128,(3,3),padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Convolution2D(256,(3,3),padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(256,(3,3),padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Convolution2D(512,(3,3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(512,(3,3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(n_outputs))
return model
nn = NeuralNetworkModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler="MinMax", verbose=True )<train_model>
model2 = create_convnet()
es = EarlyStopping(monitor='val_loss', patience=10)
 mc = ModelCheckpoint('best_model2.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True)
 model2.compile(optimizer=Adam() , loss='mean_squared_error', metrics=['mae'])
 hist2 = model2.fit(X_train, y_train, epochs=50, batch_size=128, verbose=0, validation_split=0.10, callbacks=[es, mc] )
lgbm = LgbModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler=None, verbose=True )<create_dataframe>
model2.load_weights('best_model2.h5')
 show_random_preds(model2, X_test )
catb = CatbModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler=None, verbose=True )<load_from_csv>
class DataLoader(Sequence):
def __init__(self, X, y, batch_size, augmentations=None, as_rgb=False):
self.X, self.y = X, y
self.batch_size = batch_size
self.augment = augmentations
self.shuffle = True
self.as_rgb = as_rgb
self.on_epoch_end()
def __len__(self):

 return int(np.ceil(len(self.X)/ float(self.batch_size)))
 def __getitem__(self, idx):
indexes = self.indexes[idx*self.batch_size:(idx+1)*self.batch_size]
batch_X = self.X[indexes,...]
batch_y = self.y[indexes, :]
if self.as_rgb:
batch_X = np.tile(batch_X, reps=(1,1,1,3))
if self.augment is not None:
keypoints = np.array([ tuple(zip(point[::2], point[1::2])) for point in batch_y ])
 transformed = [ self.augment(image=x, keypoints=y)for x,y in zip(batch_X, keypoints)]
batch_X = np.stack([ z['image'] for z in transformed ], axis=0)
 batch_y = np.stack([ np.array(z['keypoints'] ).flatten(order='C')for z in transformed ], axis=0)
 return batch_X, batch_y
def on_epoch_end(self):

 self.indexes = np.arange(len(self.X))
if self.shuffle == True:
np.random.shuffle(self.indexes )
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = 0.7 * lgbm.y_pred + 0.2 * catb.y_pred + 0.1 * nn.y_pred
submission_df<save_to_csv>
X_train2, X_valid, y_train2, y_valid = train_test_split(X_train, y_train, test_size=0.10, shuffle=True)
 transform = A.Compose([
A.ShiftScaleRotate(rotate_limit=30, p=0.5),
A.RandomBrightnessContrast(p=0.5),
A.GaussianBlur(p=0.5),
A.GaussNoise(var_limit=(1e-5, 1e-3), p=0.5),
], keypoint_params=A.KeypointParams(format='xy', remove_invisible=False))
train_loader = DataLoader(X_train2, y_train2, batch_size=128, augmentations=transform)
 print(X_train2.shape, y_train2.shape)
 print(X_valid.shape, y_valid.shape)
 x_batch, y_batch = train_loader[1]
show_random_samples(x_batch.squeeze() , y_batch )
submission_df.to_csv('submission.csv', index=False )<set_options>
es = EarlyStopping(monitor='val_loss', patience=20)
 mc = ModelCheckpoint('best_model3.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True)
 model3 = create_convnet()
model3.compile(optimizer=Adam() , loss='mean_squared_error', metrics=['mae'])
 hist3 = model3.fit(train_loader, steps_per_epoch=len(train_loader),
validation_data=(X_valid, y_valid),
epochs=500, verbose=0, callbacks=[es, mc] )
pd.set_option('max_columns', None)
 plt.style.use('fivethirtyeight')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 print("Libraries imported!" )<train_model>
model3.load_weights('best_model3.h5')
 show_random_preds(model3, X_test )
class BaseModel(object):

 def __init__(self, train_df, test_df, target, features, categoricals=[],
n_splits=3, cv_method="KFold", group=None, task="regression",
parameter_tuning=False, scaler=None, verbose=True):
self.train_df = train_df
self.test_df = test_df
self.target = target
self.features = features
self.n_splits = n_splits
self.categoricals = categoricals
self.cv_method = cv_method
self.group = group
self.task = task
self.parameter_tuning = parameter_tuning
self.scaler = scaler
self.cv = self.get_cv()
self.verbose = verbose
self.params = self.get_params()
self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def convert_x(self, x):
return x
def calc_metric(self, y_true, y_pred):
if self.task == "classification":
return log_loss(y_true, y_pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(y_true, y_pred))
def get_cv(self):
if self.cv_method == "KFold":
cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df)
 elif self.cv_method == "StratifiedKFold":
cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target])
 elif self.cv_method == "TimeSeriesSplit":
cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)
 return cv.split(self.train_df)
 elif self.cv_method == "GroupKFold":
cv = GroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 elif self.cv_method == "StratifiedGroupKFold":
cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 def fit(self):
oof_pred = np.zeros(( self.train_df.shape[0],))
y_vals = np.zeros(( self.train_df.shape[0],))
y_pred = np.zeros(( self.test_df.shape[0],))
if self.group is not None:
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 fi = np.zeros(( self.n_splits, len(self.features)))
 if self.scaler is not None:
numerical_features = [f for f in self.features if f not in self.categoricals]
self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())
 self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())
 self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])
 self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])
 if self.scaler == "MinMax":
scaler = MinMaxScaler()
elif self.scaler == "Standard":
scaler = StandardScaler()
df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)
 scaler.fit(df[numerical_features])
 x_test = self.test_df.copy()
x_test[numerical_features] = scaler.transform(x_test[numerical_features])
 x_test = [np.absolute(x_test[i])for i in self.categoricals] + [x_test[numerical_features]]
else:
x_test = self.test_df[self.features]
for fold,(train_idx, val_idx)in enumerate(self.cv):
x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]
y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]
if self.scaler is not None:
x_train[numerical_features] = scaler.transform(x_train[numerical_features])
 x_val[numerical_features] = scaler.transform(x_val[numerical_features])
 x_train = [np.absolute(x_train[i])for i in self.categoricals] + [x_train[numerical_features]]
x_val = [np.absolute(x_val[i])for i in self.categoricals] + [x_val[numerical_features]]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 model, importance = self.train_model(train_set, val_set)
 fi[fold, :] = importance
conv_x_val = self.convert_x(x_val)
 y_vals[val_idx] = y_val
oof_pred[val_idx] = model.predict(conv_x_val ).reshape(oof_pred[val_idx].shape)
 x_test = self.convert_x(x_test)
 y_pred += model.predict(x_test ).reshape(y_pred.shape)/ self.n_splits
print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))
 fi_df = pd.DataFrame()
for n in np.arange(self.n_splits):
tmp = pd.DataFrame()
tmp["features"] = self.features
tmp["importance"] = fi[n, :]
tmp["fold"] = n
fi_df = pd.concat([fi_df, tmp], ignore_index=True)
 gfi = fi_df[["features", "importance"]].groupby(["features"] ).mean().reset_index()
fi_df = fi_df.merge(gfi, on="features", how="left", suffixes=('', '_mean'))
loss_score = self.calc_metric(self.train_df[self.target], oof_pred)
 if self.verbose:
print('Our oof loss score is: ', loss_score)
 return y_pred, loss_score, model, oof_pred, y_vals, fi_df
def plot_feature_importance(self, rank_range=[1, 50]):
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
sorted_df = self.fi_df.sort_values(by = "importance_mean", ascending=False ).reset_index().iloc[self.n_splits *(rank_range[0]-1): self.n_splits * rank_range[1]]
sns.barplot(data=sorted_df, x ="importance", y ="features", orient='h')
 ax.set_xlabel("feature importance")
 ax.spines['top'].set_visible(False)
 ax.spines['right'].set_visible(False)
 return sorted_df<train_model>
def create_specialist(n_outputs=30, weights=None, freeze=False):
model = create_convnet()
if weights is not None:
model.load_weights(weights)
 if freeze:
for layers in model.layers[:-10]:
layers.trainable = False
if n_outputs != 30:
model.layers.pop()
model.add(Dense(n_outputs))
return model
def train_specialist(model, keypoints_range, model_name):
train_data = pd.read_csv(train_dir)
 select_col_idx = list(range(*keypoints_range[model_name])) + [-1]
subdata = train_data.iloc[:, select_col_idx]
subdata = subdata.dropna()
X_train, y_train = format_dataset(subdata)
 X_train2, X_valid, y_train2, y_valid = train_test_split(X_train, y_train, test_size=0.10, shuffle=True)
 transform = A.Compose([A.ShiftScaleRotate(rotate_limit=30, p=0.5),
A.RandomBrightnessContrast(p=0.5),
A.GaussianBlur(p=0.5),
A.GaussNoise(var_limit=(1e-5, 1e-3), p=0.5)],
keypoint_params=A.KeypointParams(format='xy', remove_invisible=False))
train_loader = DataLoader(X_train2, y_train2, batch_size=128, augmentations=transform)
 es = EarlyStopping(monitor='val_loss', patience=10)
 mc = ModelCheckpoint(f'best_model_{model_name}.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True)
 model.compile(optimizer=Adam() , loss='mean_squared_error', metrics=['mae'])
 hist = model.fit(train_loader, steps_per_epoch=len(train_loader),
validation_data=(X_valid, y_valid),
epochs=250, verbose=0, callbacks=[es, mc])
 model.load_weights(f'best_model_{model_name}.h5')
 return model, hist
class ConcatenateSpecialists:
def __init__(self, models):
self.models = models
def predict(self, X):
return np.hstack([ m.predict(X)for m in self.models ])
class LgbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
model = lgb.train(self.params, train_set, num_boost_round = 5000, valid_sets=[train_set, val_set], verbose_eval=verbosity)
 fi = model.feature_importance(importance_type="gain")
 return model, fi
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
def get_params(self):
params = {
'num_leaves': 127,
'min_data_in_leaf': 50,
'max_depth': -1,
'learning_rate': 0.005,
"boosting_type": "gbdt",
"bagging_seed": 11,
"verbosity": -1,
'random_state': 42,
}
if self.task == "regression":
params["objective"] = "regression"
params["metric"] = "rmse"
elif self.task == "classification":
params["objective"] = "binary"
params["metric"] = "binary_logloss"
if self.parameter_tuning == True:
def objective(trial):
train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features],
self.train_df[self.target],
test_size=0.3, random_state=42)
 dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)
 dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)
 hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),
'boosting_type': 'gbdt',
'objective': params["objective"],
'metric': params["metric"],
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'early_stopping_rounds': 100
}
model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)
 pred = model.predict(test_x)
 if self.task == "classification":
return log_loss(test_y, pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(test_y, pred))
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
params = trial.params
params["learning_rate"] = 0.001
plot_optimization_history(study)
 return params<train_model>
specialist_keypoints = {'eyes_centers':(0,4), 'eyes_corners':(4,12), 'eyebrows':(12,20), 'nose':(20,22), 'mouth':(22,30)}
models = {}
for region, keypoint_ids in specialist_keypoints.items() :
print(f'Training model {region}...')
 model = create_specialist(n_outputs=keypoint_ids[1]-keypoint_ids[0], weights='best_model3.h5', freeze=False)
 models[region] = train_specialist(model, specialist_keypoints, region )
class CatbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
if self.task == "regression":
model = CatBoostRegressor(**self.params)
 elif self.task == "classification":
model = CatBoostClassifier(**self.params)
 model.fit(train_set['X'], train_set['y'], eval_set=(val_set['X'], val_set['y']),
verbose=verbosity, cat_features=self.categoricals)
 return model, model.get_feature_importance()
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = {'X': x_train, 'y': y_train}
val_set = {'X': x_val, 'y': y_val}
return train_set, val_set
def get_params(self):
params = { 'task_type': "CPU",
'learning_rate': 0.01,
'iterations': 1000,
'random_seed': 42,
'use_best_model': True
}
if self.task == "regression":
params["loss_function"] = "RMSE"
elif self.task == "classification":
params["loss_function"] = "Logloss"
return params<normalization>
model = model3
layer_names = [layer.name for layer in model.layers]
intermediate_outputs = [layer.output for layer in model.layers]
visualization_model = Model(inputs = model.input, outputs = intermediate_outputs)
 n = 10
x = X_test[0:2]
feature_maps = visualization_model.predict(x)
 for layer_name, feature_map in zip(layer_names, feature_maps):
if len(feature_map.shape)== 4:
n_features = feature_map.shape[-1]
size = feature_map.shape[ 1]
display_grid = np.zeros(( size, size * n))
for i, idx in enumerate(np.random.randint(0, n_features, n)) :
x = feature_map[0, :, :, i]
x -= x.mean()
x /= x.std()
x *= 64
x += 128
x = np.clip(x, 0, 255 ).astype('uint8')
 display_grid[:, i * size :(i + 1)* size] = x
scale = 3
plt.figure(figsize=(scale * n, scale))
plt.title(layer_name)
 plt.grid(False)
 plt.imshow(display_grid, aspect='auto', cmap='viridis' )
class Mish(Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 def build(self, input_shape):
super(Mish, self ).build(input_shape)
 def call(self, x):
return x * K.tanh(K.softplus(x))
def compute_output_shape(self, input_shape):
return input_shape
class LayerNormalization(keras.layers.Layer):
def __init__(self,
center=True,
scale=True,
epsilon=None,
gamma_initializer='ones',
beta_initializer='zeros',
gamma_regularizer=None,
beta_regularizer=None,
gamma_constraint=None,
beta_constraint=None,
**kwargs):

 super(LayerNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.center = center
self.scale = scale
if epsilon is None:
epsilon = K.epsilon() * K.epsilon()
self.epsilon = epsilon
self.gamma_initializer = keras.initializers.get(gamma_initializer)
 self.beta_initializer = keras.initializers.get(beta_initializer)
 self.gamma_regularizer = keras.regularizers.get(gamma_regularizer)
 self.beta_regularizer = keras.regularizers.get(beta_regularizer)
 self.gamma_constraint = keras.constraints.get(gamma_constraint)
 self.beta_constraint = keras.constraints.get(beta_constraint)
 self.gamma, self.beta = None, None
def get_config(self):
config = {
'center': self.center,
'scale': self.scale,
'epsilon': self.epsilon,
'gamma_initializer': keras.initializers.serialize(self.gamma_initializer),
'beta_initializer': keras.initializers.serialize(self.beta_initializer),
'gamma_regularizer': keras.regularizers.serialize(self.gamma_regularizer),
'beta_regularizer': keras.regularizers.serialize(self.beta_regularizer),
'gamma_constraint': keras.constraints.serialize(self.gamma_constraint),
'beta_constraint': keras.constraints.serialize(self.beta_constraint),
}
base_config = super(LayerNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def compute_mask(self, inputs, input_mask=None):
return input_mask
def build(self, input_shape):
shape = input_shape[-1:]
if self.scale:
self.gamma = self.add_weight(
 shape=shape,
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint,
name='gamma', 
)
 if self.center:
self.beta = self.add_weight(
 shape=shape,
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint,
name='beta', 
)
 super(LayerNormalization, self ).build(input_shape)
 def call(self, inputs, training=None):
mean = K.mean(inputs, axis=-1, keepdims=True)
 variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)
 std = K.sqrt(variance + self.epsilon)
 outputs =(inputs - mean)/ std
if self.scale:
outputs *= self.gamma
if self.center:
outputs += self.beta
return outputs<train_model>
def create_submission_file(model, X_test, save_name='model_preds'):
predictions = model.predict(X_test)
 print(f'Shape: {predictions.shape} - Min: {predictions.min() } - Max: {predictions.max() }')
 predictions[predictions > 96] = 96
lookid_data = pd.read_csv('/kaggle/input/facial-keypoints-detection/IdLookupTable.csv')
 image_id = list(lookid_data['ImageId']-1)
 landmark_names = list(lookid_data['FeatureName'])
 landmark_ids = [ landmark_names.index(f)for f in landmark_names ]
expected_preds = [ predictions[x,y] for x,y in zip(image_id, landmark_ids)]
rowid = pd.Series(lookid_data['RowId'], name = 'RowId')
 loc = pd.Series(expected_preds, name = 'Location')
 submission = pd.concat([rowid, loc], axis = 1)
 submission.to_csv(f'{save_name}.csv',index = False)
 print(f'Successfully created {save_name}.csv !' )
<load_from_csv><EOS> 
create_submission_file(model1, X_test, 'model_preds1')
 create_submission_file(model2, X_test, 'model_preds2')
 create_submission_file(model3, X_test, 'model_preds3')
 create_submission_file(model4, X_test, 'model_preds4' )