Kaggle competition: Facial Keypoints Detection
<SOS> metric: RMSE Kaggle data source: facial-keypoints-detection<feature_engineering>
%matplotlib inline
for key, row in tournament_results2015_df.iterrows() :
if row['WTeamID'] < row['LTeamID']:
id_name = str(row['Season'])+ '_' + str(row['WTeamID'])+ '_' + str(row['LTeamID'])
 sample_submission.loc[sample_submission['ID'] == id_name, 'Pred'] = 1.0
else:
id_name = str(row['Season'])+ '_' + str(row['LTeamID'])+ '_' + str(row['WTeamID'])
 sample_submission.loc[sample_submission['ID'] == id_name, 'Pred'] = 0.0<save_to_csv>
def load_data(dir_path, filename):
df = pd.read_csv(os.path.join(dir_path, filename))
if filename not in ['IdLookupTable.csv','SampleSubmission.csv']:
df['Image'] = df['Image'].apply(lambda img: np.fromstring(img, sep=' '))
return df
sample_submission.to_csv('submission.csv', index=False )<define_variables>
def summary(df, test=False):
dataname = 'test' if test else 'train'
print("Length of %s data %d..
" %(dataname, len(df)))
 print("Count values for each variable:
")
 print(df.count() , '
')
 print("How many variables has missing values ?
")
 print(df.isnull().any().value_counts())
 print()
print("Pourcentage of missing values for each variable:
")
 summary_list =[100 - df[c].count() /len(df)*100 for c in df.columns]
var_list = [var for var in df.columns]
for i in range(len(summary_list)) :
print("{} : {}".format(var_list[i], np.round(summary_list[i]),2))
print()
YEAR = 2021
STAGE = 2
NCAAM = True
NCAAW = False<define_variables>
def get_mean_cols(df):

 df_mean_cols = df[df.columns[:-1]].mean(axis = 0, skipna = True ).reset_index()
mean_cols = df_mean_cols[0].mean(axis = 0, skipna = True)
 return round(mean_cols )
if NCAAM:
DATA_DIR = f'.. /input/ncaam-march-mania-2021/MDataFiles_Stage{STAGE}/'
if NCAAW:
DATA_DIR = f'.. /input/ncaaw-march-mania-2021/WDataFiles_Stage{STAGE}/'<import_modules>
def scale_data(df, test=False, random_state=42):

 img = np.vstack(df['Image'].values)/ 255.
 img = img.astype(np.float32)
 mean_cols = get_mean_cols(df)
 if not test:
keypoints = df[df.columns[:-1]].values
keypoints =(keypoints - mean_cols)/ mean_cols
keypoints = keypoints.astype(np.float32)
 else:
keypoints = None
return img, keypoints, mean_cols
import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
import lightgbm as lgb
import xgboost as xgb
from xgboost import XGBClassifier
import gc<load_from_csv>
def _random_indices(inputs, ratio, random_state=1234):

 np.random.seed(random_state);
actual_batchsize = inputs.shape[0]
size = int(actual_batchsize * ratio)
 indices = np.random.choice(actual_batchsize, size, replace=False)
 return indices
if YEAR == 2020:
tourney_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')
 elif YEAR == 2021:
tourney_result = pd.read_csv(DATA_DIR + 'MNCAATourneyCompactResults.csv')
 tourney_seed = pd.read_csv(DATA_DIR + 'MNCAATourneySeeds.csv' )<drop_column>
def rotate(y, inputs, targets, rotate_ratio, angle= None, right_left = 0):

 if angle is None:
angle = np.random.randint(10)
 if right_left != 0:
angle = 360 - angle
for i in range(inputs.shape[0]):
inputs[i, :, :, 0] = sk.rotate(inputs[i, :, :, 0], angle)
 angle = np.radians(angle)
 indices = np.arange(targets.shape[0])
 R = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
 targets = targets.reshape(len(targets), y.shape[1] // 2, 2)
 targets[indices] = np.dot(targets[indices], R)
 targets = targets.reshape(len(targets), y.shape[1])
 return inputs, targets
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result<merge>
def flipp(inputs, targets, flip_ratio, flip_indices= None, random_seed=123):

 if flip_indices is None:
flip_indices = [(0, 2),(1, 3),(4, 8),(5, 9),(6, 10),(7, 11), 
(12, 16),(13, 17),(14, 18),(15, 19),(22, 24), 
(23, 25)]
for i in range(inputs.shape[0]):
inputs[i, :, :, :] = inputs[i, :, ::-1, :]
indices = np.arange(inputs.shape[0])
 targets[indices, ::2] = targets[indices, ::2] * -1
for a, b in flip_indices:
targets[indices, a], targets[indices, b] = targets[indices, b], targets[indices, a]
return inputs, targets
tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<categorify>
def shift_image(X,y,prop=0.1):
X = X.reshape(-1,96,96)
 y = y.reshape(-1,30)
 for i in range(X.shape[0]):
x_ = X[i]
y_ = y[i]
X[i],y[i] = shift_single_image(x_,y_,prop=prop)
 return(X,y )
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
tourney_result<load_from_csv>
def add_noise(inputs, noise_ratio=0.001):
noisy_img = np.zeros(inputs.shape)
 for i in range(inputs.shape[0]):
noise = np.random.randn(96,96,1)
 noisy_img[i] = inputs[i] + noise_ratio*noise
return noisy_img
if YEAR == 2020:
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')
 elif YEAR == 2021:
season_result = pd.read_csv(DATA_DIR + 'MRegularSeasonCompactResults.csv' )<concatenate>
def split_train_validation(X, y, size=0.3, random_state = 69):
X_train, X_validation, y_train, y_validation = train_test_split(X, y, test_size=size, random_state=random_state)
 print("Splitting data into train {} and validation {}".format(X_train.shape, X_validation.shape))
return X_train, X_validation, y_train, y_validation
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_result<groupby>
train_data = load_data(dir_path, 'training.zip')
 test_data = load_data(dir_path, 'test.zip')
 idlookup = load_data(dir_path, 'IdLookupTable.csv')
 sample_submission = load_data(dir_path,'SampleSubmission.csv' )
season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
season_score<merge>
summary(test_data, test=True )
tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<drop_column>
train_data_copy = train_data.copy()
test_data_copy = test_data.copy()
tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)
 tourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True)
 tourney_win_result<prepare_output>
print("Missing values for the train set where simply NaNs were dropped:
")
 print(train_dropna.isnull().any().value_counts())
 print()
print("Missing values for the train set where NaNs filled using the forward technique:
")
 print(train_fill_nan.isnull().any().value_counts())
tourney_lose_result = tourney_win_result.copy()
tourney_lose_result['Seed1'] = tourney_win_result['Seed2']
tourney_lose_result['Seed2'] = tourney_win_result['Seed1']
tourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']
tourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']
tourney_lose_result<feature_engineering>
X_train_dropna, y_train_dropna, mean_cols = scale_data(train_dropna)
 X_test_dropna, y_test_dropna, _ = scale_data(train_dropna, test=True)
 X_train_fill_nan, y_train_fill_nan, mean_cols = scale_data(train_fill_nan)
 X_test_fill_nan, y_test_fill_nan, _ = scale_data(train_fill_nan, test=True )
tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']
tourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']<concatenate>
flip_ratio = 0.8
rotate_ratio = 0.8
contrast_ratio = 1.2
random_seed = 342
angle = 9
use_flip_transf = True
use_rotation_transf = False
use_brightness_transf = False
tourney_win_result['result'] = 1
tourney_lose_result['result'] = 0
tourney_result = pd.concat(( tourney_win_result, tourney_lose_result)).reset_index(drop=True)
 tourney_result<load_from_csv>
aug_x_train = X_train_dropna.copy()
aug_y_train = y_train_dropna.copy()
if YEAR == 2020:
test_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 elif YEAR == 2021:
test_df = pd.read_csv(DATA_DIR + f'MSampleSubmissionStage{STAGE}.csv' )<feature_engineering>
if use_flip_transf:
flipped_img, flipped_kepoints = flipp(aug_x_train, aug_y_train, flip_ratio, None, random_seed)
 print("Shape of flipped images {} and keypoints {}".format(flipped_img.shape, flipped_kepoints.shape))
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
test_df<merge>
aug_x_train = X_train_dropna.copy()
aug_y_train = y_train_dropna.copy()
use_rotation_transf = True
if use_rotation_transf:
rotated_img_l, rotated_keypoints_l = rotate(y_train_dropna, aug_x_train, aug_y_train, rotate_ratio, 9, 0 )
test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df<feature_engineering>
aug_x_train = X_train_dropna.copy()
aug_y_train = y_train_dropna.copy()
if use_rotation_transf:
rotated_img_r, rotated_keypoints_r = rotate(y_train_dropna, aug_x_train, aug_y_train, rotate_ratio, 9, 1)
 print("Shape of rotated images {} and keypoints {}".format(rotated_img_r.shape, rotated_keypoints_r.shape))
test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))
test_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']
test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)
 test_df<prepare_x_and_y>
aug_x_train = X_train_dropna.copy()
aug_y_train = y_train_dropna.copy()
use_brightness_transf = True
if use_brightness_transf:
inc_brightness_images = brightness(aug_x_train)
 print("Shape of brightned images {} ".format(inc_brightness_images.shape))
X = tourney_result.drop('result', axis=1)
 y = tourney_result.result<init_hyperparams>
aug_x_train = X_train_dropna.copy()
aug_y_train = y_train_dropna.copy()
params_lgb = {'num_leaves': 127,
'min_data_in_leaf': 10,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'logloss',
"verbosity": 0
}
params_xgb = {'colsample_bytree': 0.8,
'learning_rate': 0.0003,
'max_depth': 31,
'subsample': 1,
'objective':'binary:logistic',
'eval_metric':'logloss',
'min_child_weight':3,
'gamma':0.25,
'n_estimators':5000,
'verbosity':0
}<split>
shifted_img, shifted_keypoints = shift_image(aug_x_train, aug_y_train, prop=0.1)
 shifted_img = shifted_img[:,:,:,np.newaxis]
print("Shape of shifted images {} ".format(shifted_img.shape))
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds_lgb = np.zeros(test_df.shape[0])
 y_oof_lgb = np.zeros(X.shape[0])
 for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params_lgb, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)
 y_pred_valid = clf.predict(X_valid)
 y_oof_lgb[valid_index] = y_pred_valid
y_preds_lgb += clf.predict(test_df)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()<split>
aug_x_train = X_train_dropna.copy()
noisy_img = add_noise(aug_x_train)
 print("Shape of noisy images {} ".format(noisy_img.shape))
NFOLDS = 10
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds_xgb = np.zeros(test_df.shape[0])
 y_oof_xgb = np.zeros(X.shape[0])
 for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_set = xgb.DMatrix(X_train, y_train)
 val_set = xgb.DMatrix(X_valid, y_valid)
 test_set = xgb.DMatrix(test_df)
 clf = xgb.train(params_xgb, train_set,num_boost_round=5000, evals=[(train_set, 'train'),(val_set, 'val')], early_stopping_rounds=100, verbose_eval=100)
 y_preds_xgb += clf.predict(test_set)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()<load_from_csv>
aug_x_train_ffill = X_train_fill_nan.copy().reshape(( -1, 96,96,1))
aug_y_train_ffill = y_train_fill_nan.copy()
aug_x_train = X_train_dropna.copy().reshape(( -1, 96,96,1))
aug_y_train = y_train_dropna.copy()
aug_x_train = np.concatenate(( aug_x_train, flipped_img, rotated_img_r, rotated_img_l, inc_brightness_images, shifted_img, noisy_img))
aug_y_train = np.concatenate(( aug_y_train, flipped_kepoints, rotated_keypoints_r, rotated_keypoints_l, aug_y_train, shifted_keypoints, aug_y_train))
print("Number of images in the new train dataset using data augmentation :{} {} ".format(aug_x_train.shape, aug_y_train.shape))
if YEAR == 2020:
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 elif YEAR == 2021:
submission_df = pd.read_csv(DATA_DIR + f'MSampleSubmissionStage{STAGE}.csv')
 submission_df['Pred'] = 0.9*y_preds_lgb + 0.1*y_preds_xgb
submission_df<save_to_csv>
x_train_dna, x_validation_dna, y_train_dna, y_validation_dna = split_train_validation(X_train_dropna, y_train_dropna)
 x_train_ffill, x_validation_ffill, y_train_ffill, y_validation_ffill = split_train_validation(X_train_fill_nan, y_train_fill_nan)
 x_train_da, x_validation_da, y_train_da, y_validation_da = split_train_validation(aug_x_train, aug_y_train, 0.1 )
submission_df.to_csv('submission.csv', index=False )<import_modules>
import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV<load_from_csv>
model_06_01 = Sequential()
model_06_01.add(Convolution2D(filters=64, kernel_size=(3,3), padding='same', input_shape=(96,96,1)))
 model_06_01.add(Activation('relu'))
model_06_01.add(Dropout(0.1))
model_06_01.add(SeparableConv2D(filters=256, kernel_size=(3,3), padding='same', use_bias=False))
model_06_01.add(Activation('relu'))
model_06_01.add(MaxPooling2D(pool_size=(2, 2)))
 model_06_01.add(Dropout(0.1))
model_06_01.add(SeparableConv2D(filters=256, kernel_size=(3,3), padding='same', use_bias=False))
model_06_01.add(Activation('relu'))
model_06_01.add(BatchNormalization())
 model_06_01.add(MaxPooling2D(pool_size=(2, 2)))
 model_06_01.add(Dropout(0.25))
model_06_01.add(SeparableConv2D(filters=256, kernel_size=(3,3), padding='same', use_bias=False))
model_06_01.add(Activation('relu'))
model_06_01.add(MaxPooling2D(pool_size=(2, 2)))
 model_06_01.add(Dropout(0.25))
model_06_01.add(Flatten())
 model_06_01.add(Dense(512,activation='relu'))
model_06_01.add(Dropout(0.5))
model_06_01.add(Dense(30))
model_06_01.summary()
model_06_01.compile(optimizer = 'adam',loss = 'mean_squared_error', metrics=['mae', 'acc'] )
tourney_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv' )<drop_column>
callbacks = [
EarlyStopping(monitor='val_loss', patience=15, mode='min',restore_best_weights=True, verbose=1),
ModelCheckpoint(filepath = 'best_model_06_01.hdf5', monitor='val_mae', verbose=1, save_best_only=True, mode='min')
 ]
hist_06_01 = model_06_01.fit(x_train_da, y_train_da,
epochs= 80, batch_size=128,
validation_data=(x_validation_da, y_validation_da),
callbacks=callbacks,
verbose=1 )
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result<merge>
tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<categorify>
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
tourney_result<load_from_csv>
test_,_, _ = scale_data(test_data_copy, True)
 test_img = reshape_data(test_ )
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv' )<concatenate>
best_model = load_model('best_model_06_01.hdf5')
 pred = best_model.predict(test_img )
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_result<groupby>
feature_name = list(idlookup['FeatureName'])
 image_id = list(idlookup['ImageId']-1)
 row_id = list(idlookup['RowId'])
 feature_list = []
for feature in feature_name:
feature_list.append(feature_name.index(feature))
predictions = []
for x,y in zip(image_id, feature_list):
predictions.append(pred[x][y])
 row_id = pd.Series(row_id, name = 'RowId')
 locations = pd.Series(predictions, name = 'Location')
 locations = locations*mean_cols +mean_cols
submission_result = pd.concat([row_id,locations],axis = 1)
 submission_result.to_csv('best_perf_15_1600.csv',index = False )