Kaggle competition: Google Cloud & NCAA® ML Competition 2019-Men's
submit.to_csv('submission.csv',index=False )<import_modules>
data_dir = '.. /input/'
df_seeds = pd.read_csv(data_dir + 'stage2datafiles/NCAATourneySeeds.csv')
 df_tour = pd.read_csv(data_dir + 'stage2datafiles/NCAATourneyCompactResults.csv')
 df_massey = pd.read_csv(data_dir + 'masseyordinals/MasseyOrdinals.csv' )
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt<load_from_csv>
def seedToInt(seed):
return int(seed[1:3])
 df_seeds['int_seed'] = df_seeds['Seed'].apply(seedToInt)
 df_seeds.drop('Seed', inplace=True, axis=1)
 df_seeds.head()
old_result_df = pd.read_csv('.. /input/kmnist-data/Old_training_result.csv')
 old_result_df.head(5 )<compute_test_metric>
df_finalmassey = df_massey[df_massey['RankingDayNum']==max(df_massey['RankingDayNum'].values)]
df_gb = df_finalmassey.groupby(['TeamID', 'Season'])['OrdinalRank'].mean().unstack('TeamID')
 df_SeasonTeamID = df_gb.reset_index()
massey_seasons = list(df_SeasonTeamID['Season'].values )
display_single_fold_val_accuracy(old_result_df, 1 )<compute_test_metric>
df_tour.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], inplace=True, axis=1)
 df_tour.tail()
df_subs_tour = df_tour[df_tour['Season'] >= min(massey_seasons)]
display_single_fold_val_accuracy(old_result_df, 2 )<compute_test_metric>
WMassey = [0]*len(df_subs_tour)
 LMassey = [0]*len(df_subs_tour)
 for ind, row in df_subs_tour.iterrows() :
season = row['Season']
wid = row['WTeamID']
lid = row['LTeamID']
WMassey[ind] = df_SeasonTeamID[df_SeasonTeamID['Season']==season][wid].values[0]
LMassey[ind] = df_SeasonTeamID[df_SeasonTeamID['Season']==season][lid].values[0]
display_single_fold_val_accuracy(old_result_df, 3 )<load_from_csv>
df_subs_tour['WMassey'] = WMassey
df_subs_tour['LMassey'] = LMassey
df_subs_tour.head(10 )
new_result_df = pd.read_csv('.. /input/kmnist-data/New_training_result.csv')
 new_result_df.head(5 )<set_options>
df_winseeds = df_seeds.rename(columns={'TeamID':'WTeamID', 'int_seed':'WSeed'})
 df_loseseeds = df_seeds.rename(columns={'TeamID':'LTeamID', 'int_seed':'LSeed'})
 df_d = pd.merge(left=df_subs_tour, right=df_winseeds, how='left', on=['Season', 'WTeamID'])
 df_concat = pd.merge(left=df_d, right=df_loseseeds, on=['Season', 'LTeamID'])
 df_concat['SeedDiff'] = df_concat['WSeed'] - df_concat['LSeed']
df_concat['MasseyDiff'] = df_concat['WMassey'] - df_concat['LMassey']
df_concat.head()
warnings.filterwarnings('ignore')
 main_dir = '.. /input/Kannada-MNIST/'
tf.keras.__version__<define_variables>
df_wins = pd.DataFrame()
df_wins['SeedDiff'] = df_concat['SeedDiff']
df_wins['MasseyDiff'] = df_concat['MasseyDiff']
df_wins['Result'] = 1
df_losses = pd.DataFrame()
df_losses['SeedDiff'] = -df_concat['SeedDiff']
df_losses['MasseyDiff'] = -df_concat['MasseyDiff']
df_losses['Result'] = 0
df_predictions = pd.concat(( df_wins, df_losses))
df_predictions.head()
pretrain_weights_path = [
'.. /input/kmnist-data/0_model.hdf5',
'.. /input/kmnist-data/1_model.hdf5',
'.. /input/kmnist-data/2_model.hdf5',
'.. /input/kmnist-data/3_model.hdf5',
'.. /input/kmnist-data/4_model.hdf5',
'.. /input/kmnist-data/5_model.hdf5',
'.. /input/kmnist-data/6_model.hdf5',
'.. /input/kmnist-data/7_model.hdf5',
]
uptrain = False
submit = True
num_classes = 10
num_features =(28,28,1)
 batch_size = 1024
lr = 3e-4
epochs = 30
k_fold_split = 8<load_from_csv>
X_train = df_predictions[['SeedDiff', 'MasseyDiff']].values
y_train = df_predictions['Result'].values
X_train, y_train = shuffle(X_train, y_train )
train_df = pd.read_csv(main_dir + 'train.csv')
 valid_df = pd.read_csv(main_dir + 'Dig-MNIST.csv')
 test_df = pd.read_csv(main_dir + 'test.csv')
 train_df.head(5 )<categorify>
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
train_x = train_x.reshape(-1,28,28,1)
 train_y = tf.keras.utils.to_categorical(train_y, num_classes)
 test_x = test_x.reshape(-1,28,28,1 )<train_model>
rf = RandomForestClassifier()
rf_params = {'n_estimators': [100, 200, 300, 400, 500, 1000]}
rf_grid = GridSearchCV(rf, rf_params, scoring='neg_log_loss', refit=True)
 rf_grid.fit(X_train, y_train)
 print('Best log_loss: {:.4}, with best C: {}'.format(rf_grid.best_score_, rf_grid.best_params_['n_estimators']))
class InputGenerator(tf.keras.utils.Sequence):
def __init__(self,
x,
y=None,
aug=None,
batch_size=128,
training=True):
self.x = x
self.y = y
self.aug = aug
self.batch_size = batch_size
self.training = training
self.indices = range(len(x))
def __len__(self):
return len(self.x)// self.batch_size
def __getitem__(self,index):
batch_indices = self.indices[index * self.batch_size :(index+1)*self.batch_size]
batch_data = self.__get_batch_x(batch_indices)
 if self.training:
batch_label = self.__get_batch_y(batch_indices)
 return batch_data, batch_label
else:
return batch_data
def on_epoch_start(self):
if self.training:
np.random.shuffle(self.indices)
 def __get_batch_x(self, batch_indices):
batch_data = []
for idx in batch_indices:
cur_data = self.x[idx].astype('uint8')
 cur_data = self.aug(image = cur_data)['image']
batch_data.append(cur_data)
 return np.stack(batch_data)/255.0
def __get_batch_y(self, batch_indices):
batch_label = []
for idx in batch_indices:
batch_label.append(self.y[idx])
 return np.stack(batch_label)
 train_aug = albu.Compose([
albu.ShiftScaleRotate(scale_limit=0.2, rotate_limit=15.0, shift_limit=0.15, p=0.5, border_mode=0, value = 0)] 
)
 valid_aug = albu.Compose([] )<compute_test_metric>
logreg = LogisticRegression(solver='lbfgs')
 params = {'C': [0.001,0.01, 1, 10, 100]}
log_grid = GridSearchCV(logreg, params, scoring='neg_log_loss', refit=True)
 log_grid.fit(X_train, y_train)
 print('Best log_loss: {:.4}, with best C: {}'.format(log_grid.best_score_, log_grid.best_params_['C']))
def Precision(y_true, y_pred, epsilon=1e-7):
y_true_f = tf.keras.backend.flatten(y_true)
 y_pred_f = tf.keras.backend.flatten(y_pred)
 y_pred_f = tf.keras.backend.round(y_pred_f)
 TP = tf.keras.backend.sum(y_true_f * y_pred_f)
 FP = tf.keras.backend.sum(( 1-y_true_f)* y_pred_f)
 return TP/(TP+FP+epsilon)
 def Recall(y_true, y_pred, epsilon=1e-7):
y_true_f = tf.keras.backend.flatten(y_true)
 y_pred_f = tf.keras.backend.flatten(y_pred)
 y_pred_f = tf.keras.backend.round(y_pred_f)
 TP = tf.keras.backend.sum(y_true_f * y_pred_f)
 TN = tf.keras.backend.sum(y_true_f *(1-y_pred_f))
return TP/(TP+TN+epsilon )<compute_test_metric>
df_sub = pd.read_csv('.. /input/SampleSubmissionStage2.csv')
 df_massey_2019 = pd.read_csv('.. /input/prelim2019_masseyordinals/Prelim2019_MasseyOrdinals.csv')
 len_sub = len(df_sub )
def symmetric_cross_entropy(alpha=1.0, beta=1.0, epsilon=1e-7):
def loss(y_true, y_pred):
y_pred_ce = tf.clip_by_value(y_pred, epsilon, 1.0)
 y_true_rce = tf.clip_by_value(y_true, epsilon, 1.0)
 ce = alpha*tf.reduce_mean(-tf.reduce_sum(y_true * tf.math.log(y_pred_ce), axis = -1))
rce = beta*tf.reduce_mean(-tf.reduce_sum(y_pred * tf.math.log(y_true_rce), axis = -1))
return ce + rce
return loss<define_variables>
df_massey_2019 = df_massey_2019[df_massey_2019['Season']==2019]
df_m = df_massey_2019.groupby('TeamID' ).mean()
df_m.reset_index(drop=False, inplace=True)
 df_m.head()
permutation = np.random.RandomState(2019 ).permutation(len(train_x))
train_x = train_x[permutation]
train_y = train_y[permutation]<load_pretrained>
def getYearTeams(ID):
return(int(x)for x in ID.split('_'))
kf = KFold(n_splits=k_fold_split, random_state=2019)
 model_members = []
check_points_path = []
for idx in range(k_fold_split):
check_points_path.append(str(idx)+ '_model.hdf5')
 for model_index,(train_indices, valid_indices)in enumerate(kf.split(train_x)) :
train_aug_gen = InputGenerator(train_x[train_indices], train_y[train_indices], train_aug, batch_size)
 valid_aug_gen = InputGenerator(train_x[valid_indices], train_y[valid_indices], valid_aug, batch_size)
 model = build_model()
optimizer = tf.keras.optimizers.RMSprop(lr=lr)
 model.compile(optimizer=optimizer, metrics=['accuracy',Precision,Recall], loss=symmetric_cross_entropy())
 steps_per_epoch = len(train_x[train_indices])// batch_size
validation_steps = len(train_x[valid_indices])// batch_size
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5, min_lr=1e-8, mode='min', verbose=2)
 check_point = tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', filepath = check_points_path[model_index], mode='min', save_best_only=True, verbose=2)
 early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)
 training_callbacks = [ check_point, reduce_lr ]
if pretrain_weights_path == None or uptrain == True:
if pretrain_weights_path != None:
print('*'*10,'Load ',model_index,'-fold pretrain weights','*'*10)
 model.load_weights(pretrain_weights_path[model_index])
 print('*'*30,'Training model ', model_index+1,'*'*30)
 print('Train on ', len(train_indices),' data')
 print('Valid on ', len(valid_indices),' data')
 history = model.fit_generator(generator=train_aug_gen,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=valid_aug_gen,
validation_steps=validation_steps,
workers=-1,
verbose=2,
callbacks=training_callbacks)
 print('*'*30,'Validating model ', model_index+1,'*'*30)
 val_gen = InputGenerator(train_x[valid_indices], None, albu.Compose([]), 1, training=False)
 print('
')
 preds = model.predict_generator(val_gen, workers=-1, verbose=1)
 truths = train_y[valid_indices]
preds = np.round(np.array(preds))
truths = np.array(truths)
 valid_results = []
for c in range(num_classes):
valid_results.append(( c+1, np.sum(preds[:,c] * truths[:,c])/ np.sum(truths[:,c])))
 valid_results_df = pd.DataFrame(valid_results, columns=['Class', 'Valid_accuracy'])
 valid_results_df.to_csv('new_ValidationResults'+str(model_index+1)+'.csv', index=False)
 print(valid_results_df)
 del history, val_gen, train_aug_gen, valid_aug_gen, preds, truths, valid_results_df
gc.collect()
else:
print('Load the pretrain weights '+str(model_index+1))
model.load_weights(pretrain_weights_path[model_index])
 model_members.append(model)
 del model
print('
' )<concatenate>
X_test = np.zeros(shape=(len_sub, 2))
for ii, row in df_sub.iterrows() :
year, t1, t2 = getYearTeams(row['ID'])
 t1_seed = df_seeds[(df_seeds['TeamID'] == t1)&(df_seeds['Season'] == year)]['int_seed'].values[0]
t2_seed = df_seeds[(df_seeds['TeamID'] == t2)&(df_seeds['Season'] == year)]['int_seed'].values[0]
t1_mass = df_m[(df_m['TeamID'] == t1)]['OrdinalRank'].values[0]
t2_mass = df_m[(df_m['TeamID'] == t2)]['OrdinalRank'].values[0]
diff_seed = t1_seed - t2_seed
diff_mass = t1_mass - t2_mass
X_test[ii, 0] = diff_seed
X_test[ii, 1] = diff_mass
tta_aug = albu.Compose([
albu.ShiftScaleRotate(scale_limit=0.1, rotate_limit=10, shift_limit=0.10, p=1.0, border_mode=0, value=0)
 ] )<choose_model_class>
log_preds = log_grid.predict_proba(X_test)[:,1]
rf_preds = rf_grid.predict_proba(X_test)[:,1]
clipped_logpreds = np.clip(log_preds, 0.05, 0.95)
 clipped_rf = np.clip(rf_preds, 0.05, 0.95)
 df_sub['Pred'] = clipped_rf
len(df_sub )
<define_variables><EOS> 
df_sub.to_csv('tut_w_mass_submission.csv', index=False )