Kaggle competition: Facial Keypoints Detection
<SOS> metric: RMSE Kaggle data source: facial-keypoints-detection<load_from_csv>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
rn_seed=1
np.random.seed(rn_seed)
RegularSeason_Compact_Results = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WRegularSeasonCompactResults.csv')
 MSeasons = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WSeasons.csv')
 MTeams=pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WTeams.csv' )<merge>
train_path = Path('/tmp/train')
 test_path = Path('/tmp/test' )
Tourney_Results_Compact=pd.merge(Tourney_Compact_Results, Tourney_Seeds, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Seed':'WinningSeed'},inplace=True)
 Tourney_Results_Compact=Tourney_Results_Compact.drop(['TeamID'],axis=1)
 Tourney_Results_Compact = pd.merge(Tourney_Results_Compact, Tourney_Seeds, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Seed':'LoosingSeed'}, inplace=True)
 Tourney_Results_Compact=Tourney_Results_Compact.drop(['TeamID','NumOT','WLoc'],axis=1)
 Tourney_Results_Compact<drop_column>
root = Path('.. /input' )
Tourney_Results_Compact=Tourney_Results_Compact.drop(['WScore','LScore'],axis=1)
 Tourney_Results_Compact.head()<data_type_conversions>
id_lookup = pd.read_csv(root/'IdLookupTable.csv')
 train_csv = pd.read_csv(root/'training/training.csv')
 test_csv = pd.read_csv(root/'test/test.csv' )
Tourney_Results_Compact['WinningSeed'] = Tourney_Results_Compact['WinningSeed'].str.extract('(\d+)', expand=True)
 Tourney_Results_Compact['LoosingSeed'] = Tourney_Results_Compact['LoosingSeed'].str.extract('(\d+)', expand=True)
 Tourney_Results_Compact.WinningSeed = pd.to_numeric(Tourney_Results_Compact.WinningSeed, errors='coerce')
 Tourney_Results_Compact.LoosingSeed = pd.to_numeric(Tourney_Results_Compact.LoosingSeed, errors='coerce' )<rename_columns>
for c in train_csv.columns:
if(train_csv[c].dtype!='object'):
train_csv[c]=train_csv[c].fillna(train_csv[c].median() )
season_winning_team = RegularSeason_Compact_Results[['Season', 'WTeamID', 'WScore']]
season_losing_team = RegularSeason_Compact_Results[['Season', 'LTeamID', 'LScore']]
season_winning_team.rename(columns={'WTeamID':'TeamID','WScore':'Score'}, inplace=True)
 season_losing_team.rename(columns={'LTeamID':'TeamID','LScore':'Score'}, inplace=True)
 RegularSeason_Compact_Results = pd.concat(( season_winning_team, season_losing_team)).reset_index(drop=True)
 RegularSeason_Compact_Results<groupby>
def save_str_img(strimg,w,h,flpath):
px=255-np.array(strimg.split() ,dtype=int)
 if(len(px)==w*h and len(px)%w==0 and len(px)%h==0):
cpx = list(px.reshape(w,h))
img = Image(Tensor([cpx,cpx,cpx]))
img.save(flpath)
 return img
else:
raise Exception("Invalid height and width" )
RegularSeason_Compact_Results_Final = RegularSeason_Compact_Results.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
RegularSeason_Compact_Results_Final<merge>
for index, train_row in train_csv.iterrows() :
save_str_img(train_row.Image,96,96,train_path/(str(index)+'.jpg'))
Tourney_Results_Compact = pd.merge(Tourney_Results_Compact, RegularSeason_Compact_Results_Final, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Score':'WScoreTotal'}, inplace=True)
 Tourney_Results_Compact<save_to_csv>
for index, test_row in test_csv.iterrows() :
save_str_img(test_row.Image,96,96,test_path/(str(index)+'.jpg'))
Tourney_Results_Compact = Tourney_Results_Compact.drop('TeamID', axis=1)
 Tourney_Results_Compact = pd.merge(Tourney_Results_Compact, RegularSeason_Compact_Results_Final, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Score':'LScoreTotal'}, inplace=True)
 Tourney_Results_Compact = Tourney_Results_Compact.drop('TeamID', axis=1)
 Tourney_Results_Compact=Tourney_Results_Compact[Tourney_Results_Compact['Season'] < 2016]
Tourney_Results_Compact<drop_column>
def get_locs(flname):
index = int(flname.name[:-4])
 plist=[]
coords=list(train_csv.loc[index])
 for i in range(len(coords)//2):
plist.append([coords[i*2+1],coords[i*2]])
 return tensor(plist)
Tourney_Win_Results=Tourney_Results_Compact.drop(['Season','WTeamID','LTeamID','DayNum'],axis=1)
 Tourney_Win_Results<rename_columns>
data.show_batch(3,figsize=(6,6))
Tourney_Win_Results.rename(columns={'WinningSeed':'Seed1', 'LoosingSeed':'Seed2', 'WScoreTotal':'ScoreT1', 'LScoreTotal':'ScoreT2'}, inplace=True )<prepare_output>
def mloss(y_true, y_pred):
y_true=y_true.view(-1,15,2)
 y_true[:,:,0]=y_true[:,:,0].clone() -y_pred[:,:,0]
y_true[:,:,1]=y_true[:,:,1].clone() -y_pred[:,:,1]
y_true[:,:,0]=y_true[:,:,0].clone() **2
y_true[:,:,1]=y_true[:,:,1].clone() **2
return y_true.sum(dim=2 ).sum(dim=1 ).sum()
tourney_lose_result = Tourney_Win_Results.copy()
tourney_lose_result['Seed1'] = Tourney_Win_Results['Seed2']
tourney_lose_result['Seed2'] = Tourney_Win_Results['Seed1']
tourney_lose_result['ScoreT1'] = Tourney_Win_Results['ScoreT2']
tourney_lose_result['ScoreT2'] = Tourney_Win_Results['ScoreT1']
tourney_lose_result<feature_engineering>
learn = cnn_learner(data,models.resnet152,loss_func=mloss )
Tourney_Win_Results['Seed_diff'] = Tourney_Win_Results['Seed1'] - Tourney_Win_Results['Seed2']
Tourney_Win_Results['ScoreT_diff'] = Tourney_Win_Results['ScoreT1'] - Tourney_Win_Results['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']<save_to_csv>
learn.fit_one_cycle(10 )
Tourney_Win_Results['result'] = 1
tourney_lose_result['result'] = 0
tourney_result_Final = pd.concat(( Tourney_Win_Results, tourney_lose_result)).reset_index(drop=True)
 tourney_result_Final.to_csv('Tourneyvalidate.csv', index=False )<drop_column>
learn.save('s1' )
tourney_result_Final1 = tourney_result_Final[[
'Seed1', 'Seed2', 'ScoreT1', 'ScoreT2', 'Seed_diff', 'ScoreT_diff', 'result']]<feature_engineering>
learn.load('s1');
tourney_result_Final1.loc[lambda x:(x['Seed1'].isin([14,15,16])) &(x['Seed2'].isin([1,2,3])) ,'result'
] = 0 
<load_from_csv>
learn.lr_find()
test_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv' )<feature_engineering>
learn.fit_one_cycle(30,5e-5 )
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
test_df<merge>
def flp(npa):
for i in range(npa.shape[0]):
if(i%2==1):
tmp=npa[i]
npa[i]=npa[i-1]
npa[i-1]=tmp
return npa
test_df = pd.merge(test_df, Tourney_Seeds, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, Tourney_Seeds, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1 )<save_to_csv>
def get_coords(imgpnts):
return(( imgpnts+1)*48 ).numpy()
test_df = pd.merge(test_df, RegularSeason_Compact_Results_Final, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, RegularSeason_Compact_Results_Final, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df
test_df.to_csv('test_df_Test.csv', index=False )<data_type_conversions>
test_img = open_image(test_path/'1600.jpg')
 pred = learn.predict(test_img)
 test_img.show(y=pred[0] )
test_df['Seed1'] = test_df['Seed1'].str.extract('(\d+)', expand=True)
 test_df['Seed2'] = test_df['Seed2'].str.extract('(\d+)', expand=True)
 test_df.Seed1 = pd.to_numeric(test_df.Seed1, errors='coerce')
 test_df.Seed2 = pd.to_numeric(test_df.Seed2, errors='coerce' )<feature_engineering>
test_img = open_image(test_path/'1600.jpg')
 pred = learn.predict(test_img)
 test_img.show(y=ImagePoints(FlowField(test_img.size,torch.from_numpy(get_coords(pred[1])))) )
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']
test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)
 test_df<prepare_x_and_y>
a=list(train_csv.columns.values)
 a.remove('Image')
 a.append('ImageId' )
X = tourney_result_Final1.drop('result', axis=1)
 y = tourney_result_Final1.result<normalization>
test_preds = pd.DataFrame(columns=a )
df = pd.concat([X, test_df], axis=0, sort=False ).reset_index(drop=True)
 df_log = pd.DataFrame(
 preprocessing.MinMaxScaler().fit_transform(df),
columns=df.columns,
index=df.index 
)
 train_log, test_log = df_log.iloc[:len(X),:], df_log.iloc[len(X):,:].reset_index(drop=True )<train_on_grid>
f = IntProgress(min=0, max=test_csv.count() [0])
 display(f)
 for test_index in range(test_csv.count() [0]):
timg = open_image(test_path/(str(test_index)+'.jpg'))
pred = learn.predict(timg)
 a=np.abs(flp(get_coords(pred[1] ).reshape(1,-1)[0]))
a=np.append(a,test_csv.loc[test_index].ImageId)
 test_preds.loc[test_index]=a
f.value+=1
logreg = LogisticRegression()
logreg.fit(train_log, y)
 coeff_logreg = pd.DataFrame(train_log.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 coeff_logreg.sort_values(by='score_logreg', ascending=False )<predict_on_test>
test_preds.ImageId=test_preds.ImageId.astype('int')
 test_preds.head()
y_logreg_train = logreg.predict(train_log)
 y_logreg_pred = logreg.predict_proba(test_log )<train_model>
sub = pd.DataFrame(columns=['RowId','Location'])
 for index,row in id_lookup.iterrows() :
fname = row.FeatureName
trow=test_preds.loc[test_preds['ImageId']==row.ImageId]
sub.loc[index]=[row.RowId,trow.iloc[0][fname]]
clf = RandomForestClassifier(n_estimators=200,max_depth=50)
 clf.fit(train_log, y)
 clf_probs = clf.predict_proba(test_log )<prepare_output>
sub.RowId=sub.RowId.astype('int')
 sub.head()
<load_from_csv><EOS> 
sub.to_csv("sub.csv",index=False )