Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
<SOS> metric: MAE Kaggle data source: pubg-finish-placement-prediction-(kernels-only)<data_type_conversions>
gc.enable()
features['Title'] = df_title['Name']
features['Child'] =(full['Age'] <= 14 ).astype(int )<feature_engineering>
INPUT_DIR = ".. /input/"
def parse_surname(name):
return name.split(',')[0]
family = pd.DataFrame(full[['Parch','SibSp','Ticket']])
 family['Family_size'] = 1 + family.Parch + family.SibSp
family['Surname'] = full.Name.map(parse_surname)
 dict_scount = dict(family.groupby('Surname' ).Family_size.count())
 dict_scode = dict(zip(dict_scount.keys() , range(len(dict_scount))))
family['Surname_code'] = family['Surname'].map(dict_scode)
 family['Surname_count'] = family['Surname'].map(dict_scount)
 display(full[family.Surname == 'Smith'] )<feature_engineering>
def feature_engineering(is_train=True):
if is_train:
print("processing train_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'test_V2.csv')
 state('totalDistance')
 s = timer()
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
e = timer()
state('totalDistance', False, e - s)
 state('rankPoints')
 s = timer()
df['rankPoints'] = np.where(df['rankPoints'] <= 0 ,0 , df['rankPoints'])
 e = timer()
state('rankPoints', False, e-s)
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y
surname2chk = family[family['Family_size'] < family['Surname_count']].Surname.unique()
family['Surname_adj'] = family['Surname']
for s in surname2chk:
family_regroup = family[family['Surname'] == s]
fam_code_dict = tick2fam_gen(family_regroup)
 for idx in family_regroup.index:
curr_ticket = full.loc[idx].Ticket
fam_code = fam_code_dict[curr_ticket]
if family_regroup.loc[idx, 'Family_size'] == 1:
if family_regroup.Ticket.value_counts() [curr_ticket] > 1:
family.loc[idx, 'Surname_adj'] = s + '-hidfam' + fam_code
else:
family.loc[idx, 'Surname_adj'] = s + '-single' + fam_code
else:
family.loc[idx, 'Surname_adj'] = s + '-fam' + fam_code
display(family[family.Surname == 'Smith'] )<count_unique_values>
%%time
x_train, y = feature_engineering(True)
 scaler = preprocessing.MinMaxScaler(feature_range=(-1, 1), copy=False ).fit(x_train )
dict_fcount = dict(family.groupby('Surname_adj' ).Family_size.count())
 dict_fcode = dict(zip(dict_fcount.keys() , range(len(dict_fcount))))
family['Family_code'] = family['Surname_adj'].map(dict_fcode)
 family['Family_count'] = family['Surname_adj'].map(dict_fcount)
 print(f"No.of Family Before Regrouping: {len(family.Surname_code.unique())}")
 print(f"No.of Family After Regrouping: {len(family.Family_code.unique())}" )<feature_engineering>
print("x_train", x_train.shape, x_train.max() , x_train.min())
 scaler.transform(x_train)
 print("x_train", x_train.shape, x_train.max() , x_train.min() )
group = pd.DataFrame(family[['Surname_code','Surname_count','Family_code','Family_count']])
 dict_tcount = dict(full.groupby('Ticket' ).PassengerId.count())
 dict_tcode = dict(zip(dict_tcount.keys() ,range(len(dict_tcount))))
group['Ticket_code'] = full.Ticket.map(dict_tcode)
 group['Ticket_count'] = full.Ticket.map(dict_tcount)
 print(f"No.of Tickets Identified: {len(group['Ticket_code'].unique())}")
 display(full[(full.Ticket == 'A/4 48871')|(full.Ticket == 'A/4 48873')] )<categorify>
%%time
NN_model = Sequential()
NN_model.add(Dense(x_train.shape[1], input_dim = x_train.shape[1], activation='relu'))
NN_model.add(Dense(x_train.shape[1], activation='relu'))
NN_model.add(Dense(x_train.shape[1], activation='relu'))
NN_model.add(Dense(1, activation='linear'))
NN_model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error'])
 NN_model.summary()
def ChainCombineGroups(df, colA, colB):

 data = df.copy()
search_df = data.copy()
group_count = 0
while not search_df.empty:
pool = search_df.iloc[:1]
idx = pool.index
search_df.drop(index = idx, inplace = True)
 flag_init = 1
update = pd.DataFrame()
while(flag_init or not update.empty):
flag_init = 0
pool_A_uniq = np.unique(pool[colA])
 pool_B_uniq = np.unique(pool[colB])
 for col in [colA,colB]:
idx = []
for num in np.unique(pool[col]):
idx.extend(search_df[search_df[col] == num].index)
 update = search_df.loc[idx]
pool = pd.concat([pool, update], axis = 0)
 search_df = search_df.drop(index = idx)
 data.loc[pool.index, 'Group_'] = group_count
group_count += 1
return np.array(data['Group_'].astype(int))<groupby>
checkpoint_name = 'Weights-{epoch:03d}--{val_loss:.5f}.hdf5'
checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')
 callbacks_list = [checkpoint]
group['Group_code'] = ChainCombineGroups(group, 'Family_code', 'Ticket_code')
 dict_gcount = dict(group.groupby('Group_code' ).Family_code.count())
 group['Group_count'] = group.Group_code.map(dict_gcount)
 print(f"Family: {len(family['Family_code'].unique())}")
 print(f"Group: {len(group['Ticket_code'].unique())}")
 print(f"Combined: {len(group['Group_code'].unique())}
")
 print('An example of grouping the both friends and family under a same group:')
 display(pd.concat([full['Ticket'],family[['Surname','Family_code']],group[['Ticket_code','Group_code']]], axis = 1)[group['Group_code'] == 458] )<concatenate>
%%time
NN_model_h = NN_model.fit(x=x_train,
y=y,
batch_size=3000,
epochs=30,
verbose=1,
callbacks=callbacks_list,
validation_split=0.20,
validation_data=None,
shuffle=True,
class_weight=None,
sample_weight=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None)
 print(NN_model_h.history)
 del x_train, y
gc.collect()
group_final = pd.concat([family[['Surname_code','Surname_count','Family_code','Family_count']],
group[['Ticket_code','Ticket_count','Group_code','Group_count']],
full['Survived']], axis = 1 )<groupby>
x_test, _ = feature_engineering(False)
 scaler.transform(x_test)
 print("x_test", x_test.shape, x_test.max() , x_test.min())
 np.clip(x_test, out=x_test, a_min=-1, a_max=1)
 print("x_test", x_test.shape, x_test.max() , x_test.min() )
for param in [('Surname_code','Surname_count'), 
('Family_code','Family_count'), 
('Ticket_code','Ticket_count'), 
('Group_code','Group_count')]:
n_member_survived_by_gp = group_final.groupby(param[0] ).Survived.sum()
n_mem_survived = group_final[param[0]].map(n_member_survived_by_gp)
 n_mem_survived_adj = n_mem_survived - group_final.Survived.apply(lambda x: 1 if x == 1 else 0)
 n_member_dead_by_gp = group_final.groupby(param[0] ).Survived.count() - group_final.groupby(param[0] ).Survived.sum()
n_mem_dead = group_final[param[0]].map(n_member_dead_by_gp)
 n_mem_dead_adj = n_mem_dead - group_final.Survived.apply(lambda x: 1 if x == 0 else 0)
 unknown_factor =(group_final[param[1]] - n_mem_survived_adj - n_mem_dead_adj)/group_final[param[1]]
confidence = 1 - unknown_factor
key = 'Confidence_member_survived'+'_'+param[0]
ratio =(1/group_final[param[1]])*(n_mem_survived_adj - n_mem_dead_adj)
 group_final[key] = confidence * ratio
plt.barh(group_final.corr().Survived[-4:].index, group_final.corr().Survived[-4:])
 plt.xlabel('Correlation with Survived');
features['Cf_mem_survived'] = group_final['Confidence_member_survived_Group_code']<feature_engineering>
%%time
pred = NN_model.predict(x_test)
 del x_test
gc.collect()
features['Parch'] = full['Parch']
features['SibSp'] = full['SibSp']
features['Group_size'] = group['Group_count']
features.head()<categorify>
pred = pred.reshape(-1)
 pred =(pred + 1)/ 2
scalar = StandardScaler()
features_z_transformed = features.copy()
continuous = ['Fare']
features_z_transformed[continuous] = scalar.fit_transform(features_z_transformed[continuous])
 features_z_transformed.Sex = features_z_transformed.Sex.apply(lambda x: 1 if x == 'male' else 0)
 features_final = pd.get_dummies(features_z_transformed)
 encoded = list(features_final.columns)
 print("{} total features after one-hot encoding.".format(len(encoded)))
 features_final_train = features_final[:891]
features_final_test = features_final[891:]<split>
df_test = pd.read_csv(INPUT_DIR + 'test_V2.csv' )
X_train, X_test, y_train, y_test = train_test_split(features_final_train,
train.Survived,
test_size = 0.2,
random_state = 0 )<choose_model_class>
%%time
print("fix winPlacePerc")
 for i in range(len(df_test)) :
winPlacePerc = pred[i]
maxPlace = int(df_test.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
pred[i] = winPlacePerc
clf_A = GradientBoostingClassifier(random_state = 0)
 clf_B = LogisticRegression(random_state= 0)
 clf_C = RandomForestClassifier(random_state= 0)
 samples_100 = len(y_train)
 samples_10 = int(len(y_train)/2)
 samples_1 = int(len(y_train)/10)
 results = {}
for clf in [clf_A, clf_B, clf_C]:
clf_name = clf.__class__.__name__
results[clf_name] = {}
for i, samples in enumerate([samples_1, samples_10, samples_100]):
results[clf_name][i] = \
train_predict(clf, samples, X_train, y_train, X_test, y_test )<init_hyperparams>
df_test['winPlacePerc'] = pred
<save_to_csv><EOS> 
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )