Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [3,"sqrt", "log2"],
"min_samples_split": [n for n in range(1, 9)],
"min_samples_leaf": [5, 7],
"bootstrap": [False, True],
"n_estimators" :[200, 500],
"criterion": ["gini", "entropy"]}
rf_param_grid_best = {"max_depth": [None],
"max_features": [3],
"min_samples_split": [4],
"min_samples_leaf": [5],
"bootstrap": [False],
"n_estimators" :[200],
"criterion": ["gini"]}
gs_rf = GridSearchCV(RFC, param_grid = rf_param_grid_best, cv=kfold, scoring="roc_auc", n_jobs= 4, verbose = 1)
 gs_rf.fit(X_train, y_train)
 rf_best = gs_rf.best_estimator_
RFC.fit(X_train, y_train )<compute_test_metric>
def take_part_of_data(df, part):
match_ids = df['matchId'].unique()
match_ids_part = np.random.choice(match_ids, int(part * len(match_ids)))
 df = df[df['matchId'].isin(match_ids_part)]
del match_ids
del match_ids_part
print(f'RandomForest GridSearch best params: {gs_rf.best_params_}
')
 print(f'RandomForest GridSearch best score: {gs_rf.best_score_}')
 print(f'RandomForest score: {RFC.score(X_train,y_train)}' )<choose_model_class>
def add_new_features_1(df):
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['healsAndBoosts'] = df['heals'] + df['boosts']
df['headshotKillsOverKills'] = df['headshotKills'] / df['kills']
df['headshotKillsOverKills'].fillna(0, inplace=True)
 df['killStreaksOverKills'] = df['killStreaks'] / df['kills']
df['killStreaksOverKills'].fillna(0, inplace=True)
 df['killsAndAssists'] = df['kills'] + df['assists']
df['assistsAndRevives'] = df['assists'] + df['revives']
KNN = KNeighborsClassifier()
knn_param_grid = {'algorithm': ['auto'],
'weights': ['uniform', 'distance'],
'leaf_size': [20, 25, 30],
'n_neighbors': [12, 14, 16]}
knn_best_param_grid = {'algorithm': ['auto'],
'weights': ['uniform'],
'leaf_size': [25],
'n_neighbors': [14]}
gs_knn = GridSearchCV(KNN, param_grid = knn_best_param_grid, cv=kfold, scoring = "roc_auc", n_jobs= 4, verbose = 1)
 gs_knn.fit(X_train, y_train)
 KNN.fit(X_train, y_train)
 knn_best = gs_knn.best_estimator_<compute_test_metric>
def add_new_features_2(df):
df['playersJoined'] = df.groupby('matchId')['matchId'].transform('count')
 df['killsAndAssistsOverPlayersJoined'] = df['killsAndAssists'] *(( 100 - df['playersJoined'])/ 100 + 1)
 df['matchDurationOverPlayersJoined'] = df['matchDuration'] *(( 100 - df['playersJoined'])/ 100 + 1)
 df['damageDealtOverPlayersJoined'] = df['damageDealt'] *(( 100 - df['playersJoined'])/ 100 + 1 )
print(f'KNN GridSearch best params: {gs_knn.best_params_}')
 print()
print(f'KNN GridSearch best score: {gs_knn.best_score_}')
 print(f'KNN score: {KNN.score(X_train,y_train)}' )<train_model>
def add_new_features_3(df):
df['totalDistanceOverKillsAndAssists'] = df['totalDistance'] / df['killsAndAssists']
df['totalDistanceOverKillsAndAssists'].fillna(0, inplace=True)
 df['totalDistanceOverKillsAndAssists'].replace(np.inf, 0, inplace=True)
 df['totalDistanceOverHealsAndBoosts'] = df['totalDistance'] / df['healsAndBoosts']
df['totalDistanceOverHealsAndBoosts'].fillna(0, inplace=True)
 df['totalDistanceOverHealsAndBoosts'].replace(np.inf, 0, inplace=True )
knn1 = KNeighborsClassifier(algorithm='auto', leaf_size=26, metric='minkowski',
metric_params=None, n_jobs=1, n_neighbors=6, p=2,
weights='uniform')
 knn1.fit(X_train, y_train )<compute_test_metric>
def add_new_features_4(df):
df['headshotRate'] = df['kills'] / df['headshotKills']
df['killStreakRate'] = df['killStreaks'] / df['kills']
df['healsAndBoosts'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df['walkDistance'] + df['swimDistance']
df['killPlaceOverMaxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKillsOverKills'] = df['headshotKills'] / df['kills']
df['distanceOverWeapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistanceOverHeals'] = df['walkDistance'] / df['heals']
df['walkDistanceOverKills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df['headshotKills'] + df['roadKills']
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
df.fillna(0, inplace=True )
print(f'KNN score - 2nd model: {knn1.score(X_train, y_train)}' )<choose_model_class>
def feature_engineering(df, is_train=True):
df['rankPoints'] = np.where(df['rankPoints'] <= 0, 0, df['rankPoints'])
 features = list(df.columns)
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 if 'winPlacePerc' in features:
features.remove('winPlacePerc')
 y = None
if is_train:
y = df.groupby(['matchId','groupId'])['winPlacePerc'].agg('mean')
 elif 'winPlacePerc' in df.columns:
y = df['winPlacePerc']
agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train:
df_out = agg.reset_index() [['matchId','groupId']]
else:
df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 del agg, agg_rank
return df_out, y
GB = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [1000],
'learning_rate': [0.02, 0.05],
'min_samples_split': [15, 20, 25],
'max_depth': [4, 6],
'min_samples_leaf': [50, 60],
'max_features': ["sqrt"]
}
gb_param_grid_best = {'loss' : ["deviance"],
'n_estimators' : [1000],
'learning_rate': [0.02],
'min_samples_split': [25],
'max_depth': [4],
'min_samples_leaf': [60],
'max_features': ["sqrt"]
}
gs_gb = GridSearchCV(GB, param_grid = gb_param_grid_best, cv=kfold, scoring="roc_auc", n_jobs= 4, verbose = 1)
 gs_gb.fit(X_train,y_train)
 GB.fit(X_train, y_train)
 gb_best = gs_gb.best_estimator_<compute_test_metric>
class Estimator(object):
def fit(self, x_train, y_train, x_valid, y_valid):
raise NotImplementedException
def predict(self, x):
raise NotImplementedException
print(f'GradienBoost GridSearch best params: {gs_gb.best_params_}')
 print()
print(f'GradienBoost GridSearch best score: {gs_gb.best_score_}')
 print(f'GradienBoost score: {GB.score(X_train, y_train)}' )<define_search_model>
class ScikitLearnEstimator(Estimator):
def __init__(self, estimator):
self.estimator = estimator
def fit(self, x_train, y_train, x_valid, y_valid):
self.estimator.fit(x_train, y_train)
 def predict(self, x):
return self.estimator.predict(x )
XGB = XGBClassifier()
xgb_param_grid = {'learning_rate':[0.05, 0.1],
'reg_lambda':[0.3, 0.5],
'gamma': [0.8, 1],
'subsample': [0.8, 1],
'max_depth': [2, 3],
'n_estimators': [200, 300]
}
xgb_param_grid_best = {'learning_rate':[0.1],
'reg_lambda':[0.3],
'gamma': [1],
'subsample': [0.8],
'max_depth': [2],
'n_estimators': [300]
}
gs_xgb = GridSearchCV(XGB, param_grid = xgb_param_grid_best, cv=kfold, scoring="roc_auc", n_jobs= 4, verbose = 1)
 gs_xgb.fit(X_train,y_train)
 XGB.fit(X_train, y_train)
 xgb_best = gs_xgb.best_estimator_<compute_test_metric>
def fit_predict_step(estimator, x_train, y_train, train_idx, valid_idx, x_test, oof):
x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
x_train_valid = x_train[valid_idx]
y_train_valid = y_train[valid_idx]
estimator.fit(x_train_train, y_train_train, x_train_valid, y_train_valid)
 oof_part = estimator.predict(x_train_valid)
 print('MAE:', mean_absolute_error(y_train_valid, oof_part))
oof[valid_idx] = oof_part
y_part = estimator.predict(x_test)
 return y_part
print(f'XGB GridSearch best params: {gs_xgb.best_params_}')
 print()
print(f'XGB GridSearch best score: {gs_xgb.best_score_}')
 print(f'XGB score: {XGB.score(X_train, y_train)}' )<save_to_csv>
def fit_predict(estimator, x_train, y_train, x_test):
oof = np.zeros(x_train.shape[0])
 y = np.zeros(x_test.shape[0])
 kf = KFold(n_splits=5, random_state=42)
 for train_idx, valid_idx in kf.split(x_train):
y_part = fit_predict_step(estimator, x_train, y_train, train_idx, valid_idx, x_test, oof)
 y += y_part / kf.n_splits
print('Final MAE:', mean_absolute_error(y_train, oof))
return oof, y
results=pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':knn1.predict(X_test)})
 results.to_csv("Titanic_prediction.csv", index=False)
 print('Done!' )<set_options>
def fit_step(estimator, x_train, y_train, train_idx, valid_idx, oof):
x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
x_train_valid = x_train[valid_idx]
y_train_valid = y_train[valid_idx]
estimator.fit(x_train_train, y_train_train, x_train_valid, y_train_valid)
 oof_part = estimator.predict(x_train_valid)
 mae = mean_absolute_error(y_train_valid, oof_part)
 print('MAE:', mae)
 oof[valid_idx] = oof_part
return estimator, mae
sns.set_style('whitegrid')
 %matplotlib inline 
<load_from_csv>
def fit(estimator, x_train, y_train):
oof = np.zeros(x_train.shape[0])
 kf = KFold(n_splits=5, random_state=42)
 trained_estimators = []
for train_idx, valid_idx in kf.split(x_train):
e, mae = fit_step(estimator, x_train, y_train, train_idx, valid_idx, oof)
 trained_estimators.append(deepcopy(e))
print('Final MAE:', mean_absolute_error(y_train, oof))
return oof, trained_estimators
df_train=pd.read_csv('.. /input/train.csv')
 df_test=pd.read_csv('.. /input/test.csv')
 df_titanic=pd.concat([df_train,df_test] ).reset_index(drop=True)
 df_titanic.info() 
<feature_engineering>
def predict(trained_estimators, x_test):
y = np.zeros(x_test.shape[0])
 for estimator in trained_estimators:
y_part = estimator.predict(x_test)
 y += y_part / len(trained_estimators)
 return y
df_age=df_titanic["Age"].dropna()
mean_age=int(df_titanic["Age"].dropna().mean())
 std_age=int(df_titanic["Age"].dropna().std())
 df_titanic["Age"]=df_titanic["Age"].apply(lambda x: np.random.randint(int(mean_age-std_age),int(mean_age+std_age)) if pd.isnull(x)else x )<filter>
def pipeline_fit(estimator, df_train, scaler=None):
add_new_features_1(df_train)
 add_new_features_2(df_train)
 add_new_features_3(df_train)
 add_new_features_4(df_train)
 x_train, y_train = feature_engineering(df_train, is_train=True)
 x_train = reduce_mem_usage(x_train)
 gc.collect()
if not(scaler is None):
scaler.fit(x_train)
 scaled_x_train = scaler.transform(x_train)
 else:
scaled_x_train = x_train.values
oof, trained_estimators = fit(estimator, scaled_x_train, y_train.values)
 del x_train
del scaled_x_train
del y_train
gc.collect()
return oof, trained_estimators
print(df_titanic[pd.isnull(df_titanic["Embarked"])] )<data_type_conversions>
def pipeline_predict(trained_estimators, df_test, scaler=None):
add_new_features_1(df_test)
 add_new_features_2(df_test)
 add_new_features_3(df_test)
 add_new_features_4(df_test)
 x_test, _ = feature_engineering(df_test, is_train=False)
 x_test = reduce_mem_usage(x_test)
 gc.collect()
if not(scaler is None):
scaled_x_test = scaler.transform(x_test)
 else:
scaled_x_test = x_test.values
y = predict(trained_estimators, scaled_x_test)
 del x_test
del scaled_x_test
gc.collect()
return y
df_titanic["Embarked"]=df_titanic["Embarked"].fillna("C" )<feature_engineering>
df_train = pd.read_csv('.. /input/train_V2.csv', index_col='Id')
 df_train.shape
print(df_titanic[pd.isnull(df_titanic["Fare"])])
 mean=df_titanic[df_titanic["Embarked"]=='S']["Fare"].mean()
std=df_titanic[df_titanic["Embarked"]=='S']["Fare"].std()
df_titanic["Fare"]=df_titanic["Fare"].fillna(random.randint(int(mean-std),int(mean+std)) )<feature_engineering>
df_train = reduce_mem_usage(df_train )
df_titanic.loc[df_titanic["Age"]<=16,"Sex"]="Child"
df_titanic.loc[(df_titanic["Age"]>16)&(df_titanic["Parch"]>0)&(df_titanic["Sex"]=="female"),"Sex"]="Mother"<count_values>
gc.collect()
df_titanic["Parch"].value_counts()<count_values>
df_train.drop(df_train[df_train['winPlacePerc'].isnull() ].index, inplace=True )
df_titanic["SibSp"].value_counts()<feature_engineering>
from sklearn.linear_model import LinearRegression
df_titanic["family"]=df_titanic["Parch"]+df_titanic["SibSp"]
df_titanic.loc[(df_titanic["family"]>=1)&(df_titanic["family"]<=3),"family"]=1
df_titanic.loc[df_titanic["family"] >3 |(df_titanic["family"]==0),"family"]=0<count_values>
del df_train
del df_oof
gc.collect()
df_name=pd.DataFrame(df_titanic["Name"].str.extract('([A-Za-z]+\.) '))
print(df_name["Name"].value_counts() )<feature_engineering>
df_test = pd.read_csv('.. /input/test_V2.csv', index_col = 'Id')
 df_test.shape
VIP=["Master.","Rev.","Dr.","Col.","Major.","Jonkheer.","Dona.","Capt.","Don.","Sir.","Lady.","Countess."]
df_name.loc[df_name["Name"].isin(VIP),"Name"]="VIP"
df_name.loc[df_name["Name"].isin(["Mlle.","Ms."]),"Name"]="Miss."
df_name.loc[df_name["Name"]=="Mme.","Name"]="Mrs."<feature_engineering>
df_test = reduce_mem_usage(df_test )
df_titanic["Name"]=df_name["Name"]<create_dataframe>
df_test_id = pd.DataFrame(index=df_test.index )
df_cabin=pd.DataFrame(df_titanic[["Cabin","Pclass","Fare"]].dropna())
 df_cabin["Cabin"]=df_cabin["Cabin"].astype(str ).str[0]<count_values>
gc.collect()
print(df_cabin["Cabin"].value_counts() )<split>
%%time
df_y = pd.DataFrame(index=df_test.index)
 df_y['y'] = 0
for matchType in matchTypes:
print("----------", matchType, "----------")
 df_test_part = df_test[df_test['matchType'] == matchType]
y_part = pipeline_predict(trained_estimators[matchType], df_test_part, trained_scalers[matchType])
 df_y_part = pd.DataFrame(index=df_test_part.index)
 df_y_part['y'] = y_part
df_y.update(df_y_part)
 y = df_y['y'].values
df_train=df_titanic.loc[0:890,]
df_test=df_titanic.loc[891:1308,]<prepare_x_and_y>
del df_test
del df_y
gc.collect()
X=df_train.drop(["PassengerId","Cabin","Ticket","Survived","SibSp","Parch"],axis=1)
 y=df_train["Survived"]
Submission_set=df_test.drop(["PassengerId","Cabin","Ticket","Survived","SibSp","Parch"],axis=1 )<categorify>
df_oof = pd.DataFrame()
df_oof['linear_oof'] = oof
df_oof.to_csv('linear_oof.csv', index_label='id' )
X=pd.get_dummies(X)
 Submission_set=pd.get_dummies(Submission_set)
 X.info()
X.describe()<normalization>
df_submission = pd.DataFrame(index=df_test_id.index)
 df_submission['winPlacePerc'] = y
df_submission.to_csv('linear_raw.csv', index_label='Id' )
X=scale(X)
 Submission_set=scale(Submission_set )<split>
df_test = pd.read_csv('.. /input/test_V2.csv')
 df_test.shape
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.2, random_state=42 )<choose_model_class>
df_submission = df_submission.merge(df_test[['Id', 'matchId', 'groupId', 'maxPlace', 'numGroups']], on='Id', how='left')
 df_submission.head()
Neighbors=KNeighborsClassifier()
n_neighbors = np.arange(1,50)
 param_grid = {'n_neighbors': n_neighbors,
'metric':["cityblock","euclidean"]}<train_on_grid>
df_submission_group = df_submission.groupby(['matchId', 'groupId'] ).first().reset_index()
df_submission_group['rank'] = df_submission_group.groupby(['matchId'])['winPlacePerc'].rank()
df_submission_group = df_submission_group.merge(df_submission_group.groupby('matchId')['rank'].max().to_frame('max_rank' ).reset_index() , on='matchId', how='left')
 df_submission_group['adjusted_perc'] =(df_submission_group['rank'] - 1)/(df_submission_group['numGroups'] - 1)
 df_submission = df_submission.merge(df_submission_group[['adjusted_perc', 'matchId', 'groupId']], on=['matchId', 'groupId'], how='left')
 df_submission['winPlacePerc'] = df_submission['adjusted_perc']
df_submission.head()
Neighbors_cv = GridSearchCV(Neighbors,param_grid, cv=5)
 Neighbors_cv.fit(X_train,y_train)
 print("Tuned k-NN classifier Parameters: {}".format(Neighbors_cv.best_params_))
print("Best score is {}".format(Neighbors_cv.best_score_)) 
<predict_on_test>
df_submission.loc[df_submission.maxPlace == 0, 'winPlacePerc'] = 0
df_submission.loc[df_submission.maxPlace == 1, 'winPlacePerc'] = 1
y_pred=Neighbors_cv.predict(X_test )<compute_test_metric>
t = df_submission.loc[df_submission.maxPlace > 1]
gap = 1.0 /(t.maxPlace.values - 1)
 fixed_perc = np.around(t.winPlacePerc.values / gap)* gap
df_submission.loc[df_submission.maxPlace > 1, 'winPlacePerc'] = fixed_perc
print("Confusion Matrix :")
 print(confusion_matrix(y_pred,y_test))
print("Classification report :")
 print(classification_report(y_pred,y_test))<predict_on_test>
df_submission.loc[(df_submission.maxPlace > 1)&(df_submission.numGroups == 1), 'winPlacePerc'] = 0
assert df_submission['winPlacePerc'].isnull().sum() == 0
<save_to_csv><EOS> 
df_submission[['Id', 'winPlacePerc']].to_csv('linear_adjusted.csv', index=False )