Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
<SOS> metric: MAE Kaggle data source: pubg-finish-placement-prediction-(kernels-only)<categorify>
%matplotlib inline
%pylab inline
print(os.listdir(".. /input"))
warnings.filterwarnings(action='ignore' )
for col in categorical_columns:
if col in X_train.columns:
le = LabelEncoder()
le.fit(list(X_train[col].astype(str ).values)+ list(X_test[col].astype(str ).values))
X_train[col] = le.transform(list(X_train[col].astype(str ).values))
X_test[col] = le.transform(list(X_test[col].astype(str ).values))<prepare_x_and_y>
df_train = pd.read_csv('.. /input/train_V2.csv')
 df = reduce_mem_usage(df_train)
 df.info()
train0, test0 = X_train, X_test
target0 = Y<split>
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor
from IPython.display import display
from sklearn import metrics
from scipy.cluster import hierarchy as hc
from pdpbox import pdp
from plotnine import *
from fastai.imports import *
from fastai.structured import *
scaler = StandardScaler()
train0 = pd.DataFrame(scaler.fit_transform(train0), columns = train0.columns)
 test0 = pd.DataFrame(scaler.transform(test0), columns = test0.columns)
 train0b = train0.copy()
test0b = test0.copy()
trainb, testb, targetb, target_testb = train_test_split(train0b, target0, test_size=test_train_split_part, random_state=random_state )<normalization>
df[df['winPlacePerc'].isnull() ]
scaler = MinMaxScaler()
train0 = pd.DataFrame(scaler.fit_transform(train0), columns = train0.columns)
 test0 = pd.DataFrame(scaler.fit_transform(test0), columns = test0.columns )<split>
df.drop(2744604, inplace=True)
 df[df['winPlacePerc'].isnull() ]
train, test, target, target_test = train_test_split(train0, target0, test_size=test_train_split_part, random_state=random_state )<define_variables>
train = df
num_models = 20
acc_train = []
acc_test = []
acc_all = np.empty(( len(metrics_now)*2, 0)).tolist()
acc_all<define_variables>
train['killsNorm'] = train['kills']*(( 100-train['playersJoined'])/100 + 1)
 train['damageDealtNorm'] = train['damageDealt']*(( 100-train['playersJoined'])/100 + 1)
 train['maxPlaceNorm'] = train['maxPlace']*(( 100-train['playersJoined'])/100 + 1)
 train['matchDurationNorm'] = train['matchDuration']*(( 100-train['playersJoined'])/100 + 1)
 to_show = ['Id', 'kills','killsNorm','damageDealt', 'damageDealtNorm', 'maxPlace', 'maxPlaceNorm', 'matchDuration', 'matchDurationNorm']
train[to_show][0:11]
acc_all_pred = np.empty(( len(metrics_now), 0)).tolist()
acc_all_pred<split>
train['healsandboosts'] = train['heals'] + train['boosts']
train[['heals', 'boosts', 'healsandboosts']].tail()
cv_train = ShuffleSplit(n_splits=cv_n_split, test_size=test_train_split_part, random_state=random_state )<compute_test_metric>
train['totalDistance'] = train['rideDistance'] + train['walkDistance'] + train['swimDistance']
train['killsWithoutMoving'] =(( train['kills'] > 0)&(train['totalDistance'] == 0))
def acc_d(y_meas, y_pred):
return mean_absolute_error(y_meas, y_pred)*len(y_meas)/sum(abs(y_meas))
def acc_rmse(y_meas, y_pred):
return(mean_squared_error(y_meas, y_pred)) **0.5<compute_train_metric>
train['headshot_rate'] = train['headshotKills'] / train['kills']
train['headshot_rate'] = train['headshot_rate'].fillna(0 )
def acc_metrics_calc(num,model,train,test,target,target_test):
global acc_all
ytrain = model.predict(train ).astype(int)
 ytest = model.predict(test ).astype(int)
 print('target = ', target[:5].values)
 print('ytrain = ', ytrain[:5])
 print('target_test =', target_test[:5].values)
 print('ytest =', ytest[:5])
 num_acc = 0
for x in metrics_now:
if x == 1:
acc_train = round(r2_score(target, ytrain)* 100, 2)
 acc_test = round(r2_score(target_test, ytest)* 100, 2)
 elif x == 2:
acc_train = round(metrics.accuracy_score(target, ytrain)* 100, 2)
 acc_test = round(metrics.accuracy_score(target_test, ytest)* 100, 2)
 elif x == 3:
acc_train = round(acc_rmse(target, ytrain)* 100, 2)
 acc_test = round(acc_rmse(target_test, ytest)* 100, 2)
 elif x == 4:
acc_train = round(acc_d(target, ytrain)* 100, 2)
 acc_test = round(acc_d(target_test, ytest)* 100, 2)
 print('acc of', metrics_all[x], 'for train =', acc_train)
 print('acc of', metrics_all[x], 'for test =', acc_test)
 acc_all[num_acc].append(acc_train)
 acc_all[num_acc+1].append(acc_test)
 num_acc += 2<compute_train_metric>
train.drop(train[train['killsWithoutMoving'] == True].index, inplace=True )
def acc_metrics_calc_pred(num,model,name_model,train,test,target):
global acc_all_pred
ytrain = model.predict(train ).astype(int)
 ytest = model.predict(test ).astype(int)
 print('**********')
 print(name_model)
 print('target = ', target[:15].values)
 print('ytrain = ', ytrain[:15])
 print('ytest =', ytest[:15])
 num_acc = 0
for x in metrics_now:
if x == 1:
acc_train = round(r2_score(target, ytrain)* 100, 2)
 elif x == 2:
acc_train = round(metrics.accuracy_score(target, ytrain)* 100, 2)
 elif x == 3:
acc_train = round(acc_rmse(target, ytrain)* 100, 2)
 elif x == 4:
acc_train = round(acc_d(target, ytrain)* 100, 2)
 print('acc of', metrics_all[x], 'for train =', acc_train)
 acc_all_pred[num_acc].append(acc_train)
 num_acc += 1
submission[target_name] = ytest
submission.to_csv('submission_' + name_model + '.csv', index=False )<train_on_grid>
train[train['roadKills'] > 10]
linreg = LinearRegression()
linreg_CV = GridSearchCV(linreg, param_grid={}, cv=cv_train, verbose=False)
 linreg_CV.fit(train, target)
 print(linreg_CV.best_params_)
 acc_metrics_calc(0,linreg_CV,train,test,target,target_test )<train_on_grid>
train.drop(train[train['roadKills'] > 10].index, inplace=True )
svr = SVC()
svr_CV = GridSearchCV(svr, param_grid={'kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
'tol': [1e-4]},
cv=cv_train, verbose=False)
 svr_CV.fit(train, target)
 print(svr_CV.best_params_)
 acc_metrics_calc(1,svr_CV,train,test,target,target_test )<train_on_grid>
display(train[train['kills'] > 30].shape)
 train[train['kills'] > 30]
linear_svc = LinearSVC()
param_grid = {'dual':[False],
'C': np.linspace(1, 15, 15)}
linear_svc_CV = GridSearchCV(linear_svc, param_grid=param_grid, cv=cv_train, verbose=False)
 linear_svc_CV.fit(train, target)
 print(linear_svc_CV.best_params_)
 acc_metrics_calc(2,linear_svc_CV,train,test,target,target_test )<train_on_grid>
train.drop(train[train['kills'] > 30].index, inplace=True )
%%time
mlp = MLPClassifier()
param_grid = {'hidden_layer_sizes': [i for i in range(2,5)],
'solver': ['sgd'],
'learning_rate': ['adaptive'],
'max_iter': [1000]
}
mlp_GS = GridSearchCV(mlp, param_grid=param_grid, cv=cv_train, verbose=False)
 mlp_GS.fit(train, target)
 print(mlp_GS.best_params_)
 acc_metrics_calc(3,mlp_GS,train,test,target,target_test )<train_on_grid>
train.drop(train[train['longestKill'] >= 1000].index, inplace=True )
sgd = SGDClassifier(early_stopping=True)
 param_grid = {'alpha': [0.0001, 0.001, 0.01, 0.1, 1]}
sgd_CV = GridSearchCV(sgd, param_grid=param_grid, cv=cv_train, verbose=False)
 sgd_CV.fit(train, target)
 print(sgd_CV.best_params_)
 acc_metrics_calc(4,sgd_CV,train,test,target,target_test )<train_on_grid>
train.drop(train[train['walkDistance'] >= 10000].index, inplace=True )
decision_tree = DecisionTreeClassifier()
param_grid = {'min_samples_leaf': [i for i in range(2,10)]}
decision_tree_CV = GridSearchCV(decision_tree, param_grid=param_grid, cv=cv_train, verbose=False)
 decision_tree_CV.fit(train, target)
 print(decision_tree_CV.best_params_)
 acc_metrics_calc(5,decision_tree_CV,train,test,target,target_test )<train_on_grid>
train[train['rideDistance'] >= 20000]
%%time
random_forest = RandomForestClassifier()
param_grid = {'n_estimators': [300, 400, 500, 600], 'min_samples_split': [60], 'min_samples_leaf': [20, 25, 30, 35, 40],
'max_features': ['auto'], 'max_depth': [5, 6, 7, 8, 9, 10], 'criterion': ['gini'], 'bootstrap': [False]}
random_forest_CV = GridSearchCV(estimator=random_forest, param_grid=param_grid,
cv=cv_train, verbose=False)
 random_forest_CV.fit(train, target)
 print(random_forest_CV.best_params_)
 acc_metrics_calc(6,random_forest_CV,train,test,target,target_test )<train_on_grid>
train.drop(train[train['rideDistance'] >= 20000].index, inplace=True )
%%time
xgb_clf = xgb.XGBClassifier(objective='reg:squarederror')
 parameters = {'n_estimators': [200, 300, 400],
'learning_rate': [0.001, 0.003, 0.005, 0.006, 0.01],
'max_depth': [4, 5, 6]}
xgb_reg = GridSearchCV(estimator=xgb_clf, param_grid=parameters, cv=cv_train ).fit(trainb, targetb)
 print("Best score: %0.3f" % xgb_reg.best_score_)
 print("Best parameters set:", xgb_reg.best_params_)
 acc_metrics_calc(7,xgb_reg,trainb,testb,targetb,target_testb )<split>
train[train['swimDistance'] >= 2000]
Xtrain, Xval, Ztrain, Zval = train_test_split(trainb, targetb, test_size=test_train_split_part, random_state=random_state)
 modelL = lgb.LGBMClassifier(n_estimators=1000, num_leaves=50)
 modelL.fit(Xtrain, Ztrain, eval_set=[(Xval, Zval)], early_stopping_rounds=50, verbose=True )<compute_test_metric>
train.drop(train[train['swimDistance'] >= 2000].index, inplace=True )
acc_metrics_calc(8,modelL,trainb,testb,targetb,target_testb )<train_on_grid>
train[train['weaponsAcquired'] >= 80]
gradient_boosting = GradientBoostingClassifier()
param_grid = {'learning_rate' : [0.001, 0.01, 0.1],
'max_depth': [i for i in range(2,5)],
'min_samples_leaf': [i for i in range(2,5)]}
gradient_boosting_CV = GridSearchCV(estimator=gradient_boosting, param_grid=param_grid,
cv=cv_train, verbose=False)
 gradient_boosting_CV.fit(train, target)
 print(gradient_boosting_CV.best_params_)
 acc_metrics_calc(9,gradient_boosting_CV,train,test,target,target_test )<train_on_grid>
train.drop(train[train['weaponsAcquired'] >= 80].index, inplace=True )
ridge = RidgeClassifier()
ridge_CV = GridSearchCV(estimator=ridge, param_grid={'alpha': np.linspace (.1, 1.5, 15)}, cv=cv_train, verbose=False)
 ridge_CV.fit(train, target)
 print(ridge_CV.best_params_)
 acc_metrics_calc(10,ridge_CV,train,test,target,target_test )<train_on_grid>
train.drop(train[train['heals'] >= 40].index, inplace=True )
%%time
bagging = BaggingClassifier(base_estimator=linear_svc_CV)
 param_grid={'max_features': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
'n_estimators': [3, 5, 10],
'warm_start' : [True],
'random_state': [random_state]}
bagging_CV = GridSearchCV(estimator=bagging, param_grid=param_grid, cv=cv_train, verbose=False)
 bagging_CV.fit(train, target)
 print(bagging_CV.best_params_)
 acc_metrics_calc(11,bagging_CV,train,test,target,target_test )<train_on_grid>
train = train.drop(columns = ['matchType', 'Id', 'groupId', 'matchId'] )
etr = ExtraTreesClassifier()
etr_CV = GridSearchCV(estimator=etr, param_grid={'min_samples_leaf' : [10, 20, 30, 40, 50]}, cv=cv_train, verbose=False)
 etr_CV.fit(train, target)
 acc_metrics_calc(12,etr_CV,train,test,target,target_test )<train_on_grid>
df = df_sample.drop(columns = ['winPlacePerc'])
 y = df_sample['winPlacePerc']
Ada_Boost = AdaBoostClassifier()
Ada_Boost_CV = GridSearchCV(estimator=Ada_Boost, param_grid={'learning_rate' : [.01,.1,.5, 1]}, cv=cv_train, verbose=False)
 Ada_Boost_CV.fit(train, target)
 acc_metrics_calc(13,Ada_Boost_CV,train,test,target,target_test )<train_on_grid>
def split_vals(a, n : int):
return a[:n].copy() , a[n:].copy()
val_perc = 0.12
n_valid = int(val_perc * sample)
 n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(df_sample, n_trn)
 X_train, X_valid = split_vals(df, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )
logreg = LogisticRegression()
logreg_CV = GridSearchCV(estimator=logreg, param_grid={'C' : [.1,.3,.5,.7, 1]}, cv=cv_train, verbose=False)
 logreg_CV.fit(train, target)
 acc_metrics_calc(14,logreg_CV,train,test,target,target_test )<train_on_grid>
def print_score(m : RandomForestRegressor):
res = ['mae train: ', mean_absolute_error(m.predict(X_train), y_train),
'mae val: ', mean_absolute_error(m.predict(X_valid), y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )
knn = KNeighborsClassifier()
param_grid={'n_neighbors': range(2, 7)}
knn_CV = GridSearchCV(estimator=knn, param_grid=param_grid,
cv=cv_train, verbose=False ).fit(train, target)
 print(knn_CV.best_params_)
 acc_metrics_calc(15,knn_CV,train,test,target,target_test )<train_on_grid>
m1 = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features='sqrt',
n_jobs=-1)
 m1.fit(X_train, y_train)
 print_score(m1 )
gaussian = GaussianNB()
param_grid={'var_smoothing': [1e-8, 1e-9, 1e-10]}
gaussian_CV = GridSearchCV(estimator=gaussian, param_grid=param_grid, cv=cv_train, verbose=False)
 gaussian_CV.fit(train, target)
 print(gaussian_CV.best_params_)
 acc_metrics_calc(16,gaussian_CV,train,test,target,target_test )<train_on_grid>
fi = rf_feat_importance(m1, df); fi[:10]
perceptron = Perceptron()
param_grid = {'penalty': [None, 'l2', 'l1', 'elasticnet']}
perceptron_CV = GridSearchCV(estimator=perceptron, param_grid=param_grid, cv=cv_train, verbose=False)
 perceptron_CV.fit(train, target)
 print(perceptron_CV.best_params_)
 acc_metrics_calc(17,perceptron_CV,train,test,target,target_test )<train_on_grid>
to_keep = fi[fi.imp>0.005].cols
print('Significant features: ', len(to_keep))
to_keep
gpc = GaussianProcessClassifier()
param_grid = {'max_iter_predict': [100, 200],
'warm_start': [True, False],
'n_restarts_optimizer': range(3)}
gpc_CV = GridSearchCV(estimator=gpc, param_grid=param_grid, cv=cv_train, verbose=False)
 gpc_CV.fit(train, target)
 print(gpc_CV.best_params_)
 acc_metrics_calc(18,gpc_CV,train,test,target,target_test )<train_on_grid>
df_keep = df[to_keep].copy()
X_train, X_valid = split_vals(df_keep, n_trn )
Voting_ens = VotingClassifier(estimators=[('log', logreg_CV),('mlp', mlp_GS),('svc', linear_svc_CV)])
 Voting_ens.fit(train, target)
 acc_metrics_calc(19,Voting_ens,train,test,target,target_test )<create_dataframe>
m2 = RandomForestRegressor(n_estimators=80, min_samples_leaf=3, max_features='sqrt',
n_jobs=-1)
 m2.fit(X_train, y_train)
 print_score(m2 )
models = pd.DataFrame({
'Model': ['Linear Regression', 'Support Vector Machines', 'Linear SVC',
'MLPClassifier', 'Stochastic Gradient Decent',
'Decision Tree Classifier', 'Random Forest', 'XGBClassifier', 'LGBMClassifier',
'GradientBoostingClassifier', 'RidgeClassifier', 'BaggingClassifier', 'ExtraTreesClassifier',
'AdaBoostClassifier', 'Logistic Regression',
'KNN', 'Naive Bayes', 'Perceptron', 'Gaussian Process Classification',
'VotingClassifier']} )<feature_engineering>
val_perc_full = 0.12
n_valid_full = int(val_perc_full * len(train))
n_trn_full = len(train)-n_valid_full
df_full = train.drop(columns = ['winPlacePerc'])
 y = train['winPlacePerc']
df_full = df_full[to_keep]
X_train, X_valid = split_vals(df_full, n_trn_full)
 y_train, y_valid = split_vals(y, n_trn_full)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )
for x in metrics_now:
xs = metrics_all[x]
models[xs + '_train'] = acc_all[(x-1)*2]
models[xs + '_test'] = acc_all[(x-1)*2+1]
if xs == "acc":
models[xs + '_diff'] = models[xs + '_train'] - models[xs + '_test']
models<sort_values>
m3 = RandomForestRegressor(n_estimators=60, min_samples_leaf=3, max_features=0.5, n_jobs=-1)
 m3.fit(X_train, y_train)
 print_score(m3 )
print('Prediction accuracy for models')
 ms = metrics_all[metrics_now[1]]
models.sort_values(by=[(ms + '_test'),(ms + '_train')], ascending=False )<set_options>
test = pd.read_csv('.. /input/test_V2.csv' )
pd.options.display.float_format = '{:,.2f}'.format<sort_values>
test['headshot_rate'] = test['headshotKills'] / test['kills']
test['headshot_rate'] = test['headshot_rate'].fillna(0)
 test['totalDistance'] = test['rideDistance'] + test['walkDistance'] + test['swimDistance']
test['playersJoined'] = test.groupby('matchId')['matchId'].transform('count')
 test['killsNorm'] = test['kills']*(( 100-test['playersJoined'])/100 + 1)
 test['damageDealtNorm'] = test['damageDealt']*(( 100-test['playersJoined'])/100 + 1)
 test['maxPlaceNorm'] = test['maxPlace']*(( 100-train['playersJoined'])/100 + 1)
 test['matchDurationNorm'] = test['matchDuration']*(( 100-test['playersJoined'])/100 + 1)
 test['healsandboosts'] = test['heals'] + test['boosts']
test['killsWithoutMoving'] =(( test['kills'] > 0)&(test['totalDistance'] == 0))
test_pred = test[to_keep].copy()
test_pred.fillna(0, inplace=True)
 test_pred.head()
metrics_main = 2
xs = metrics_all[metrics_main]
xs_train = metrics_all[metrics_main] + '_train'
xs_test = metrics_all[metrics_main] + '_test'
print('The best models by the',xs,'criterion:')
 direct_sort = False if(metrics_main >= 2)else True
models_sort = models.sort_values(by=[xs_test, xs_train], ascending=direct_sort )<sort_values>
predictions = np.clip(a = m3.predict(test_pred), a_min = 0.0, a_max = 1.0)
 pred_df = pd.DataFrame({'Id' : test['Id'], 'winPlacePerc' : predictions})
 pred_df.to_csv("submission.csv", index=False )