Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
<SOS> metric: MAE Kaggle data source: pubg-finish-placement-prediction-(kernels-only)<data_type_conversions>
warnings.filterwarnings(action='ignore')
 plt.style.use('fivethirtyeight')
 gc.enable()
X_train_analysis = X_train.copy()
X_train_analysis['Sex'] = X_train_analysis['Sex'].map({'male': 0, 'female': 1} ).astype(int)
 X_train_analysis['Embarked'] = X_train_analysis['Embarked'].map({'C': 0, 'Q': 1, 'S': 2} ).astype(int)
 X_train_analysis['Family'] = X_train_analysis['Family'].map({'alone': 0, 'small': 1, 'medium': 2, 'large': 3} ).astype(int)
 agerange_dict = dict(zip(age_labels, list(range(len(age_labels)))))
 X_train_analysis['AgeRange'] = X_train_analysis['AgeRange'].map(agerange_dict ).astype(int)
 farerange_dict = dict(zip(fare_labels, list(range(len(fare_labels)))))
 X_train_analysis['FareRange'] = X_train_analysis['FareRange'].map(farerange_dict ).astype(int)
 titlegroup_labels = list(set(title_dict.values()))
titlegroup_dict = dict(zip(titlegroup_labels, list(range(len(titlegroup_labels)))))
 X_train_analysis['TitleGroup'] = X_train_analysis['TitleGroup'].map(titlegroup_dict ).astype(int )<train_model>
%time
train_d=dd.read_csv('.. /input/train_V2.csv',blocksize=25e6)
rforest_checker = RandomForestClassifier(random_state = 0)
 rforest_checker.fit(X_train_analysis, y_train)
 importances_df = pd.DataFrame(rforest_checker.feature_importances_, columns=['Feature_Importance'],
index=X_train_analysis.columns)
 importances_df.sort_values(by=['Feature_Importance'], ascending=False, inplace=True)
 print(importances_df )<create_dataframe>
train=pd.read_csv('.. /input/train_V2.csv',nrows=100000)
 test=pd.read_csv(".. /input/test_V2.csv",)
 test_id=test['Id']
my_imp_dict = {'Feature Importance' : pd.Series([0.360313, 0.113686, 0.109495, 0.103845, 0.100966, 0.099818, 0.056429, 0.055449],
index=['TitleGroup', 'Family', 'Pclass', 'Sex','FareRange', 'AgeRange', 'HasCabin', 'Embarked'])}
my_imp_df = pd.DataFrame(my_imp_dict)
 print(my_imp_df )<drop_column>
train.isnull().sum()
X_train = X_train.drop(['HasCabin','Embarked'], axis=1)
 X_test = X_test.drop(['HasCabin','Embarked'], axis=1 )<categorify>
train.isnull().sum()
X_train = pd.get_dummies(X_train, columns=['TitleGroup','Family','Pclass','Sex','AgeRange','FareRange'])
 X_test = pd.get_dummies(X_test, columns=['TitleGroup','Family','Pclass','Sex','AgeRange','FareRange'] )<drop_column>
train_group=train.groupby('groupId',as_index=False)['winPlacePerc'].agg('mean' )
X_train = X_train.drop(['Pclass_1','Sex_female','TitleGroup_Master','AgeRange_15-','FareRange_10-','Family_alone'], axis=1)
 X_test = X_test.drop(['Pclass_1','Sex_female','TitleGroup_Master','AgeRange_15-','FareRange_10-','Family_alone'], axis=1 )<train_on_grid>
train[train['groupId']==2]['winPlacePerc']
params_logreg = [{'C': [0.01, 0.1, 1, 10, 100], 'penalty': ['l1','l2']}]
grid_logreg = GridSearchCV(estimator = LogisticRegression() ,
param_grid = params_logreg,
scoring = 'accuracy',
cv = 10)
 grid_logreg = grid_logreg.fit(X_train, y_train)
 best_acc_logreg = grid_logreg.best_score_
best_params_logreg = grid_logreg.best_params_<define_search_space>
train_corr=train.drop(columns=['Id',"matchId","groupId"])
params_ksvm = [{'C': [0.1, 1, 10, 100], 'kernel': ['linear']},
{'C': [0.1, 1, 10, 100], 'kernel': ['rbf'],
'gamma': [0.1, 0.2, 0.3, 0.4, 0.5]},
{'C': [0.1, 1, 10, 100], 'kernel': ['poly'],
'degree': [1, 2, 3],
'gamma': [0.1, 0.2, 0.3, 0.4, 0.5]}]
grid_ksvm = GridSearchCV(estimator = SVC(random_state = 0),
param_grid = params_ksvm,
scoring = 'accuracy',
cv = 10,
n_jobs=-1)
 grid_ksvm = grid_ksvm.fit(X_train, y_train)
 best_acc_ksvm = grid_ksvm.best_score_
best_params_ksvm = grid_ksvm.best_params_<choose_model_class>
train_dbno=pd.DataFrame(train['DBNOs'].value_counts() ,columns=['DBNOs'])
 dbno=train_dbno.iloc[:9,:]
dbno.iloc[8]['DBNOs']=train_dbno.iloc[8:,:].sum() ['DBNOs']
params_dtree = [{'min_samples_split': [5, 10, 15, 20],
'min_samples_leaf': [1, 2, 3],
'max_features': ['auto', 'log2']}]
grid_dtree = GridSearchCV(estimator = DecisionTreeClassifier(criterion = 'gini',
random_state = 0),
param_grid = params_dtree,
scoring = 'accuracy',
cv = 10,
n_jobs=-1)
 grid_dtree = grid_dtree.fit(X_train, y_train)
 best_acc_dtree = grid_dtree.best_score_
best_params_dtree = grid_dtree.best_params_<choose_model_class>
set1=set(i for i in train[(train['kills']>40)&(train['heals']==0)].index.tolist() )
params_rforest = [{'n_estimators': [200, 300],
'max_depth': [5, 7, 10],
'min_samples_split': [2, 4]}]
grid_rforest = GridSearchCV(estimator = RandomForestClassifier(criterion = 'gini',
random_state = 0, n_jobs=-1),
param_grid = params_rforest,
scoring = 'accuracy',
cv = 10,
n_jobs=-1)
 grid_rforest = grid_rforest.fit(X_train, y_train)
 best_acc_rforest = grid_rforest.best_score_
best_params_rforest = grid_rforest.best_params_<create_dataframe>
set2=set(i for i in train[(train['move']==0)&(train['kills']>20)].index.tolist() )
grid_score_dict = {'Best Score': [best_acc_logreg,best_acc_ksvm,best_acc_dtree,best_acc_rforest],
'Optimized Parameters': [best_params_logreg,best_params_ksvm,best_params_dtree,best_params_rforest],
}
pd.DataFrame(grid_score_dict, index=['Logistic Regression','Kernel SVM','Decision Tree','Random Forest'] )<predict_on_test>
set3=set(i for i in train[(train['damageDealt']>4000)&(train['heals']<2)].index.tolist() )
logreg = LogisticRegression(C = 1, penalty = 'l1')
 logreg.fit(X_train, y_train)
 y_pred_train_logreg = cross_val_predict(logreg, X_train, y_train)
 y_pred_test_logreg = logreg.predict(X_test )<find_best_model_class>
set4=set(i for i in train[(train['rideDistance']>25000)&(train['roadkillsrate']>.4)].index.tolist() )
ksvm = SVC(C = 1, gamma = 0.2, kernel = 'rbf', random_state = 0)
 ksvm.fit(X_train, y_train)
 y_pred_train_ksvm = cross_val_predict(ksvm, X_train, y_train)
 y_pred_test_ksvm = ksvm.predict(X_test )<find_best_model_class>
set5=set(i for i in train[(train['killStreaks']>3)&(train['weaponsAcquired']> 30)].index.tolist() )
dtree = DecisionTreeClassifier(criterion = 'gini', max_features='auto', min_samples_leaf=1, min_samples_split=5, random_state = 0)
 dtree.fit(X_train, y_train)
 y_pred_train_dtree = cross_val_predict(dtree, X_train, y_train)
 y_pred_test_dtree = dtree.predict(X_test )<find_best_model_class>
set6=set(i for i in train[(train['killStreaks']>3)&(train['weaponsAcquired']> 30)].index.tolist() )
rforest = RandomForestClassifier(max_depth = 7, min_samples_split=4, n_estimators = 200, random_state = 0)
 rforest.fit(X_train, y_train)
 y_pred_train_rforest = cross_val_predict(rforest, X_train, y_train)
 y_pred_test_rforest = rforest.predict(X_test )<create_dataframe>
sets=set1 | set2 | set3 | set4 | set5 | set6
second_layer_train = pd.DataFrame({'Logistic Regression': y_pred_train_logreg.ravel() ,
'Kernel SVM': y_pred_train_ksvm.ravel() ,
'Decision Tree': y_pred_train_dtree.ravel() ,
'Random Forest': y_pred_train_rforest.ravel()
})
 second_layer_train.head()
X_train_second = np.concatenate(( y_pred_train_logreg.reshape(-1, 1), y_pred_train_ksvm.reshape(-1, 1),
y_pred_train_dtree.reshape(-1, 1), y_pred_train_rforest.reshape(-1, 1)) ,
axis=1)
 X_test_second = np.concatenate(( y_pred_test_logreg.reshape(-1, 1), y_pred_test_ksvm.reshape(-1, 1),
y_pred_test_dtree.reshape(-1, 1), y_pred_test_rforest.reshape(-1, 1)) ,
axis=1)
 xgb = XGBClassifier(
 n_estimators= 800,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
scale_pos_weight=1 ).fit(X_train_second, y_train)
 y_pred = xgb.predict(X_test_second )<save_to_csv>
train.drop(train.index[list(sets)],inplace=True )
passengerId = np.array(test_set['PassengerId'] ).astype(int)
 submission = pd.DataFrame({ 'PassengerId' : passengerId, 'Survived' : y_pred })
 print(submission.shape)
 submission.to_csv('submission.csv', index=False )<set_options>
train.drop(['roadkillsrate'],axis=1,inplace=True)
 train.columns.drop('winPlacePerc')==test.columns
np.random.seed(13 )<define_variables>
def new_features(df):
df['items']=df['boosts']+df['heals']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['headshotKills_over_kills'].fillna(0, inplace=True)
 df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['killPlace_over_maxPlace'].fillna(0, inplace=True)
 df['killPlace_over_maxPlace'].replace(np.inf, 0, inplace=True)
 df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_heals'].fillna(0, inplace=True)
 df['walkDistance_over_heals'].replace(np.inf, 0, inplace=True)
 df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['walkDistance_over_kills'].fillna(0, inplace=True)
 df['walkDistance_over_kills'].replace(np.inf, 0, inplace=True)
 df['teamwork'] = df['assists'] + df['revives']
return df
num_classes = 10
batch_size = 128
epochs = 700
img_rows, img_cols = 28, 28
input_shape =(img_rows, img_cols,1 )<load_from_csv>
train=new_features(train)
 test=new_features(test )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
def feature_engineering(is_train=True):
if is_train:
print("processing train.csv")
 df = train
else:
print("processing test.csv")
 df = test
target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove('matchType')
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y
x_train, y = feature_engineering(True)
y_train = train["label"]
x_train = train.drop(labels = ["label"],axis = 1 )<feature_engineering>
test,z=feature_engineering(False )
x_train /= 255
test /= 255<data_type_conversions>
scaler=MinMaxScaler(feature_range=(-1,1),copy=False ).fit(x_train)
 x_train=scaler.transform(x_train )
x_train = x_train.values.reshape(-1,img_rows,img_cols,1 ).astype('float32')
 test = test.values.reshape(-1,img_rows,img_cols,1 ).astype('float32' )<categorify>
x_test=scaler.transform(test )
y_train = keras.utils.to_categorical(y_train, num_classes = num_classes )<split>
from xgboost.sklearn import XGBRegressor
from sklearn.model_selection import cross_val_score,KFold
import numpy as np
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Lasso,LinearRegression
from sklearn import preprocessing
from sklearn.model_selection import RandomizedSearchCV,train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
x_train, x_test, y_train, y_test = train_test_split(x_train, y_train, test_size = 0.1 )<choose_model_class>
n_folds=2
def mbs(model):
kf=KFold(n_folds,shuffle=True,random_state=42 ).get_n_splits(x_train)
 score=-cross_val_score(model,x_train,y,scoring='neg_mean_absolute_error',cv=kf)
 return score
def model_cnn(input_shape=input_shape, num_classes=num_classes):
model = Sequential()
model.add(Conv2D(32, kernel_size =(3,3), activation='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size =(3,3), activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size =(5,5), strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size =(3,3), activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size =(3,3), activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size =(5,5), strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(num_classes, activation = "softmax"))
model.compile(optimizer = Adam() , loss = "categorical_crossentropy", metrics=["accuracy"])
 return model<choose_model_class>
lasso=Lasso(alpha=.005,random_state=1 )
def LeNet5(input_shape=input_shape,num_classes=num_classes):
model = Sequential()
model.add(Conv2D(6, kernel_size=(5, 5), strides=(1, 1), activation='relu', input_shape=input_shape, padding="same"))
model.add(AveragePooling2D(pool_size=(2, 2), strides=(1, 1), padding='valid'))
model.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='valid'))
model.add(AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
model.add(Conv2D(120, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='valid'))
model.add(Flatten())
 model.add(Dense(84, activation='relu'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(optimizer = Adam() , loss = "categorical_crossentropy", metrics=["accuracy"])
 return model<train_model>
params={'learning_rate':[.02,.03,.1],'min_child_weight':[4,6,8],'max_depth':[8,10],"subsample":[.6,.4],"n_estimators":[300,500]}
datagen = ImageDataGenerator(rotation_range=10, zoom_range = 0.1, width_shift_range=0.1, height_shift_range=0.1)
 datagen.fit(x_train )<train_model>
xgb=XGBRegressor(learning_rate=.03,min_child_weight=4,max_depth=10,subsample=.4,n_estimators=500,n_jobs=-1)
models = []
for i in range(len(model)) :
model[i].fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),
epochs = epochs, steps_per_epoch=x_train.shape[0] // batch_size,
validation_data =(x_test,y_test),
callbacks=[ReduceLROnPlateau(monitor='loss', patience=3, factor=0.1)],
verbose=2)
 models.append(model[i] )<predict_on_test>
GBoost = GradientBoostingRegressor(n_estimators=600, learning_rate=0.05,
max_depth=4, max_features='auto',
min_samples_leaf=15, min_samples_split=5,
loss='huber', random_state =5 )
labels = []
for m in models:
predicts = np.argmax(m.predict(test), axis=1)
 labels.append(predicts)
 labels = np.array(labels)
 labels = np.transpose(labels,(1, 0))
labels = scipy.stats.mode(labels, axis=-1)[0]
labels = np.squeeze(labels )<save_to_csv>
class Average_models(BaseEstimator,RegressorMixin,TransformerMixin):
def __init__(self,models):
self.models=models
def fit(self,X,y):
self.models_=[clone(x)for x in self.models]
for model in self.models_:
model.fit(X,y)
 return self
def predict(self,X):
predictions= np.column_stack([model.predict(X)for model in self.models_])
 return np.mean(predictions,axis=1 )
pd.DataFrame({'ImageId' : np.arange(1, predicts.shape[0] + 1), 'Label' : labels } ).to_csv('submission.csv', index=False )<import_modules>
training,valid,ytraining,yvalid=train_test_split(x_train,y)
 xgb.fit(training,ytraining)
 lasso.fit(training,ytraining)
 pred1=xgb.predict(valid)
 pred2=lasso.predict(valid)
 test_pred1=xgb.predict(test)
 test_pred2=lasso.predict(test)
 stacked_predictions=np.column_stack(( pred1,pred2))
stacked_test_prediction=np.column_stack(( test_pred1,test_pred2))
meta_model=LinearRegression()
meta_model.fit(stacked_predictions,yvalid)
 final=meta_model.predict(stacked_test_prediction)
import plotly.graph_objects as go
import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from datetime import datetime
from pathlib import Path
from sklearn import preprocessing
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, LSTM, RNN, Dropout
from keras.callbacks import EarlyStopping
from keras import optimizers
from sklearn.preprocessing import StandardScaler, MinMaxScaler<load_from_csv>
df=pd.DataFrame({'id':test_id,'winPlacePerc':final},columns=['id','winPlacePerc'])
 csv=df[['id','winPlacePerc']].to_csv('submission.csv',index=False )