Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = RandomForestClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
from lightgbm import LGBMRegressor
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.metrics import mean_absolute_error
import gc
from sklearn.model_selection import GridSearchCV
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = ExtraTreesClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
n_estimators = [10, 15, 20, 25, 50, 75, 100, 150]
max_samples = [1, 2, 3, 5, 7, 10, 15, 20, 25, 30, 50]
max_features = [1, 3, 5, 7]
hyperparams = {'n_estimators': n_estimators, 'max_samples': max_samples, 'max_features': max_features}
gd=GridSearchCV(estimator = BaggingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<choose_model_class>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
ran = RandomForestClassifier(n_estimators=25,
max_depth=3,
max_features=3,
min_samples_leaf=2,
min_samples_split=8,
random_state=1)
 knn = KNeighborsClassifier(algorithm='auto',
leaf_size=1,
n_neighbors=5,
weights='uniform')
 log = LogisticRegression(C=2.7825594022071245,
penalty='l2')
 xgb = XGBClassifier(learning_rate=0.0001,
n_estimators=10,
random_state=1)
 gbc = GradientBoostingClassifier(learning_rate=0.0005,
n_estimators=1250,
random_state=1)
 svc = SVC(probability=True)
 ext = ExtraTreesClassifier(max_depth=None,
max_features=3,
min_samples_leaf=2,
min_samples_split=8,
n_estimators=10,
random_state=1)
 ada = AdaBoostClassifier(learning_rate=0.1,
n_estimators=50,
random_state=1)
 gpc = GaussianProcessClassifier()
bag = BaggingClassifier(random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v3 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v3.append(acc.mean() )<choose_model_class>
concat = pd.concat([train, test])
 del train
del test
gc.collect()
grid_hard = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_train, y_train, cv = 10)
 grid_hard.fit(X_train, y_train)
 print("Hard voting on test set score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))<choose_model_class>
concat = reduce_mem_usage(concat )
grid_soft = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, X_train, y_train, cv = 10)
 grid_soft.fit(X_train, y_train)
 print("Soft voting on test set score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))<save_to_csv>
def count_transform(df, cols):
for c in cols:
df[c + "_count"] = df.groupby(c)[c].transform('count')
 return df
predictions = grid_soft.predict(X_test)
 submission = pd.concat([pd.DataFrame(passId), pd.DataFrame(predictions)], axis = 'columns')
 submission.columns = ["PassengerId", "Survived"]
submission.to_csv('titanic_submission.csv', header = True, index = False )<set_options>
concat = count_transform(concat, ['groupId', 'matchId'] )
%matplotlib inline<import_modules>
per_dist_stats = ['assists', 'boosts', 'damageDealt', 'DBNOs',
'headshotKills', 'heals', 'kills',
'teamKills', 'vehicleDestroys', 'weaponsAcquired']
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import seaborn as sns<import_modules>
concat['LogWalk'] = np.log1p(concat['walkDistance'] )
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.optimizers import RMSprop, Adam
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools<define_variables>
for stat in per_dist_stats:
concat[stat + '_perLogWalk'] = concat[stat] / concat['LogWalk']
PATH = '.. /input/'<load_from_csv>
concat['grpSizeMult'] = concat['groupId_count'] /(concat['matchId_count'] / concat['numGroups'] )
df_train = pd.read_csv(f'{PATH}train.csv')
 df_test = pd.read_csv(f'{PATH}test.csv' )<count_unique_values>
match_stats = ['DBNOs',
'assists',
'boosts',
'damageDealt',
'headshotKills',
'heals',
'killPlace',
'killPoints',
'killStreaks',
'kills',
'longestKill',
'revives',
'rideDistance',
'roadKills',
'swimDistance',
'vehicleDestroys',
'walkDistance',
'weaponsAcquired',
'winPoints',
'LogWalk',
'assists_perLogWalk',
'boosts_perLogWalk',
'damageDealt_perLogWalk',
'DBNOs_perLogWalk',
'headshotKills_perLogWalk',
'heals_perLogWalk',
'kills_perLogWalk',
'teamKills_perLogWalk',
'vehicleDestroys_perLogWalk',
'weaponsAcquired_perLogWalk',]
X_train = df_train.drop('label', axis=1 ).values
X_test = df_test.values
Y_train = df_train.label
print(f'Number of training Examples, {X_train.shape[0]}')
 print(f'Number of test Examples, {X_test.shape[0]}')
 print(f'Number of classes, {np.unique(Y_train)}' )<train_model>
for stat in match_stats:
concat['matchRel_' + stat] = concat[stat] / concat.groupby('matchId')[stat].transform('mean' )
X_train = X_train.reshape(-1, 28, 28, 1)
 X_test = X_test.reshape(-1, 28, 28, 1 )<categorify>
drop_features = ["winPlacePerc", "Id", "groupId", "matchId"]
feats = [c for c in concat.columns if c not in drop_features]
print('Integer Valued Labels')
 print(Y_train[:10])
 Y_train = np_utils.to_categorical(Y_train, 10)
 print('One Hot Labels')
 print(Y_train[:10] )<data_type_conversions>
aggs = {
'grpSizeMult' : ['mean'],
'groupId_count' : ['mean'],
'matchId_count' : ['mean'],
'winPlacePerc' : ['mean'],
}
for c in feats:
if c not in aggs:
aggs[c] = ['mean', 'min', 'max', 'std']
new_cols = [k + '_' + agg for k in aggs.keys() for agg in aggs[k]]
X_train = X_train.astype('float32')/ 255
X_test = X_test.astype('float32')/ 255<split>
groups = concat.groupby('groupId' ).agg(aggs)
X_train, X_valid = X_train[7000:], X_train[:7000]
Y_train, Y_valid = Y_train[7000:], Y_train[:7000]<choose_model_class>
groups.columns = new_cols
model = Sequential()
model.add(Flatten(input_shape=X_train.shape[1:]))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()<choose_model_class>
del concat
gc.collect()
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )<train_model>
groups = reduce_mem_usage(groups )
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5',
verbose=1, save_best_only=True)
 hist = model.fit(X_train, Y_train, batch_size=128, epochs=10,
validation_split=0.2, callbacks=[checkpointer],
verbose=1, shuffle=True )<load_pretrained>
params = {
'num_leaves': 144,
'learning_rate': 0.1,
'n_estimators': 800,
'max_depth':13,
'max_bin':55,
'bagging_fraction':0.8,
'bagging_freq':5,
'feature_fraction':0.9
}
model.load_weights('mnist.model.best.hdf5' )<compute_test_metric>
def kfold_lightgbm(df, num_folds, stratified = False, debug= False):
train_df = df[df['winPlacePerc_mean'].notnull() ]
test_df = df[df['winPlacePerc_mean'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
drop_features = ["winPlacePerc_mean", "Id", "groupId", "matchId"]
feats = [f for f in train_df.columns if f not in drop_features]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['winPlacePerc_mean'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['winPlacePerc_mean'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['winPlacePerc_mean'].iloc[valid_idx]
reg = LGBMRegressor(num_leaves=params['num_leaves'], learning_rate=params['learning_rate'],
n_estimators=params['n_estimators'], max_depth=params['max_depth'],
max_bin = params['max_bin'], bagging_fraction = params['bagging_fraction'],
bagging_freq = params['bagging_freq'], feature_fraction = params['feature_fraction'] 
)
 reg.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'mae', verbose= 50, early_stopping_rounds= 200)
 oof_preds[valid_idx] = reg.predict(valid_x)
 sub_preds += reg.predict(test_df[feats])/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = reg.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d MAE : %.6f' %(n_fold + 1, mean_absolute_error(valid_y, oof_preds[valid_idx])))
 del reg, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full MAE score %.6f' % mean_absolute_error(train_df['winPlacePerc_mean'], oof_preds))
if not debug:
test_df['winPlacePerc'] = sub_preds
return feature_importance_df, test_df[['winPlacePerc']]
score = model.evaluate(X_valid, Y_valid, verbose=0)
 accuracy = 100 * score[1]
print(f'Test Accuracy {accuracy:.4f}' )<save_to_csv>
feat_importances, test_df = kfold_lightgbm(groups, num_folds=5, stratified=False, debug=False )
def submit_result(model, X_test):
results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("mnist_datagen.csv",index=False)
<choose_model_class>
display_importances(feat_importances )
cnn_model = Sequential()
cnn_model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same',
activation='relu', input_shape=X_train.shape[1:]))
cnn_model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same',
activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.2))
cnn_model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same',
activation='relu'))
cnn_model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same',
activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.2))
cnn_model.add(Flatten())
 cnn_model.add(Dense(512, activation='relu'))
cnn_model.add(Dropout(0.2))
cnn_model.add(Dense(512, activation='relu'))
cnn_model.add(Dropout(0.2))
cnn_model.add(Dense(10, activation='softmax'))
cnn_model.summary()<choose_model_class>
feat_gb = feat_importances.groupby('feature' ).mean().sort_values(by="importance", ascending=False )
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 cnn_model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy'] )<train_model>
feat_gb.to_csv("feature_importance.csv" )
checkpointer = ModelCheckpoint(filepath='mnist.model.best.cnn.hdf5',
verbose=1, save_best_only=True)
 hist = cnn_model.fit(X_train, Y_train, batch_size=128, epochs=10,
validation_split=0.2, callbacks=[checkpointer],
verbose=1, shuffle=True )<load_pretrained>
test = pd.read_csv(".. /input/test.csv" )
cnn_model.load_weights('mnist.model.best.cnn.hdf5' )<compute_test_metric>
test = test.merge(test_df, right_index=True, left_on='groupId' )
<split><EOS> 
test[['Id', 'winPlacePerc']].to_csv("submission.csv", index=False )