Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))<categorify>
training_df = pd.read_csv(".. /input/train_V2.csv")
 testing_df = pd.read_csv(".. /input/test_V2.csv" )
all_data = pd.get_dummies(all_data)
 all_data.shape<split>
training_df = training_df.drop(['Id', 'longestKill', 'rankPoints', 'numGroups', 'matchType'], axis=1)
 testing_df = testing_df.drop(['Id', 'longestKill', 'rankPoints', 'numGroups', 'matchType'], axis=1 )
train = all_data[:ntrain]
test = all_data[ntrain:]<compute_train_metric>
training_df['kills'].value_counts().tail(10 )
X_train = train
X_test = test
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 10))
return(rmse )<find_best_params>
training_df = training_df.drop(training_df[training_df.kills > 20].index)
 training_df['kills'].value_counts()
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean() for alpha in alphas]<train_model>
training_df = training_df.drop(training_df[training_df.DBNOs > 15].index)
 training_df['DBNOs'].value_counts()
model_ridge = Ridge(alpha = 5 )<choose_model_class>
training_df = training_df.drop(training_df[training_df.weaponsAcquired > 30].index)
 training_df['weaponsAcquired'].value_counts()
model_lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0005))<choose_model_class>
training_df.isnull().sum()
training_df = training_df.dropna(how='any',axis=0)
 training_df.winPlacePerc.isnull().sum()
model_elastic = make_pipeline(RobustScaler() , ElasticNet(alpha = 0.0005))<choose_model_class>
features = list(training_df.columns)
 features.remove("matchId")
 features.remove("groupId")
 features.remove("winPlacePerc")
 print("get match mean feature")
 match_mean = training_df.groupby(['matchId'])[features].agg('mean' ).reset_index()
training_df = training_df.merge(match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 match_mean = testing_df.groupby(['matchId'])[features].agg('mean' ).reset_index()
testing_df = testing_df.merge(match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 match_size = training_df.groupby(['matchId'] ).size().reset_index(name='match_size')
 training_df = training_df.merge(match_size, how='left', on=['matchId'])
 match_size = testing_df.groupby(['matchId'] ).size().reset_index(name='match_size')
 testing_df = testing_df.merge(match_size, how='left', on=['matchId'])
 del match_mean, match_size
gc.collect()
model_krr = make_pipeline(RobustScaler() , KernelRidge(alpha=6, kernel='polynomial', degree=2.65, coef0=6.9))<choose_model_class>
print("get group size feature")
 group_size = training_df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 training_df = training_df.merge(group_size, how='left', on=['matchId', 'groupId'])
 group_size = testing_df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 testing_df = testing_df.merge(group_size, how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 group_max = training_df.groupby(['matchId','groupId'])[features].agg('max')
 group_max_rank = group_max.groupby('matchId')[features].rank(pct=True ).reset_index()
training_df = training_df.merge(group_max.reset_index() , suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 training_df = training_df.merge(group_max_rank, suffixes=["", "_max_rank"], how='left', on=['matchId', 'groupId'])
 group_max = testing_df.groupby(['matchId','groupId'])[features].agg('max')
 group_max_rank = group_max.groupby('matchId')[features].rank(pct=True ).reset_index()
testing_df = testing_df.merge(group_max.reset_index() , suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 testing_df = testing_df.merge(group_max_rank, suffixes=["", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 group_min = training_df.groupby(['matchId','groupId'])[features].agg('min')
 group_min_rank = group_min.groupby('matchId')[features].rank(pct=True ).reset_index()
training_df = training_df.merge(group_min.reset_index() , suffixes=["", "_min"], how='left', on=['matchId', 'groupId'])
 training_df = training_df.merge(group_min_rank, suffixes=["", "_min_rank"], how='left', on=['matchId', 'groupId'])
 group_min = testing_df.groupby(['matchId','groupId'])[features].agg('min')
 group_min_rank = group_min.groupby('matchId')[features].rank(pct=True ).reset_index()
testing_df = testing_df.merge(group_min.reset_index() , suffixes=["", "_min"], how='left', on=['matchId', 'groupId'])
 testing_df = testing_df.merge(group_min_rank, suffixes=["", "_min_rank"], how='left', on=['matchId', 'groupId'])
 del group_size, group_max, group_max_rank, group_min, group_min_rank
gc.collect()
model_gbr = GradientBoostingRegressor(n_estimators=1200,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=5 )<choose_model_class>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.2,
learning_rate=0.06,
max_depth=3,
n_estimators=1150 )<choose_model_class>
training_df.drop(["matchId", "groupId"], axis=1, inplace=True)
 testing_df.drop(["matchId", "groupId"], axis=1, inplace=True )
model_lgb = lgb.LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.05,
n_estimators=1080,
max_bin=75,
bagging_fraction=0.80,
bagging_freq=5,
feature_fraction=0.232,
feature_fraction_seed=9,
bagging_seed=9,
min_data_in_leaf=6,
min_sum_hessian_in_leaf=11 )<choose_model_class>
training_df = reduce_mem_usage(training_df)
 print('_'*40)
 testing_df = reduce_mem_usage(testing_df )
model_cat = CatBoostRegressor(iterations=2000,
learning_rate=0.10,
depth=3,
l2_leaf_reg=4,
border_count=15,
loss_function='RMSE',
verbose=200)
 params = {'iterations':2000,
'learning_rate':0.10,
'depth':3,
'l2_leaf_reg':4,
'border_count':15,
'loss_function':'RMSE',
'verbose':200}<find_best_params>
training_df['headshotRate'] = training_df['headshotKills'] / training_df['kills']
training_df['headshotRate'].fillna(0, inplace=True)
 training_df['headshotRate'].replace(np.inf, 0, inplace=True)
 testing_df['headshotRate'] = testing_df['headshotKills'] / training_df['kills']
testing_df['headshotRate'].fillna(0, inplace=True)
 testing_df['headshotRate'].replace(np.inf, 0, inplace=True)
 training_df['totalDistance'] = training_df['rideDistance'] + training_df['swimDistance'] + training_df['walkDistance']
testing_df['totalDistance'] = testing_df['rideDistance'] + testing_df['swimDistance'] + testing_df['walkDistance']
training_df['items'] = training_df['heals'] + training_df['boosts']
testing_df['items'] = testing_df['heals'] + testing_df['boosts']
training_df['healsPerWalkDistance'] = training_df['heals'] / training_df['walkDistance']
training_df['healsPerWalkDistance'].fillna(0, inplace=True)
 training_df['healsPerWalkDistance'].replace(np.inf, 0, inplace=True)
 testing_df['healsPerWalkDistance'] = testing_df['heals'] / testing_df['walkDistance']
testing_df['healsPerWalkDistance'].fillna(0, inplace=True)
 testing_df['healsPerWalkDistance'].replace(np.inf, 0, inplace=True)
 training_df['killsPerWalkDistance'] = training_df['kills'] / training_df['walkDistance']
training_df['killsPerWalkDistance'].fillna(0, inplace=True)
 training_df['killsPerWalkDistance'].replace(np.inf, 0, inplace=True)
 testing_df['killsPerWalkDistance'] = testing_df['kills'] / testing_df['walkDistance']
testing_df['killsPerWalkDistance'].fillna(0, inplace=True)
 testing_df['killsPerWalkDistance'].replace(np.inf, 0, inplace=True)
 training_df.head()
cv_ridge = rmse_cv(model_ridge ).mean()
cv_lasso = rmse_cv(model_lasso ).mean()
cv_elastic = rmse_cv(model_elastic ).mean()
cv_krr = rmse_cv(model_krr ).mean()
cv_gbr = rmse_cv(model_gbr ).mean()
cv_xgb = rmse_cv(model_xgb ).mean()
cv_lgb = rmse_cv(model_lgb ).mean()<train_on_grid>
feature_MI = [
'assists',
'boosts',
'damageDealt',
'DBNOs',
'headshotKills',
'heals',
'killPlace',
'killPoints',
'kills',
'killStreaks',
'matchDuration',
'maxPlace',
'revives',
'rideDistance',
'roadKills',
'swimDistance',
'teamKills',
'vehicleDestroys',
'walkDistance',
'weaponsAcquired',
'winPoints',
'winPlacePerc',
'headshotRate',
'totalDistance',
'items',
'healsPerWalkDistance',
'killsPerWalkDistance'
]
mutual_info_df = training_df.truncate(after=-1)
pool = Pool(X_train, y_train)
 cv_cat = cv(pool=pool, params=params, fold_count=10, shuffle=True )<create_dataframe>
results = pd.DataFrame({
'Model': ['Ridge',
'Lasso',
'ElasticNet',
'Kernel Ridge',
'Gradient Boosting Regressor',
'XGBoost Regressor',
'Light Gradient Boosting Regressor',
'CatBoost'],
'Score': [cv_ridge,
cv_lasso,
cv_elastic,
cv_krr,
cv_gbr,
cv_xgb,
cv_lgb,
cv_cat]})
 result_df = results.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 result_df.head(8 )<train_model>
cross_validation_split = model_selection.ShuffleSplit(n_splits = 5, test_size =.3, train_size =.6, random_state = 0)
 lasso_results = pd.DataFrame(data = {'degree': [], 'test_score_mean': [], 'fit_time_mean': []})
 for degree in range(1,6):
X_train_truncated = np.asarray(training_df_truncated[['walkDistance','killPlace', 'damageDealt', 'boosts', 'weaponsAcquired']])
 X_train_truncated = np.float32(X_train_truncated)
 X_train_truncated = PolynomialFeatures(degree, interaction_only=False ).fit_transform(X_train_truncated)
 cross_validation_results = model_selection.cross_validate(Lasso(alpha = 0.00001, max_iter=10000, normalize=True), X_train_truncated, y_train_truncated, cv = cross_validation_split, return_train_score = True)
 lasso_results = lasso_results.append({'degree' : degree,
'test_score_mean' : cross_validation_results['test_score'].mean() ,
'fit_time_mean' : cross_validation_results['fit_time'].mean() }, ignore_index=True)
 print('OK degree ' + str(degree))
sns.pointplot(x = lasso_results.degree, y = lasso_results.test_score_mean)
model_lasso.fit(X_train, y_train)
 lasso_pred = np.expm1(model_lasso.predict(X_test))
model_elastic.fit(X_train, y_train)
 elastic_pred = np.expm1(model_elastic.predict(X_test))
model_ridge.fit(X_train, y_train)
 ridge_pred = np.expm1(model_ridge.predict(X_test))
model_xgb.fit(X_train, y_train)
 xgb_pred = np.expm1(model_xgb.predict(X_test))
model_gbr.fit(X_train, y_train)
 gbr_pred = np.expm1(model_gbr.predict(X_test))
model_lgb.fit(X_train, y_train)
 lgb_pred = np.expm1(model_lgb.predict(X_test))
model_krr.fit(X_train, y_train)
 krr_pred = np.expm1(model_krr.predict(X_test))
model_cat.fit(X_train, y_train)
 cat_pred = np.expm1(model_cat.predict(X_test))<compute_test_metric>
training_df_truncated = training_df.truncate(after=100000)
 X_train_truncated = np.asarray(training_df_truncated.drop(['winPlacePerc'], axis = 1))
X_train_truncated[0:1]
stacked =(lasso_pred + elastic_pred + ridge_pred + xgb_pred + lgb_pred + krr_pred + gbr_pred)/ 7<save_to_csv>
y_train_truncated = np.asarray(training_df_truncated[['winPlacePerc']])
 y_train_truncated[0:5]
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = stacked
sub.to_csv('house_price_predictions.csv',index=False )<load_from_csv>
del training_df
gc.collect()
df = pd.read_csv('.. /input/train.csv')
 df.head(2 )<filter>
X_test = np.asarray(testing_df)
 X_test[0:1]
df[df['Embarked'].isna() ]<create_dataframe>
del testing_df
gc.collect()
titles = df['Name'].str.extract('([A-za-z]+)\.', expand=False)
 titles_df = pd.DataFrame()
titles_df['Title'] = titles
titles_df['Survived'] = df['Survived']<count_values>
print('Train set truncated:', X_train_truncated.shape,y_train_truncated.shape)
 print('Train set:', X_train.shape,y_train.shape)
 print('Test set:', X_test.shape )
titles_df['Title'].value_counts().head(6 )<groupby>
RFR = RandomForestRegressor(criterion = 'mse', oob_score = True, random_state = 1)
 param_grid = {'min_samples_leaf' : [5, 10, 20, 40, 70, 100],
'min_samples_split' : [10, 20, 40, 60, 80, 100],
'max_depth': [10, 20, 30, 40, 50, None],
'n_estimators': [30, 40, 50]}
param_distributions = param_grid,
n_iter = 100,
cv = cross_validation_split, verbose = 5, random_state = 0, n_jobs = -1)
 print(RS.best_score_)
 print(RS.best_params_ )
age_eda = df.groupby(['Survived', 'Age'] ).size().rename('count' ).reset_index()<feature_engineering>
param_grid = {'min_samples_leaf' : [5, 10, 15],
'min_samples_split' : [10, 15, 20],
'max_depth': [30, 35, 40, None],
'n_estimators': [40]}
print(GS.best_score_)
 print(GS.best_params_ )
del age_eda
age_eda = df.copy()
age_eda['Age'].dropna(inplace=True)
 age_eda['Age'] = age_eda['Age'].astype(int)
 age_eda.loc[ age_eda['Age'] <= 11, 'Age'] = 0
age_eda.loc[(age_eda['Age'] > 11)&(age_eda['Age'] <= 18), 'Age'] = 1
age_eda.loc[(age_eda['Age'] > 18)&(age_eda['Age'] <= 22), 'Age'] = 2
age_eda.loc[(age_eda['Age'] > 22)&(age_eda['Age'] <= 27), 'Age'] = 3
age_eda.loc[(age_eda['Age'] > 27)&(age_eda['Age'] <= 32), 'Age'] = 4
age_eda.loc[(age_eda['Age'] > 32)&(age_eda['Age'] <= 40), 'Age'] = 5
age_eda.loc[(age_eda['Age'] > 40), 'Age'] = 6<import_modules>
oob_score = True,
min_samples_leaf = 5,
min_samples_split = 15,
max_depth = 30,
random_state = 1 ).fit(X_train_truncated,y_train_truncated)
 print("%.4f" % best_model.oob_score_)
 print("%.4f" % mean_absolute_error(y_train, y_hat))
importance_df = pd.concat(( pd.DataFrame(training_df_truncated.drop(['winPlacePerc'], axis=1 ).columns, columns = ['variable']),
pd.DataFrame(best_model.feature_importances_, columns = ['importance'])) ,
axis = 1 ).sort_values(by='importance', ascending = False)
 importance_df
from sklearn import linear_model
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train_test = train.append(test, sort=False)
 train_test.reset_index(inplace=True)
 train_test.drop(['index'], inplace=True, axis=1)
 train_test.head(2 )<feature_engineering>
print('Train set truncated:', X_train_truncated.shape,y_train_truncated.shape)
 print('Train set:', X_train.shape,y_train.shape)
 print('Test set:', X_test.shape )
def add_titles(data):
data['Title'] = data['Name'].str.extract('([A-za-z]+)\.', expand=False)
 title_mapping = {"Mr": 'Mr',
"Miss": 'Miss',
"Mrs": 'Mrs',
"Master": 'Master',
"Rev": 'Rev',
"Dr": 'Other',
"Col": 'Other',
"Major": 'Other',
"Mlle": 'Other',
"Countess": 'Other',
"Ms": 'Other',
"Lady": 'Other',
"Jonkheer": 'Other',
"Don": 'Other',
"Dona" : 'Other',
"Mme": 'Other',
"Capt": 'Other',
"Sir": 'Other'}
data['Title'] = data['Title'].map(title_mapping )<define_variables>
models = [
]
model_results = pd.DataFrame(data = {'test_score_mean': [], 'fit_time_mean': [], 'mean_absolute_error': []})
 cross_validation_split = model_selection.ShuffleSplit(n_splits = 4, test_size =.3, train_size =.6, random_state = 0)
 for model in models:
cross_validation_results = model_selection.cross_validate(model, X_train_truncated, y_train_truncated, cv= cross_validation_split,
scoring = 'neg_mean_absolute_error', return_train_score=True)
 y_hat = model.fit(X_train_truncated, y_train_truncated ).predict(X_train)
 model_results = model_results.append({'test_score_mean' : cross_validation_results['test_score'].mean() ,
'fit_time_mean' : cross_validation_results['fit_time'].mean() ,
'mean_absolute_error' : mean_absolute_error(y_train, y_hat)}, ignore_index=True)
 model_results
add_titles(train_test )<data_type_conversions>
LGBM = lgb.LGBMRegressor(learning_rate=0.003, metric = 'mae', n_estimators = 100, n_jobs=-1)
 param_grid = {'boosting_type' : ['gbdt', 'dart', 'goss'],
'max_depth' : [10, 20, 30, -1],
'min_data_in_leaf' : [10, 50, 100, 500, 1000],
'num_leaves' : [50, 100, 200, 500, 1000]
}
n_iter = 50, scoring = 'neg_mean_absolute_error',
cv = cross_validation_split, verbose = 10, random_state = 0, n_jobs = -1)
 print(RS.best_score_)
 print(RS.best_params_ )
def clean_fare(data):
data['Fare'] = data['Fare'].fillna(data['Fare'].dropna().median() )<drop_column>
param_grid = {'boosting_type' : ['goss'],
'max_depth' : [20, 30, 40, -1],
'min_data_in_leaf' : [10, 20, 50, 100],
'num_leaves' : [400, 500, 600]}
print(GS.best_score_)
 print(GS.best_params_ )
clean_fare(train_test )<feature_engineering>
boosting_type = 'gbdt',
max_depth = 30,
min_data_in_leaf = 10,
num_leaves = 200 ).fit(X_train_truncated, y_train_truncated ).predict(X_test)
 testing_df = pd.read_csv(".. /input/test_V2.csv")
 submission = testing_df.copy()
submission['winPlacePerc'] = yhat_test
submission.to_csv('submission.csv', columns=['Id', 'winPlacePerc'], index=False)
 submission[['Id', 'winPlacePerc']].head()