Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
<SOS> metric: MAE Kaggle data source: pubg-finish-placement-prediction-(kernels-only)<categorify>
def memory_reduce(df):
start_memory = df.memory_usage().sum() /1024**2
for col in df.columns:
col_type = df[col].dtype
if(col_type != object):
min_val = min(df[col])
 max_val = max(df[col])
 if(str(col_type)[:3] == 'int'):
if(min_val > np.iinfo(np.int8 ).min and max_val < np.iinfo(np.int8 ).max):
df[col] = df[col].astype(np.int8)
 elif(min_val > np.iinfo(np.int16 ).min and max_val < np.iinfo(np.int16 ).max):
df[col] = df[col].astype(np.int16)
 elif(min_val > np.iinfo(np.int32 ).min and max_val < np.iinfo(np.int32 ).max):
df[col] = df[col].astype(np.int32)
 elif(min_val > np.iinfo(np.int64 ).min and max_val < np.iinfo(np.int64 ).max):
df[col] = df[col].astype(np.int64)
 else:
if(min_val > np.finfo(np.float16 ).min and max_val < np.finfo(np.float16 ).max):
df[col] = df[col].astype(np.float16)
 elif(min_val > np.finfo(np.float32 ).min and max_val < np.finfo(np.float32 ).max):
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_memory = df.memory_usage().sum() /1024**2
return df
all_data = all_data.replace({
"BsmtCond": {"NoBa": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4,
"Ex": 5},
"BsmtExposure": {"NoBa": 0, "Mn": 1, "Av": 2, "Gd": 3},
"BsmtFinType1" : {"NoBa" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"NoBa" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"NoBa" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4,
"Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"Functional" : {"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5,
"Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"NoGa" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"NoGa" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PoolQC" : {"NoPo" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}
} )<categorify>
print("NAs for numerical features in train : " + str(all_data_num.isnull().values.sum()))
all_data_num = all_data_num.fillna(all_data_num.mean())
 print("Remaining NAs for numerical features in train : " + str(all_data_num.isnull().values.sum()))<feature_engineering>
startTime = time.time()
def feature_engineering(is_train = True , debug = True):
test_Idx = None
if(is_train):
print('processing train data')
 if(debug):
df = memory_reduce(pd.read_csv('.. /input/train_V2.csv' , nrows=10000))
else:
df = memory_reduce(pd.read_csv('.. /input/train_V2.csv'))
df = df[pd.notnull(df['winPlacePerc'])]
else:
print('processing test data')
 if(debug):
df = memory_reduce(pd.read_csv('.. /input/test_V2.csv' , nrows = 10000))
else:
df = memory_reduce(pd.read_csv('.. /input/test_V2.csv'))
test_Idx = df.Id
print('remove many feature')
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove('Id')
 features.remove('groupId')
 features.remove('matchId')
 features.remove('matchType')
 y = None
if(is_train):
print('get target')
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print('get group mean featuers')
 agg = df.groupby(['matchId' , 'groupId'])[features].agg('mean')
 agg_rank = agg.groupby(['matchId'])[features].rank(pct = True ).reset_index()
if(is_train):
df_out = agg.reset_index() [['matchId','groupId']]
else:
df_out = df[['matchId' , 'groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes = ['',''] , how = 'left' , on = ['matchId' , 'groupId'])
 df_out = df_out.merge(agg_rank , suffixes = ['_mean','_mean_rank'] , how = 'left' , on = ['matchId' , 'groupId'])
 print('get group max features')
 agg = df.groupby(['matchId' , 'groupId'])[features].agg('max')
 agg_rank = agg.groupby(['matchId'])[features].rank(pct = True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes = ['',''] , how = 'left' , on = ['matchId' , 'groupId'])
 df_out = df_out.merge(agg_rank , suffixes = ['_max','_max_rank'] , how = 'left' , on = ['matchId' , 'groupId'])
 print('get group min features')
 agg = df.groupby(['matchId' , 'groupId'])[features].agg('min')
 agg_rank = agg.groupby(['matchId'])[features].rank(pct = True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes = ['',''] , how = 'left' , on = ['matchId' , 'groupId'])
 df_out = df_out.merge(agg_rank , suffixes = ['_min','_min_rank'] , how = 'left' , on = ['matchId' , 'groupId'])
 print('get group size feature')
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name = 'group_size')
 df_out = df_out.merge(agg , how = 'left' , on = ['matchId','groupId'])
 print('get group matchId mean')
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg , suffixes = ['','_match_mean'], how = 'left' , on = 'matchId')
 print('get match size feature')
 agg = df.groupby(['matchId'] ).size().reset_index(name = 'matchSize')
 df_out = df_out.merge(agg , on = 'matchId' , how = 'left')
 df_out.drop(['matchId' , 'groupId'] , axis = 1 ,inplace = True)
 X = df_out
columnsName = list(df_out.columns)
 del df , df_out , agg , agg_rank
gc.collect()
return X , y , columnsName , test_Idx
x_train , y_train , train_columns , _ = feature_engineering(True , False)
 x_test , _ , _ , test_Idx = feature_engineering(False , False)
 print('This time is {:.2f}'.format(time.time() - startTime))
skewness = all_data_num.apply(lambda x: skew(x))
skewness = skewness[abs(skewness)> 0.5]
print(str(skewness.shape[0])+ " skewed numerical features to log transform")
 skewed_features = skewness.index
all_data_num[skewed_features] = np.log1p(all_data_num[skewed_features] )<categorify>
x_train['totalDistance'] = x_train['swimDistance'] + x_train['rideDistance'] + x_train['walkDistance']
x_test['totalDistance'] = x_test['swimDistance'] + x_test['rideDistance'] + x_test['walkDistance']
x_train['healthItem'] = x_train['heals'] + x_train['boosts']
x_test['healthItem'] = x_test['heals'] + x_test['boosts']
x_train['headshotRate'] = x_train['kills'] / x_train['headshotKills']
x_test['headshotRate'] = x_test['kills'] / x_test['headshotKills']
x_train['killStreakRate'] = x_train['killStreaks']/x_train['kills']
x_test['killStreakRate'] = x_test['killStreaks']/x_test['kills']
x_train['killMinute'] = x_train['kills'] / x_train['matchDuration']
x_test['killMinute'] = x_test['kills'] / x_test['matchDuration']
x_train['damageDealtMinute'] = x_train['damageDealt'] / x_train['matchDuration']
x_test['damageDealtMinute'] = x_test['damageDealt'] / x_test['matchDuration']
x_train['participateKills'] = x_train['kills'] + x_train['assists'] + x_train['DBNOs']
x_test['participateKills'] = x_test['kills'] + x_test['assists'] + x_test['DBNOs']
x_train['vehicleDestroysMinute'] = x_train['vehicleDestroys'] / x_train['matchDuration']
x_test['vehicleDestroysMinute'] = x_test['vehicleDestroys'] / x_test['matchDuration']
x_train['killsMiter'] = x_train['roadKills'] / x_train['rideDistance']
x_test['killsMiter'] = x_test['roadKills'] / x_test['rideDistance']
del x_train['heals']
del x_test['heals']
train_columns.append('totalDistance')
 train_columns.append('healthItems')
 train_columns.append('headshotRate')
 train_columns.append('killStreakRate')
 train_columns.append('killMinute')
 train_columns.append('damageDealtMinute')
 train_columns.append('participateKills')
 train_columns.append('vehicleDestroysMinute')
 train_columns.append('killsMiter')
 train_columns.remove('heals')
 print(x_train.shape)
 print('This time is {:.2f}'.format(time.time() - startTime))
print("NAs for categorical features in train : " + str(all_data_cat.isnull().values.sum()))
all_data_cat = pd.get_dummies(all_data_cat)
 print("Remaining NAs for categorical features in train : " + str(all_data_cat.isnull().values.sum()))<categorify>
startTime = time.time()
x_train = memory_reduce(x_train)
 x_test = memory_reduce(x_test)
 print(time.time() - startTime )
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<split>
train = all_data[:ntrain]
test = all_data[ntrain:]<split>
warnings.filterwarnings('ignore')
 startTime = time.time()
print(startTime)
 train_index = round(int(x_train.shape[0]*0.8))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
gc.collect() ;
def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':20000, 'early_stopping_rounds':200,
"num_leaves" : 31, "learning_rate" : 0.05, "bagging_fraction" : 0.7,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.7
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y, model
pred_test, model = run_lgb(dev_X, dev_y, val_X, val_y, x_test)
 print(time.time() - startTime )
all_data = pd.concat([all_data_num, all_data_cat], axis=1)
 print("New number of features : " + str(all_data.shape[1]))
train = all_data[:ntrain]
test = all_data[ntrain:]
print(len(train))
print(len(y_train))<compute_train_metric>
startTime = time.time()
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission.csv", index=False)
 print(time.time() - startTime )
scorer = make_scorer(mean_squared_error, greater_is_better=False)
 def rmse_cv_train(model):
rmse = np.sqrt(-cross_val_score(model, train.values, y_train, scoring=scorer, cv=10))
return(rmse )<define_search_space>
l1 = [0.00001, 0.00003, 0.00006, 0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6, 10, 30, 60, 100]
l2 = np.arange(100)+ 1
l3 = l2 * 0.1
l4 = l2 * 0.01
l5 = l2 * 0.001
l6 = l2 * 0.0001
l7 = l2 * 0.00001
l8 = l2 * 0.000001
print(l3 )<define_variables>
l2_1 = np.arange(1000)+ 1
l3_1 = l2_1 * 0.1
l4_1 = l2_1 * 0.01 
<train_on_grid>
ridge = RidgeCV(alphas = [5.63])
 ridge.fit(train.values, y_train)
 alpha = ridge.alpha_
print("Best alpha: ", alpha)
 print("Ridge RMSE on Training set :", rmse_cv_train(ridge ).mean() )<train_on_grid>
lasso = LassoCV(alphas = [0.00031], max_iter=50000, cv=10)
 lasso.fit(train.values, y_train)
 print("Lasso RMSE on Training set: ", rmse_cv_train(lasso ).mean() )<train_on_grid>
elasticNet = ElasticNetCV(l1_ratio = [0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
 elasticNet.fit(train.values, y_train)
 alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("Try again for more precision with l1_ratio centered around " + str(ratio))
elasticNet = ElasticNetCV(l1_ratio = [ratio *.85, ratio *.9, ratio *.95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
 elasticNet.fit(train.values, y_train)
 if(elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio)+
" and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio = ratio,
alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8, alpha *.85, alpha *.9,
alpha *.95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3,
alpha * 1.35, alpha * 1.4],
max_iter = 50000, cv = 10)
 elasticNet.fit(train.values, y_train)
 if(elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("ElasticNet RMSE on Training set :", rmse_cv_train(elasticNet ).mean())
 y_train_ela = elasticNet.predict(train.values )<train_on_grid>
KRR = KernelRidge(alpha = [600], kernel="polynomial", degree=1.94, coef0=40)
 KRR.fit(train.values, y_train)
 print("Kernel Ridge Regression RMSE on Training set :", rmse_cv_train(KRR ).mean() )<train_on_grid>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 GBoost.fit(train.values, y_train)
 print("XGB RMSE on Training set: ", rmse_cv_train(GBoost ).mean() )<train_model>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([model.predict(X)for model in self.models_])
 return np.mean(predictions, axis=1 )<train_on_grid>
averaged_models = AveragingModels(models =(lasso, elasticNet, KRR, GBoost))
print("Averaged Models RMSE on Training set :", rmse_cv_train(averaged_models ).mean())
<compute_test_metric>
def rmse(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
averaged_models.fit(train.values, y_train)
 averaged_train_pred = averaged_models.predict(train.values)
 averaged_pred = np.expm1(averaged_models.predict(test.values))
print(rmse(y_train, averaged_train_pred))<save_to_csv>
sub = pd.DataFrame()
sub["Id"] = test_ID
sub["SalePrice"] = averaged_pred
sub.to_csv("submission2.csv", index=False )<define_variables>
print(keras.__version__)
 print(tensorflow.__version__)
 EMBEDDING_FILES = [
'.. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim',
'.. /input/gensim-embeddings-dataset/glove.840B.300d.gensim'
]
N_ROWS = None
NUM_MODELS = 3
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
LOCAL_EPOCHS = 2
MAX_LEN = 220
AUX_COLUMNS = ['target']
TEXT_COLUMN = 'text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<choose_model_class>
def build_matrix(word_index, path):
embedding_index = KeyedVectors.load(path, mmap='r')
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
for candidate in [word, word.lower() ]:
if candidate in embedding_index:
embedding_matrix[i] = embedding_index[candidate]
break
return embedding_matrix
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<prepare_x_and_y>
print("Read Data")
 train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv', nrows = N_ROWS)
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv', nrows = N_ROWS)
 x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str)
 tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE, lower=False)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 checkpoint_predictions = []
weights = []
model = build_model(embedding_matrix, y_aux_train.shape[-1] )<train_model>
print("Start Modeling")
 for model_idx in range(1, NUM_MODELS + 1):
print("Start Model: {}".format(str(model_idx)))
 model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=LOCAL_EPOCHS,
verbose=1 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 print("Modeling Complete" )<save_to_csv>
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
TARGET_COLUMN:(predictions >.5 ).astype(int)
 })
 submission.to_csv('lstm_submission.csv', index=False)
 submission.head()<set_options>
np.set_printoptions(suppress=True)
 sys.path.append('./transformers')
 sys.path.append('./sacremoses')
 tf.keras.backend.clear_session()
print(tf.__version__ )<define_variables>
<load_from_csv><EOS>