Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
<SOS> metric: MAE Kaggle data source: pubg-finish-placement-prediction-(kernels-only)<import_modules>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from catboost import CatBoostRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.preprocessing import StandardScaler,MinMaxScaler
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, LSTM, Bidirectional
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization<categorify>
train=pd.read_csv('.. /input/train_V2.csv')
 test=pd.read_csv('.. /input/test_V2.csv')
 ID=test['Id']
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ['[CLS]'] + text + ['[SEP]']
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
train=train.dropna(axis=0 )
def build_model(bert_layer, max_len=512):
def inner_build_model() :
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name='input_word_ids')
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name='input_mask')
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name='segment_ids')
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy', 'mse'])
 return model
return inner_build_model<choose_model_class>
y_train=train['winPlacePerc']
train=train.drop(['winPlacePerc'],axis=1 )
%%time
module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1'
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
train["players_Match"] = train.groupby("matchId")["Id"].transform("count")
 train["players_Group"] = train.groupby("groupId")["Id"].transform("count")
 test["players_Match"] = test.groupby("matchId")["Id"].transform("count")
 test["players_Group"] = test.groupby("groupId")["Id"].transform("count" )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<load_from_csv>
train['Total_Kills'] = train.groupby('groupId')['kills'].transform('sum')
 test['Total_Kills'] = test.groupby('groupId')['kills'].transform('sum' )
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )<feature_engineering>
train['First_Man'] = train.groupby('groupId')['matchDuration'].transform('min')
 test['First_Man'] = test.groupby('groupId')['matchDuration'].transform('min' )
train['token_len'] = train['text'].apply(lambda x: len(tokenizer.tokenize(x)))
 test['token_len'] = test['text'].apply(lambda x: len(tokenizer.tokenize(x)) )<define_variables>
train['Last_Man'] = train.groupby('groupId')['matchDuration'].transform('max')
 test['Last_Man'] = test.groupby('groupId')['matchDuration'].transform('max' )
token_max_len = max(train['token_len'].max() , test['token_len'].max())+ 2
display(token_max_len )<split>
train['Time_Survival'] = train['Last_Man'] - train['First_Man']
test['Time_Survival'] = test['Last_Man'] - test['First_Man']
def get_kfold_sets(train, k=5):
kf = KFold(n_splits=k, shuffle=True)
 for train_texts, train_labels in kf.split(train.text.values, train.target.values):
train_texts, valid_texts, train_labels, valid_labels = train_test_split(train.text.values, train.target.values, test_size=0.2)
 train_input = bert_encode(train_texts, tokenizer, max_len=token_max_len)
 valid_input = bert_encode(valid_texts, tokenizer, max_len=token_max_len)
 yield train_input, train_labels, valid_input, valid_labels
def get_train_sets(train):
train_input = bert_encode(train.text.values, tokenizer, max_len=token_max_len)
 train_labels = train.target.values
return train_input, train_labels<categorify>
train['Kill_Percentile'] = train['killPlace'] /(train['maxPlace'] + 1e-9)
 test['Kill_Percentile'] = test['killPlace'] /(test['maxPlace'] + 1e-9 )
test_input = bert_encode(test.text.values, tokenizer, max_len=token_max_len )<predict_on_test>
train.drop(["matchId","groupId",'Id','killPoints', 'maxPlace', 'winPoints','vehicleDestroys'],axis=1,inplace=True)
 test.drop(["matchId","groupId",'Id','killPoints', 'maxPlace', 'winPoints','vehicleDestroys'],axis=1,inplace=True )
class F1Callback(Callback):
def __init__(self, model, X_val, y_val):
self.model = model
self.X_val = X_val
self.y_val = y_val
def on_epoch_end(self, epoch, logs):
pred = self.model.predict(self.X_val)
 f1_val = f1_score(self.y_val, np.round(pred))
print('
f1_val = ', f1_val )<find_best_model_class>
train['Headshot_rate'] = train['kills'] /(train['headshotKills'] + 1e-9)
 test['Headshot_rate'] = test['kills'] /(test['headshotKills'] + 1e-9)
 train['KillStreak_rate'] = train['killStreaks'] /(train['kills'] + 1e-9)
 test['KillStreak_rate'] = test['killStreaks'] /(test['kills'] + 1e-9 )
def cross_val_score(train, k=3, epochs=10, batch_size=16):
f1_vals = []
models = []
i = 0
for train_input, train_labels, valid_input, valid_labels in get_kfold_sets(train, k=k):
model = clone_model(model_template)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy', 'mse'])
 train_history = model.fit(
 train_input, train_labels,
validation_data=(valid_input, valid_labels),
epochs=epochs,
batch_size=batch_size,
callbacks=[EarlyStopping(patience=1, monitor='val_mse', mode='min', verbose=True)] 
)
 pred = model.predict(valid_input)
 f1_val = f1_score(valid_labels, np.round(pred))
print(f'f1-val: {f1_val}')
 f1_vals.append(f1_val)
 models.append(model)
 df = pd.DataFrame(train_history.history)
 df['f1-val'] = f1_val
df.to_csv(f'history_{i}.csv')
 i += 1
return np.array(f1_vals ).mean() , models
k = 5
f1_val, models = cross_val_score(train, k=k)
 print(f'f1-mean: {f1_val}' )<define_variables>
train['Total_Damage'] = train['damageDealt'] + train['teamKills']*100
test['Total_Damage'] = test['damageDealt'] + test['teamKills']*100
train_input, train_labels = get_train_sets(train )<find_best_params>
train['New']=(train['matchDuration'] < train['matchDuration'].mean())
 test['New']=(test['matchDuration'] < test['matchDuration'].mean() )
def calc_best_threshold(pred, labels):
f1_vals = []
ts = []
for t in np.arange(0.1, 1, 0.1):
f1_val = f1_score(train_labels, [1 if p >= t else 0 for p in train_pred])
 f1_vals.append(f1_val)
 ts.append(t)
 return ts[np.argmax(f1_vals)]
best_ts = []
for model in models:
train_pred = model.predict(train_input)
 tmp = calc_best_threshold(train_pred, train_labels)
 best_ts.append(tmp)
 print(f'best ts: {best_ts}' )<predict_on_test>
train['ProKiller']=(train['headshotKills']/train['kills']+1e-9)
 test['ProKiller']=(test['headshotKills']/test['kills']+1e-9 )
test_preds = []
for model in models:
test_pred = model.predict(test_input)
 test_preds.append(test_pred)
 test_preds = np.array(test_preds)
 print(test_preds.shape )<define_variables>
train['Is_Sniper']=(train['longestKill']>=250)
 test['Is_Sniper']=(test['longestKill']>=250 )
test_size = test_preds.shape[1]
mean_pred = []
for s in range(test_size):
tmp = []
for i in range(k):
tmp.append(1 if test_preds[i][s][0] >= best_ts[i] else 0)
 mean = np.mean(tmp)
 mean_pred.append(mean)
 mean_pred = np.array(mean_pred)
 print(mean_pred.shape)
 print(mean_pred[20:])
 print(mean_pred[:20] )<save_to_csv>
train['killsOverWalkDistance'] = train['kills'] /(train['walkDistance'] + 1e-9)
 test['killsOverWalkDistance'] = test['kills'] /(test['walkDistance'] + 1e-9 )
submission['target'] = mean_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )<define_variables>
train['Total_Distance'] =(train['rideDistance']+train['swimDistance']+train['walkDistance'])
 test['Total_Distance'] =(test['rideDistance']+test['swimDistance']+test['walkDistance'] )
root = Path('.. /input')
 train_path = Path('train')
 rseed = 7
val_size = 0.05<normalization>
train['killsOverDistance'] = train['kills'] /(train['distance'] + 1e-9)
 test['killsOverDistance'] = test['kills'] /(test['distance'] + 1e-9 )
def save_imgs(path:Path, data, labels):
path.mkdir(parents=True,exist_ok=True)
 for label in np.unique(labels): 
(path/str(label)).mkdir(parents=True,exist_ok=True)
 for i in range(len(data)) :
if(len(labels)!=0):
imageio.imsave(str(path/str(labels[i])/(str(i)+'.jpg')) , data[i])
 else:
imageio.imsave(str(path/(str(i)+'.jpg')) , data[i] )<load_from_csv>
train['Total_enemies'] = train['playersInMatch'] - train['playersInGroup']
test['Total_enemies'] = test['playersInMatch'] - test['playersInGroup']
train_csv = pd.read_csv(root/'train.csv' )<load_from_csv>
train['Team_Spirit'] = train['heals'] + train['revives'] + train['boosts']
test['Team_Spirit'] = test['heals'] + test['revives'] + test['boosts']
test_csv = pd.read_csv(root/'test.csv' )<prepare_x_and_y>
set1=set(i for i in train[(train['kills']>40)&(train['heals']==0)].index.tolist())
 set2=set(i for i in train[(train['distance']==0)&(train['kills']>20)].index.tolist())
 set3=set(i for i in train[(train['damageDealt']>4000)&(train['heals']<2)].index.tolist())
 set4=set(i for i in train[(train['rideDistance']>25000)].index.tolist())
 set5=set(i for i in train[(train['killStreaks']>3)&(train['weaponsAcquired']> 30)].index.tolist())
 sets=set1 | set2 | set3 | set4 | set5
data_X, data_y = train_csv.loc[:,'pixel0':'pixel783'], train_csv['label']<split>
train=train.drop(list(sets))
y_train=y_train.drop(list(sets))
train_X, val_X, train_y, val_y = train_test_split(data_X, data_y, test_size=val_size,random_state=rseed,stratify=data_y )<train_model>
fpp=['crashfpp','duo-fpp','flare-fpp','normal-duo-fpp','normal-solo-fpp','normal-squad-fpp','solo-fpp','squad-fpp']
train["fpp"] = np.where(train["matchType"].isin(fpp),1,0)
 test["fpp"] = np.where(test["matchType"].isin(fpp),1,0 )
def to_img_shape(data_X, data_y=[]):
data_X = np.array(data_X ).reshape(-1,28,28)
 data_X = np.stack(( data_X,)*3, axis=-1)
 data_y = np.array(data_y)
 return data_X,data_y<train_model>
change={'crashfpp':'crash',
'crashtpp':'crash',
'duo':'duo',
'duo-fpp':'duo',
'flarefpp':'flare',
'flaretpp':'flare',
'normal-duo':'duo',
'normal-duo-fpp':'duo',
'normal-solo':'solo',
'normal-solo-fpp':'solo',
'normal-squad':'squad',
'normal-squad-fpp':'squad',
'solo-fpp':'solo',
'squad-fpp':'squad',
'solo':'solo',
'squad':'squad'
}
train['matchType']=train['matchType'].map(change)
 test['matchType']=test['matchType'].map(change )
train_X,train_y = to_img_shape(data_X, data_y )<train_model>
modes={'crash':1,
'duo':2,
'flare':3,
'solo':4,
'squad':5
}
train['matchType']=train['matchType'].map(modes)
 test['matchType']=test['matchType'].map(modes )
val_X,val_y = to_img_shape(val_X,val_y )<load_pretrained>
d1=pd.get_dummies(train['matchType'])
 train=train.drop(['matchType'],axis=1)
 train=train.join(d1)
 d2=pd.get_dummies(test['matchType'])
 test=test.drop(['matchType'],axis=1)
 test=test.join(d2 )
save_imgs(Path('/data/train'),train_X,train_y )<train_model>
scaler = MinMaxScaler()
scaler.fit(train)
 train=scaler.transform(train)
 test=scaler.transform(test )
save_imgs(Path('/data/valid'),val_X,val_y )<load_pretrained>
X_train,X_test,y_train,y_test= train_test_split(train,y_train,test_size=0.3 )
data = ImageDataBunch.from_folder('/data/',bs=256,size=28,ds_tfms=get_transforms(do_flip=False),num_workers=0 ).normalize(imagenet_stats )<define_variables>
train_pool = Pool(X_train, y_train)
 test_pool = Pool(X_test, y_test )
data.show_batch(3,figsize=(6,6))<choose_model_class>
model = CatBoostRegressor(
 iterations=5000,
depth=10,
learning_rate=0.1,
l2_leaf_reg= 2,
loss_function='RMSE',
eval_metric='MAE',
random_strength=0.1,
bootstrap_type='Bernoulli',
leaf_estimation_method='Gradient',
leaf_estimation_iterations=1,
boosting_type='Plain'
,task_type = "GPU"
,feature_border_type='GreedyLogSum'
,random_seed=1234
)
learn = cnn_learner(data,models.resnet18,metrics=accuracy,path='.')
 learn.lr_find()
learn.recorder.plot()<train_model>
model.fit(train_pool, eval_set=test_pool, plot=True )
learn.fit_one_cycle(1,1e-02 )<save_model>
train_mse =(mean_absolute_error(y_train, model.predict(X_train)))
 test_mse =(mean_absolute_error(y_test, model.predict(X_test)))
 print('Train error= ',train_mse)
 print('Test error= ',test_mse)
learn.save('s1' )<load_pretrained>
subm = pd.read_csv('.. /input/sample_submission_V2.csv')
 predictions = model.predict(test)
 test = pd.read_csv('.. /input/test_V2.csv')
 test['winPlacePerc'] = predictions
test['winPlacePerc'] = test.groupby('groupId')['winPlacePerc'].transform('median')
 subm['winPlacePerc'] = test['winPlacePerc']
subm['Id']=ID
subm.to_csv('submission.csv', index = False )