Kaggle competition: Japanese Review Rating Prediction
<SOS> metric: auc Kaggle data source: japanese-review-rating-prediction<feature_engineering>
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
import re
import torch
from sklearn.model_selection import train_test_split
from tqdm import tqdm_notebook
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
from scipy import stats
from sklearn.model_selection import KFold
import os,urllib,glob,tarfile
from transformers import BertJapaneseTokenizer, BertModel, BertConfig
cols = [col for col in all_data.columns if col not in ['Id','matchId','groupId']]
for i, t in all_data.loc[:, cols].dtypes.iteritems() :
if t == object:
all_data[i] = pd.factorize(all_data[i])[0]
all_data = reduce_mem_usage(all_data)
 all_data.head()<prepare_x_and_y>
!pip install mecab-python3==0.996.2 fugashi ipadic
X_train = all_data[all_data['winPlacePerc'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['winPlacePerc'].isnull() ].drop(['winPlacePerc'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
Y_train = X_train.pop('winPlacePerc')
 X_test_grp = X_test[['matchId','groupId']].copy()
train_matchId = X_train['matchId']
X_train.drop(['matchId','groupId'], axis=1, inplace=True)
 X_test.drop(['matchId','groupId'], axis=1, inplace=True)
 print(X_train.shape, X_test.shape )<create_dataframe>
train_df = pd.read_csv(".. /input/japanesereviewratingprediction/yahoo_review_train.csv")
 test_df = pd.read_csv(".. /input/japanesereviewratingprediction/yahoo_review_test.csv")
 sample_submission = pd.read_csv(".. /input/japanesereviewratingprediction/sample_submission.csv" )
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )<train_model>
train_df = train_df[train_df["Label"]!=-1]
params={'learning_rate': 0.1,
'objective':'mae',
'metric':'mae',
'num_leaves': 31,
'verbose': 1,
'random_state':42,
'bagging_fraction': 0.7,
'feature_fraction': 0.7
}
reg = lgb.LGBMRegressor(**params, n_estimators=5000)
 reg.fit(X_train, Y_train)
 pred = reg.predict(X_test, num_iteration=reg.best_iteration_ )<merge>
tokenizer = BertJapaneseTokenizer.from_pretrained("cl-tohoku/bert-base-japanese-whole-word-masking")
 for i,item in enumerate(tokenizer.vocab.items()):
print(item)
 if i > 20:
break
X_test_grp['_nofit.winPlacePerc'] = pred
group = X_test_grp.groupby(['matchId'])
 X_test_grp['winPlacePerc'] = pred
X_test_grp['_rank.winPlacePerc'] = group['winPlacePerc'].rank(method='min')
 X_test = pd.concat([X_test, X_test_grp], axis=1)
 sub_match = X_test_grp[['matchId','_rank.winPlacePerc']].groupby(['matchId'])
 sub_group = group.count().reset_index() ['matchId'].to_frame()
X_test = pd.merge(X_test, sub_group )<feature_engineering>
pad = tokenizer.vocab["[PAD]"]
max_lengths=512
fullgroup =(X_test['numGroups'] == X_test['maxPlace'])
 subset = X_test.loc[fullgroup]
X_test.loc[fullgroup, 'winPlacePerc'] =(subset['_rank.winPlacePerc'].values - 1)/(subset['maxPlace'].values - 1)
 subset = X_test.loc[~fullgroup]
gap = 1.0 /(subset['maxPlace'].values - 1)
 new_perc = np.around(subset['winPlacePerc'].values / gap)* gap
X_test.loc[~fullgroup, 'winPlacePerc'] = new_perc
X_test['winPlacePerc'] = X_test['winPlacePerc'].clip(lower=0,upper=1 )<groupby>
text = "今日は東京ディズニーランドに行った"
tokenizer.tokenize(text )
_=<feature_engineering>
token = ["[CLS]"]+tokenizer.tokenize(text)[:max_lengths-2]+["[SEP]"]
input_id = tokenizer.convert_tokens_to_ids(token)
 segment_id = [0]*max_lengths
attention_mask = [1]*len(input_id)+[0]*(max_lengths - len(input_id))
input_id = input_id+[pad]*(max_lengths-len(input_id))
X_test.loc[X_test['maxPlace'] == 0, 'winPlacePerc'] = 0
X_test.loc[X_test['maxPlace'] == 1, 'winPlacePerc'] = 1
X_test.loc[(X_test['maxPlace'] > 1)&(X_test['numGroups'] == 1), 'winPlacePerc'] = 0
X_test['winPlacePerc'].describe()<save_to_csv>
class ReviewDataset(torch.utils.data.Dataset):
def __init__(self, texts, labels=[]):
self.input_ids, self.segment_ids, self.attention_masks = [],[],[]
for text in tqdm(texts):
token = ["[CLS]"]+tokenizer.tokenize(text)[:max_lengths-2]+["[SEP]"]
input_id = tokenizer.convert_tokens_to_ids(token)
 segment_id = [0]*max_lengths
attention_mask = [1]*len(input_id)+[0]*(max_lengths - len(input_id))
input_id = input_id+[pad]*(max_lengths-len(input_id))
self.input_ids.append(input_id)
 self.segment_ids.append(segment_id)
 self.attention_masks.append(attention_mask)
 self.input_ids = np.array(self.input_ids)
 self.segment_ids = np.array(self.segment_ids)
 self.attention_masks = np.array(self.attention_masks)
 self.labels = labels
def __len__(self):
return len(self.input_ids)
 def __getitem__(self, idx):
if len(self.labels):
return self.input_ids[idx], self.segment_ids[idx], self.attention_masks[idx], self.labels[idx]
else:
return self.input_ids[idx], self.segment_ids[idx], self.attention_masks[idx]
test = pd.read_csv('.. /input/test_V2.csv')
 test['matchId'] = test['matchId'].apply(lambda x: int(x,16))
test['groupId'] = test['groupId'].apply(lambda x: int(x,16))
submission = pd.merge(test, X_test[['matchId','groupId','winPlacePerc']])
 submission = submission[['Id','winPlacePerc']]
submission.to_csv("submission.csv", index=False )<load_from_csv>
batch_size=8
X,y = train_df["Body"].values, train_df["Label"].values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=100,stratify=y)
 train_ds = ReviewDataset(texts=X_train, labels=y_train)
 train_dl = torch.utils.data.DataLoader(
 train_ds, batch_size=batch_size, shuffle=True)
 val_ds = ReviewDataset(texts=X_val, labels=y_val)
 val_dl = torch.utils.data.DataLoader(
 val_ds, batch_size=batch_size, shuffle=False )
dataset_train = pd.read_csv('.. /input/train_V2.csv')
 dataset_test = pd.read_csv('.. /input/test_V2.csv' )<drop_column>
class BertClassification(nn.Module):
def __init__(self, bert):
super(BertClassification, self ).__init__()
self.bert = bert
self.cls = nn.Linear(in_features=768, out_features=2)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids, token_type_ids=None, attention_mask=None):
pooled_output = self.bert(
 input_ids, token_type_ids, attention_mask)[1]
pooled_output = pooled_output.view(-1, 768)
 out = self.cls(pooled_output)
 return out
dataset_train = dataset_train.dropna()
dataset_train = dataset_train.drop(labels=['Id','groupId','matchId'],axis=1 )<prepare_x_and_y>
cuda = torch.cuda.is_available()
bert = BertModel.from_pretrained("cl-tohoku/bert-base-japanese-whole-word-masking")
 model = BertClassification(bert)
 torch.backends.cudnn.benchmark = True
if cuda:
model.cuda()
optimizer = optim.Adam(model.parameters() ,lr = 4e-4 ,betas=(0.9, 0.999))
criterion = nn.CrossEntropyLoss()
dataset_train = pd.get_dummies(dataset_train,drop_first=True)
 dataset_train_y = dataset_train.loc[:,'winPlacePerc']
dataset_train_x = dataset_train.drop(labels='winPlacePerc',axis=1)
 scaler = StandardScaler()
dataset_train_x = scaler.fit_transform(dataset_train_x )<choose_model_class>
for param in model.bert.encoder.layer[:-1].parameters() :
param.requires_grad = False
reduceLr = ReduceLROnPlateau(monitor='loss', factor=0.01,patience=5, min_lr=0.00001)
 input_layer = InputLayer(input_shape =(39,))
model = Sequential()
model.add(input_layer)
 model.add(Dense(128,activation = "relu"))
model.add(Dense(256,activation = "relu"))
model.add(Dense(128,activation = "relu"))
model.add(Dense(256,activation = "relu"))
model.add(Dense(128,activation = "relu"))
model.add(Dense(1,activation='sigmoid'))
model.compile(optimizer = "adadelta",loss="binary_crossentropy",metrics=["mae","accuracy"])
 model.summary()
history = model.fit(dataset_train_x,dataset_train_y,epochs=10,batch_size= 5000,callbacks=[reduceLr] )<normalization>
epochs = 2
for epoch in range(epochs):
total_loss = 0
accuracy = 0
model.train()
print("epoch {} start!".format(epoch+1))
for iter_num,(input_ids, segment_ids, attention_masks, labels)in tqdm(enumerate(train_dl),total = len(train_dl)) :
optimizer.zero_grad()
if cuda:
input_ids, segment_ids, attention_masks, labels =\
input_ids.cuda() , segment_ids.cuda() , attention_masks.cuda() , labels.cuda()
outputs = model(input_ids = input_ids,
token_type_ids = segment_ids,
attention_mask = attention_masks)
 pred_proba = outputs.softmax(dim=1)[:,1]
pred =(pred_proba>=0.5 ).type(torch.int)
 loss = criterion(outputs,labels)
 loss.backward()
optimizer.step()
total_loss += loss.item()
accuracy +=(pred==labels ).sum().item()
if(iter_num+1)% 50 == 0:
size = batch_size*(iter_num+1)
 print("{} iter loss:{:.4f} accuracy:{:.4f}".format(
 iter_num+1,total_loss/(iter_num+1),accuracy/size))
total_loss /= len(train_dl)
 accuracy /= len(train_ds)
 val_total_loss = 0
val_accuracy = 0
model.eval()
for input_ids, segment_ids, attention_masks, labels in tqdm(val_dl):
if cuda:
input_ids, segment_ids, attention_masks, labels =\
input_ids.cuda() , segment_ids.cuda() , attention_masks.cuda() , labels.cuda()
with torch.no_grad() :
outputs = model(input_ids = input_ids,
token_type_ids = segment_ids,
attention_mask = attention_masks)
 pred_proba = outputs.softmax(dim=1)[:,1]
pred =(pred_proba>=0.5 ).type(torch.int)
 loss = criterion(outputs,labels)
 val_total_loss += loss.item()
val_accuracy +=(pred==labels ).sum().item()
val_total_loss /= len(val_dl)
 val_accuracy /= len(val_ds)
 print("epoch{} total loss:{:.4f}, accuracy:{:.4f}, val_total loss:{:.4f}, val_accuracy:{:.4f}"\ 
.format(epoch+1,total_loss,accuracy,val_total_loss,val_accuracy))
dataset_test_id = dataset_test.loc[:,"Id"]
dataset_test = dataset_test.dropna()
dataset_test = dataset_test.drop(labels=['Id','groupId','matchId'],axis=1)
 dataset_test = pd.get_dummies(dataset_test,drop_first=True)
 display(dataset_test.head())
 scaler = StandardScaler()
dataset_test = scaler.fit_transform(dataset_test )<predict_on_test>
test_ds = ReviewDataset(texts=test_df["Body"].values)
 test_dl = torch.utils.data.DataLoader(
 test_ds, batch_size=8, shuffle=False )
output = model.predict(dataset_test)
<create_dataframe>
lists = []
for input_ids, segment_ids, attention_masks in tqdm(test_dl):
if cuda:
input_ids, segment_ids, attention_masks =\
input_ids.cuda() , segment_ids.cuda() , attention_masks.cuda()
outputs = model(input_ids = input_ids,
token_type_ids = segment_ids,
attention_mask = attention_masks)
 pred_proba = outputs.softmax(dim=1)[:,1]
lists += list(pred_proba.cpu().detach().numpy() )
data_file = pd.DataFrame({"Id":dataset_test_id,"winPlacePerc":pd.Series(output.reshape(-1,)) } )<save_to_csv>
test_df["Label"]=lists
submission_df=test_df[["Id","Label"]]
submission_df.to_csv("sample_submission.csv",index=False )
data_file.to_csv("pubg_predictions.csv",index = False )<correct_missing_values>
!pip install mecab-python3==0.996.2
def fillInf(df, val):
numcols = df.select_dtypes(include='number' ).columns
cols = numcols[numcols != 'winPlacePerc']
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
for c in cols: df[c].fillna(val, inplace=True)
 def reduce_mem_usage(df):

 start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB --> {:.2f} MB(Decreased by {:.1f}%)'.format(
 start_mem, end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<load_from_csv>
tokenizer = BertJapaneseTokenizer.from_pretrained("bert-base-japanese")
 for i,item in enumerate(tokenizer.vocab.items()):
print(item)
 if i > 20:
break
<import_modules><EOS> 
cuda = torch.cuda.is_available()
bert = BertModel.from_pretrained("bert-base-japanese")
 model = BertClassification(bert)
 torch.backends.cudnn.benchmark = True
if cuda:
model.cuda()
optimizer = optim.Adam(model.parameters() ,lr = 4e-4 ,betas=(0.9, 0.999))
criterion = nn.CrossEntropyLoss()