Kaggle competition: Word vectors
<SOS> metric: categorizationaccuracy Kaggle data source: word-vectors<prepare_x_and_y>
from types import SimpleNamespace
from collections import Counter
import os
import re
import pathlib
import array
import pickle
import numpy as np
import torch
import torch.nn as nn
import pandas as pd
target='winPlacePerc'
features=list(train.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 features.remove("killsWithoutMoving")
 features.remove("headshot_rate")
 features.remove(target)
 sample = 500000
df_sample = train.sample(sample)
 x_train=df_sample[features]
y_train=df_sample[target]<split>
DATASET_VERSION = 'ca-100'
COMPETITION_ROOT = '.. /input/vectors'
DATASET_ROOT = f'.. /input/cbow-preprocessing/data/{DATASET_VERSION}'
WORKING_ROOT = f'data/{DATASET_VERSION}'
DATASET_PREFIX = 'ca.wiki'
random_seed=1
sample = 500000
x_train,x_val,y_train,y_val=train_test_split(x_train,y_train,test_size=0.1,random_state=random_seed)
 model=RandomForestRegressor(n_estimators=70,min_samples_leaf=3,max_features=0.5,n_jobs=-1 )<train_model>
params = SimpleNamespace(
 embedding_dim = 100,
window_size = 5,
batch_size = 1000,
epochs = 4,
preprocessed = f'{DATASET_ROOT}/{DATASET_PREFIX}',
working = f'{WORKING_ROOT}/{DATASET_PREFIX}',
modelname = f'{WORKING_ROOT}/{DATASET_VERSION}.pt',
train = True
)
%%time
model.fit(x_train,y_train )<compute_test_metric>
class Vocabulary(object):
def __init__(self, pad_token='<pad>', unk_token='<unk>', eos_token='<eos>'):
self.token2idx = {}
self.idx2token = []
self.pad_token = pad_token
self.unk_token = unk_token
self.eos_token = eos_token
if pad_token is not None:
self.pad_index = self.add_token(pad_token)
 if unk_token is not None:
self.unk_index = self.add_token(unk_token)
 if eos_token is not None:
self.eos_index = self.add_token(eos_token)
 def add_token(self, token):
if token not in self.token2idx:
self.idx2token.append(token)
 self.token2idx[token] = len(self.idx2token)- 1
return self.token2idx[token]
def get_index(self, token):
if isinstance(token, str):
return self.token2idx.get(token, self.unk_index)
 else:
return [self.token2idx.get(t, self.unk_index)for t in token]
def __len__(self):
return len(self.idx2token)
 def save(self, filename):
with open(filename, 'wb')as f:
pickle.dump(self.__dict__, f)
 def load(self, filename):
with open(filename, 'rb')as f:
self.__dict__.update(pickle.load(f))
print('mae train:',mean_absolute_error(model.predict(x_train),y_train))
print('mae train:',mean_absolute_error(model.predict(x_val),y_val))<feature_engineering>
def batch_generator(idata, target, batch_size, shuffle=True):
nsamples = len(idata)
 if shuffle:
perm = np.random.permutation(nsamples)
 else:
perm = range(nsamples)
 for i in range(0, nsamples, batch_size):
batch_idx = perm[i:i+batch_size]
if target is not None:
yield idata[batch_idx], target[batch_idx]
else:
yield idata[batch_idx], None
test['totalDistance']=test['rideDistance']+test['walkDistance']+test['swimDistance']
test['headshot_rate']=test['headshotKills']/test['kills']
test['playersJoined'] = test.groupby('matchId')['matchId'].transform('count')
 test['killsNorm'] = test['kills']*(( 100-test['playersJoined'])/100 + 1)
 test['headshotKillsNorm'] = test['headshotKills']*(( 100-test['playersJoined'])/100 + 1)
 test['killPlaceNorm'] = test['killPlace']*(( 100-test['playersJoined'])/100 + 1)
 test['killPointsNorm'] = test['killPoints']*(( 100-test['playersJoined'])/100 + 1)
 test['killStreaksNorm'] = test['killStreaks']*(( 100-test['playersJoined'])/100 + 1)
 test['longestKillNorm'] = test['longestKill']*(( 100-test['playersJoined'])/100 + 1)
 test['roadKillsNorm'] = test['roadKills']*(( 100-test['playersJoined'])/100 + 1)
 test['teamKillsNorm'] = test['teamKills']*(( 100-test['playersJoined'])/100 + 1)
 test['damageDealtNorm'] = test['damageDealt']*(( 100-test['playersJoined'])/100 + 1)
 test['DBNOsNorm'] = test['DBNOs']*(( 100-test['playersJoined'])/100 + 1)
 test['revivesNorm'] = test['revives']*(( 100-test['playersJoined'])/100 + 1)
 test = test.drop([ 'kills', 'headshotKills', 'killPlace', 'killPoints', 'killStreaks',
'longestKill', 'roadKills', 'teamKills', 'damageDealt', 'DBNOs', 'revives'],axis=1 )<predict_on_test>
class CBOW(nn.Module):
def __init__(self, num_embeddings, embedding_dim):
super().__init__()
self.emb = nn.Embedding(num_embeddings, embedding_dim, padding_idx=0)
 self.lin = nn.Linear(embedding_dim, num_embeddings, bias=False)
 def forward(self, input):
e = self.emb(input)
 u = e.sum(dim=1)
 v = self.lin(u)
 return v
x_test=test[features]
pred=model.predict(x_test)
 test['winPlacePerc']=pred
submission=test[['Id','winPlacePerc']]
submission.head()<save_to_csv>
def load_preprocessed_dataset(prefix):
token_vocab = Vocabulary()
token_vocab.load(f'{prefix}.vocab')
 data = []
for part in ['train', 'valid', 'test']:
with np.load(f'{prefix}.{part}.npz')as set_data:
idata, target = set_data['idata'], set_data['target']
data.append(( idata, target))
print(f'Number of samples({part}): {len(target)}')
 print("Using precomputed vocabulary and data files")
 print(f'Vocabulary size: {len(token_vocab)}')
 return token_vocab, data
submission.to_csv('submission.csv',index=False )<load_from_csv>
def train(model, criterion, optimizer, idata, target, batch_size, device, log=False):
model.train()
total_loss = 0
ncorrect = 0
ntokens = 0
niterations = 0
for X, y in batch_generator(idata, target, batch_size, shuffle=True):
X = torch.tensor(X, dtype=torch.long, device=device)
 y = torch.tensor(y, dtype=torch.long, device=device)
 model.zero_grad()
output = model(X)
 loss = criterion(output, y)
 loss.backward()
optimizer.step()
total_loss += loss.item()
ncorrect +=(torch.max(output, 1)[1] == y ).sum().item()
ntokens += y.numel()
niterations += 1
if niterations == 200 or niterations == 500 or niterations % 1000 == 0:
print(f'Train: wpb={ntokens//niterations}, num_updates={niterations}, accuracy={100*ncorrect/ntokens:.1f}, loss={total_loss/ntokens:.2f}')
 total_loss = total_loss / ntokens
accuracy = 100 * ncorrect / ntokens
if log:
print(f'Train: wpb={ntokens//niterations}, num_updates={niterations}, accuracy={accuracy:.1f}, loss={total_loss:.2f}')
 return accuracy, total_loss
%load_ext autoreload
%autoreload 2
%matplotlib inline
KAGGLE_DIR = '.. /input/'
train = pd.read_csv(KAGGLE_DIR + 'train_V2.csv')
 test = pd.read_csv(KAGGLE_DIR + 'test_V2.csv' )<feature_engineering>
def validate(model, criterion, idata, target, batch_size, device):
model.eval()
total_loss = 0
ncorrect = 0
ntokens = 0
niterations = 0
y_pred = []
with torch.no_grad() :
for X, y in batch_generator(idata, target, batch_size, shuffle=False):
X = torch.tensor(X, dtype=torch.long, device=device)
 output = model(X)
 if target is not None:
y = torch.tensor(y, dtype=torch.long, device=device)
 loss = criterion(output, y)
 total_loss += loss.item()
ncorrect +=(torch.max(output, 1)[1] == y ).sum().item()
ntokens += y.numel()
niterations += 1
else:
pred = torch.max(output, 1)[1].detach().to('cpu' ).numpy()
y_pred.append(pred)
 if target is not None:
total_loss = total_loss / ntokens
accuracy = 100 * ncorrect / ntokens
return accuracy, total_loss
else:
return np.concatenate(y_pred )
train.drop(2744604, inplace=True)
 train['playersJoined'] = train.groupby('matchId')['matchId'].transform('count')

train['killsNorm'] = train['kills']*(( 100-train['playersJoined'])/100 + 1)
 train['damageDealtNorm'] = train['damageDealt']*(( 100-train['playersJoined'])/100 + 1)
 train['maxPlaceNorm'] = train['maxPlace']*(( 100-train['playersJoined'])/100 + 1)
 train['matchDurationNorm'] = train['matchDuration']*(( 100-train['playersJoined'])/100 + 1)
 train['DBNOsNorm'] = train['DBNOs']*(( 100-train['playersJoined'])/100 + 1)
 train['healsandboosts'] = train['heals'] + train['boosts'] 
<feature_engineering>
if torch.cuda.is_available() :
device = torch.device('cuda')
 else:
device = torch.device('cpu')
 print("WARNING: Training without GPU can be very slow!" )
train['totalDistance'] = train['rideDistance'] + train['walkDistance'] + train['swimDistance']
train['killsWithoutMoving'] =(( train['kills'] > 0)&(train['totalDistance'] == 0))
train['headshot_rate'] = train['headshotKills'] / train['kills']
train['headshot_rate'] = train['headshot_rate'].fillna(0)
 train.drop(train[train['killsWithoutMoving'] == True].index, inplace=True)
 train.drop('killsWithoutMoving', axis=1, inplace=True)
 train.drop(train[train['roadKills'] > 10].index, inplace=True)
 train.drop(train[train['kills'] > 35].index, inplace=True)
 train.drop(train[(train['headshot_rate'] == 1)&(train['kills'] > 11)].index,inplace=True)
 train.drop(train[train['walkDistance'] >= 10000].index, inplace=True)
 train.drop(train[train['rideDistance'] >= 20000].index, inplace=True)
 train.drop(train[train['swimDistance'] >= 2000].index, inplace=True)
 train.drop(train[train['weaponsAcquired'] >= 80].index, inplace=True)
 train.drop(train[train['heals'] >= 40].index, inplace=True )<feature_engineering>
vocab, data = load_preprocessed_dataset(params.preprocessed )
top = ['walkDistance','walkDistance','damageDealt']
bot = ['boosts','kills','kills']
for t,b in zip(top,bot):
train.loc[(train[b]!=0),t+'/'+b] = train[t]/train[b]
train.loc[(train[b]==0),t+'/'+b] = 0
print('a')
 features_to_agg = ['groupId','DBNOs','kills','totalDistance','headshot_rate',
'damageDealt','killPlace','walkDistance/boosts','walkDistance/kills','damageDealt/kills']
aggs = train[features_to_agg].groupby('groupId' ).agg([np.sum, np.mean, np.std])
 train_slice = train[['groupId']]
for col in list(aggs.columns.levels[0]):
for ag in list(aggs.columns.levels[1]):
train_slice = train_slice.join(aggs[col,ag], on='groupId')
 train_slice.columns = train_slice.columns.map(lambda x: x[0]+'_'+x[1] if type(x)==tuple else x)
 train_slice.fillna(0, inplace=True)
 del(aggs )<drop_column>
model = CBOW(len(vocab), params.embedding_dim ).to(device )
train_slice.drop(columns='groupId', inplace=True)
 train = pd.concat([train, train_slice], axis=1)
 print(len(train))
train.head()<drop_column>
valid_x_df = pd.read_csv(f'{COMPETITION_ROOT}/x_valid.csv')
 tokens = valid_x_df.columns[1:]
valid_x = valid_x_df[tokens].apply(vocab.get_index ).to_numpy(dtype='int32')
 valid_y_df = pd.read_csv(f'{COMPETITION_ROOT}/y_valid.csv')
 valid_y = valid_y_df['token'].apply(vocab.get_index ).to_numpy(dtype='int32' )
del(train_slice)
<concatenate>
optimizer = torch.optim.Adam(model.parameters())
 criterion = nn.CrossEntropyLoss(reduction='sum')
 train_accuracy = []
wiki_accuracy = []
valid_accuracy = []
for epoch in range(params.epochs):
acc, loss = train(model, criterion, optimizer, data[0][0], data[0][1], params.batch_size, device, log=True)
 train_accuracy.append(acc)
 print(f'| epoch {epoch:03d} | train accuracy={acc:.1f}%, train loss={loss:.2f}')
 acc, loss = validate(model, criterion, data[1][0], data[1][1], params.batch_size, device)
 wiki_accuracy.append(acc)
 print(f'| epoch {epoch:03d} | valid accuracy={acc:.1f}%, valid loss={loss:.2f}(wikipedia)')
 acc, loss = validate(model, criterion, valid_x, valid_y, params.batch_size, device)
 valid_accuracy.append(acc)
 print(f'| epoch {epoch:03d} | valid accuracy={acc:.1f}%, valid loss={loss:.2f}(El Periódico)')
 torch.save(model.state_dict() , params.modelname )
train = pd.concat([train, ranked], axis=1)
 del(ranked)
<categorify>
valid_x_df = pd.read_csv(f'{COMPETITION_ROOT}/x_test.csv')
 test_x = valid_x_df[tokens].apply(vocab.get_index ).to_numpy(dtype='int32')
 y_pred = validate(model, None, test_x, None, params.batch_size, device)
 y_token = [vocab.idx2token[index] for index in y_pred]
<normalization><EOS> 
submission = pd.DataFrame({'id':valid_x_df['id'], 'token': y_token}, columns=['id', 'token'])
 print(submission.head())
 submission.to_csv('submission.csv', index=False )