Kaggle competition: Sentiment Analysis on Movie Reviews
<SOS> metric: CategorizationAccuracy Kaggle data source: sentiment-analysis-on-movie-reviews<feature_engineering>
def seed_all(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all(2020 )
for dataset in combine:
df = dataset.groupby(['Sex', 'SibSp'] ).size().unstack(0 ).fillna(0)
 df['male_perc'] =(df[df.columns[1]]/(df[df.columns[0]] + df[df.columns[1]]))
df['fem_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]]))
print(df)
 print("_"*40 )<groupby>
!pip install transformers
train_df[["Sex", "SibSp", "Survived"]].groupby(['Sex', 'SibSp'],
as_index=True ).mean()<feature_engineering>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased' )
for dataset in combine:
df = dataset.groupby(['Sex', 'Embarked'] ).size().unstack(0)
 df['male_perc'] =(df[df.columns[1]]/(df[df.columns[0]] + df[df.columns[1]]))
df['fem_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]]))
print(df)
 print("_"*40 )<groupby>
def tokenize_and_cut(sentence):
tokens = tokenizer.tokenize(sentence)
 max_input_length = tokenizer.max_model_input_sizes['bert-base-uncased']
tokens = tokens[:max_input_length-2]
return tokens
train_df[["Embarked", "Sex", "Survived"]].groupby(['Sex', 'Embarked'],
as_index=True ).mean()<feature_engineering>
TEXT = data.Field(batch_first = True,
use_vocab = False,
tokenize = tokenize_and_cut,
preprocessing = tokenizer.convert_tokens_to_ids,
init_token = tokenizer.cls_token_id,
eos_token = tokenizer.sep_token_id,
pad_token = tokenizer.pad_token_id,
unk_token = tokenizer.unk_token_id)
 LABEL = data.Field(sequential=False, use_vocab=False )
for dataset in combine:
df = dataset.groupby(['Pclass', 'Embarked'] ).size().unstack(0)
 df['first_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]] + df[df.columns[2]]))
df['second_perc'] =(df[df.columns[1]]/(df[df.columns[0]] + df[df.columns[1]] + df[df.columns[2]]))
df['third_perc'] =(df[df.columns[2]]/(df[df.columns[0]] + df[df.columns[1]] + df[df.columns[2]]))
print(df)
 print("_"*40 )<groupby>
!mkdir./data
!unzip -d./data.. /input/sentiment-analysis-on-movie-reviews/train.tsv.zip
!unzip -d./data.. /input/sentiment-analysis-on-movie-reviews/test.tsv.zip
train_df[["Embarked", "Pclass", "Survived"]].groupby(['Embarked', 'Pclass'],
as_index=True ).mean()<data_type_conversions>
train = pd.read_csv('./data/train.tsv', sep='\t')
 test = pd.read_csv('./data/test.tsv', sep='\t')
 train, valid = train_test_split(train, test_size=0.2)
 train.to_csv('./data/train.csv', index=False)
 valid.to_csv('./data/validation.csv', index=False)
 train, valid = data.TabularDataset.splits(
 path='./data', train='train.csv', validation='validation.csv', format='csv', skip_header=True,
fields=[('PhraseId', None),('SentenceId', None),('Phrase', TEXT),('Sentiment', LABEL)])
 test = data.TabularDataset('./data/test.tsv', format='tsv', skip_header=True,
fields=[('PhraseId', None),('SentenceId', None),('Phrase', TEXT)] )
for dataset in combine:
fil1 =(dataset.Cabin.isnull())
 fil2 =(dataset.Cabin.notnull())
 dataset.loc[fil1, 'Cabin'] = 0
dataset.loc[fil2, 'Cabin'] = 1
dataset.Cabin = pd.to_numeric(dataset['Cabin'])
 print(train_df.Cabin.value_counts())
 print("_"*40)
 print(test_df.Cabin.value_counts() )<sort_values>
print(f"Number of training examples: {len(train)}")
 print(f"Number of validation examples: {len(valid)}")
 print(f"Number of testing examples: {len(test)}" )
train_df[['Cabin', 'Survived']].groupby(['Cabin'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
print(tokenizer.convert_ids_to_tokens(vars(train[6])['Phrase']))
for dataset in combine:
df = dataset.groupby(['Sex', 'Cabin'] ).size().unstack(0)
 df['fem_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]]))
df['male_perc'] =(df[df.columns[1]]/(df[df.columns[0]] + df[df.columns[1]]))
print(df)
 print("_"*40 )<groupby>
BATCH_SIZE = 256
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 train_iter = data.BucketIterator(train, batch_size=BATCH_SIZE, shuffle=True, device=DEVICE)
 valid_iter = data.BucketIterator(valid, batch_size=BATCH_SIZE, shuffle=True, device=DEVICE)
 test_iter = data.Iterator(test, batch_size=BATCH_SIZE, train=False, sort=False, device=DEVICE )
train_df[["Cabin", "Sex", "Survived"]].groupby(['Sex', 'Cabin'],
as_index=True ).mean()<groupby>
bert = BertModel.from_pretrained('bert-base-uncased' )
for dataset in combine:
df = dataset.groupby(['Cabin', 'Pclass'] ).size().unstack(0)
 df['miss_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]]))
print(df)
 print("_"*40 )<groupby>
class BERTGRUSentiment(nn.Module):
def __init__(self, bert, output_dim):
super().__init__()
self.bert = bert
self.embedding_dim = bert.config.to_dict() ['hidden_size']
self.gru11 = nn.GRU(self.embedding_dim, 512, num_layers=1, batch_first=True)
 self.gru12 = nn.GRU(512, 256, num_layers=1, batch_first=True)
 self.gru13 = nn.GRU(256, 128, num_layers=1, batch_first=True)
 self.gru21 = nn.GRU(self.embedding_dim, 512, num_layers=1, batch_first=True)
 self.gru22 = nn.GRU(512, 256, num_layers=1, batch_first=True)
 self.gru23 = nn.GRU(256, 128, num_layers=1, batch_first=True)
 self.fc = nn.Linear(256, output_dim)
 self.dropout1 = nn.Dropout(0.5)
 self.dropout2 = nn.Dropout(0.3)
 self.DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def forward(self, text):
with torch.no_grad() :
embedding = self.bert(text)[0]
output1, _ = self.gru11(embedding)
 output1 = self.dropout1(output1)
 output1, _ = self.gru12(output1)
 output1 = self.dropout2(output1)
 _, hidden1 = self.gru13(output1)
 reversed_embedding = torch.from_numpy(embedding.detach().cpu().numpy() [:, ::-1, :].copy() ).to(self.DEVICE)
 output2, _ = self.gru21(reversed_embedding)
 output2 = self.dropout1(output2)
 output2, _ = self.gru22(output2)
 output2 = self.dropout2(output2)
 _, hidden2 = self.gru23(output2)
 hidden = self.dropout2(torch.cat(( hidden1[-1, :, :], hidden2[-1, :, :]), dim=1))
output = self.fc(hidden)
 return output
train_df[["Cabin", "Pclass", "Survived"]].groupby(['Pclass', 'Cabin'],
as_index=True ).mean()<filter>
OUTPUT_DIM = 5
model = BERTGRUSentiment(bert, OUTPUT_DIM )
train_df[train_df.Embarked.isnull() ]<filter>
test_df[test_df.Fare.isnull() ]<feature_engineering>
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
 print(f'The model has {count_parameters(model):,} trainable parameters' )
fil =(( train_df.Pclass == 1)&(train_df.SibSp == 0)&(train_df.Parch == 0)
 &(train_df.Sex == 'female'))
mis = train_df[fil].Embarked.mode()
print(mis)
 fil = train_df.Embarked.isnull()
train_df.loc[fil, 'Embarked'] = 'C'
print("_"*40)
 print(train_df.Embarked.value_counts(dropna = False))<feature_engineering>
for name, param in model.named_parameters() :
if name.startswith('bert'):
param.requires_grad = False
print(f'The model has {count_parameters(model):,} trainable parameters' )
fil =(( test_df.Pclass == 3)&(test_df.SibSp == 0)&(test_df.Parch == 0)
 &(test_df.Cabin == 0)&(test_df.Sex == 'male')&(test_df.Embarked == 'S'))
mis = round(test_df[fil].Fare.median() , 4)
 print(mis)
 fil = test_df.Fare.isnull()
test_df.loc[fil, 'Fare'] = mis
print("_"*40)
 print(test_df.Fare.isnull().value_counts(dropna = False))<count_values>
for name, param in model.named_parameters() :
if param.requires_grad:
print(name )
for dataset in combine:
dataset['MisAge'] = 0
fil =(dataset.Age.isnull())
 dataset.loc[fil, 'MisAge'] = 1
print(train_df.MisAge.value_counts())
 print("_"*40)
 print(test_df.MisAge.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['MisAge', 'Survived']].groupby(['MisAge'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
optimizer = optim.Adam(model.parameters() )
for dataset in combine:
df = dataset.groupby(['MisAge', 'Sex'] ).size().unstack(0)
 df['miss_perc'] =(df[df.columns[1]]/(df[df.columns[0]] + df[df.columns[1]]))
df['nomiss_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]]))
print(df)
 print("_"*40 )<groupby>
criterion = nn.CrossEntropyLoss()
train_df[["Sex", "MisAge", "Survived"]].groupby(['Sex', 'MisAge'],
as_index=True ).mean()<feature_engineering>
model = model.to(DEVICE)
 criterion = criterion.to(DEVICE )
for dataset in combine:
df = dataset.groupby(['MisAge', 'Pclass'] ).size().unstack(0)
 df['miss_perc'] =(df[df.columns[1]]/(df[df.columns[0]] + df[df.columns[1]]))
df['nomiss_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]]))
print(df)
 print("_"*40 )<groupby>
def accuracy(prediction, label):

 prediction = torch.argmax(nn.functional.softmax(prediction, dim=1), dim=1)
 acc = torch.sum(prediction == label ).float() / len(prediction == label)
 return acc
train_df[["Pclass", "MisAge", "Survived"]].groupby(['Pclass', 'MisAge'],
as_index=True ).mean()<groupby>
def train(model, iterator, optimizer, criterion):
epoch_loss = 0
epoch_acc = 0
model.train()
for batch in iterator:
optimizer.zero_grad()
data = batch.Phrase
label = batch.Sentiment
prediction = model(data)
 loss = criterion(prediction, label)
 acc = accuracy(prediction, label)
 loss.backward()
optimizer.step()
epoch_loss += loss.item()
epoch_acc += acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator )
train_df[["Cabin", "MisAge", "Survived"]].groupby(['Cabin', 'MisAge'],
as_index=True ).mean()<count_values>
def evaluate(model, iterator, criterion):
epoch_loss = 0
epoch_acc = 0
model.eval()
with torch.no_grad() :
for batch in iterator:
data = batch.Phrase
label = batch.Sentiment
prediction = model(data)
 loss = criterion(prediction, label)
 acc = accuracy(prediction, label)
 epoch_loss += loss.item()
epoch_acc += acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator )
fil =(train_df.Age.isnull())
 print("By class:")
 print(train_df[fil].Pclass.value_counts())
 print("_"*40)
 print(train_df[train_df.MisAge == 0].Pclass.value_counts())
 print("_"*40)
 print("_"*40)
 print("By sex:")
 print(train_df[fil].Sex.value_counts())
 print("_"*40)
 print(train_df[train_df.MisAge == 0].Sex.value_counts())
 print("_"*40)
 print("_"*40)
 print("By parents and children:")
 print(train_df[fil].Parch.value_counts())
 print("_"*40)
 print(train_df[train_df.MisAge == 0].Parch.value_counts())
 print("_"*40)
 print("_"*40)
 print("By spouse and siblings:")
 print(train_df[fil].SibSp.value_counts())
 print("_"*40)
 print(train_df[train_df.MisAge == 0].SibSp.value_counts() )<count_values>
def epoch_time(start_time, end_time):
elapsed_time = end_time - start_time
elapsed_mins = int(elapsed_time / 60)
 elapsed_secs = int(elapsed_time -(elapsed_mins * 60))
return elapsed_mins, elapsed_secs
fil =(test_df.Age.isnull())
 print("By class:")
 print(test_df[fil].Pclass.value_counts())
 print("_"*40)
 print(test_df[test_df.MisAge == 0].Pclass.value_counts())
 print("_"*40)
 print("_"*40)
 print("By sex:")
 print(test_df[fil].Sex.value_counts())
 print("_"*40)
 print(test_df[test_df.MisAge == 0].Sex.value_counts())
 print("_"*40)
 print("_"*40)
 print("By parents and children:")
 print(test_df[fil].Parch.value_counts())
 print("_"*40)
 print(test_df[test_df.MisAge == 0].Parch.value_counts())
 print("_"*40)
 print("_"*40)
 print("By spouse and siblings:")
 print(test_df[fil].SibSp.value_counts())
 print("_"*40)
 print(test_df[test_df.MisAge == 0].SibSp.value_counts() )<feature_engineering>
N_EPOCHS = 10
best_epoch = 0
best_valid_loss = float('inf')
 for epoch in range(N_EPOCHS):
start_time = time.time()
train_loss, train_acc = train(model, train_iter, optimizer, criterion)
 valid_loss, valid_acc = evaluate(model, valid_iter, criterion)
 end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
 if valid_loss < best_valid_loss:
best_epoch = epoch + 1
best_valid_loss = valid_loss
torch.save(model.state_dict() , 'model.pt')
 print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
 print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
 print(f'\t Val.Loss: {valid_loss:.3f} | Val.Acc: {valid_acc*100:.2f}%' )
for df in combine:
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )<feature_engineering>
train_full = data.TabularDataset('./data/train.tsv', format='tsv', skip_header=True,
fields=[('PhraseId', None),('SentenceId', None), 
('Phrase', TEXT),('Sentiment', LABEL)])
 train_full_iter = data.BucketIterator(train_full, batch_size=BATCH_SIZE, shuffle=True, device=DEVICE)
 model = BERTGRUSentiment(bert, OUTPUT_DIM)
 optimizer = optim.Adam(model.parameters())
 criterion = nn.CrossEntropyLoss()
model = model.to(DEVICE)
 criterion = criterion.to(DEVICE)
 for epoch in range(best_epoch + 1):
start_time = time.time()
train_loss, train_acc = train(model, train_full_iter, optimizer, criterion)
 end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
 if epoch == best_epoch:
torch.save(model.state_dict() , 'model_full.pt')
 print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
 print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%' )
for df in combine:
df['Title'] = df['Title'].replace(['Mme', 'Countess','Dona'], 'Mrs')
 df['Title'] = df['Title'].replace(['Capt', 'Col','Don', 'Jonkheer', 'Rev',
'Major', 'Sir'], 'Mr')
 df['Title'] = df['Title'].replace(['Mlle', 'Lady','Ms'], 'Miss')
 df.loc[(df.Sex == 'male')&(df.Title == 'Dr'), 'Title'] = 'Mr'
df.loc[(df.Sex == 'female')&(df.Title == 'Dr'), 'Title'] = 'Mrs'
pd.crosstab(train_df['Title'], train_df['Sex'] )<count_values>
def predict(model, iterator):
model.eval()
predictions = []
with torch.no_grad() :
for batch in iterator:
data = batch.Phrase
prediction = model(data)
 prediction = torch.argmax(nn.functional.softmax(prediction, dim=1), dim=1)
 predictions.extend(prediction.tolist())
 return predictions
<define_variables><EOS> 
predictions = predict(model, test_iter)
 submission = pd.read_csv('.. /input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv')
 submission['Sentiment'] = predictions
submission.to_csv('submissionBERTGRU.csv', index=False )