Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<train_model>
import os
import numpy as np
import pandas as pd
from kaggle_datasets import KaggleDatasets
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
import plotly.express as px
def fit(model, X, y, epochs=250, optim='adam', lr=0.001, verbose=0):
valid_optims = ['sgd', 'rmsprop', 'adam']
optim = optim.lower()
if optim.lower() not in valid_optims:
raise ValueError("invalid optimizer got '{0}' and expect one of {1}" 
.format(optim, valid_optims))
loss_fn = torch.nn.BCEWithLogitsLoss()
optim = optim.lower()
if optim == 'sgd':
optimizer = torch.optim.SGD(model.parameters() , lr=lr)
 elif optim == 'rmsprop':
optimizer = torch.optim.RMSprop(model.parameters() , lr=lr)
 elif optim == 'adam':
optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 for t in range(epochs):
logits = model(X)
 loss = loss_fn(logits, y)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
pred = torch.sigmoid(logits)
 acc = torch.eq(y, pred.round_() ).cpu().float().mean().data[0]
if verbose > 1:
print("Epoch {0:>{2}}/{1}: Loss={3:.4f}, Accuracy={4:.4f}" 
.format(t + 1, epochs, len(str(epochs)) , loss.data[0], acc))
if verbose > 0:
print("Training complete! Loss={0:.4f}, Accuracy={1:.4f}".format(loss.data[0], acc))
return {'loss': loss.data[0], 'acc': acc}<compute_train_metric>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def cross_val_score(model, X, y, cv=3, epochs=250, optim='adam', lr=0.001, use_cuda=True, verbose=0):
kfolds = KFold(cv, False ).split(X)
 score = []
for fold,(train_idx, val_idx)in enumerate(kfolds):
X_train = X[train_idx]
y_train = y[train_idx]
X_val = X[val_idx]
y_val = y[val_idx]
X_train = Variable(torch.Tensor(X_train), requires_grad=True)
 y_train = Variable(torch.Tensor(y_train), requires_grad=False ).unsqueeze_(-1)
 X_val = Variable(torch.Tensor(X_val), requires_grad=False)
 y_val = Variable(torch.Tensor(y_val), requires_grad=False ).unsqueeze_(-1)
 model_train = copy.deepcopy(model)
 if(use_cuda):
X_train = X_train.cuda()
y_train = y_train.cuda()
X_val = X_val.cuda()
y_val = y_val.cuda()
model_train = model_train.cuda()
metrics = fit(model_train, X_train, y_train, epochs=epochs, optim=optim,
lr=lr, verbose=0)
 y_val_pred = torch.sigmoid(model_train(X_val))
acc = torch.eq(y_val, y_val_pred.round_() ).cpu().float().mean().data[0]
score.append(acc)
 if verbose > 1:
print("Fold {0:>{2}}/{1}: Validation accuracy={3:.4f}" 
.format(fold + 1, cv, len(str(cv)) , acc))
if verbose > 0:
print("Mean k-fold accuracy: {0:.4f}".format(np.mean(score)))
 return score<train_on_grid>
model_name = 'jplu/tf-xlm-roberta-large'
n_epochs = 10
max_len = 80
batch_size = 16 * strategy.num_replicas_in_sync
def titanic_net_grid_search(X, y, param_grid, cv=3, epochs=250, use_cuda=True, verbose=0):
grid = list(( dict(zip(param_grid, param))
for param in itertools.product(*param_grid.values())))
 n_candidates = len(grid)
 if verbose > 0:
print("Fitting {0} folds for each of {1} candidates, totaling {2} fits" 
.format(n_folds, n_candidates, n_folds * n_candidates))
print()
best_params = None
best_model = None
best_score = 0
for candidate, params in enumerate(grid):
if verbose == 1:
progress = "Candidate {0:>{2}}/{1}".format(candidate + 1, n_candidates,
len(str(n_candidates)))
 print(progress, end="\r", flush=True)
 elif verbose > 1:
print("Candidate", candidate + 1)
 print("Parameters: {}".format(params))
d_in = X_train.shape[-1]
d_hidden = params['d_hidden']
n_hidden = params['n_hidden']
d_out = 1
model = titanic_net(d_in, d_hidden, n_hidden, d_out)
 cv_score = cross_val_score(model, X_train, Y_train, cv = n_folds, epochs=epochs,
use_cuda=use_cuda, verbose=0)
 cv_mean_acc = np.mean(cv_score)
 if verbose > 1:
print("Mean CV accuracy: {0:.4f}".format(cv_mean_acc))
print()
if cv_mean_acc > best_score:
best_params = params
best_model = model
best_score = cv_mean_acc
if verbose > 0:
if verbose == 1:
print()
print("Best model")
 print("Parameters: {}".format(best_params))
print("Mean CV accuracy: {0:.4f}".format(best_score))
return {'best_model': best_model, 'best_params': best_params, 'best_score': best_score}<train_model>
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv' )
X_train = np.array(train.drop(columns='Survived'))
Y_train = np.array(train['Survived'].astype(int))
X_test = np.array(test)
 print("Training samples shape: {}".format(X_train.shape))
print("Training targets shape: {}".format(Y_train.shape))
print("Test samples shape: {}".format(X_test.shape))<train_on_grid>
tokenizer = AutoTokenizer.from_pretrained(model_name )
n_folds = 10
grid = {
'n_hidden': [0, 3, 7, 10, 15],
'd_hidden': [3, 7, 10],
'lr': [0.001, 0.005, 0.01],
'optim': ['Adam']
}
best_candidate = titanic_net_grid_search(X_train, Y_train, grid, cv=n_folds,
epochs=500, verbose=1)
 best_model = best_candidate['best_model']<train_model>
train_text = train[['premise', 'hypothesis']].values.tolist()
test_text = test[['premise', 'hypothesis']].values.tolist()
train_encoded = tokenizer.batch_encode_plus(
 train_text,
pad_to_max_length=True,
max_length=max_len 
)
 test_encoded = tokenizer.batch_encode_plus(
 test_text,
pad_to_max_length=True,
max_length=max_len
)
X_train_t = Variable(torch.Tensor(X_train), requires_grad=True)
 y_train_t = Variable(torch.Tensor(Y_train), requires_grad=False ).unsqueeze_(-1)
 X_test_t = Variable(torch.Tensor(X_test), requires_grad=False)
 best_params = best_candidate["best_params"]
_ = fit(best_model, X_train_t, y_train_t, epochs=500,optim=best_params['optim'],
lr=best_params['lr'] )<save_to_csv>
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded['input_ids'], train.label.values,
test_size=0.2, random_state=2020 
)
 x_test = test_encoded['input_ids']
prediction = torch.sigmoid(best_model(X_test_t)).data.round_().numpy().flatten()
_test = pd.read_csv(".. /input/titanic/test.csv")
 submission_df = pd.DataFrame({'PassengerId': _test['PassengerId'], 'Survived': prediction.astype(int)})
 submission_df.to_csv("submission.csv", index=False)
 train.to_csv("submission_train.csv", index=False)
 test.to_csv("submission_test.csv", index=False )<import_modules>
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder, MinMaxScaler 
<load_from_csv>
with strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(model_name)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer_encoder(input_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('The size of the training set: ', train.shape)
 print('The size of the test set is: ' ,test.shape )<define_variables>
n_steps = len(x_train)// batch_size
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)
Passenger_Id = test['PassengerId']<define_variables>
test_preds = model.predict(test_dataset, verbose=1)
 submission['prediction'] = test_preds.argmax(axis=1 )
<count_values><EOS> 
submission.to_csv('submission.csv', index=False)
 submission.head()