Kaggle competition: Contradictory, My Dear Watson
nlp_train_df[['text','target']].to_csv('train.csv', index=False, header=False )<load_from_csv>
!pip install --upgrade pip
!pip install --upgrade allennlp
!pip install transformers==4.0.1
training_gcs_path = 'uploads/kaggle_getstarted/full_train.csv'
upload_blob(bucket_name, 'train.csv', training_gcs_path )<choose_model_class>
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev
amw = AutoMLWrapper(client=client,
project_id=PROJECT_ID,
bucket_name=bucket_name,
region='us-central1',
dataset_display_name=dataset_display_name,
model_display_name=model_display_name)
<prepare_x_and_y>
import transformers
import pandas as pd
import torch
print(f'Getting dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_dataset_by_display_name(dataset_display_name):
print('dataset not found')
 amw.create_dataset()
amw.import_gcs_data(training_gcs_path)
 amw.dataset
print(f'Dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
<train_model>
import torch_xla
import torch_xla.core.xla_model as xm
print(f'Getting model trained at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_model_by_display_name() :
print(f'Training model at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.train_model()
print(f'Model trained.Ensuring model is deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.deploy_model()
amw.model
print(f'Model trained and deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
<find_best_params>
print('Transformers version: ', transformers.__version__)
 print('Pytorch version: ', torch.__version__ )
amw.model_full_path<predict_on_test>
data_dir = '/kaggle/input/contradictory-my-dear-watson/'
train_df = pd.read_csv(data_dir+'train.csv' ).sample(frac=1, random_state=100)
 test_df = pd.read_csv(data_dir+'test.csv')
 print(train_df['label'].value_counts())
 train_df.head(5 )
print(f'Begin getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 prediction_client = automl.PredictionServiceClient()
amw.set_prediction_client(prediction_client)
 predictions_df = amw.get_predictions(nlp_test_df,
input_col_name='text',
limit=None,
threshold=0.5,
verbose=False)
 print(f'Finished getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )<concatenate>
PRE_TRAINED_MODEL = 'xlm-roberta-large'
tokenizer = transformers.AutoTokenizer.from_pretrained(PRE_TRAINED_MODEL )
submission_df = pd.concat([nlp_test_df['id'], predictions_df['class']], axis=1)
 submission_df.head()<rename_columns>
MAX_LEN = 84
split_idx = int(train_df.shape[0] * 0.8 )
submission_df = submission_df.rename(columns={'class':'target'})
 submission_df.head()<save_to_csv>
def get_encode(data):
tokenized_data=tokenizer(
 text=list(data['premise']), text_pair=list(data['hypothesis']),
max_length=MAX_LEN,
pad_to_max_length=True,
add_special_tokens=True,
truncation=True,
return_attention_mask=True,
return_token_type_ids=True,
return_tensors='pt')
 return tokenized_data
submission_df.to_csv("submission.csv", index=False, header=True )<load_from_csv>
seed=2
tokenized_train=get_encode(train_df[:split_idx])
 labels_train=torch.tensor(train_df.label.values[:split_idx])
 tokenized_valid=get_encode(train_df[split_idx:])
 labels_valid=torch.tensor(train_df.label.values[split_idx:])
 tokenized_test=get_encode(test_df )
train, test = pd.read_csv('.. /input/train.csv'), pd.read_csv('.. /input/test.csv')
 train['Boy'], test['Boy'] = [(df.Name.str.split().str[1] == 'Master.' ).astype('int')for df in [train, test]]
train['Surname'], test['Surname'] = [df.Name.str.split(',' ).str[0] for df in [train, test]]
model = catboost.CatBoostClassifier(one_hot_max_size=4, iterations=100, random_seed=0, verbose=False)
 model.fit(train[['Sex', 'Pclass', 'Embarked', 'Boy', 'Surname']].fillna(''), train['Survived'], cat_features=[0, 2, 4])
 pred = model.predict(test[['Sex', 'Pclass', 'Embarked', 'Boy', 'Surname']].fillna('')).astype('int')
 pd.concat([test['PassengerId'], pd.DataFrame(pred, columns=['Survived'])], axis=1 ).to_csv('submission.csv', index=False )<train_on_grid>
batch_size=64
train_data=TensorDataset(torch.tensor(tokenized_train['input_ids']),
torch.tensor(tokenized_train['token_type_ids']),torch.tensor(tokenized_train['attention_mask'])
 ,labels_train)
 train_sampler=RandomSampler(train_data)
 train_dataloader=DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 valid_data=TensorDataset(torch.tensor(tokenized_valid['input_ids']),
torch.tensor(tokenized_valid['token_type_ids']),
torch.tensor(tokenized_valid['attention_mask'])
 ,labels_valid)
 valid_sampler=SequentialSampler(valid_data)
 valid_dataloader=DataLoader(valid_data, sampler=valid_sampler, batch_size=batch_size)
 test_data=TensorDataset(torch.tensor(tokenized_test['input_ids']),
torch.tensor(tokenized_test['token_type_ids']),
torch.tensor(tokenized_test['attention_mask']))
test_dataloader=DataLoader(test_data, batch_size=batch_size )
features = ['Sex', 'Pclass', 'Embarked']
X_train = train[features].fillna('')
 y_train = train['Survived']
X_test = test[features].fillna('')
 model = catboost.CatBoostClassifier(
 one_hot_max_size=4,
iterations=100,
random_seed=0,
verbose=False,
eval_metric='Accuracy' 
)
 pool = catboost.Pool(X_train, y_train, cat_features=[0, 2])
 print('To see the Catboost plots, fork this kernel and run it in the editing mode.')
 cv_scores = catboost.cv(pool, model.get_params() , fold_count=10, plot=True)
 print('CV score: {:.5f}'.format(cv_scores['test-Accuracy-mean'].values[-1]))<save_to_csv>
model = AutoModelForSequenceClassification.from_pretrained(PRE_TRAINED_MODEL,
num_labels = 3,
output_attentions = False,
output_hidden_states = False)
model.fit(pool)
 pred = model.predict(X_test ).astype('int')
 output = pd.concat([test['PassengerId'], pd.DataFrame(pred, columns=['Survived'])], axis=1)
 output.to_csv('submission2.csv', index=False )<train_on_grid>

 device = xm.xla_device()
torch.set_default_tensor_type('torch.FloatTensor')
 print(model.to(device))
print('TPU available')
features = ['Sex', 'Pclass', 'Embarked', 'Boy']
X_train = train[features].fillna('')
 y_train = train['Survived']
X_test = test[features].fillna('')
 model = catboost.CatBoostClassifier(
 one_hot_max_size=4,
iterations=100,
random_seed=0,
verbose=False,
eval_metric='Accuracy' 
)
 pool = catboost.Pool(X_train, y_train, cat_features=[0, 2])
 print('To see the Catboost plots, fork this kernel and run it in the editing mode.')
 cv_scores = catboost.cv(pool, model.get_params() , fold_count=10, plot=True)
 print('CV score: {:.5f}'.format(cv_scores['test-Accuracy-mean'].values[-1]))<save_to_csv>
def accuracy(predictions, labels):
prediction_flat = np.argmax(predictions, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(prediction_flat == labels_flat)/ len(labels_flat )
model.fit(pool)
 pred = model.predict(X_test ).astype('int')
 output = pd.concat([test['PassengerId'], pd.DataFrame(pred, columns=['Survived'])], axis=1)
 output.to_csv('submission3.csv', index=False )<train_on_grid>
optimizer = AdamW(model.parameters() ,
lr = 2e-5 
)
 epochs = 20
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
features = ['Sex', 'Pclass', 'Embarked', 'Boy', 'Surname']
X_train = train[features].fillna('')
 y_train = train['Survived']
X_test = test[features].fillna('')
 model = catboost.CatBoostClassifier(
 one_hot_max_size=4,
iterations=100,
random_seed=0,
verbose=False,
eval_metric='Accuracy' 
)
 pool = catboost.Pool(X_train, y_train, cat_features=[0, 2, 4])
 print('To see the Catboost plots, fork this kernel and run it in the editing mode.')
 cv_scores = catboost.cv(pool, model.get_params() , fold_count=10, plot=True)
 print('CV score: {:.5f}'.format(cv_scores['test-Accuracy-mean'].values[-1]))<save_to_csv>
random.seed(10)
 np.random.seed(10)
 torch.manual_seed(10)
 torch.cuda.manual_seed_all(10)
 losses = []
for i in range(0, epochs):
print('Epoch {:} of {:} Training...'.format(i+1, epochs))
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 10 == 0 and step != 0:
print('[{}] Batch {:>5,} of {:>5,}' 
.format(datetime.datetime.now().strftime('%H:%M:%S'), step, len(train_dataloader)))
 train_batch_input = batch[0].to(device)
 train_batch_input_types = batch[1].to(device)
 train_batch_mask = batch[2].to(device)
 train_batch_label = batch[3].to(device)
 model.zero_grad()
outputs = model(train_batch_input, token_type_ids = train_batch_input_types,
attention_mask = train_batch_mask, labels = train_batch_label)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 xm.optimizer_step(optimizer, barrier=True)
 scheduler.step()
average_train_loss = total_loss / len(train_dataloader)
 losses.append(average_train_loss)
 print('Training loss={:.2f}'.format(average_train_loss))
model.eval()
eval_accuracy = 0
eval_count = 0
for batch in valid_dataloader:
valid_batch_input = batch[0].to(device)
 valid_batch_input_types = batch[1].to(device)
 valid_batch_mask = batch[2].to(device)
 valid_batch_labels = batch[3].to(device)
 with torch.no_grad() :
outputs = model(valid_batch_input, token_type_ids = valid_batch_input_types,
attention_mask = valid_batch_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = valid_batch_labels.to('cpu' ).numpy()
batch_accuracy = accuracy(logits, label_ids)
 eval_accuracy += batch_accuracy
eval_count += 1
print('Validation accuracy={:.2f}'.format(eval_accuracy / eval_count))
print('')
 if i == 4:
break
print("Training done")
model.fit(pool)
 pred = model.predict(X_test ).astype('int')
 output = pd.concat([test['PassengerId'], pd.DataFrame(pred, columns=['Survived'])], axis=1)
 output.to_csv('submission4.csv', index=False )<set_options>
model.eval()
submissions = []
for batch in test_dataloader:
test_batch_input = batch[0].to(device)
 test_batch_input_types = batch[1].to(device)
 test_batch_mask = batch[2].to(device)
 with torch.no_grad() :
outputs = model(test_batch_input, token_type_ids = test_batch_input_types,
attention_mask = test_batch_mask)
 logits = outputs[0]
submissions.extend(np.argmax(logits.detach().cpu().numpy() , axis=1 ).flatten())
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn 
<load_from_csv>
output = pd.DataFrame({'id': test_df.id,
'prediction': submissions})
 output.to_csv('submission.csv', index=False )