Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<define_variables>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
features = ['Title_2.0', 'Sex_1.0', 'Fare']<import_modules>
!pip install nlp
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev
clfs = {
'tree' : DecisionTreeClassifier() ,
'rf' : RandomForestClassifier() ,
'extra' : ExtraTreesClassifier() ,
'grad' : GradientBoostingClassifier() ,
}<find_best_model_class>
%%time
%autosave 60
os.environ['XLA_USE_BF16'] = "1"
os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'
gc.enable()
XLMRobertaTokenizer,
XLMRobertaModel,
get_cosine_schedule_with_warmup)
 warnings.filterwarnings("ignore")
 print('PYTORCH:', xv.__torch_gitrev__)
 print('XLA:', xv.__xla_gitrev__ )
clf_result = []
for model_name, model in clfs.items() :
model.fit(X_train[features], y_train)
 preds = model.predict(X_test[features])
 cv_score = cross_val_score(model, X_train[features], y_train, scoring = 'accuracy', cv = 5)
 clf_result.append({
'model' : model_name,
'mean acc' : cv_score.mean() ,
'std acc' : cv_score.std()
})
 clf_result = pd.DataFrame(clf_result )<train_on_grid>
train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv')
 sample_submission = pd.read_csv('.. /input/contradictory-my-dear-watson/sample_submission.csv' )
params = {
'n_estimators' : [50, 100, 150, 200],
'learning_rate' : [0.01, 0.03, 0.05, 0.07, 0.1, 0.3],
'subsample' : [0.4, 0.45, 0.5, 0.55, 0.7],
'max_features' : ['auto', 'sqrt'],
'n_iter_no_change' : [0, 5, 10],
'max_depth' : [2, 3, 4],
}
base_clf = GradientBoostingClassifier(criterion = 'mse', validation_fraction = 0.25, random_state = 0)
 grid_search = RandomizedSearchCV(base_clf, params, scoring = 'accuracy', random_state = 0, n_iter = 40)
 grid_search.fit(X_train[features], y_train)
 print('40 random searches complete' )<create_dataframe>
TRAIN_BATCH_SIZE = 16
VALID_BATCH_SIZE = 16
EPOCHS = 4
MAX_LEN = 80
LR = 2e-5 * xm.xrt_world_size()
METRICS_DEBUG = True
tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large' )
grid_search_result = pd.DataFrame(grid_search.cv_results_)
 grid_search_result.loc[grid_search_result['rank_test_score'] < 5].sort_values('rank_test_score' )<choose_model_class>
mnli = nlp.load_dataset(path='glue', name='mnli', split='train[:5%]')
 xnli = nlp.load_dataset(path='xnli')
 xnli = nlp.concatenate_datasets([xnli['test'], xnli['validation']])
 snli = nlp.load_dataset(path='snli', split='train[:5%]' )
clf_1 = GradientBoostingClassifier(n_estimators = 200,
subsample = 0.45,
max_features = 'auto',
criterion = 'mse',
n_iter_no_change = 10,
learning_rate = 0.03,
validation_fraction = 0.25,
max_depth = 3,
random_state = 0)
 clf_2 = GradientBoostingClassifier(n_estimators = 50,
subsample = 0.7,
max_features = 'sqrt',
criterion = 'mse',
n_iter_no_change = 10,
learning_rate = 0.03,
validation_fraction = 0.25,
max_depth = 4,
random_state = 0)
 clf_3 = GradientBoostingClassifier(n_estimators = 150,
subsample = 0.7,
max_features = 'auto',
criterion = 'mse',
n_iter_no_change = 5,
learning_rate = 0.1,
validation_fraction = 0.25,
max_depth = 2,
random_state = 0 )<choose_model_class>
def convert_to_features(batch):
input_pairs = list(zip(batch['premise'], batch['hypothesis']))
encodings = tokenizer.batch_encode_plus(input_pairs,
add_special_tokens=True,
pad_to_max_length=True,
max_length=MAX_LEN,
truncation=True,
return_attention_mask=True,
return_token_type_ids=True)
 return encodings
final_clf = GradientBoostingClassifier(n_estimators = 150,
subsample = 0.5,
max_features = 'auto',
criterion = 'mse',
n_iter_no_change = 5,
learning_rate = 0.07,
validation_fraction = 0.25,
max_depth = 3,
random_state = 0)
 plot_learning_curve(final_clf )<train_model>
def preprocess_xnli(example):
premise_output = []
hypothesis_output = []
label_output = []
for prem, hyp, lab in zip(example['premise'], example['hypothesis'], example["label"]):
label = lab
langs = hyp['language']
translations = hyp['translation']
hypothesis = {k: v for k, v in zip(langs, translations)}
for lang in prem:
if lang in hypothesis:
premise_output += [prem[lang]]
hypothesis_output += [hypothesis[lang]]
label_output += [label]
return {'premise':premise_output, 'hypothesis':hypothesis_output, 'label':label_output}
final_clf.fit(X_train[features], y_train)
 preds = final_clf.predict(X_val[features])
 print('Number of estimators after early stopping: {}'.format(final_clf.n_estimators_))
print('Accuracy: {}'.format(accuracy_score(preds, y_val)))
 print('F1: {}'.format(f1_score(preds, y_val)) )<load_from_csv>
mnli_encoded = mnli.map(convert_to_features, batched=True, remove_columns=['idx', 'premise', 'hypothesis'])
 mnli_encoded.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'] )
test = pd.read_csv('.. /input/titanic/test.csv')
 X_test = test.copy()
X_test['Fare'] = np.log1p(X_test.Fare)
 get_titles(X_test )<drop_column>
xnli_processed = xnli.map(preprocess_xnli, batched=True)
 xnli_encoded = xnli_processed.map(convert_to_features, batched=True, remove_columns=['premise', 'hypothesis'])
 xnli_encoded.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'] )
X_test_imp = pd.DataFrame(missing_transformer.transform(X_test))
X_test_imp.columns = numerical + categorical
X_test.drop(numerical + categorical, axis = 1, inplace = True)
 X_test = pd.concat([X_test, X_test_imp], axis = 1 )<categorify>
snli_encoded = snli.map(convert_to_features, batched=True, remove_columns=['premise', 'hypothesis'])
 snli_encoded.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'] )
X_test_enc = pd.DataFrame(ordinal_encoder.fit_transform(X_test[categorical]))
X_test_enc.columns = categorical
X_test.drop(categorical, axis = 1, inplace = True)
 X_test = pd.concat([X_test, X_test_enc], axis = 1)
 X_test = pd.get_dummies(X_test, columns = categorical, dtype = np.int64, drop_first = True )<save_to_csv>
train_dataset = nlp.load_dataset('csv', data_files=['.. /input/contradictory-my-dear-watson/train.csv'])['train']
print(train_dataset.num_rows)
 print(train_dataset.column_names)
 drop_columns = train_dataset.column_names[:-1]
encoded_train_dataset = train_dataset.map(convert_to_features, batched=True, remove_columns=drop_columns)
 encoded_train_dataset.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'])
 print(encoded_train_dataset.num_rows)
 print(encoded_train_dataset.column_names )
final_clf.fit(X[features], y)
 final_preds = final_clf.predict(X_test[features])
 final_submission = pd.DataFrame({'PassengerId' : X_test.PassengerId,
'Survived' : final_preds})
 final_submission.to_csv('may12_finalpreds.csv', index = False )<load_from_csv>
train_dataset = nlp.concatenate_datasets([mnli_encoded,
xnli_encoded,
snli_encoded,
encoded_train_dataset
])
 print(train_dataset.num_rows)
 print(train_dataset.column_names )
train_data_initial = pd.read_csv('.. /input/titanic/train.csv', index_col='PassengerId')
 test_data = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId')
 train_data_initial.head()<drop_column>
train_dataset.cleanup_cache_files()
del mnli, mnli_encoded
del xnli, xnli_encoded, xnli_processed
del snli, snli_encoded
gc.collect()
train_data_initial.drop(columns = ['Name','Ticket','Cabin'], inplace=True )<create_dataframe>
class DatasetRetriever(Dataset):
def __init__(self, dataset:nlp.arrow_dataset.Dataset):
self.dataset = dataset
self.ids = self.dataset['input_ids']
self.mask = self.dataset['attention_mask']
self.type_ids = self.dataset['token_type_ids']
self.targets = self.dataset["label"]
def __len__(self):
return self.dataset.num_rows
def __getitem__(self, index):
ids = self.ids[index]
mask = self.mask[index]
type_ids = self.type_ids[index]
targets = self.targets[index]
return {
'ids':torch.tensor(ids),
'mask':torch.tensor(mask),
'type_ids':torch.tensor(type_ids),
'targets':targets
}
train_data = train_data_initial.copy()<count_values>
class XLMRoberta(nn.Module):
def __init__(self, num_labels, multisample):
super(XLMRoberta, self ).__init__()
output_hidden_states = False
self.num_labels = num_labels
self.multisample= multisample
self.roberta = XLMRobertaModel.from_pretrained("xlm-roberta-large",
output_hidden_states=output_hidden_states,
num_labels=1)
 self.layer_norm = nn.LayerNorm(1024*2)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 self.classifier = nn.Linear(1024*2, self.num_labels)
 def forward(self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None):
outputs = self.roberta(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds)
 average_pool = torch.mean(outputs[0], 1)
 max_pool, _ = torch.max(outputs[0], 1)
 concatenate_layer = torch.cat(( average_pool, max_pool), 1)
 normalization = self.layer_norm(concatenate_layer)
 if self.multisample:
logits = torch.mean(
 torch.stack(
 [self.classifier(self.dropout(normalization)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 else:
logits = self.dropout(normalization)
 logits = self.classifier(logits)
 outputs = logits
return outputs
train_data['Age'].fillna(value = train_data['Age'].mean() , inplace = True)
 train_data['Embarked'].fillna(value = train_data['Embarked'].value_counts().idxmax() , inplace = True )<define_variables>
class AverageMeter(object):
def __init__(self, name, fmt=':f'):
self.name = name
self.fmt = fmt
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def __str__(self):
fmtstr = '{name} {val' + self.fmt + '}({avg' + self.fmt + '})'
return fmtstr.format(**self.__dict__)
 class ProgressMeter(object):
def __init__(self, num_batches, meters, prefix=""):
self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
 self.meters = meters
self.prefix = prefix
def display(self, batch):
entries = [self.prefix + self.batch_fmtstr.format(batch)]
entries += [str(meter)for meter in self.meters]
print('\t'.join(entries))
def _get_batch_fmtstr(self, num_batches):
num_digits = len(str(num_batches // 1))
fmt = '{:' + str(num_digits)+ 'd}'
return '[' + fmt + '/' + fmt.format(num_batches)+ ']'
def accuracy(output, target, topk=(1,)) :
with torch.no_grad() :
maxk = max(topk)
 batch_size = target.size(0)
 _, pred = output.topk(maxk, 1, True, True)
 pred = pred.t()
correct = pred.eq(target.view(1, -1 ).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1 ).float().sum(0, keepdim=True)
 res.append(correct_k.mul_(100.0 / batch_size))
return res
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index)
 print("Categorical variables:")
 print(object_cols )<categorify>
def get_model_optimizer(model):
def is_backbone(name):
return "roberta" in name
optimizer_grouped_parameters = [
{'params': [param for name, param in model.named_parameters() if is_backbone(name)], 'lr': LR},
{'params': [param for name, param in model.named_parameters() if not is_backbone(name)], 'lr': 1e-3}
]
optimizer = AdamW(
 optimizer_grouped_parameters, lr=LR, weight_decay=1e-2 
)
 return optimizer
label_train_data = train_data.copy()
label_encoder = LabelEncoder()
for col in object_cols:
label_train_data[col] = label_encoder.fit_transform(train_data[col] )<prepare_x_and_y>
def loss_fn(outputs, targets):
return nn.CrossEntropyLoss()(outputs, targets )
target_col = 'Survived'
y = label_train_data[target_col]
X = label_train_data.drop(columns=[target_col])
 X.head()<split>
def train_loop_fn(train_loader, model, optimizer, device, scheduler, epoch=None):
batch_time = AverageMeter('Time', ':6.3f')
 data_time = AverageMeter('Data', ':6.3f')
 losses = AverageMeter('Loss', ':.4e')
 top1 = AverageMeter('Acc@1', ':6.2f')
 progress = ProgressMeter(
 len(train_loader),
[batch_time, data_time, losses, top1],
prefix="[xla:{}]Train: Epoch: [{}]".format(xm.get_ordinal() , epoch)
)
 model.train()
end = time.time()
for i, data in enumerate(train_loader):
data_time.update(time.time() -end)
 ids, mask, type_ids, targets = data["input_ids"], data["attention_mask"], data['token_type_ids'], data["label"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 type_ids = type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids = ids,
attention_mask = mask,
token_type_ids = type_ids 
)
 loss = loss_fn(outputs, targets)
 loss.backward()
xm.optimizer_step(optimizer)
 loss = loss_fn(outputs, targets)
 acc1= accuracy(outputs, targets, topk=(1,))
losses.update(loss.item() , ids.size(0))
top1.update(acc1[0].item() , ids.size(0))
scheduler.step()
batch_time.update(time.time() - end)
 end = time.time()
if i % 50 == 0:
progress.display(i)
 del loss
del outputs
del ids
del mask
del targets
gc.collect()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0 )<define_search_space>
def eval_loop_fn(validation_loader, model, device):
model.eval()
batch_time = AverageMeter('Time', ':6.3f')
 losses = AverageMeter('Loss', ':.4e')
 top1 = AverageMeter('Acc@1', ':6.2f')
 learning_rate = AverageMeter('LR',':2.8f')
 progress = ProgressMeter(
 len(validation_loader),
[batch_time, losses, top1],
prefix='[xla:{}]Validation: '.format(xm.get_ordinal()))
with torch.no_grad() :
end = time.time()
for i, data in enumerate(validation_loader):
ids, mask, type_ids, targets = data["input_ids"], data["attention_mask"], data['token_type_ids'], data["label"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 type_ids = type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids = ids,
attention_mask = mask,
token_type_ids = type_ids 
)
 loss = loss_fn(outputs, targets)
 acc1= accuracy(outputs, targets, topk=(1,))
losses.update(loss.item() , ids.size(0))
top1.update(acc1[0].item() , ids.size(0))
batch_time.update(time.time() - end)
 end = time.time()
if i % 50 == 0:
progress.display(i)
 del loss
del outputs
del ids
del mask
del targets
gc.collect()
preds_dict = []
for n_estimators in range(100,1001,100):
for max_depth in range(6, 70,10):
for max_leaf_nodes in range(5, 500, 50):
parameters = {'n_estimators': n_estimators,
'max_depth': max_depth,
'max_leaf_nodes': max_leaf_nodes
}
model = RandomForestClassifier(**parameters)
 model.fit(X_train, y_train)
 preds = model.predict(X_valid)
 prediction = {}
prediction['n_estimators'] = n_estimators
prediction['max_depth'] = max_depth
prediction['max_leaf_nodes'] = max_leaf_nodes
prediction['accuracy_score'] = accuracy_score(y_true=y_valid, y_pred=preds)
 preds_dict.append(prediction)
 print(preds_dict )<define_variables>
WRAPPED_MODEL = xmp.MpModelWrapper(XLMRoberta(num_labels=3, multisample=False))
dataset = train_dataset.train_test_split(test_size=0.1)
 train_dataset = dataset['train']
valid_dataset = dataset['test']
train_dataset.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'])
 valid_dataset.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'] )
count = 0
indexIs = 0
maxValue = preds_dict[0]['accuracy_score']
for i in preds_dict:
if maxValue < i['accuracy_score']:
print(count , ' :', i['accuracy_score'])
 maxValue = i['accuracy_score']
indexIs = count
count = count + 1
print(count ,': Max Val is :',maxValue)
 print(preds_dict[indexIs] )<load_from_csv>
def _run() :
xm.master_print('Starting Run...')
 train_sampler = DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 train_data_loader = DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=0 
)
 xm.master_print('Train Loader Created.')
 valid_sampler = DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 valid_data_loader = DataLoader(
 valid_dataset,
batch_size=VALID_BATCH_SIZE,
sampler=valid_sampler,
drop_last=True,
num_workers=0 
)
 xm.master_print('Valid Loader Created.')
 num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size())
 device = xm.xla_device()
model = WRAPPED_MODEL.to(device)
 xm.master_print('Done Model Loading.')
 optimizer = get_model_optimizer(model)
 scheduler = get_cosine_schedule_with_warmup(
 optimizer,
num_warmup_steps = 0,
num_training_steps = num_train_steps * EPOCHS 
)
 xm.master_print(f'Num Train Steps= {num_train_steps}, XRT World Size= {xm.xrt_world_size() }.')
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 xm.master_print('Parallel Loader Created.Training...')
 train_loop_fn(para_loader.per_device_loader(device),
model,
optimizer,
device,
scheduler,
epoch 
)
 xm.master_print("Finished training epoch {}".format(epoch))
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 xm.master_print('Parallel Loader Created.Validating...')
 eval_loop_fn(para_loader.per_device_loader(device),
model,
device 
)
 if epoch == EPOCHS-1:
xm.master_print('Saving Model.. ')
 xm.save(model.state_dict() , "model.bin")
 xm.master_print('Model Saved.')
 if METRICS_DEBUG:
xm.master_print(met.metrics_report() , flush=True )
<drop_column><EOS> 
def _mp_fn(rank, flags):
_run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )