Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<data_type_conversions>
os.environ["WANDB_API_KEY"] = "0"
!pip uninstall -y transformers
!pip install transformers
!pip install nlp
strategy = None
def seed_all(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed_all(2020 )
all_data = all_data.fillna(all_data.mean() )<prepare_x_and_y>
original_train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 original_train = shuffle(original_train)
 original_valid = original_train[:len(original_train)// 5]
original_train = original_train[len(original_train)// 5:]
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice<compute_train_metric>
print(f"original - training: {len(original_train)} examples")
 original_train.head(10 )
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )<choose_model_class>
print(f"original - validation: {len(original_valid)} examples")
 original_valid.head(10 )
model_ridge = Ridge()<define_search_space>
original_test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 print(f"original - test: {len(original_test)} examples")
 original_test.head(10 )
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]<train_on_grid>
mnli = nlp.load_dataset(path='glue', name='mnli' )
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(X_train, y )<find_best_params>
print(mnli, '
')
 print('The split names in MNLI dataset:')
 for k in mnli:
print(' ', k)
 print("
mnli['train'] is ", type(mnli['train']))
mnli['train']
rmse_cv(model_lasso ).mean()<define_variables>
print('The number of training examples in mnli dataset:', mnli['train'].num_rows)
 print('The number of validation examples in mnli dataset - part 1:', mnli['validation_matched'].num_rows)
 print('The number of validation examples in mnli dataset - part 2:', mnli['validation_mismatched'].num_rows, '
')
 print('The class names in mnli dataset:', mnli['train'].features['label'].names)
 print('The feature names in mnli dataset:', list(mnli['train'].features.keys()), '
')
 for elt in mnli['train']:
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', mnli['train'].features['label'].names[elt['label']])
 print('idx', elt['idx'])
 print('-' * 80)
 if elt['idx'] >= 10:
break
print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables" )<concatenate>
snli = nlp.load_dataset(path='snli')
 print('The number of training examples in snli dataset:', snli['train'].num_rows)
 print('The number of validation examples in snli dataset:', snli['validation'].num_rows, '
')
 print('The class names in snli dataset:', snli['train'].features['label'].names)
 print('The feature names in snli dataset:', list(snli['train'].features.keys()), '
')
 for idx, elt in enumerate(snli['train']):
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', snli['train'].features['label'].names[elt['label']])
 print('-' * 80)
 if idx >= 10:
break
imp_coef = pd.concat([coef.sort_values().head(10),
coef.sort_values().tail(10)] )<import_modules>
xnli = nlp.load_dataset(path='xnli')
 print('The number of validation examples in xnli dataset:', xnli['validation'].num_rows, '
')
 print('The class names in xnli dataset:', xnli['validation'].features['label'].names)
 print('The feature names in xnli dataset:', list(xnli['validation'].features.keys()), '
')
 for idx, elt in enumerate(xnli['validation']):
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', xnli['validation'].features['label'].names[elt['label']])
 print('-' * 80)
 if idx >= 3:
break
import xgboost as xgb<train_on_grid>
for k in raw_ds_mapping:
for idx, x in enumerate(get_raw_dataset(k)) :
print(x)
 if idx >= 3:
break
dtrain = xgb.DMatrix(X_train, label = y)
 dtest = xgb.DMatrix(X_test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100 )<train_model>
def get_unbatched_dataset(ds_names, model_name, max_len=64):

 if type(ds_names)== list:
ds_names = {k: None for k in ds_names}
ds_names = {k: v for k, v in ds_names.items() if k in raw_ds_mapping}
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, use_fast=True)
 raw_datasets = [get_raw_dataset(x)for x in ds_names]
nb_examples = 0
sentence_pairs = []
labels = []
for name in ds_names:
raw_ds = get_raw_dataset(name)
 nb_examples_to_use = raw_ds_mapping[name][2]
if ds_names[name]:
nb_examples_to_use = min(ds_names[name], nb_examples_to_use)
 nb_examples += nb_examples_to_use
n = 0
for x in raw_ds:
sentence_pairs.append(( x['premise'], x['hypothesis']))
labels.append(x['label'])
 n += 1
if n >= nb_examples_to_use:
break
r = dict(tokenizer.batch_encode_plus(batch_text_or_text_pairs=sentence_pairs, max_length=max_len, padding='max_length', truncation=True))
dataset = tf.data.Dataset.from_tensor_slices(( r, labels))
return dataset, nb_examples
def get_batched_training_dataset(dataset, nb_examples, batch_size=16, shuffle_buffer_size=1, repeat=False):
if repeat:
dataset = dataset.repeat()
if not shuffle_buffer_size:
shuffle_buffer_size = nb_examples
dataset = dataset.shuffle(shuffle_buffer_size)
 dataset = dataset.batch(batch_size, drop_remainder=True)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset
def get_prediction_dataset(dataset, batch_size=16):
dataset = dataset.batch(batch_size, drop_remainder=False)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset
model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=2, learning_rate=0.1)
 model_xgb.fit(X_train, y )<predict_on_test>
for k in raw_ds_mapping.keys() :
ds, nb_examples = get_unbatched_dataset({k: 100}, model_name='distilbert-base-uncased')
 ds_batched = get_batched_training_dataset(ds, nb_examples, batch_size=16, shuffle_buffer_size=1, repeat=False)
 print('{} - select {} examples'.format(k, nb_examples))
for x in ds_batched:
break
xgb_preds = np.expm1(model_xgb.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))<create_dataframe>
class Classifier(tf.keras.Model):
def __init__(self, model_name):
super(Classifier, self ).__init__()
self.transformer = transformers.TFAutoModel.from_pretrained(model_name)
 self.dropout = tf.keras.layers.Dropout(rate=0.05)
 self.global_pool = tf.keras.layers.GlobalAveragePooling1D()
self.classifier = tf.keras.layers.Dense(3)
 def call(self, inputs, training=False):
x = self.transformer(inputs, training=training)[0]
x = self.dropout(x, training=training)
 x = self.global_pool(x)
 return self.classifier(x)
 class Trainer:
def __init__(
 self, ds_names, model_name, max_len=64,
batch_size_per_replica=16, prediction_batch_size_per_replica=64,
shuffle_buffer_size=1 
):
global strategy
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 self.ds_names = ds_names
self.model_name = model_name
self.max_len = max_len
self.batch_size_per_replica = batch_size_per_replica
self.prediction_batch_size_per_replica = prediction_batch_size_per_replica
self.batch_size = batch_size_per_replica * strategy.num_replicas_in_sync
self.prediction_batch_size = prediction_batch_size_per_replica * strategy.num_replicas_in_sync
self.shuffle_buffer_size = shuffle_buffer_size
train_ds, self.nb_examples = get_unbatched_dataset(
 ds_names=ds_names, model_name=model_name, max_len=max_len 
)
 self.train_ds = get_batched_training_dataset(
 train_ds, self.nb_examples, batch_size=self.batch_size,
shuffle_buffer_size=self.shuffle_buffer_size, repeat=True 
)
 valid_ds, self.nb_valid_examples = get_unbatched_dataset(
 ds_names=['original valid'], model_name=model_name, max_len=max_len 
)
 self.valid_ds = get_prediction_dataset(valid_ds, self.prediction_batch_size)
 self.valid_labels = next(iter(self.valid_ds.map(lambda inputs, label: label ).unbatch().batch(len(original_valid))))
test_ds, self.nb_test_examples = get_unbatched_dataset(
 ds_names=['original test'], model_name=model_name, max_len=max_len 
)
 self.test_ds = get_prediction_dataset(test_ds, self.prediction_batch_size)
 self.steps_per_epoch = self.nb_examples // self.batch_size
def get_model(self, model_name, lr, verbose=False):
with strategy.scope() :
model = Classifier(model_name)
 model.trainable = True
dummy = model(tf.constant(1, shape=[1, 64]))
if verbose:
model.summary()
optimizer = tf.keras.optimizers.Adam(lr=lr)
 loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.SUM)
 metrics = {
'train loss': tf.keras.metrics.Sum() ,
'train acc': tf.keras.metrics.SparseCategoricalAccuracy()
}
return model, loss_fn, optimizer, metrics
def get_routines(self, model, loss_fn, optimizer, metrics):
def train_1_step(batch):
inputs, labels = batch
with tf.GradientTape() as tape:
logits = model(inputs, training=True)
 loss = loss_fn(labels, logits)/ self.batch_size
grads = tape.gradient(loss, model.trainable_variables)
 optimizer.apply_gradients(zip(grads, model.trainable_variables))
metrics['train loss'].update_state(loss)
 metrics['train acc'].update_state(labels, logits)
 @tf.function
def dist_train_1_epoch(data_iter):

 for _ in tf.range(self.steps_per_epoch):
strategy.run(train_1_step, args=(next(data_iter),))
@tf.function
def predict_step(batch):
inputs, _ = batch
logits = model(inputs, training=False)
 return logits
def predict_fn(dist_test_ds):
all_logits = []
for batch in dist_test_ds:
logits = strategy.run(predict_step, args=(batch,))
logits = strategy.experimental_local_results(logits)
 logits = tf.concat(logits, axis=0)
 all_logits.append(logits)
 logits = tf.concat(all_logits, axis=0)
 return logits
return dist_train_1_epoch, predict_fn
def train(self, train_name, model_name, epochs, verbose=False):
global strategy
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 model, loss_fn, optimizer, metrics = self.get_model(model_name, 1e-5, verbose=verbose)
 dist_train_1_epoch, predict_fn = self.get_routines(model, loss_fn, optimizer, metrics)
 train_dist_ds = strategy.experimental_distribute_dataset(self.train_ds)
 train_dist_iter = iter(train_dist_ds)
 dist_valid_ds = strategy.experimental_distribute_dataset(self.valid_ds)
 dist_test_ds = strategy.experimental_distribute_dataset(self.test_ds)
 history = {}
best_acc=0.5
for epoch in range(epochs):
s = datetime.datetime.now()
dist_train_1_epoch(train_dist_iter)
 train_loss = metrics['train loss'].result() / self.steps_per_epoch
train_acc = metrics['train acc'].result()
metrics['train loss'].reset_states()
metrics['train acc'].reset_states()
print('epoch: {}
'.format(epoch + 1))
print('train loss: {}'.format(train_loss))
print('train acc: {}
'.format(train_acc))
e = datetime.datetime.now()
elapsed =(e - s ).total_seconds()
logits = predict_fn(dist_valid_ds)
 valid_loss = tf.reduce_mean(tf.keras.losses.sparse_categorical_crossentropy(self.valid_labels, logits, from_logits=True, axis=-1))
valid_acc = tf.reduce_mean(tf.keras.metrics.sparse_categorical_accuracy(self.valid_labels, logits))
if valid_acc>best_acc:
best_acc=valid_acc
model.save_weights('best.h5')
 print('valid loss: {}'.format(valid_loss))
print('valid acc: {}
'.format(valid_acc))
print('train timing: {}
'.format(elapsed))
history[epoch] = {
'train loss': float(train_loss),
'train acc': float(train_acc),
'valid loss': float(valid_loss),
'valid acc': float(valid_acc),
'train timing': elapsed
}
print('-' * 40)
 print('best acc:{}'.format(best_acc))
model.load_weights('best.h5')
 logits = predict_fn(dist_test_ds)
 preds = tf.math.argmax(logits, axis=-1)
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 submission['prediction'] = preds.numpy()
submission.to_csv(f'submission-{train_name}.csv', index=False)
 return history, submission,logits
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds})
 predictions.plot(x = "xgb", y = "lasso", kind = "scatter" )<prepare_output>
def print_config(trainer):
print('nb.of training examples used: {}'.format(trainer.nb_examples))
print('nb.of valid examples used: {}'.format(trainer.nb_valid_examples))
print('nb.of test examples used: {}'.format(trainer.nb_test_examples))
print('per replica batch size for training: {}'.format(trainer.batch_size_per_replica))
print('batch size for training: {}'.format(trainer.batch_size))
print('per replica batch size for prediction: {}'.format(trainer.prediction_batch_size_per_replica))
print('batch size for prediction: {}'.format(trainer.prediction_batch_size))
print('steps per epoch: {}'.format(trainer.steps_per_epoch))
preds = 0.7*lasso_preds + 0.3*xgb_preds<save_to_csv>
trainer = Trainer(
 ds_names={'original train': None, 'xnli valid': None, 'mnli train': 60000, 'mnli valid 1': None, 'mnli valid 2': None}, model_name=model_name,
max_len=208, batch_size_per_replica=16, prediction_batch_size_per_replica=64,
shuffle_buffer_size=None 
)
 print_config(trainer)
 train_name = f'{model_name} + extra-xnli-mnli'.replace('/', '-')
 history_3, submission_3,preds = trainer.train(train_name=train_name, model_name=model_name, epochs=epochs, verbose=True )
solution = pd.DataFrame({"id":test.Id, "SalePrice":preds})
 solution.to_csv("ridge_sol.csv", index = False )<import_modules>
np.savez_compressed('preds',a=preds )
<normalization><EOS> 
s = pd.read_csv(f'submission-{train_name}.csv')
 s.to_csv(f'submission.csv', index=False)
 s.head(20 )