Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<feature_engineering>
!pip install -q transformers==3.0.2
sample_train['Linear_Yhat'] = beta_0 + beta_1*sample_train['GrLivArea']<compute_test_metric>
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 train
print("Mean Squared Error(MSE)for regression line is : {}".format(np.square(sample_train['SalePrice'] - sample_train['Linear_Yhat'] ).mean()))<compute_test_metric>
test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 test
mean_squared_error(sample_train['SalePrice'], sample_train.Linear_Yhat )<train_model>
sub = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 sub
lin_reg = LinearRegression(normalize=True, n_jobs=-1)
 lin_reg.fit(X_train, y_train)
 y_pred = lin_reg.predict(X_test )<compute_test_metric>
model_name = 'jplu/tf-xlm-roberta-large'
epochs = 4
maxlen = 80
AUTO = tf.data.experimental.AUTOTUNE
print('%.2f'%mean_squared_error(y_test, y_pred))<import_modules>
tokenizer = AutoTokenizer.from_pretrained(model_name )
lin_reg = LinearRegression()
cv = KFold(shuffle=True, random_state=2, n_splits=10)
 scores = cross_val_score(lin_reg, X,y,cv = cv, scoring = 'neg_mean_absolute_error' )<train_model>
def get_training_dataset(idx, df = train, is_train = True):
text = df[['premise', 'hypothesis']].values[idx].tolist()
text_enc = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = True,
max_length = maxlen 
)
 dataset = tf.data.Dataset.from_tensor_slices(( text_enc['input_ids'], df['label'][idx].values))
dataset = dataset.repeat()
dataset = dataset.shuffle(2020)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_valid_dataset(idx, df = train, is_train = False):
text = df[['premise', 'hypothesis']].values[idx].tolist()
text_enc = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = True,
max_length = maxlen 
)
 dataset = tf.data.Dataset.from_tensor_slices(( text_enc['input_ids'], df['label'][idx].values))
dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(df = test, is_train = False):
text = df[['premise', 'hypothesis']].values.tolist()
text_enc = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = True,
max_length = maxlen 
)
 dataset = tf.data.Dataset.from_tensor_slices(text_enc['input_ids'])
 dataset = dataset.batch(batch_size)
 return dataset
alpha_ridge = [-3,-2,-1,1e-15, 1e-10, 1e-8,1e-5,1e-4, 1e-3,1e-2,0.5,1,1.5, 2,3,4, 5, 10, 20, 30, 40]
temp_rss = {}
temp_mse = {}
for i in alpha_ridge:
ridge = Ridge(alpha= i, normalize=True)
 ridge.fit(X_train, y_train)
 y_pred = ridge.predict(X_test)
 mse = mean_squared_error(y_test, y_pred)
 rss = sum(( y_pred-y_test)**2)
 temp_mse[i] = mse
temp_rss[i] = rss<sort_values>
def build_model(maxlen, model_name):
with strategy.scope() :
base_model = TFAutoModel.from_pretrained(model_name)
 input_word_ids = tf.keras.Input(shape =(maxlen,), dtype = tf.int32, name = "input_word_ids")
 embedding = base_model(input_word_ids)[0]
out_tokens = embedding[:, 0, :]
output = tf.keras.layers.Dense(3, activation = 'softmax' )(out_tokens)
 model = tf.keras.Model(inputs = input_word_ids, outputs = output)
 model.compile(tf.keras.optimizers.Adam(lr = 1e-5),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 return model
for key, value in sorted(temp_mse.items() , key=lambda item: item[1]):
print("%s: %s" %(key, value))<sort_values>
folds = 3
kf = KFold(n_splits = folds, shuffle = True, random_state = 777)
 models = []
histories = []
predictions = np.zeros(( test.shape[0], 3))
for fold,(trn_idx, val_idx)in enumerate(kf.split(np.arange(train['label'].shape[0]))):
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 train_dataset = get_training_dataset(trn_idx, df = train, is_train = True)
 valid_dataset = get_valid_dataset(val_idx, df = train, is_train = False)
 K.clear_session()
model = build_model(maxlen, model_name)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(
 'XLM-R_fold-%i.h5'%fold, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'min', save_freq = 'epoch' 
)
 print('Model Training.....')
 STEPS_PER_EPOCH = len(trn_idx)// batch_size
history = model.fit(
 train_dataset, epochs = epochs, verbose = 1,
steps_per_epoch = STEPS_PER_EPOCH,
batch_size = batch_size,
validation_data = valid_dataset 
)
 display_training_curves(
 history.history['loss'],
history.history['val_loss'],
'loss', 311 
)
 display_training_curves(
 history.history['accuracy'],
history.history['val_accuracy'],
'accuracy', 312 
)
 histories.append(history)
 models.append(model)
 print('Prediting on test data.. ')
 test_dataset = get_test_dataset(test, is_train = False)
 pred = model.predict(test_dataset, verbose = 1)
 predictions += pred / folds
del history, train_dataset, valid_dataset, model
gc.collect()
print('
')
 print('-'*50 )
for key, value in sorted(temp_rss.items() , key=lambda item: item[1]):
print("%s: %s" %(key, value))<predict_on_test>
sub['prediction'] = np.argmax(predictions, axis = 1 )
<sort_values><EOS> 
sub.to_csv('./submission.csv', index = False)
 sub