Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<compute_test_metric>
!pip install --upgrade pip > /dev/null
!pip install --upgrade transformers > /dev/null
!pip install nlp > /dev/null
cm = confusion_matrix(y_test2['Survived'], y_pred2['Predicted'])
 print(cm)
<predict_on_test>
import os
import gc
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import plotly.express as px
from tensorflow.keras.layers import Dense, Input, GlobalAveragePooling1D, GlobalMaxPooling1D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
import tensorflow as tf
from transformers import TFAutoModel, AutoTokenizer
import nlp
y_pred = mlpr_model.predict(test_dataframe)
 threshold = 0.5
tansform_survived = np.vectorize(lambda x: 1 if x >= threshold else 0)
 y_pred2 = tansform_survived(y_pred)
 y_pred2 = pd.DataFrame({'Predicted': y_pred2})
<load_from_csv>
def init_strategy() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Init TPU strategy")
 except ValueError:
strategy = tf.distribute.get_strategy()
print("Init CPU/GPU strategy")
 return strategy
def build_model(model_name, maxlen, head="avg_pooling"):
input_ids = Input(shape=(maxlen,), dtype=tf.int32, name="input_ids")
 encoder = TFAutoModel.from_pretrained(model_name)
 encoder_output = encoder(input_ids)[0]
if head == "cls":
features = encoder_output[:, 0, :]
elif head == "avg_pooling":
features = GlobalAveragePooling1D()(encoder_output)
 elif head == "max_pooling":
features = GlobalMaxPooling1D()(encoder_output)
 else:
raise NotImplementedError
out = Dense(3, activation='softmax' )(features)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model
def tokenize_dataframe(df, tokenizer, max_length):
text = df[['premise', 'hypothesis']].values.tolist()
encoded = tokenizer.batch_encode_plus(text, padding=True, max_length=max_length, truncation=True)
 x = encoded['input_ids']
y = None
if 'label' in df.columns:
y = df.label.values
return x, y
def build_dataset(x, y, mode, batch_size):
if mode == "train":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 elif mode == "valid":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 elif mode == "test":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(x)
.batch(batch_size)
)
 else:
raise NotImplementedError
return dataset
def load_mnli(use_validation=True):
result = []
dataset = nlp.load_dataset(path='glue', name='mnli')
 keys = ['train', 'validation_matched','validation_mismatched'] if use_validation else ['train']
for k in keys:
for record in dataset[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result.append(( c1,c2,c3,'en'))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
def load_xnli() :
result = []
dataset = nlp.load_dataset(path='xnli')
 for k in dataset.keys() :
for record in dataset[k]:
hp, pr, lb = record['hypothesis'], record['premise'], record['label']
if hp and pr and lb in {0,1,2}:
for lang, translation in zip(hp['language'], hp['translation']):
pr_lang = pr.get(lang, None)
 if pr_lang is None:
continue
result.append(( pr_lang, translation, lb,lang))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
y_test = pd.read_csv(".. /input/gender_submission.csv", sep=",")
 cm = confusion_matrix(y_test['Survived'], y_pred2['Predicted'])
 print(cm)
<save_to_csv>
MODEL = 'jplu/tf-xlm-roberta-large'
MAXLEN = 120
strategy = init_strategy()
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 auto = tf.data.experimental.AUTOTUNE
def preprocess(df):
return tokenize_dataframe(df, tokenizer, MAXLEN )
test_csv['Survived'] = y_pred2['Predicted']
test_csv.to_csv('submission_all_nn_0.5.csv', index=False)
<train_model>
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 x, y = preprocess(train)
 x_test, _ = preprocess(test)
 test_dataset = build_dataset(x_test, None, "test", BATCH_SIZE)
 mnli = load_mnli()
xnli = load_xnli()
print('Keras NN: ')
 nn = Sequential()
nn.add(
 Dense(
 units=3, kernel_initializer='uniform', activation='relu', input_dim=2))
nn.add(Dense(units=1, kernel_initializer='uniform', activation='sigmoid'))
nn.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
 nn.fit(X_train, y_train, batch_size=10, epochs=100)
<predict_on_test>
punct = '[' + ''.join([c for c in string.punctuation if c != "'"])+ ']'
def preprocess_query(q):
q = q.lower()
q = re.sub(punct, ' ', q)
 q = re.sub('[ ]{2,}', ' ', q)
 return q
def search_in_base(q, kb):
q = preprocess_query(q)
 return int(q in kb)
y_pred = nn.predict(X_test)
<prepare_output>
premises = pd.concat([train[['premise', 'lang_abv']], test[['premise', 'lang_abv']]] )
threshold = 0.5
tansform_survived = np.vectorize(lambda x: 1 if x >= threshold else 0)
 y_pred2 = tansform_survived(y_pred)
 y_pred2 = pd.DataFrame({'Predicted': y_pred2[:,0]})
 y_pred2.head(10)
<prepare_output>
knowledge_base = set(mnli['premise'].apply(preprocess_query))
premises['mnli'] = premises['premise'].apply(lambda q: search_in_base(q, knowledge_base))
print(f"fraction of train set english premises occurence in MNLI = {premises.loc[premises.lang_abv=='en', 'mnli'].mean() * 100}%" )
y_test2 = pd.DataFrame({'Survived': y_test})
 y_test2.head(10)
<compute_test_metric>
knowledge_base = set(xnli['premise'].apply(preprocess_query))
premises['xnli'] = premises['premise'].apply(lambda q: search_in_base(q, knowledge_base))
print(f"fraction of train set non-english premises occurence in XNLI = {premises.loc[premises.lang_abv!='en', 'xnli'].mean() * 100}%" )
<predict_on_test><EOS> 
strategy = init_strategy()
with strategy.scope() :
model = build_model(MODEL, MAXLEN)
 model.load_weights(".. /input/watson-xlmr-models/XLMR_mnlixnli_ep6.h5")
 dataset = build_dataset(x, y, "valid", BATCH_SIZE)
 pr = np.argmax(model.predict(dataset), axis=1)
 print(f"accuracy {accuracy_score(y, pr):.4f}")
 test_preds = model.predict(test_dataset, verbose=0)
 submission['prediction'] = test_preds.argmax(axis=1)
 submission.to_csv('submission.csv', index=False )