Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<create_dataframe>
import os
import numpy as np
import pandas as pd
from kaggle_datasets import KaggleDatasets
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
import plotly.express as px
column_name = 'Age'
dataframe = transform_age(dataframe)
 dataframe.head(10)
<create_dataframe>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def transform_pclass(dataframe):
column_name = 'Pclass'
dataframe = pd.concat(
 [
dataframe,
pd.get_dummies(dataframe[column_name], prefix=column_name)
 ],
axis=1)
 dataframe = dataframe.drop(columns=[column_name, 'Pclass_2'])
 return dataframe 
<prepare_output>
model_name = 'jplu/tf-xlm-roberta-large'
n_epochs = 10
max_len = 80
batch_size = 16 * strategy.num_replicas_in_sync
column_name = 'Pclass'
dataframe = transform_pclass(dataframe)
 dataframe.head(10)
<create_dataframe>
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv' )
def transform_relatives(dataframe):
column_name = 'Relatives'
dataframe[column_name] = dataframe['SibSp'] + dataframe['Parch']
dataframe = dataframe.drop(columns=['SibSp', 'Parch'])
 dataframe[column_name] = dataframe[column_name].astype(int)
 dataframe.loc[dataframe[column_name] >= 6, column_name] = 6
return dataframe 
<create_dataframe>
twitter = pd.read_csv('/kaggle/input/sentiment140/training.1600000.processed.noemoticon.csv',
encoding = 'latin1',
names=['polarity','id','date','flag','user','text'] )
column_name = 'Relatives'
dataframe = transform_relatives(dataframe)
 dataframe.head(10)
<set_options>
twitter_premise, twitter_hypothesis = train_test_split(twitter['text'], test_size=0.5, random_state=2020 )
corr.style.background_gradient() 
<prepare_x_and_y>
twitter_df = pd.DataFrame()
twitter_df['premise'] = twitter_premise.values
twitter_df['hypothesis'] = twitter_hypothesis.values
twitter_df.head()
minimal_features = [
"Sex", "Pclass_3"
]
X = dataframe[minimal_features]
y = dataframe['Survived'] 
<categorify>
tokenizer = AutoTokenizer.from_pretrained(model_name )
def preprocessing_data(source):
columns = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'Fare']
target = source.drop(columns=columns)
 column_name = 'Sex'
target[column_name] = transform_sex(target)
 column_name = 'Embarked'
target = transform_embarked(target)
 column_name = 'Age'
target = transform_age(target)
 column_name = 'Pclass'
target = transform_pclass(target)
 column_name = 'Relatives'
target = transform_relatives(target)
 return target 
<train_model>
%%time
train_text = train[['premise', 'hypothesis']].values.tolist()
test_text = test[['premise', 'hypothesis']].values.tolist()
twitter_text = twitter_df[['premise', 'hypothesis']].values.tolist()
train_encoded = tokenizer.batch_encode_plus(
 train_text,
pad_to_max_length=True,
max_length=max_len 
)
 test_encoded = tokenizer.batch_encode_plus(
 test_text,
pad_to_max_length=True,
max_length=max_len 
)
 twitter_encoded = tokenizer.batch_encode_plus(
 twitter_text,
pad_to_max_length=True,
max_length=max_len
)
print('LinearRegression: ')
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
 linreg = LinearRegression()
linreg.fit(X_train, y_train)
<predict_on_test>
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded['input_ids'], train.label.values,
test_size=0.2, random_state=2020 
)
 x_test = test_encoded['input_ids']
x_twitter_train1, x_twitter_train2 = train_test_split(
 twitter_encoded['input_ids'],
test_size=0.5, random_state=2020
)
y_pred = linreg.predict(X_test)
 y_pred<categorify>
%%time
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
)
 train_twitter_dataset1 =(
 tf.data.Dataset 
.from_tensor_slices(x_twitter_train1)
.batch(batch_size)
)
 train_twitter_dataset2 =(
 tf.data.Dataset 
.from_tensor_slices(x_twitter_train2)
.batch(batch_size)
 )
def transform_predicted(dataframe, threshold = 0.5):
column_name = 'Predicted'
dataframe.loc[dataframe[column_name] >= threshold, column_name] = 1
dataframe.loc[dataframe[column_name] < threshold, column_name] = 0
dataframe[column_name] = dataframe[column_name].astype(int)
 return dataframe<predict_on_test>
with strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(model_name)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 last_hidden_states = transformer_encoder(input_ids)[0]
cls_token = last_hidden_states[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()
threshold = 0.5
y_pred2 = pd.DataFrame({'Predicted': y_pred})
 y_pred2 = transform_predicted(y_pred2, threshold)
 y_pred2.head(10)
<prepare_output>
n_steps = len(x_train)// batch_size
train_history1 = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)
y_test2 = pd.DataFrame({'Survived': y_test})
 y_test2.head(10)
<compute_test_metric>
test_preds1 = model.predict(test_dataset, verbose=1 )
cm = confusion_matrix(y_test2['Survived'], y_pred2['Predicted'])
 print(cm)
<train_model>
twitter_preds = model.predict(train_twitter_dataset1, verbose=1)
 twitter_pseudo_labels_train1 = twitter_preds.argmax(axis=1)
 twitter_preds = model.predict(train_twitter_dataset2, verbose=1)
 twitter_pseudo_labels_train2 = twitter_preds.argmax(axis=1 )
def display_cm_report(cm):
TN = cm[0][0]
FP = cm[0][1]
FN = cm[1][0]
TP = cm[1][1]
print()
print("TP: {}".format(TP))
print("FP: {}".format(FP))
print("FN: {}".format(FN))
print("TN: {}
".format(TN))
accuracy =(TP + TN)/(TP + TN + FP + FN)
 precision = TP /(TP + FP)
 recall = TP /(TP + FN)
 f1_score = 2 *(precision * recall)/(precision + recall)
 print("accuracy: {}".format(accuracy))
print("precision: {}".format(precision))
print("recall: {}".format(recall))
print("f1_score: {}
".format(f1_score))
plt.clf()
plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Wistia)
 plt.ylabel('True label')
 plt.xlabel('Predicted label')
 classNames = ['Negative', 'Positive']
tick_marks = np.arange(len(classNames))
plt.xticks(tick_marks, classNames)
 plt.yticks(tick_marks, classNames)
 s = [['TN', 'FP'], ['FN', 'TP']]
for i in range(2):
for j in range(2):
plt.text(j, i, str(s[i][j])+ " = " + str(cm[i][j]))
plt.show() 
<load_from_csv>
twitter_dataset_ssl1 =(
 tf.data.Dataset 
.from_tensor_slices(( x_twitter_train1, twitter_pseudo_labels_train1)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 twitter_dataset_ssl2 =(
 tf.data.Dataset 
.from_tensor_slices(( x_twitter_train2, twitter_pseudo_labels_train2)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
 )
test_dataframe = pd.read_csv(".. /input/test.csv", sep=",")
<create_dataframe>
n_steps = len(x_twitter_train1)// batch_size
n_steps_val = len(x_valid)// batch_size
model.fit(
 twitter_dataset_ssl1,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
validation_steps=n_steps_val,
epochs=2 
)
 test_preds2 = model.predict(test_dataset, verbose=1 )
test_csv = pd.DataFrame()
test_csv['PassengerId'] = test_dataframe['PassengerId'] 
<prepare_output>
n_steps = len(x_twitter_train2)// batch_size
n_steps_val = len(x_valid)// batch_size
model.fit(
 twitter_dataset_ssl2,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
validation_steps=n_steps_val,
epochs=2 
)
 test_preds3 = model.predict(test_dataset, verbose=1 )
test_dataframe = preprocessing_data(test_dataframe)
 test_dataframe = test_dataframe[minimal_features]
test_dataframe.head(10)
<predict_on_test>
test_preds =(0.92)*test_preds1 +(0.05)*test_preds2 +(0.03)*test_preds3
test_preds = test_preds.argmax(axis=1)
 submission['prediction'] = test_preds
<load_from_csv><EOS> 
submission.to_csv('submission.csv', index=False)
 submission.head()