Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<prepare_x_and_y>
!pip install --quiet googletrans
warnings.filterwarnings('ignore')
 print("Currently using Tensorflow version " + tf.__version__ )
y_train = train.iloc[:, 1].values<drop_column>
SEED = 34
def seed_everything(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything(SEED )
extra_eda_cols = ['SibSp', 'Parch', 'Family_Size', 'Fare_Range', 'Alone']
train = train.drop(extra_eda_cols, axis = 1, inplace = False)
 train.head()<drop_column>
DEVICE = 'TPU'
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}')
 BATCH_SIZE = 16 * REPLICAS
extra_cols = ['PassengerId', 'Name', 'Ticket', 'Fare', 'Cabin']
train = train.drop(extra_cols, axis = 1, inplace = False)
 train.head()<drop_column>
train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 print(f'Train shape: {train.shape}')
 train.head()
x_train = train.drop('Survived', axis = 1, inplace = False)
 print(x_train )<count_missing_values>
print(f"Premise: {train['premise'].values[0]}")
 print(f"Hypothesis: {train['hypothesis'].values[0]}")
 print(f"Label: {train['label'].values[0]}" )
train.isnull().sum()<count_missing_values>
print(f"Premise: {train['premise'].values[1]}")
 print(f"Hypothesis: {train['hypothesis'].values[1]}")
 print(f"Label: {train['label'].values[1]}" )
train.isnull().sum()<categorify>
def back_translate(sequence, PROB = 1):
languages = ['en', 'fr', 'th', 'tr', 'ur', 'ru', 'bg', 'de', 'ar', 'zh-cn', 'hi',
'sw', 'vi', 'es', 'el']
translator = Translator()
org_lang = translator.detect(sequence ).lang
random_lang = np.random.choice([lang for lang in languages if lang is not org_lang])
 if org_lang in languages:
translated = translator.translate(sequence, dest = random_lang ).text
translated_back = translator.translate(translated, dest = org_lang ).text
if np.random.uniform(0, 1)<= PROB:
output_sequence = translated_back
else:
output_sequence = sequence
else:
output_sequence = sequence
return output_sequence
for i in range(5):
output = back_translate('I genuinely have no idea what the output of this sequence of words will be')
 print(output )
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
 imputer.fit(x_train[['Age']])
 x_train[['Age']]= imputer.transform(x_train[['Age']])
 imputers = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
 imputers.fit(x_train[['Embarked']])
 x_train[['Embarked']]= imputers.transform(x_train[['Embarked']] )<count_missing_values>
train_aug = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/translation_aug_train.csv')
 train_aug.head()
x_train.isnull().sum().any()<categorify>
train_twice_aug = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/twice_translated_aug_train.csv')
 train_twice_aug.head()
label_encoder = preprocessing.LabelEncoder()
x_train['Sex']= label_encoder.fit_transform(x_train['Sex'])
 x_train['Embarked']= label_encoder.fit_transform(x_train['Embarked'] )<normalization>
train_vi = pd.read_csv(".. /input/contradictorytranslatedtrain/train_vi.csv")
 train_hi = pd.read_csv(".. /input/contradictorytranslatedtrain/train_hi.csv")
 train_bg = pd.read_csv(".. /input/contradictorytranslatedtrain/train_bg.csv" )
sc = StandardScaler()
x_train = sc.fit_transform(x_train )<count_missing_values>
!pip install --quiet transformers
roberta_base = "jplu/tf-xlm-roberta-base"
roberta_large = 'jplu/tf-xlm-roberta-large'
test.isnull().sum().any()<categorify>
test_bt = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/translation_aug_test.csv')
 test_bt_twice = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/translation_aug_test.csv' )
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
 imputer.fit(test[['Age']])
 test[['Age']]= imputer.transform(test[['Age']])
 imputers = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
 imputers.fit(test[['Embarked']])
 test[['Embarked']]= imputers.transform(test[['Embarked']] )<drop_column>
TOKENIZER = AutoTokenizer.from_pretrained(roberta_large)
 def to_tf_dataset(dataset, max_len, repeat = False, shuffle = False, labeled = True, batch_size = BATCH_SIZE):
dataset_text = dataset[['premise', 'hypothesis']].values.tolist()
dataset_enc = TOKENIZER.batch_encode_plus(dataset_text, pad_to_max_length = True, max_length = max_len)
 if labeled:
tf_dataset = tf.data.Dataset.from_tensor_slices(( dataset_enc['input_ids'], dataset['label']))
else:
tf_dataset = tf.data.Dataset.from_tensor_slices(( dataset_enc['input_ids']))
if repeat: tf_dataset = tf_dataset.repeat()
if shuffle:
tf_dataset = tf_dataset.shuffle(2048)
 opt = tf.data.Options()
opt.experimental_deterministic = False
tf_dataset = tf_dataset.with_options(opt)
 tf_dataset = tf_dataset.batch(batch_size)
 tf_dataset = tf_dataset.prefetch(AUTO)
 return tf_dataset
extra_cols_test = ['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Fare', 'Cabin']
test = test.drop(extra_cols_test, axis = 1, inplace = False)
 test.head()<categorify>
LR_START = 1e-6
LR_MAX = 1e-6 * 8
LR_MIN = 1e-6
LR_RAMPUP_EPOCHS = 2
LR_SUSTAIN_EPOCHS = 0
LR_DECAY =.8
def lrfn_step(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr = LR_MAX * LR_DECAY**(( epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//2)
 return lr
def lrfn_smooth(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback_step = tf.keras.callbacks.LearningRateScheduler(lrfn_step, verbose = True)
 lr_callback_smooth = tf.keras.callbacks.LearningRateScheduler(lrfn_smooth, verbose = True)
 rng = [i for i in range(25)]
y1 = [lrfn_step(x)for x in rng]
y2 = [lrfn_smooth(x)for x in rng]
fix, ax = plt.subplots(1,2, figsize =(15, 5))
ax[0].plot(rng, y1)
 ax[1].plot(rng, y2)
 plt.tight_layout()
print("Learning rate schedule for step schedule: {:.3g} to {:.3g} to {:.3g}".format(y1[0], max(y1), y1[-1]))
print("Learning rate schedule for smooth schedule: {:.3g} to {:.3g} to {:.3g}".format(y2[0], max(y2), y2[-1]))
label_encoder = preprocessing.LabelEncoder()
test['Sex']= label_encoder.fit_transform(test['Sex'])
 test['Embarked']= label_encoder.fit_transform(test['Embarked'] )<normalization>
def build_model(transformer_layer, max_len, learning_rate):
with strategy.scope() :
input_ids = tf.keras.Input(shape =(max_len,), dtype = tf.int32)
 roberta = TFAutoModel.from_pretrained(transformer_layer)
 roberta = roberta(input_ids)[0]
out = roberta[:, 0, :]
out = tf.keras.layers.Dense(3, activation = 'softmax' )(out)
 model = tf.keras.Model(inputs = input_ids, outputs = out)
 model.compile(
 optimizer = tf.keras.optimizers.Adam(lr = learning_rate),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 return model
sc_x = StandardScaler()
test = sc_x.fit_transform(test )<train_model>
LR_RATE = 1e-5
EPOCHS = 10
FOLDS = 4
MAX_LEN = 85
STEPS_PER_EPOCH = len(train)// BATCH_SIZE
TTA = 3
VERBOSE = 2
preds = np.zeros(( len(test), 3))
preds_tta = np.zeros(( len(test), 3))
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 for fold,(train_index,val_index)in enumerate(skf.split(train)) :
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 K.clear_session()
model = build_model(roberta_large, max_len = MAX_LEN, learning_rate = LR_RATE)
 sv = tf.keras.callbacks.ModelCheckpoint(f'fold-{fold}.h5', monitor = 'val_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 train_ds = to_tf_dataset(train.loc[train_index], labeled = True, shuffle = True, repeat = True, max_len = MAX_LEN)
 val_ds = to_tf_dataset(train.loc[val_index], labeled = True, shuffle = False, repeat = False, max_len = MAX_LEN)
 print('
print('Training...'); print('')
 history = model.fit(train_ds, validation_data = val_ds, callbacks = [sv],
epochs = EPOCHS, steps_per_epoch = STEPS_PER_EPOCH,
verbose = VERBOSE); print('')
 print('Loading best model...')
 model.load_weights(f'fold-{fold}.h5')
 print('Predicting validation with TTA...')
 val_df = train.loc[val_index]
val_df_bt = train_aug.loc[val_index]
val_df_bt_twice = train_twice_aug.loc[val_index]
val_tta1 = to_tf_dataset(val_df, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 val_tta2 = to_tf_dataset(val_df_bt, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 val_tta3 = to_tf_dataset(val_df_bt_twice, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 val_pred1 = model.predict(val_tta1, verbose = VERBOSE)
 val_pred2 = model.predict(val_tta2, verbose = VERBOSE)
 val_pred3 = model.predict(val_tta3, verbose = VERBOSE)
 val_preds =(val_pred1 + val_pred2 + val_pred3)/ TTA
print(f"Without TTA: {accuracy_score(val_pred1.argmax(axis = 1), val_df['label'])}")
 print(f"With TTA: {accuracy_score(val_preds.argmax(axis = 1), val_df['label'])}")
 print('')
 print('Predicting OOF with TTA...')
 test_tta1 = to_tf_dataset(test, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 test_tta2 = to_tf_dataset(test_bt, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 test_tta3 = to_tf_dataset(test_bt_twice, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 pred1 = model.predict(test_tta1, verbose = VERBOSE)
 pred2 = model.predict(test_tta2, verbose = VERBOSE)
 pred3 = model.predict(test_tta3, verbose = VERBOSE)
 preds_tta +=(pred1 + pred2 + pred3)/ TTA / FOLDS
preds += pred1 / FOLDS
os.remove(f"/kaggle/working/fold-{fold}.h5")
 del model ; z = gc.collect()
classifier = LogisticRegression(random_state = 0)
 classifier.fit(x_train, y_train )<predict_on_test>
USE_TTA = False
y_pred = classifier.predict(test)
 acc_Tree = cross_val_score(classifier, x_train, y_train, cv=10, scoring='accuracy' ).mean()
acc_Tree<train_model>
if USE_TTA:
submission = pd.DataFrame()
submission['id'] = test['id']
submission['prediction'] = preds_tta.argmax(axis = 1)
 else:
submission = pd.DataFrame()
submission['id'] = test['id']
submission['prediction'] = preds.argmax(axis = 1)
 submission.head()
<predict_on_test><EOS> 
submission.to_csv('submission.csv', index = False)
 print('Submission saved' )