Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<import_modules>
plt.style.use('fivethirtyeight')
 warnings.filterwarnings('ignore')
sns.__version__<load_from_csv>
df_train=pd.read_csv(os.path.join(path,"train.csv"))
df_test=pd.read_csv(os.path.join(path,"test.csv"))
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
<groupby>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
df_train.groupby(['Survived','Sex'])['Survived'].count()<count_values>
MODEL = 'jplu/tf-xlm-roberta-large'
EPOCHS = 10
MAX_LEN = 96
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE
tokenizer = AutoTokenizer.from_pretrained(MODEL )
print("% of women survived: " , df_train[df_train.Sex == 'female'].Survived.sum() /df_train[df_train.Sex == 'female'].Survived.count())
 print("% of men survived: " , df_train[df_train.Sex == 'male'].Survived.sum() /df_train[df_train.Sex == 'male'].Survived.count() )<save_to_csv>
def quick_encode(df,maxlen=100):
values = df[['premise','hypothesis']].values.tolist()
tokens=tokenizer.batch_encode_plus(values,max_length=maxlen,pad_to_max_length=True)
 return np.array(tokens['input_ids'])
 x_train = quick_encode(df_train)
 x_test = quick_encode(df_test)
 y_train = df_train.label.values
<count_values>
def create_dist_dataset(X, y,val,batch_size= BATCH_SIZE):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(len(X))
if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(AUTO)
 else:
dataset = dataset.batch(batch_size ).prefetch(AUTO)
 return dataset
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test)) 
.batch(BATCH_SIZE)
)
print("% of survivals in")
 print("Pclass=1 : ", df_train.Survived[df_train.Pclass == 1].sum() /df_train[df_train.Pclass == 1].Survived.count())
 print("Pclass=2 : ", df_train.Survived[df_train.Pclass == 2].sum() /df_train[df_train.Pclass == 2].Survived.count())
 print("Pclass=3 : ", df_train.Survived[df_train.Pclass == 3].sum() /df_train[df_train.Pclass == 3].Survived.count() )<save_to_csv>
def build_model(transformer,max_len):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
cls_token = Dropout(0.2 )(cls_token)
 cls_token = Dense(32,activation='relu' )(cls_token)
 out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model
df_test['Survived'] = 0
df_test.loc[(df_test.Sex == 'female'), 'Survived'] = 1
df_test.loc[(df_test.Sex == 'female')&(df_test.Pclass == 3)&(df_test.Embarked == 'S'), 'Survived'] = 0 
<feature_engineering>
def build_lrfn(lr_start=0.00001, lr_max=0.00003,
lr_min=0.000001, lr_rampup_epochs=3,
lr_sustain_epochs=0, lr_exp_decay=.6):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
for df in [df_train, df_test]:
df['Age_bin']=np.nan
for i in range(8,0,-1):
df.loc[ df['Age'] <= i*10, 'Age_bin'] = i<feature_engineering>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1 )
df_test.loc[(df_test.Sex == 'male')&(df_test.Pclass == 1)&(df_test.Age_bin == 1), 'Survived'] = 1
df_test.loc[(df_test.Sex == 'male')&(df_test.Pclass == 2)&(df_test.Age_bin == 1), 'Survived'] = 1<feature_engineering>
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 val_score=[]
history=[]
for fold,(train_ind,valid_ind)in enumerate(skf.split(x_train,y_train)) :
if fold < 4:
print("fold",fold+1)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 train_data = create_dist_dataset(x_train[train_ind],y_train[train_ind],val=False)
 valid_data = create_dist_dataset(x_train[valid_ind],y_train[valid_ind],val=True)
 Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"roberta_base.h5", monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min')
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 n_steps = len(train_ind)//BATCH_SIZE
print("training model {} ".format(fold+1))
train_history = model.fit(
 train_data,
steps_per_epoch=n_steps,
validation_data=valid_data,
epochs=EPOCHS,callbacks=[Checkpoint],verbose=1)
 print("Loading model...")
 model.load_weights(f"roberta_base.h5")
 print("fold {} validation accuracy {}".format(fold+1,np.mean(train_history.history['val_accuracy'])))
 print("fold {} validation loss {}".format(fold+1,np.mean(train_history.history['val_loss'])))
 val_score.append(train_history.history['val_accuracy'])
 history.append(train_history)
 val_score.append(np.mean(train_history.history['val_accuracy']))
print('predict on test.... ')
 preds=model.predict(test_dataset,verbose=1)
 pred_test+=preds/4
print("Mean Validation accuracy : ",np.mean(val_score))
df_test.loc[(df_test.Sex == 'female')&(df_test.SibSp > 7), 'Survived'] = 0<feature_engineering>
submission = pd.read_csv(os.path.join(path,'sample_submission.csv'))
submission['prediction'] = np.argmax(pred_test,axis=1)
 submission.head()
<feature_engineering><EOS> 
submission.to_csv('submission.csv',index=False )