Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<split>
plt.style.use('fivethirtyeight')
 warnings.filterwarnings('ignore')
mask =(train[target] == 1)
 t_result = []
for num in numerical:
t_stat, p_val = ttest_ind(( train[mask])[num], 
(train[~mask])[num],
equal_var = True,
nan_policy = 'omit')
 t_result.append({
'group_1' : 'Survived',
'group_2' : 'Died',
'variable' : num,
't_stat' : t_stat,
'p_value' : p_val
})
 t_result = pd.DataFrame(t_result )<feature_engineering>
df_train=pd.read_csv(os.path.join(path,"train.csv"))
df_test=pd.read_csv(os.path.join(path,"test.csv"))
def get_titles(data):
title_re = re.compile(r'(?:^.+),(\w+)')
 titles = []
for name in data['Name']:
titles.append(title_re.findall(name)[0])
 data['Title'] = titles
for i, title in enumerate(data['Title']):
if title in ['Miss', 'Ms', 'Mlle', 'Mrs', 'Mme']:
cat = 'Mrs_Ms'
elif title in ['Mr', 'Don']:
cat = 'Mr'
elif title in ['Master']:
cat = 'Master'
else:
cat = 'Honorable'
data.loc[i, 'Title'] = cat<create_dataframe>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
chi2_result = []
for cat in categorical:
crosstab = pd.crosstab(train[cat], train[target])
 chi2_stat, p_val, dof, ex = chi2_contingency(crosstab)
 chi2_result.append({
'var_1' : cat,
'var_2' : target,
'chi2' : chi2_stat,
'dof' : dof,
'p_value' : p_val
})
 chi2_result = pd.DataFrame(chi2_result )<count_missing_values>
MODEL = 'jplu/tf-xlm-roberta-large'
EPOCHS = 10
MAX_LEN = 96
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE
tokenizer = AutoTokenizer.from_pretrained(MODEL )
train.isnull().sum()<count_missing_values>
def quick_encode(df,maxlen=100):
values = df[['premise','hypothesis']].values.tolist()
tokens=tokenizer.batch_encode_plus(values,max_length=maxlen,pad_to_max_length=True)
 return np.array(tokens['input_ids'])
 x_train = quick_encode(df_train)
 x_test = quick_encode(df_test)
 y_train = df_train.label.values
train.isnull().sum()<prepare_x_and_y>
def create_dist_dataset(X, y,val,batch_size= BATCH_SIZE):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(len(X))
if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(AUTO)
 else:
dataset = dataset.batch(batch_size ).prefetch(AUTO)
 return dataset
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test)) 
.batch(BATCH_SIZE)
)
X = train.copy()
y = X.pop(target )<prepare_x_and_y>
def build_model(transformer,max_len):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
cls_token = Dropout(0.4 )(cls_token)
 cls_token = Dense(32,activation='relu' )(cls_token)
 cls_token = Dropout(0.4 )(cls_token)
 out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model
median_imputer = SimpleImputer(strategy = 'median')
 mode_imputer = SimpleImputer(strategy = 'most_frequent')
 missing_transformer = ColumnTransformer([('num', median_imputer, numerical), 
('cat', mode_imputer, categorical)])
 missing_transformer.fit(X)
 X_imp = pd.DataFrame(missing_transformer.transform(X))
X_imp.columns = numerical + categorical
X.drop(numerical + categorical, axis = 1, inplace = True)
 X = pd.concat([X, X_imp], axis = 1 )<categorify>
pred_test=np.zeros(( df_test.shape[0],3))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 val_score=[]
history=[]
for fold,(train_ind,valid_ind)in enumerate(skf.split(x_train,y_train)) :
if fold < 4:
print("fold",fold+1)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 train_data = create_dist_dataset(x_train[train_ind],y_train[train_ind],val=False)
 valid_data = create_dist_dataset(x_train[valid_ind],y_train[valid_ind],val=True)
 Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"roberta_base.h5", monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min')
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 n_steps = len(train_ind)//BATCH_SIZE
print("training model {} ".format(fold+1))
train_history = model.fit(
 train_data,
steps_per_epoch=n_steps,
validation_data=valid_data,
epochs=EPOCHS,callbacks=[Checkpoint],verbose=1)
 print("Loading model...")
 model.load_weights(f"roberta_base.h5")
 print("fold {} validation accuracy {}".format(fold+1,np.mean(train_history.history['val_accuracy'])))
 print("fold {} validation loss {}".format(fold+1,np.mean(train_history.history['val_loss'])))
 history.append(train_history)
 val_score.append(np.mean(train_history.history['val_accuracy']))
print('predict on test.... ')
 preds=model.predict(test_dataset,verbose=1)
 pred_test+=preds/4
ordinal_encoder = OrdinalEncoder()
X_enc = pd.DataFrame(ordinal_encoder.fit_transform(X[categorical]))
X_enc.columns = categorical
X.drop(categorical, axis = 1, inplace = True)
 X = pd.concat([X, X_enc], axis = 1)
 X = pd.get_dummies(X, columns = categorical, dtype = np.int64, drop_first = True )<compute_train_metric>
submission = pd.read_csv(os.path.join(path,'sample_submission.csv'))
submission['prediction'] = np.argmax(pred_test,axis=1)
 submission.head()
<split><EOS> 
submission.to_csv('submission.csv',index=False )