Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<import_modules>
pip install pyspellchecker
from sklearn.linear_model import Lasso<import_modules>
import pandas as pd
from spellchecker import SpellChecker
import regex as re
from sklearn.linear_model import Lasso<train_on_grid>
warnings.filterwarnings("ignore" )
ls = LogisticRegression(C=0.000002)
 ls.fit(data.drop('Survived', axis=1), data.Survived)
 ls.coef_<import_modules>
train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv' )
import numpy as np
import pandas as pd
import tensorflow as tf
from skimage import io, transform
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import accuracy_score<import_modules>
eng = train.loc[train.language == 'English']
from keras.models import Sequential
from keras.layers import Conv2D, MaxPool2D, Dropout, Flatten, Dense, BatchNormalization
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import RMSprop
from keras.callbacks import ReduceLROnPlateau
from keras.datasets import mnist<load_from_csv>
spell = SpellChecker()
spell.word_frequency.load_text_file('.. /input/english-words/words_alpha.txt')
 spell.word_frequency.load_words([''] )
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv")
 print(train_data.shape,
test_data.shape )<concatenate>
eng['premise_misspelled'] = eng.premise.apply(lambda sentence: tuple(spell.unknown(re.split('[!\:;,.\-\% \b\s() "/$0-9]',re.sub('['\[\]]', '', sentence)))) )
( x_train1, y_train1),(x_test1, y_test1)= mnist.load_data()
x_train1 = np.concatenate(( x_test1, x_train1))
y_train1 = np.concatenate(( y_test1, y_train1))
x_train1 = x_train1.reshape(( x_train1.shape[0], 28, 28, 1))
print(x_train1.shape, y_train1.shape )<prepare_x_and_y>
misspelled_df = eng.loc[eng.premise_misspelled !=() ]
list(misspelled_df.premise_misspelled )
x = np.array(train_data.drop(['label'], axis = 1))
y = np.array(train_data['label'])
 test_data = np.array(test_data)
 x = x.reshape(( x.shape[0], 28, 28, 1))
test_data = test_data.reshape(test_data.shape[0], 28, 28, 1)
 x = np.concatenate(( x, x_train1))
y = np.concatenate(( y, y_train1))
x = x/255
test_data = test_data/255
y = to_categorical(y, num_classes = 10)
 print(x.shape, y.shape )<split>
pd.set_option('display.max_colwidth', -1 )
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.10, shuffle = True)
 print(x_train.shape, y_train.shape, x_test.shape, y_test.shape )<choose_model_class>
err1 = misspelled_df.loc[eng.premise.str.contains('\?\?', case = False)]
err2 = misspelled_df.loc[eng.premise.str.contains("\xad", case = False)]
print(len(err1), len(err2))
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(3,3), activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(3,3), activation ='relu'))
model.add(MaxPool2D(( 2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3), activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3), activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3), activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3), activation ='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(128, activation = "relu"))
model.add(Dropout(0.30))
model.add(Dense(10, activation = "softmax"))
optimizer = RMSprop(lr = 0.01, rho = 0.9, epsilon = 1e-08, decay = 0.0)
 model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics = ['accuracy'])
 model.summary()<define_variables>
reduced_1 = pd.concat([misspelled_df, err1, err1] ).drop_duplicates(keep=False)
 reduced_2 = pd.concat([reduced_1, err2, err2] ).drop_duplicates(keep=False )
datagen = ImageDataGenerator(
 rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1,)
 train_batch = datagen.flow(x, y, batch_size = 64)
 val_batch = datagen.flow(x_test, y_test, batch_size = 64 )<choose_model_class>
correction = err1.premise.apply(lambda sentence: re.sub('\?\?', 'e', sentence))
correct_rhone = correction.apply(lambda sentence: re.sub('Rhene', 'Rhone', sentence))
correct_ataturk = correct_rhone.apply(lambda sentence: re.sub('Ataterk', 'Ataturk', sentence))
correct_madrileno = correct_ataturk.apply(lambda sentence: re.sub('Madrileeo', 'Madrileno', sentence))
correct_alacahoyuk = correct_madrileno.apply(lambda sentence: re.sub('Alacaheyek', 'Alacahoyuk', sentence))
correct_alcudia = correct_alacahoyuk.apply(lambda sentence: re.sub('Alcedia', 'Alcudia', sentence))
correction1 = correct_alcudia
learning_rate_reduction = ReduceLROnPlateau(monitor = 'val_loss',
patience = 3,
verbose = 1,
factor = 0.1,
min_lr = 0.00001 )<train_model>
for i in correction1.index:
eng.loc[eng.index == i, 'premise'] = correction1.loc[correction1.index == i]
history = model.fit_generator(generator = train_batch,
epochs = 100,
steps_per_epoch = len(train_batch),
validation_data = val_batch,
validation_steps = len(val_batch),
verbose = 1,
callbacks = [learning_rate_reduction] )<save_to_csv>
err2 = eng.loc[eng.premise.str.contains("\?\xad", case = False)]
err2
res = model.predict_classes(test_data, batch_size = 64)
 result = pd.Series(res, name = 'Label')
 submission = pd.concat([pd.Series(range(1, 28001), name = 'ImageId'), result], axis = 1)
 submission.to_csv('Submission.csv', index = False )<compute_test_metric>
correction2 = err2.premise.apply(lambda sentence: re.sub('\?\xad', '', sentence))
for i in correction2.index:
eng.loc[eng.index == i, 'premise'] = correction2.loc[correction2.index == i]
res = model.evaluate(x, y, batch_size = 1024)
 print(res[1]*100 )<set_options>
eng['hypothesis_misspelled'] = eng.hypothesis.apply(lambda sentence: tuple(spell.unknown(re.split('[\?!\:;,.\-\% \b\s() "/$0-9]',re.sub('['\[\]]', '', sentence)))))
 misspelled_hyp_df = eng.loc[eng.hypothesis_misspelled !=() ]
list(misspelled_hyp_df.hypothesis_misspelled )
%matplotlib inline<import_modules>
eng.loc[eng.hypothesis.str.contains('Ile de R', case = False)]
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import RMSprop
from keras.callbacks import ReduceLROnPlateau
from sklearn.model_selection import train_test_split<load_from_csv>
eng.loc[eng.hypothesis == 'Ile de R is no longer part of the attraction.', 'hypothesis'] = 'Ile de Re is no longer part of the attraction.'
eng.loc[eng.hypothesis == 'Ile de R.', 'hypothesis'] = 'Ile de Re.'
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
square_brackets = eng.loc[eng.premise.str.contains('[\[\]]', case = False)]
for i in square_brackets.index:
eng.loc[eng.index == i, 'premise'] = re.sub('[\[\]]', '', str(eng.loc[eng.index == i].premise.values[0]))
ampersands = eng.loc[eng.premise.str.contains('\&amp', case = False)]
for i in ampersands.index:
eng.loc[eng.index == i, 'premise'] = re.sub('\&amp', ' and ', str(eng.loc[eng.index == i].premise.values[0]))
X_train=train.drop('label',axis=1)
 y_train=train['label']<count_values>
eng.drop(columns=['premise_misspelled', 'hypothesis_misspelled'])
 for i in eng.index:
train.loc[train.index == i] = eng.loc[eng.index == i]
sns.countplot(x=y_train)
 y_train.value_counts()<count_missing_values>
eng_test = test.loc[test.language == 'English']
X_train.isnull().any().value_counts()<count_missing_values>
err1 = eng_test.loc[eng_test.premise.str.contains('\?\?', case = False)]
correction = err1.premise.apply(lambda sentence: re.sub('\?\?', 'e', sentence))
correct_alacahoyuk = correction.apply(lambda sentence: re.sub('Alacaheyek', 'Alacahoyuk', sentence))
correct_madrileno = correct_alacahoyuk.apply(lambda sentence: re.sub('Madrileeo', 'Madrileno', sentence))
correct_alcudia = correct_madrileno.apply(lambda sentence: re.sub('Alcedia', 'Alcudia', sentence))
correct_ataturk = correct_alcudia.apply(lambda sentence: re.sub('Ataterk', 'Ataturk', sentence))
for i in correct_ataturk.index:
eng_test.loc[eng_test.index == i, 'premise'] = correct_ataturk.loc[correct_ataturk.index == i]
test.isnull().any().value_counts()<normalization>
err2 = eng_test.loc[eng_test.premise.str.contains('\?\xad', case = False)]
correction = err2.premise.apply(lambda sentence: re.sub('\?\xad', '', sentence))
for i in correction.index:
eng_test.loc[eng_test.index == i, 'premise'] = correction.loc[correction.index == i]
X_train = X_train / 255.0
test = test / 255.0<categorify>
square_brackets = eng_test.loc[eng_test.premise.str.contains('[\[\]]', case = False)]
for i in square_brackets.index:
eng_test.loc[eng_test.index == i, 'premise'] = re.sub('[\[\]]', '', str(eng_test.loc[eng_test.index == i].premise.values[0]))
X_train = X_train.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1 )<categorify>
ampersands = eng_test.loc[eng_test.premise.str.contains('\&amp', case = False)]
for i in ampersands.index:
eng_test.loc[eng_test.index == i, 'premise'] = re.sub('\&amp', ' and ', str(eng_test.loc[eng_test.index == i].premise.values[0]))
y_train=to_categorical(y_train,num_classes=10 )<split>
for i in eng_test.index:
test.loc[test.index == i] = eng_test.loc[eng_test.index == i]
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=42 )<choose_model_class>
train.to_csv('train_cleaned.csv',index=False)
 test.to_csv('test_cleaned.csv',index=False )
model=Sequential()
model.add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
!pip install git+https://github.com/ssut/py-googletrans.git
optimizer = RMSprop()
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1,
factor=0.5, min_lr=0.00001 )<init_hyperparams>
from googletrans import Translator
from dask import bag, diagnostics
import numpy as np
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<define_variables>
def translate(words, dest):
dest_choices = ['zh-cn',
'ar',
'fr',
'sw',
'ur',
'vi',
'ru',
'hi',
'el',
'th',
'es',
'de',
'tr',
'bg'
]
if not dest:
dest = np.random.choice(dest_choices)
 translator = Translator()
decoded = translator.translate(words, dest=dest ).text
return decoded
def trans_parallel(df, dest):
premise_bag = bag.from_sequence(df.premise.tolist() ).map(translate, dest)
 hypo_bag = bag.from_sequence(df.hypothesis.tolist() ).map(translate, dest)
 with diagnostics.ProgressBar() :
premises = premise_bag.compute()
hypos = hypo_bag.compute()
df[['premise', 'hypothesis']] = list(zip(premises, hypos))
return df
eng_trans = train.loc[train.lang_abv == "en"].copy() \ 
.pipe(trans_parallel, dest=None)
 non_eng_trans = train.loc[train.lang_abv != "en"].copy() \ 
.pipe(trans_parallel, dest='en')
 eng_trans[['lang_abv', 'language']] = [['mx', 'Mixed']]*len(eng)
 non_eng_trans[['lang_abv', 'language']] = [['en', 'English']]*len(non_eng_trans)
 train = train.append([eng_trans, non_eng_trans])
 train.reset_index
train.shape
batch_size=30
nb_train=len(X_train)
 nb_val=len(X_val )<train_model>
!pip install git+https://github.com/makcedward/nlpaug
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = 20,verbose = 1,
steps_per_epoch=nb_train // batch_size,
validation_data =(X_val,y_val),
validation_steps=nb_val//batch_size
,callbacks=[learning_rate_reduction])
 model.save('Digit_recognizer.h5')
 model.save_weights('Weight_file_for_Digit_recognizer.h5' )<predict_on_test>
import nlpaug.augmenter.word as naw
import nlpaug.flow as nafc
from nlpaug.util import Action
prob_pred= model.predict(test)
 results = np.argmax(prob_pred,axis = 1 )<save_to_csv>
text = train.premise.values[0]
model = 'distilbert-base-uncased'
ins_aug = naw.ContextualWordEmbsAug(
 model_path=model, action="insert")
 print("Original:")
 print(text)
 print("Augmented Text:")
 for i in range(10):
augmented_text = ins_aug.augment(text)
 print(augmented_text )
results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("my_submission.csv",index=False )<import_modules>
syn_aug = naw.SynonymAug(aug_src='wordnet')
 augmented_text = syn_aug.augment(text)
 print("Original:")
 print(text)
 print("Augmented Text:")
 for i in range(10):
augmented_text = syn_aug.augment(text)
 print(augmented_text )
import numpy as np
import pandas as pd
from scipy.stats import skew
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.linear_model import Ridge
from sklearn.linear_model import LinearRegression
from mlxtend.regressor import StackingRegressor
from sklearn.metrics import mean_squared_error
from math import sqrt<load_from_csv>
def insert_augment(words, model):
ins_aug = naw.ContextualWordEmbsAug(
 model_path=model, action="insert")
 augmented_text = ins_aug.augment(words)
 return augmented_text
def ins_aug_parallel(df, model):
premise_bag = bag.from_sequence(df.premise.tolist() ).map(insert_augment, model)
 hypo_bag = bag.from_sequence(df.hypothesis.tolist() ).map(insert_augment, model)
 with diagnostics.ProgressBar() :
premises = premise_bag.compute()
hypos = hypo_bag.compute()
df[['premise', 'hypothesis']] = list(zip(premises, hypos))
return df
eng_ins_aug = train.loc[train.lang_abv == "en"].copy() \ 
.pipe(ins_aug_parallel, model='distilbert-base-uncased')
 train = train.append([eng_ins_aug])
 train.to_csv('train_cleaned_ins.csv',index=False)
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<drop_column>
def synonym_augment(words):
syn_aug = naw.SynonymAug(
 aug_src = 'wordnet')
 augmented_text = syn_aug.augment(words)
 return augmented_text
def syn_aug_parallel(df):
premise_bag = bag.from_sequence(df.premise.tolist() ).map(synonym_augment)
 hypo_bag = bag.from_sequence(df.hypothesis.tolist() ).map(synonym_augment)
 with diagnostics.ProgressBar() :
premises = premise_bag.compute()
hypos = hypo_bag.compute()
df[['premise', 'hypothesis']] = list(zip(premises, hypos))
return df
eng_syn_aug = train.loc[train.lang_abv == "en"].copy() \ 
.pipe(syn_aug_parallel)
 train = train.append([eng_syn_aug])
 train.to_csv('train_cleaned_ins_syn.csv',index=False)
df_train = df_train.drop(
 df_train[(df_train['GrLivArea']>4000)&(df_train['SalePrice']<300000)].index )<concatenate>
plt.style.use('fivethirtyeight')
 warnings.filterwarnings('ignore')
all_data = pd.concat(( df_train.loc[:,'MSSubClass':'SaleCondition'],
df_test.loc[:,'MSSubClass':'SaleCondition']))<drop_column>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
MODEL = 'jplu/tf-xlm-roberta-large'
EPOCHS = 8
MAX_LEN = 96
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE
all_data["PoolQC"] = all_data["PoolQC"].fillna("None")
 all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None")
 all_data["Alley"] = all_data["Alley"].fillna("None")
 all_data["Fence"] = all_data["Fence"].fillna("None")
 all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None")
 for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None')
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )<categorify>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0)
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )<data_type_conversions>
def quick_encode(df,maxlen=100):
values = df[['premise','hypothesis']].values.tolist()
tokens=tokenizer.batch_encode_plus(values,max_length=maxlen,pad_to_max_length=True)
 return np.array(tokens['input_ids'])
 x_train = quick_encode(train)
 x_test = quick_encode(test)
 y_train = train.label.values
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<feature_engineering>
def create_dist_dataset(X, y,val,batch_size= BATCH_SIZE):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(len(X))
if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(AUTO)
 else:
dataset = dataset.batch(batch_size ).prefetch(AUTO)
 return dataset
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test)) 
.batch(BATCH_SIZE)
 )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<feature_engineering>
def build_model(transformer,max_len):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model
df_train["SalePrice"] = np.log1p(df_train["SalePrice"] )<feature_engineering>
n_steps = len(x_train)// batch_size
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )<categorify>
pred_test=np.zeros(( test.shape[0],3))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 val_score=[]
history=[]
for fold,(train_ind,valid_ind)in enumerate(skf.split(x_train,y_train)) :
if fold < 4:
print("fold",fold+1)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 train_data = create_dist_dataset(x_train[train_ind],y_train[train_ind],val=False)
 valid_data = create_dist_dataset(x_train[valid_ind],y_train[valid_ind],val=True)
 Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"roberta_base.h5", monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min')
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 n_steps = len(train_ind)//BATCH_SIZE
print("training model {} ".format(fold+1))
train_history = model.fit(
 train_data,
steps_per_epoch=n_steps,
validation_data=valid_data,
epochs=EPOCHS,callbacks=[Checkpoint],verbose=0)
 print("Loading model...")
 model.load_weights(f"roberta_base.h5")
 print("fold {} validation acc {}".format(fold+1,np.mean(train_history.history['val_accuracy'])))
 print("fold {} validation acc {}".format(fold+1,np.mean(train_history.history['val_loss'])))
 history.append(train_history)
 val_score.append(np.mean(train_history.history['val_accuracy']))
print('predict on test.... ')
 preds=model.predict(test_dataset,verbose=1)
 pred_test+=preds/4
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))<categorify>
submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 submission['prediction'] = np.argmax(pred_test,axis=1)
 submission.head()
<prepare_x_and_y><EOS> 
submission.to_csv('submission.csv',index=False )