Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<drop_column>
!pip install transformers==3.0.2
!pip install nlp
df = df.drop(["Name","Ticket"], axis=1 )<filter>
mnli = load_dataset(path='glue', name='mnli')
missing_columns = df.columns[df.isnull().any() ]
df.isnull().sum(axis=0 ).loc[missing_columns]<drop_column>
xnli = pd.read_csv('.. /input/xnli-organized/xnli_df.csv')
 xnli = xnli.rename(columns = {'Unnamed: 0': 'lang_abv', '0' : 'premise', '1': 'hypothesis', '0.1': 'label' })
 xnli.head()
df = df.drop(["Cabin"], axis=1 )<load_from_csv>
train_df = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 print('Traning Data, the size of the dataset is: {}
'.format(train_df.shape))
test_df = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv' )
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 X_test = test_data.copy()
X_test = X_test.drop(["Name","Ticket","Cabin","PassengerId"], axis=1)
 X_test.head()<categorify>
train_df = pd.concat([train_df, xnli, mnli_df])
 train_df = train_df[train_df['label'] != -1]
mnli_df = None
snli_df = None
print('the shape of the whole DF to be used is: ' + str(train_df.shape))
y = df.Survived
X = df.drop(['Survived'], axis=1)
 numerical_cols = [cname for cname in X if X[cname].dtype in ['int64', 'float64']]
categorical_cols = [cname for cname in X if X[cname].dtype == "object"]
numerical_transformer = SimpleImputer(strategy='mean')
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore', sparse=False))
])
 preprocessor_temp = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ])
 preprocessor = Pipeline(steps=[ 
('preprocessor_temp', preprocessor_temp), 
('scaler', StandardScaler())])
 preprocessor.fit(X )<filter>
train_df = train_df[train_df.duplicated() == False]
print('the shape of the whole DF to be used is: ' + str(train_df.shape))
X.iloc[0,:]<normalization>
np.random.seed(123)
 max_len = 50
Bert_model = "bert-large-uncased"
Bert_tokenizer = BertTokenizer.from_pretrained(Bert_model)
 def tokeniZer(dataset,tokenizer):
encoded_list = []
type_id_list = np.zeros(( dataset.shape[0], max_len))
mask_list = np.zeros(( dataset.shape[0], max_len))
for i in range(dataset.shape[0]):
datapoint = '[CLS] ' + dataset['premise'][i] + ' [SEP]' + dataset['hypothesis'][i] + ' [SEP]'
datapoint = tokenizer.tokenize(datapoint)
 datapoint = tokenizer.convert_tokens_to_ids(datapoint)
 encoded_list.append(datapoint)
 encoded_list = pad_sequences(encoded_list, maxlen = max_len, padding = 'post')
 for i in range(encoded_list.shape[0]):
flag = 0
a = encoded_list[i]
for j in range(len(a)) :
if flag == 0:
type_id_list[i,j] = 0
else:
type_id_list[i,j] = 1
if encoded_list[i,j] == 102:
flag = 1
if encoded_list[i,j] == 0:
mask_list[i,j] = 0
else:
mask_list[i,j] = 1
return encoded_list,mask_list,type_id_list
dim_trans = preprocessor.transform(X ).shape[1]
preprocessor.transform(X)[0,:]<train_on_grid>
def mish(x):
return x*tanh(softplus(x))
get_custom_objects() ["mish"] = Activation(mish )
params_knn = {"kNN__n_neighbors": np.arange(3,10), "kNN__weights":["uniform", "distance"]}
kNN = KNeighborsClassifier()
pipe = Pipeline(steps=[('preprocessor', preprocessor),('kNN', kNN)])
 knnpipe = GridSearchCV(pipe, params_knn, n_jobs=-1)
 knnpipe.fit(X, y)
 print("Best parameter(CV score=%0.3f):" % knnpipe.best_score_)
 print(knnpipe.best_params_)
 print()
print("Accuracy training set %0.3f" % accuracy_score(y, knnpipe.predict(X)) )<train_on_grid>
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )
params_logit = {"logit__penalty": ["l1","l2"], "logit__C":np.arange(0.5,1.5,0.1)}
logit = LogisticRegression()
pipe = Pipeline(steps=[('preprocessor', preprocessor),('logit', logit)])
 logitpipe = GridSearchCV(pipe, params_logit, n_jobs=-1)
 logitpipe.fit(X, y)
 print("Best parameter(CV score=%0.3f):" % logitpipe.best_score_)
 print(logitpipe.best_params_)
 print()
print("Accuracy training set %0.3f" % accuracy_score(y, logitpipe.predict(X)) )<train_on_grid>
def create_BERT(random_seed):
tf.random.set_seed(random_seed)
 with tpu_strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(Bert_model)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_layer")
 input_masks = Input(shape =(max_len,), dtype = tf.int32, name = 'input_mask')
 input_type_id = Input(shape =(max_len,), dtype = tf.int32, name = 'input_type_id')
 sequence_output = transformer_encoder([input_ids, input_masks, input_type_id])[0]
cls_token = sequence_output[:, 0, :]
output_layer = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=[input_ids, input_masks, input_type_id], outputs = output_layer)
 model.summary()
model.compile(Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model
params_svm = {"svm__kernel": ["linear","poly","rbf"], "svm__C":np.arange(0.5,1.5,0.1)}
svm = SVC()
pipe = Pipeline(steps=[('preprocessor', preprocessor),('svm', svm)])
 svmpipe = GridSearchCV(pipe, params_svm, n_jobs=-1)
 svmpipe.fit(X, y)
 print("Best parameter(CV score=%0.3f):" % svmpipe.best_score_)
 print(svmpipe.best_params_)
 print()
print("Accuracy training set %0.3f" % accuracy_score(y, svmpipe.predict(X)) )<train_model>
callbacks = [tf.keras.callbacks.EarlyStopping(patience = 2, monitor = 'val_loss', \
restore_best_weights = True, mode = 'min')]
shuffled_data = shuffle(train_df ).reset_index(drop = True)
 train_df = None
batch_size = 128
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state = 1)
 preprocessor.fit(X_train)
 X_train = preprocessor.transform(X_train)
 X_valid = preprocessor.transform(X_valid)
 network = Sequential([
Dense(16, activation='relu', input_shape=(dim_trans,)) ,
Dense(16, activation='relu'),
Dense(1, activation='sigmoid'),
])
 network.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
 es = EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=10)
 mc = ModelCheckpoint('best_model.h5', monitor='val_accuracy', mode='max', verbose=1, save_best_only=True)
 history = network.fit(X_train,
y_train,
epochs=1000,
verbose=1,
validation_data=(X_valid,y_valid),
callbacks=[es, mc])
 saved_model = load_model('best_model.h5')
 print("Accuracy training set %0.3f" % accuracy_score(y, saved_model.predict(preprocessor.transform(X)) >0.5))<train_model>
XLM_model = "jplu/tf-xlm-roberta-large"
xlm_tokenizer = AutoTokenizer.from_pretrained(XLM_model)
 X_train_ids, X_train_masks, _ = tokeniZer(shuffled_data,xlm_tokenizer )
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state = 1)
 preprocessor.fit(X_train)
 X_train = preprocessor.transform(X_train)
 X_valid = preprocessor.transform(X_valid)
 xgb = XGBClassifier(n_estimators=1000, learning_rate=0.05, n_jobs=-1)
 xgb.fit(X_train, y_train,
early_stopping_rounds=5,
eval_set=[(X_valid, y_valid)],
verbose=True)
 print("Accuracy training set %0.3f" % accuracy_score(y, xgb.predict(preprocessor.transform(X),ntree_limit=xgb.best_ntree_limit)) )<save_to_csv>
def create_xlm(transformer_layer, random_seed, learning_rate = 1e-5):
tf.keras.backend.clear_session()
tf.random.set_seed(random_seed)
 with tpu_strategy.scope() :
input_ids = Input(shape =(max_len,), dtype = tf.int32)
 input_masks = Input(shape =(max_len,), dtype = tf.int32)
 roberta = TFAutoModel.from_pretrained(transformer_layer)
 roberta = roberta([input_ids, input_masks])[0]
out = GlobalAveragePooling1D()(roberta)
 out = Dense(3, activation = 'softmax' )(out)
 model = Model(inputs = [input_ids, input_masks], outputs = out)
 model.compile(
 optimizer = Adam(lr = learning_rate),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 model.summary()
return model
Xlm = create_xlm(XLM_model ,123443334, 1e-5 )
predictions = xgb.predict(preprocessor.transform(X_test),ntree_limit=xgb.best_ntree_limit)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
history_xlm = Xlm.fit([X_train_ids, X_train_masks], shuffled_data['label'],
batch_size = batch_size,
validation_split = 0.2,
epochs = 39, callbacks = callbacks )
<load_from_csv><EOS> 
input_ids_test_xml, input_masks_test_xml, _ = tokeniZer(test_df, xlm_tokenizer)
 predictions_xlm = Xlm.predict([input_ids_test_xml, input_masks_test_xml])
 predictions = predictions_xlm
final = np.argmax(predictions, axis = 1)
 submission = pd.DataFrame()
submission['id'] = test_df['id']
submission['prediction'] = final.astype(np.int32)
 submission.to_csv('submission.csv', index = False )