Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<feature_engineering>
!pip install -q transformers==3.0.2
!pip install -q nlp
data.loc[ data['FamilySize'] == 1, 'FSize'] = 'Single family'
data.loc[(data['FamilySize'] > 1)&(data['FamilySize'] <= 5), 'FSize'] = 'Small Family'
data.loc[(data['FamilySize'] > 5), 'FSize'] = ' Extended Family'<categorify>
np.random.seed(12345 )
le = LabelEncoder()
data['Sex'] = le.fit_transform(data['Sex'])
 data['Embarked'] = le.fit_transform(data['Embarked'])
 data['Deck'] = le.fit_transform(data['Deck'])
 data['FSize'] = le.fit_transform(data['FSize'] )<count_values>
pd.set_option('display.max_colwidth', 100)
 os.environ["WANDB_API_KEY"] = "0"
sns.set_context("talk", font_scale=1.05 )
data.dtypes.value_counts()<drop_column>
train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv" )
data = data.drop(['Name', 'Ticket','Cabin',], axis = 1 )<feature_engineering>
print("Any duplicate rows(train or test): ",
max(train['id'].nunique() != train.shape[0], test['id'].nunique() != test.shape[0]))
skewed = skew_cols.index.tolist()
data[skewed] = data[skewed].apply(np.log1p )<import_modules>
print("Train and test datasets have unique, non-overlapping ids: ",
pd.merge(train['id'], test['id'], on = 'id', how = 'inner' ).shape[0] == 0 )
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.ensemble import VotingClassifier 
<count_values>
print("% distribution by language - training dataset")
 train['language'].value_counts(normalize=True)* 100 .
train_new.dtypes.value_counts()<count_values>
print("% distribution by language - test dataset")
 test['language'].value_counts(normalize=True)* 100 .
test_new.dtypes.value_counts()<choose_model_class>
train['label_str'] = train['label'].map({0 : "entailment", 1 : "neutral", 2 : "contradiction"} )
n_folds = 5
kf = KFold(n_folds, shuffle = True, random_state = 42 ).get_n_splits(train_new)
<compute_train_metric>
def _get_word_count(snt):
return len(str(snt ).split() )
y_train = train.Survived
n_folds = 5
def f1_score(model):
kf = KFold(n_folds, shuffle = True, random_state = 42 ).get_n_splits(train_new)
 rmse = np.sqrt(cross_val_score(model, train_new, y_train, scoring = 'f1', cv = kf))
return(rmse )<define_search_model>
train['premise_len'] = train['premise'].apply(lambda x: _get_word_count(x))
train['hypothesis_len'] = train['hypothesis'].apply(lambda x: _get_word_count(x))
train['relative_diff'] =(train['hypothesis_len'] - train['premise_len'])* 1./ train['premise_len']
logreg = LogisticRegression()
rf = RandomForestClassifier()
gboost = GradientBoostingClassifier()
xgb = XGBClassifier()
lgbm = LGBMClassifier()<import_modules>
snli = load_dataset(path='snli' )
<compute_test_metric>
result_snli = []
for k in ['train', 'validation']:
for record in snli[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result_snli.append(( c1,c2,'en','English',c3))
snli_df = pd.DataFrame(result_snli, columns=['premise','hypothesis','lang_abv', 'language', 'label'] )
score = f1_score(logreg)
 print("
Logistic regression score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
pd.merge(train['premise'], snli_df['premise'], on = 'premise', how='inner' ).shape[0] != 0
score = f1_score(rf)
 print("
Random Forest score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
final_df = shuffle(train.drop(columns=['id', 'label_str', 'premise_len', 'hypothesis_len', 'relative_diff'])).reset_index(drop = True )
score = f1_score(gboost)
 print("
Gradient Boosting Classifier score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
X, y = final_df[['premise', 'hypothesis']].values.tolist() , final_df['label']
x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25, random_state=12345 )
score = f1_score(xgb)
 print("
eXtreme Gradient BOOSTing score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
del snli
gc.collect()
score = f1_score(lgbm)
 print("
Light Gradient Boosting score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))<train_model>
encoder_handle = 'jplu/tf-xlm-roberta-large'
all_classifier = VotingClassifier(estimators=[('logreg', logreg),('rf', rf), 
('gboost', gboost),('xgb', xgb), 
('lgbm', lgbm)], voting='soft')
 VC = all_classifier.fit(train_new, y_train )<compute_test_metric>
tokenizer = AutoTokenizer.from_pretrained(encoder_handle )
score = f1_score(VC)
 print("
Voting Classifier score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
max_len = 64
random_seed = 11887
learning_rate = 1e-5
epochs = 5
batch_size = 16 * strategy.num_replicas_in_sync
loss = 'sparse_categorical_crossentropy'
metrics = ['accuracy']
steps_per_epoch = 1000
auto = tf.data.experimental.AUTOTUNE
prediction = VC.predict(test_new )<save_to_csv>
def encode_sentence(s, tokenizer):

 tokens = list(tokenizer.tokenize(s))
tokens.append(tokenizer.sep_token)
 return tokenizer.convert_tokens_to_ids(tokens)
 def tokenize(data, tokenizer, max_len):

 PAD_ID = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)
 tokens1 = tf.ragged.constant([encode_sentence(s[0], tokenizer)for s in data], dtype=tf.int32)
 tokens2 = tf.ragged.constant([encode_sentence(s[1], tokenizer)for s in data], dtype=tf.int32)
 cls_label = [tokenizer.convert_tokens_to_ids([tokenizer.cls_token])]*tokens1.shape[0]
tokens = tf.concat([cls_label, tokens1, tokens2], axis=-1)
 tokens = tokens[:, :max_len]
tokens = tokens.to_tensor(default_value=PAD_ID)
 pad = max_len - tf.shape(tokens)[1]
tokens = tf.pad(tokens, [[0, 0], [0, pad]], constant_values=PAD_ID)
 input_word_ids = tf.reshape(tokens, [-1, max_len])
 input_mask = tf.cast(input_word_ids != PAD_ID, tf.int32)
 input_mask = tf.reshape(input_mask, [-1, max_len])
 input_type_ids = tf.concat([tf.zeros_like(cls_label), tf.zeros_like(tokens1), tf.ones_like(tokens2)], axis=-1 ).to_tensor()
inputs = {
'input_word_ids': input_word_ids,
'input_mask': input_mask,
'input_type_ids': input_type_ids}
return inputs
titanic_submission = pd.DataFrame({"PassengerId": test["PassengerId"],
"Survived": prediction})
 titanic_submission.to_csv('Titanic_Submission.csv', index = False)
 titanic_submission.sample(10)
<import_modules>
def build_dataset(x, y, mode, batch_size):

 if mode == "train":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.repeat() 
.shuffle(5678)
.batch(batch_size)
.prefetch(auto)
)
 elif mode == "valid":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 elif mode == "test":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(x)
.batch(batch_size)
)
 else:
raise NotImplementedError
return dataset
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier<load_from_csv>
x_train_ = tokenize(x_train, tokenizer, max_len)
 x_valid_ = tokenize(x_valid, tokenizer, max_len )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 full_data = [train, test]<count_missing_values>
train_dataset = build_dataset(x_train_, y_train, "train", batch_size)
 valid_dataset = build_dataset(x_valid_, y_valid, "valid", batch_size )
print(train.isnull().sum())
 print('_'*40)
 print(test.isnull().sum() )<feature_engineering>
def build_model(encoder_handle, random_seed, learning_rate, loss, metrics, max_len):
tf.keras.backend.clear_session()
tf.random.set_seed(random_seed)
 with strategy.scope() :
input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 roberta = TFXLMRobertaModel.from_pretrained(encoder_handle)
 roberta = roberta([input_word_ids, input_mask])[0]
out = GlobalAveragePooling1D()(roberta)
 out = Dense(3, activation='softmax' )(out)
 model = Model(inputs=[input_word_ids, input_mask], outputs = out)
 model.compile(optimizer=Adam(lr=learning_rate), loss=loss, metrics=metrics)
 model.summary()
return model
PassengerId = test['PassengerId']
for dataset in full_data:
dataset['Has_Cabin'] = dataset["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
dataset['Sex'] = dataset['Sex'].replace(['male', 'female'], [0, 1])
 master_ave = dataset.loc[dataset.Name.str.contains('Master'), 'Age'].mean()
mr_ave = dataset.loc[dataset.Name.str.contains('Mr'), 'Age'].mean()
miss_ave = dataset.loc[dataset.Name.str.contains('Miss'), 'Age'].mean()
mrs_ave = dataset.loc[dataset.Name.str.contains('Mrs'), 'Age'].mean()
dataset.loc[dataset.Name.str.contains('Mraster')& dataset.Age.isnull() , 'Age'] = master_ave
dataset.loc[dataset.Name.str.contains('Mr')& dataset.Age.isnull() , 'Age'] = mr_ave
dataset.loc[dataset.Name.str.contains('Miss')& dataset.Age.isnull() , 'Age'] = miss_ave
dataset.loc[dataset.Name.str.contains('Mrs')& dataset.Age.isnull() , 'Age'] = mrs_ave
dataset['Age'] = dataset['Age'].fillna(dataset['Age'].mean())
 dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
dataset['Age'] = dataset['Age'].astype(int)
 dataset['Fare'] = dataset['Fare'].fillna(dataset['Fare'].mean())
 dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset['Embarked'] = dataset['Embarked'].fillna('S')
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True )<count_missing_values>
model = build_model(encoder_handle, random_seed, learning_rate, loss, metrics, max_len )
print(train.isnull().sum())
 print('_'*40)
 print(test.isnull().sum() )<prepare_x_and_y>
early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
verbose=1,
patience=2,
mode='min',
restore_best_weights=True )
trn_x = train.drop(['Survived'], axis=1, inplace=False)
 trn_y = train['Survived']<train_model>
x_small_train, y_small_train = X[:batch_size], y[:batch_size]
pred = np.zeros(( test.shape[0], 10))
for i in range(10):
model = RandomForestClassifier(
 max_depth = 10,
max_features =10,
min_samples_split = 15,
n_estimators = 10,
n_jobs = -1,
random_state = i)
 model.fit(trn_x, trn_y)
 pred[:,i] = model.predict_proba(test)[:,1].reshape(test.shape[0],)
 output = pred.mean(axis=1)
 output[output >= 0.5] = 1
output[output < 0.5] = 0
output = output.astype(int )<save_to_csv>
x_small_train = tokenize(x_small_train, tokenizer, max_len)
 small_train_dataset = build_dataset(x_small_train, y_small_train, "train", batch_size )
df_out = pd.DataFrame({ 'PassengerId': PassengerId,
'Survived': output })
 df_out.to_csv("submission.csv", index=False )<set_options>
history_small_train = model.fit(small_train_dataset,
steps_per_epoch=100,
epochs=3 )
warnings.filterwarnings("ignore" )<load_from_csv>
model = build_model(encoder_handle, random_seed, learning_rate, loss, metrics, max_len )
traindata = pd.read_csv('/kaggle/input/titanic/train.csv')
 testdata = pd.read_csv('/kaggle/input/titanic/test.csv' )<categorify>
history = model.fit(train_dataset,
validation_data=valid_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=[early_stopping] )
le = LabelEncoder()
traindata['Sex'] = le.fit_transform(traindata['Sex'])
 testdata['Sex'] = le.fit_transform(testdata['Sex'])
 traindata['Embarked'] = le.fit_transform(traindata['Embarked'].astype(str))
testdata['Embarked'] = le.fit_transform(testdata['Embarked'].astype(str))<prepare_x_and_y>
x_test = tokenize(test[['premise', 'hypothesis']].values.tolist() , tokenizer, max_len)
 test_dataset = build_dataset(x_test, None, "test", batch_size )
X = traindata[['Pclass','Sex','Age','Fare','Parch','SibSp','Embarked']].values
y = traindata.iloc[:,1].values
X_real_test = testdata[['Pclass','Sex','Age','Fare','Parch','SibSp','Embarked']].values<count_missing_values>
predictions_prob = model.predict(test_dataset)
 final = predictions_prob.argmax(axis=-1)
 submission = pd.DataFrame()
submission['id'] = test['id']
submission['prediction'] = final.astype(np.int32 )
<categorify><EOS> 
submission.to_csv("submission.csv", index = False )