Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<groupby>
%matplotlib inline
rcParams['figure.figsize'] = [10,5]
plt.style.use('fivethirtyeight')
 sns.set_style('whitegrid')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 50)
 pd.set_option('display.max_columns', 150)
 pd.options.display.float_format = '{:.4f}'.format
df.groupby('age_quantile' ).agg({'Monocytes': np.median} )<categorify>
!pip install googletrans
!pip install p_tqdm
!pip install transformers==3.0.2
clear_output()
df['Monocytes_gr'] = df['Monocytes'] - df.groupby('age_quantile')['Monocytes'].transform(np.median)
 df['Leukocytes_gr'] = df['Leukocytes'] - df.groupby('age_quantile')['Leukocytes'].transform(np.median )<correct_missing_values>
df_train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 df_test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv' )
df.fillna(-999, inplace = True)
 df<drop_column>
translator = Translator()
def translate_to_en(text):
global translator
translated_text = translator.translate(text, dest = 'en' ).text
return translated_text
def pool_translated_en(list_text):
threads = 8*multiprocessing.cpu_count()
with Pool(threads)as executor:
result = list(tqdm(executor.imap(translate_to_en, list_text), total=len(list_text)))
 return result
def dataset_translated_en(df):
df_trans = df.copy()
list_idx = list(df_trans[df_trans['lang_abv']!='en'].index)
 list_prem = list(df_trans.loc[list_idx]['premise'])
 list_hyp = list(df_trans.loc[list_idx]['hypothesis'])
 print('
Premise Translation - En')
 df_trans.loc[list_idx,'premise'] = pool_translated_en(list_prem)
 print('
Premise Hypotheses - En')
 df_trans.loc[list_idx,'hypothesis'] = pool_translated_en(list_hyp)
 df_trans['lang_abv'] = 'en'
df_trans['language'] = 'English'
return df_trans
df_train_en = dataset_translated_en(df_train)
 df_test_en = dataset_translated_en(df_test )
df = df.drop('covid_19', axis='columns' )<drop_column>
list_non_en_idx = list(df_train[df_train['lang_abv']!='en'].index)
 for i in range(5):
idx = random.randint(0,len(list_non_en_idx))
print('
ORIGINAL TEXT :',df_train['premise'][list_non_en_idx[idx]])
 print('TRANSLATED TEXT :',df_train_en['premise'][list_non_en_idx[idx]] )
df = df.drop('id', axis='columns' )<prepare_x_and_y>
print('Proportion for each languange in train dataset')
 df_train['language'].value_counts() / len(df_train)* 100
train_X = df.iloc[:4000, :].select_dtypes(exclude=['object'] ).values
test_X = df.iloc[4000:, :].select_dtypes(exclude=['object'] ).values<import_modules>
def remove_punctuation(text):
no_punct = ''.join([c for c in text if c not in string.punctuation])
 return no_punct
def quick_clean_data(dataset, var):
df = dataset.copy()
df[var] = df[var].str.lower()
df[var] = df[var].str.strip()
df[var] = df.apply(lambda x : remove_punctuation(x[var]), axis=1)
 df[var] = df.apply(lambda x : " ".join(x[var].split()), axis=1)
 return df
list_var = ['premise','hypothesis']
for var in list_var :
df_train = quick_clean_data(df_train, var)
 df_test = quick_clean_data(df_test, var)
 df_train_en = quick_clean_data(df_train_en, var)
 df_test_en = quick_clean_data(df_test_en, var )
<import_modules>
tqdm_notebook.pandas()
def remove_stop_words(text):
en_stop_words = stopwords.words('english')
 text = ' '.join([c for c in text.split() if c not in en_stop_words])
 return text
df_train_no_stop = df_train_en.copy()
df_test_no_stop = df_test_en.copy()
list_var = ['premise','hypothesis']
for var in list_var :
df_train_no_stop[var] = df_train_no_stop.progress_apply(lambda x : remove_stop_words(x[var]), axis=1)
 df_test_no_stop[var] = df_test_no_stop.progress_apply(lambda x : remove_stop_words(x[var]), axis=1 )
<train_model>
list_idx = list(df_train_no_stop.index)
 for i in range(5):
idx = random.randint(0,len(list_idx))
print('
ORIGINAL TEXT :',df_train_en['premise'][list_idx[idx]])
 print('NO STOPWORDS TEXT :',df_train_no_stop['premise'][list_idx[idx]] )
<train_model>
LANGUAGE_MAP = {
"English" : 0,
"Chinese" : 1,
"Arabic" : 2,
"French" : 3,
"Swahili" : 4,
"Urdu" : 5,
"Vietnamese": 6,
"Russian" : 7,
"Hindi" : 8,
"Greek" : 9,
"Thai" : 10,
"Spanish" : 11,
"German" : 12,
"Turkish" : 13,
"Bulgarian" : 14
}
df_train['language'] = df_train['language'].map(LANGUAGE_MAP)
 df_train['language_label'] = df_train['language'].astype(str)+ "_" + df_train['label'].astype(str )
tree_clf = DecisionTreeClassifier(
 max_depth=10,
min_samples_leaf=28,
max_features=0.94,
criterion="gini",
random_state=1)
 dt = tree_clf.fit(train_X, y)
 y_pred = dt.predict_proba(test_X )<create_dataframe>
def initialize_accelerator(ACCELERATOR):
if ACCELERATOR == "TPU":
print("Connecting to TPU")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f"Running on TPU {tpu.master() }")
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("Initializing TPU")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except :
print("Failed to initialize TPU")
 strategy = tf.distribute.get_strategy()
else:
print("Unable to initialize TPU")
 ACCELERATOR = "GPU"
if ACCELERATOR != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if ACCELERATOR == "GPU":
print(f"GPUs Available: {len(tf.config.experimental.list_physical_devices('GPU')) }")
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f"REPLICAS: {REPLICAS}")
 return strategy, AUTO, REPLICAS
STRATEGY, AUTO, REPLICAS = initialize_accelerator('TPU' )
imp = pd.DataFrame({
'colnames': df.columns,
'importances': tree_clf.feature_importances_
})
 good_columns = imp.loc[imp['importances'] != 0, :].colnames.values<prepare_x_and_y>
def run_experiments(df, var_stratified, encode_text):
skf = StratifiedKFold(n_splits = CV_SPLIT, shuffle = True, random_state = SEED)
 acc_oof = []
for(fold,(train_index, valid_index)) in enumerate(skf.split(df, df[var_stratified])) :
STRATEGY, AUTO, REPLICAS = initialize_accelerator('TPU')
 K.clear_session()
with STRATEGY.scope() :
model = build_model(MODEL_NAME, MAX_LENGTH, METRICS)
 if fold == 0:
print(model.summary())
 print("
")
 print("
print(f"
print("
X_train = df.iloc[train_index].sample(frac=1)
 X_valid = df.iloc[valid_index]
y_train = to_categorical(X_train['label'].values)
 y_valid = to_categorical(X_valid['label'].values)
 print("
Tokenizing")
 X_train_encoded = encode_text(texts = X_train, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING)
 X_valid_encoded = encode_text(texts = X_valid, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING)
 ds_train =(
 tf.data.Dataset 
.from_tensor_slices(( X_train_encoded, y_train)) 
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 ds_valid =(
 tf.data.Dataset 
.from_tensor_slices(( X_valid_encoded, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 n_train = X_train.shape[0]
sv = tf.keras.callbacks.ModelCheckpoint(
 "model.h5",
monitor = 'val_'+METRICS[0],
verbose = 0,
save_best_only = True,
save_weights_only = True,
mode = "max",
save_freq = "epoch" 
)
 print("
Training")
 history = model.fit(
 ds_train,
epochs = EPOCHS,
callbacks = [sv],
steps_per_epoch = n_train // BATCH_SIZE,
validation_data = ds_valid,
verbose = VERBOSE 
)
 model.load_weights("model.h5")
 pred = model.predict(ds_valid)
 acc = accuracy_score(X_valid['label'].values, np.argmax(pred, axis=1))
acc_oof.append(acc)
 print(f"
Fold {fold + 1} Accuracy: {round(acc, 4)}
")
 g = gc.collect()
print(f"
CV Mean Accuracy: {round(np.mean(acc_oof), 4)}")
 print(f"CV StdDev Accuracy: {round(np.std(acc_oof), 4)}
" )
train_X = df.iloc[:4000, :].loc[:, good_columns].values
test_X = df.iloc[4000:, :].loc[:, good_columns].values<train_model>
def build_model(model_name, max_len, metrics):
input_ids = Input(shape =(max_len,), dtype = tf.int32, name = "input_ids")
 transformer_model = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings = transformer_model(input_ids)[0]
transformer_token = transformer_embeddings[:, 0, :]
output_values = Dense(3, activation = "softmax" )(transformer_token)
 model = Model(inputs = input_ids, outputs = output_values)
 model.compile(optimizer = Adam(learning_rate = 1e-5),
loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),
metrics = metrics)
 return model
<init_hyperparams>
df_train['predictor'] = " [CLS] " + df_train['premise'] + " [SEP] " + df_train['hypothesis']
df_test['predictor'] = " [CLS] " + df_test['premise'] + " [SEP] " + df_test['hypothesis']
import lightgbm as lgb 
<choose_model_class>
MAX_LENGTH = 75
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]
bt = lgb.LGBMClassifier(colsample_bytree=0.85, min_child_samples=20, min_split_gain=0, n_estimators=250,
num_leaves=10, reg_alpha=0.0, reg_lambda=0.1, subsample=1, n_jobs=-1, objective='binary',
boosting_type='gbdt', learning_rate=0.01, random_state=1 )<train_model>
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )
bt.fit(train_X, y )<predict_on_test>
def original_encode(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts['predictor'],
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]
bt.predict_proba(test_X)[:,1]<predict_on_test>
start = time.time()
run_experiments(df_train, 'language_label', original_encode)
 end = time.time()
print('Time used :',(end-start)/60 )
submission['covid_19'] = bt.predict_proba(test_X)[:,1]
submission<save_to_csv>
df_train_en['predictor'] = " [CLS] " + df_train_en['premise'] + " [SEP] " + df_train_en['hypothesis']
df_test_en['predictor'] = " [CLS] " + df_test_en['premise'] + " [SEP] " + df_test_en['hypothesis']
submission.to_csv('submission.csv', index=False )<save_to_csv>
MAX_LENGTH = 75
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]
submission.to_csv('submission.csv', index=False )<load_from_csv>
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'])
 train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax' )<feature_engineering>
def original_encode(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts['predictor'],
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 feature_day = [1,20,50,100,200,500,1000,5000,10000,15000,20000,50000,100000,200000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature] 
<install_modules>
start = time.time()
run_experiments(df_train_en, 'label', original_encode)
 end = time.time()
print('Time used :',(end-start)/60 )
!pip install pmdarima<save_to_csv>
df_train_no_stop['predictor'] = " [CLS] " + df_train_no_stop['premise'] + " [SEP] " + df_train_no_stop['hypothesis']
df_test_no_stop['predictor'] = " [CLS] " + df_test_no_stop['premise'] + " [SEP] " + df_test_no_stop['hypothesis']
df_val = df_val_2
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission = submission.round({'ConfirmedCases': 0, 'Fatalities': 0})
 submission.to_csv('submission.csv', index=False)
 submission<set_options>
MAX_LENGTH = 50
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]
plt.style.use('classic')
 %matplotlib inline
sb.set()
warnings.filterwarnings('ignore' )<load_from_csv>
def original_encode(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts['predictor'],
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 train_len = len(train)
 data_all = train.append(test, sort=False, ignore_index=True)
 data_all.isnull().sum()<feature_engineering>
start = time.time()
run_experiments(df_train_no_stop, 'label', original_encode)
 end = time.time()
print('Time used :',(end-start)/60 )
data_all['Age'] = data_all.groupby(['Pclass', 'Sex'])['Age'].apply(lambda x: x.fillna(x.median()))
data_all['Fare'] = data_all.groupby('Pclass')['Fare'].apply(lambda x: x.fillna(x.median()))
data_all['Embarked'] = data_all['Embarked'].fillna('S' )<categorify>
def build_model_multimodal(model_name, max_len, metrics):
input_ids_prem = Input(shape =(MAX_LENGTH_PREM,), dtype = tf.int32, name = "input_ids_prem")
 input_ids_hyp = Input(shape =(MAX_LENGTH_HYP,), dtype = tf.int32, name = "input_ids_hyp")
 transformer_model_prem = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings_prem = transformer_model_prem(input_ids_prem)[0]
transformer_token_prem = transformer_embeddings_prem[:, 0, :]
transformer_model_hyp = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings_hyp = transformer_model_hyp(input_ids_hyp)[0]
transformer_token_hyp = transformer_embeddings_hyp[:, 0, :]
transformer_concat = Concatenate()([transformer_token_prem, transformer_token_hyp])
 output_values = Dense(3, activation = "softmax" )(transformer_concat)
 model = Model(inputs = [input_ids_prem, input_ids_hyp], outputs = output_values)
 model.compile(optimizer = Adam(learning_rate = 1e-5),
loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.2),
metrics = metrics)
 return model
data_all['Pclass'].replace([1, 2, 3], [3, 2, 1],inplace=True )<feature_engineering>
def original_encode_multimodal(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]
for i in data_all:
data_all['Title'] = [x[1].split(".")[0].strip(" ")for x in data_all['Name'].str.split(",")]
print(pd.crosstab(data_all['Title'], data_all['Sex']))<feature_engineering>
def run_experiments_multimodal(df, var_stratified, encode_text):
skf = StratifiedKFold(n_splits = CV_SPLIT, shuffle = True, random_state = SEED)
 acc_oof = []
for(fold,(train_index, valid_index)) in enumerate(skf.split(df, df[var_stratified])) :
STRATEGY, AUTO, REPLICAS = initialize_accelerator('TPU')
 K.clear_session()
with STRATEGY.scope() :
model = build_model_multimodal(MODEL_NAME, MAX_LENGTH_PREM, METRICS)
 if fold == 0:
print(model.summary())
 print("
")
 print("
print(f"
print("
X_train = df.iloc[train_index]
X_valid = df.iloc[valid_index]
y_train = to_categorical(X_train['label'].values)
 y_valid = to_categorical(X_valid['label'].values)
 print("
Tokenizing")
 X_train_encoded_prem = encode_text(texts = X_train['premise'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_PREM, padding = PADDING)
 X_train_encoded_hyp = encode_text(texts = X_train['hypothesis'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_HYP, padding = PADDING)
 X_valid_encoded_prem = encode_text(texts = X_valid['premise'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_PREM, padding = PADDING)
 X_valid_encoded_hyp = encode_text(texts = X_valid['hypothesis'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_HYP, padding = PADDING)
 ds_train =(
 tf.data.Dataset 
.from_tensor_slices(( {"input_ids_prem": X_train_encoded_prem, "input_ids_hyp": X_train_encoded_hyp}, y_train)) 
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 ds_valid =(
 tf.data.Dataset 
.from_tensor_slices(( {"input_ids_prem": X_valid_encoded_prem, "input_ids_hyp": X_valid_encoded_hyp}, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 n_train = X_train.shape[0]
sv = tf.keras.callbacks.ModelCheckpoint(
 "model.h5",
monitor = 'val_'+METRICS[0],
verbose = 0,
save_best_only = True,
save_weights_only = True,
mode = "max",
save_freq = "epoch" 
)
 print("
Training")
 history = model.fit(
 ds_train,
epochs = EPOCHS,
callbacks = [sv],
steps_per_epoch = n_train // BATCH_SIZE,
validation_data = ds_valid,
verbose = VERBOSE 
)
 model.load_weights("model.h5")
 pred = model.predict(ds_valid)
 acc = accuracy_score(X_valid['label'].values, np.argmax(pred, axis=1))
acc_oof.append(acc)
 print(f"
Fold {fold + 1} Accuracy: {round(acc, 4)}
")
 g = gc.collect()
print(f"
CV Mean Accuracy: {round(np.mean(acc_oof), 4)}")
 print(f"CV StdDev Accuracy: {round(np.std(acc_oof), 4)}
" )
for i in data_all:
data_all['Title'] = data_all['Title'].replace(['Capt', 'Col', 'Don', 'Major', 'Rev', 'Sir', 'Jonkheer'], 'Mr')
 data_all['Title'] = data_all['Title'].replace(['Mlle', 'Ms'], 'Miss')
 data_all['Title'] = data_all['Title'].replace(['Dona', 'Lady', 'the Countess', 'Mme'], 'Mrs')
 for i in np.where(pd.notnull(data_all.Title)&(data_all['Title'] == 'Dr')&(data_all['Sex'] == 'M')) :
data_all.at[i, 'Title'] = 'Mr'
for i in np.where(pd.notnull(data_all.Title)&(data_all['Title'] == 'Dr')&(data_all['Sex'] == 'F')) :
data_all.at[i, 'Title'] = 'Mrs'
bar = sb.barplot(x='Title', y='Survived' , data=data_all )<define_variables>
MAX_LENGTH_PREM = 50
MAX_LENGTH_HYP = 25
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]
numeric = ['Pclass', 'Fare', 'FamilySize']
data_num = data_all[numeric]<categorify>
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )
categorical = ['SexAge', 'Embarked', 'Sex', 'Title']
data_cat = data_all[categorical]
data_cat = pd.get_dummies(data_cat )<prepare_x_and_y>
start = time.time()
run_experiments_multimodal(df_train_en, 'label', original_encode_multimodal)
 end = time.time()
print('Time used :',(end-start)/60 )
y = train.Survived
df_all = pd.concat([data_num, data_cat], axis = 1)
 features =(data_num.columns ).append(data_cat.columns)
 df_train = df_all[:891]
df_test = df_all[891:]
target = 'Survived'
X = df_train[features]
X_test = df_test[features]<compute_train_metric>
MAX_LENGTH = 75
MODEL_NAME = "jplu/tf-xlm-roberta-large"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]
kfold = StratifiedKFold(n_splits=5)
 def auc_cv(model):
roc_auc = cross_val_score(model, X, y, scoring='roc_auc', cv=kfold)
 return(roc_auc )<train_on_grid>
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )
logreg2 = LogisticRegression(C =.4, penalty='l2')
 auc_cv_logreg = auc_cv(logreg2)
 logreg2.fit(X, y)
 print('LogisticRegression CV score min: ' + str(auc_cv_logreg.min())+ ' mean: ' + str(auc_cv_logreg.mean())
 + ' max: ' + str(auc_cv_logreg.max()))<train_on_grid>
y_train = to_categorical(df_train_en['label'].values)
 X_train_encoded = original_encode(texts = df_train_en, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING)
 X_test_encoded = original_encode(texts = df_test_en, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING )
gbct = GradientBoostingClassifier(n_estimators=1000, learning_rate=.01, max_depth=3, max_features=3, min_samples_split=4,
min_samples_leaf=7, loss='exponential', subsample=.6, random_state=0)
 auc_cv_gbct = auc_cv(gbct)
 gbct.fit(X, y)
 print('GBoost CV score min: ' + str(auc_cv_gbct.min())+ ' mean: ' + str(auc_cv_gbct.mean())
 + ' max: ' + str(auc_cv_gbct.max()))<save_to_csv>
ds_train =(
 tf.data.Dataset 
.from_tensor_slices(( X_train_encoded, y_train)) 
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 ds_test =(
 tf.data.Dataset 
.from_tensor_slices(( X_test_encoded)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
def blend_models(X):
return(( logreg2.predict(X)) +(gbct.predict(X)))/2
logreg2_preds = logreg2.predict(X_test)
 gbct_preds = gbct.predict(X_test)
 logreg2_gbct_preds = blend_models(X_test ).astype(int)
 titanic_logreg2_gbct = pd.DataFrame({'PassengerId':test.PassengerId, 'Survived':logreg2_gbct_preds})
 titanic_logreg2_gbct.to_csv('solution.csv', index = False)
 print('Version 3 submitted on', datetime.now() )<load_from_csv>
K.clear_session()
with STRATEGY.scope() :
model = build_model(MODEL_NAME, MAX_LENGTH, METRICS )
data = pd.read_csv("/kaggle/input/titanic/train.csv")
 data = data.set_index("PassengerId" )<prepare_x_and_y>
n_train = df_train_en.shape[0]
history = model.fit(
 ds_train,
epochs = EPOCHS,
steps_per_epoch = n_train // BATCH_SIZE,
verbose = VERBOSE
)
Y = data.Survived
data = data.drop(columns= ['Survived','Name', 'Ticket', 'Cabin'])
<categorify>
pred = model.predict(ds_test )
<concatenate><EOS> 
sub = pd.read_csv('.. /input/contradictory-my-dear-watson/sample_submission.csv')
 sub['prediction'] = np.argmax(pred, axis=1)
 sub.to_csv('submission.csv', index=False )