Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
<SOS> metric: LogLoss Kaggle data source: dogs-vs.-cats-redux:-kernels-edition<categorify>
!pip install --use-feature=2020-resolver https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/xgboost-1.3.0_SNAPSHOT%2B00b0ad1293b4fa74d6aca5da4e9ab7a9d16777f0-py3-none-manylinux2010_x86_64.whl
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
xgb.__version__
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=6e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_from_csv>
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
import gc
import os
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<choose_model_class>
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()
model = build_model(bert_layer, max_len=160 )<train_model>
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(train_input, train_labels, validation_split=0.2,
epochs=3, callbacks=[checkpoint], batch_size=16 )<predict_on_test>
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )
model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras.utils.np_utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split 
<load_from_csv>
train_EB7 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/train_EB7.npy')
 test_EB7 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/test_EB7.npy' )
train = pd.read_csv(".. /input/train.csv" )<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(train_EB7, target, test_size=0.1, random_state=42 )
test = pd.read_csv(".. /input/test.csv")
 print(test.info())
 test.head()<data_type_conversions>
xgb_params = {
'eta': 0.05,
'max_depth': 4,
'subsample': 0.8,
'colsample_bytree': 0.6,
'alpha': 0.01,
'lambda': 1.00,
'gamma' : 0.02,
'max_bin': 256,
'objective': 'reg:logistic',
'eval_metric': 'auc',
'verbosity': 0,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'
}
dtrain = xgb.DMatrix(X_train, y_train)
 dval = xgb.DMatrix(X_val, y_val)
 dtest = xgb.DMatrix(test_EB7 )
X_train = train.drop(['label'],axis=1 ).astype('float32')
 y_train = train['label'].astype('float32')
 X_test = test.values.astype('float32' )<categorify>
del train_EB7, test_EB7, X_train, X_val
gc.collect()
def normalize(m):
return m / 255
X_train = normalize(X_train)
 X_test = normalize(X_test )<categorify>
%%time
model = xgb.train(xgb_params, dtrain, num_boost_round=106 )
print('Labels')
 print(y_train[:5])
 y_train = to_categorical(y_train, 10)
 print('Encoded labels')
 print(y_train[:5] )<choose_model_class>
val_preds = model.predict(dval )
checkpoint = ModelCheckpoint(filepath='mnist.model.best.hdf5',
verbose=1,
save_best_only=True,
monitor='val_acc')
 def build_model() :
model = Sequential([
Convolution2D(16,(3,3), activation='relu', input_shape=(28, 28, 1)) ,
BatchNormalization() ,
Convolution2D(16,(3,3), activation='relu'),
BatchNormalization() ,
MaxPooling2D() ,
Dropout(0.25),
Convolution2D(32,(3,3), activation='relu'),
Convolution2D(32,(3,3), activation='relu'),
BatchNormalization() ,
MaxPooling2D() ,
Dropout(0.25),
Flatten() ,
Dense(256, activation='relu'),
BatchNormalization() ,
Dropout(0.25),
Dense(10, activation='softmax')
 ])
 model.compile(optimizer=Adam() ,
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model
model = build_model() 
<split>
roc_auc_score(y_val, val_preds )
X_train, X_val, y_train, y_val = train_test_split(X_train,
y_train,
test_size=0.1,
random_state=42 )<choose_model_class>
log_loss(y_val, val_preds )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<choose_model_class>
%%time
val_shap_preds = model.predict(dval, pred_contribs=True )
image_generator = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
<define_variables>
%%time
test_shap_preds = model.predict(dtest, pred_contribs=True)
batch_size = 96
epochs = 60
steps_per_epoch = X_train.shape[0] / batch_size
batches = image_generator.flow(X_train, y_train, batch_size=batch_size )<train_model>
%%time
train_shap_preds = model.predict(dtrain, pred_contribs=True )
history = model.fit_generator(generator=batches,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=(X_val, y_val),
callbacks=[checkpoint, learning_rate_reduction] )<load_pretrained>
test_preds = model.predict(dtest )
<compute_test_metric><EOS> 
sample_submission['id'] = ids
sample_submission['label'] = test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()