Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
for dataset in all_df:
dataset.loc[ dataset['FamilySize'] <= 1, 'FamilySize'] = 0
dataset.loc[(dataset['FamilySize'] > 1)&(dataset['FamilySize'] <= 4), 'FamilySize'] = 1
dataset.loc[ dataset['FamilySize'] > 4, 'FamilySize'] = 2<feature_engineering>
%matplotlib inline
for dataset in all_df:
dataset['Class*Age'] = dataset['Pclass'] * dataset['Age']<split>
zip_file = glob.glob('.. /input/dogs-vs-cats-redux-kernels-edition/*.zip')
 print(zip_file)
 def extract_zip(file):
with ZipFile(file,'r')as zip_ref:
zip_ref.extractall('.')
 for files in zip_file:
extract_zip(files )
VALID_SIZE = 0.2
RANDOM_STATE = 2018
train, valid = train_test_split(train_df, test_size=VALID_SIZE, random_state=RANDOM_STATE, shuffle=True )<define_variables>
def get_path(directory):
path = []
for files in os.listdir(directory):
path.append(files)
 return path
train_path = get_path('.. /working/train')
 test_path = get_path('.. /working/test' )
predictors = ['Sex', 'Age']
target = 'Survived'<prepare_x_and_y>
def label_img(img):
category = img.split('.')[-3]
if category == 'cat': return [1,0]
elif category == 'dog': return [0,1]
def process_data(img_data, data_dir, isTrain=True):
data_df = []
for img in tqdm(img_data):
path = os.path.join(data_dir,img)
 if(isTrain):
label = label_img(img)
 else:
label = img.split('.')[0]
img = cv2.imread(path,cv2.IMREAD_COLOR)
 img = cv2.resize(img,(224,224))
data_df.append([np.array(img),np.array(label)])
 shuffle(data_df)
 return data_df
train_X = train[predictors]
train_Y = train[target].values
valid_X = valid[predictors]
valid_Y = valid[target].values<init_hyperparams>
def get_label(directory):
label = []
for file in os.listdir(directory):
if(file.split('.')[0] == 'dog'):
label.append(str(1))
elif(file.split('.')[0] == 'cat'):
label.append(str(0))
return label
label = get_label('.. /working/train')
 df = pd.DataFrame({'filename': train_path, 'label': label})
 df.head()
RFC_METRIC = 'gini'
NUM_ESTIMATORS = 100
NO_JOBS = 4<choose_model_class>
train_df, valid_df = train_test_split(df, test_size = 0.2, stratify = df['label'], random_state = 123)
 print(train_df.shape)
 print(valid_df.shape )
clf = RandomForestClassifier(n_jobs=NO_JOBS,
random_state=RANDOM_STATE,
criterion=RFC_METRIC,
n_estimators=NUM_ESTIMATORS,
verbose=False )<train_model>
train_datagen = ImageDataGenerator(rotation_range = 10, zoom_range = 0.1, horizontal_flip = True, fill_mode = 'nearest',
width_shift_range = 0.1, height_shift_range = 0.1, preprocessing_function = preprocess_input)
 train_gen = train_datagen.flow_from_dataframe(train_df, directory = '.. /working/train', x_col = 'filename', y_col = 'label', target_size =(224,224),
batch_size = 64, class_mode='binary')
 valid_datagen = ImageDataGenerator(preprocessing_function = preprocess_input)
 valid_gen = valid_datagen.flow_from_dataframe(valid_df, directory = '.. /working/train', x_col = 'filename', y_col = 'label', target_size =(224,224),
batch_size = 64, class_mode='binary' )
clf.fit(train_X, train_Y )<predict_on_test>
model = Sequential()
model.add(ResNet50(include_top = False, pooling = 'max', weights = 'imagenet'))
model.add(Dense(1, activation = 'sigmoid'))
model.layers[0].trainable = False
preds = clf.predict(valid_X )<compute_test_metric>
model.compile(optimizer = 'adam', metrics = ['accuracy'], loss = 'binary_crossentropy' )
clf.score(train_X, train_Y)
 acc = round(clf.score(train_X, train_Y)* 100, 2)
 print("RandomForest accuracy(train set):", acc )<compute_test_metric>
checkpointer = ModelCheckpoint(filepath = 'dogcat.weights.best.hdf5', save_best_only = True, save_weights_only = True )
clf.score(valid_X, valid_Y)
 acc = round(clf.score(valid_X, valid_Y)* 100, 2)
 print("RandomForest accuracy(validation set):", acc )<compute_test_metric>
model.fit_generator(train_gen, epochs = 10, validation_data = valid_gen, callbacks = [checkpointer] )
print(metrics.classification_report(valid_Y, preds, target_names=['Not Survived', 'Survived']))<define_variables>
model.load_weights('dogcat.weights.best.hdf5' )
predictors = ['Sex', 'Age', 'Pclass', 'Fare']
target = 'Survived'<prepare_x_and_y>
test_df = pd.DataFrame({'filename': test_path} )
train_X = train[predictors]
train_Y = train[target].values
valid_X = valid[predictors]
valid_Y = valid[target].values<train_model>
test_sample = test_df.sample(n = 25, random_state = 123)
 label_pred = []
for path in test_sample['filename'].to_numpy() :
full_path = '.. /working/test/'+path
x = load_img(full_path, target_size=(224,224))
img_array = img_to_array(x)
 img_array = np.expand_dims(img_array, axis=0)
 out = model.predict(img_array)
 out = 'Dog' if float(out)>0.5 else 'Cat'
label_pred.append(out )
clf.fit(train_X, train_Y )<predict_on_test>
test_datagen = ImageDataGenerator(preprocessing_function = preprocess_input)
 test_gen = test_datagen.flow_from_dataframe(test_df, directory = '.. /working/test', batch_size = 64, x_col = 'filename', y_col = None, class_mode = None, shuffle = False,
img_size =(224, 224))
preds = clf.predict(valid_X )<compute_test_metric>
prediction = model.predict_generator(test_gen)
 prediction = prediction.clip(min = 0.005, max = 0.995 )
clf.score(train_X, train_Y)
 acc = round(clf.score(train_X, train_Y)* 100, 2)
 print("RandomForest accuracy(train set):", acc )<compute_test_metric>
submission_df = pd.read_csv('.. /input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 for i, fname in enumerate(test_path):
index = int(fname[fname.rfind('/')+1:fname.rfind('.')])
 submission_df.at[index-1, 'label'] = prediction[i]
submission_df.to_csv('Cats&DogsSubmission.csv', index=False )