Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
class SklearnBasicClassifier(object):
def __init__(self, clf, seed=2018, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y)
 def feature_importances(self,x,y):
print(self.clf.fit(x,y ).feature_importances_)
 def get_feature_importances(self,x,y):
return(self.clf.fit(x,y ).feature_importances_)
<predict_on_test>
%matplotlib inline
ntrain = train_df.shape[0]
ntest = test_df.shape[0]
def get_oof_predictions(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NUMBER_KFOLDS, ntest))
for i,(train_idx, valid_idx)in enumerate(kf.split(train_df)) :
clf.train(x_train[train_idx], y_train[train_idx])
 oof_train[valid_idx] = clf.predict(x_train[valid_idx])
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )<init_hyperparams>
base_dir = '.. /input/dogs-vs-cats-redux-kernels-edition'
train_dir = '.. /data/train'
test_dir = '.. /data/test'
ada_params = {
'n_estimators': 200,
'learning_rate' : 0.75
}
cat_params = {
'iterations': 150,
'learning_rate': 0.02,
'depth': 12,
'bagging_temperature':0.2,
'od_type':'Iter',
'metric_period':400,
}
ext_params = {
'n_jobs': -1,
'n_estimators':100,
'max_depth': 8,
'min_samples_leaf': 3,
'verbose': 0
}
gbm_params = {
'n_estimators': 200,
'max_depth': 5,
'min_samples_leaf': 3,
'verbose': 0
}
rfo_params = {
'n_jobs': -1,
'n_estimators': 50,
'max_depth': 5,
'min_samples_leaf': 5,
'max_features' : 'auto',
'verbose': 0
}
svc_params = {
'kernel' : 'linear',
'C' : 0.02
}<choose_model_class>
with zipfile.ZipFile(os.path.join(base_dir, 'train.zip')) as train_zip:
train_zip.extractall('.. /data')
 with zipfile.ZipFile(os.path.join(base_dir, 'test.zip')) as test_zip:
test_zip.extractall('.. /data' )
ada = SklearnBasicClassifier(clf=AdaBoostClassifier, seed=RANDOM_STATE, params=ada_params)
 cat = SklearnBasicClassifier(clf=CatBoostClassifier, seed=RANDOM_STATE, params=cat_params)
 ext = SklearnBasicClassifier(clf=ExtraTreesClassifier, seed=RANDOM_STATE, params=ext_params)
 gbm = SklearnBasicClassifier(clf=GradientBoostingClassifier, seed=RANDOM_STATE, params=gbm_params)
 rfo = SklearnBasicClassifier(clf=RandomForestClassifier, seed=RANDOM_STATE, params=rfo_params)
 svc = SklearnBasicClassifier(clf=SVC, seed=RANDOM_STATE, params=svc_params )<define_variables>
train_list = glob.glob(os.path.join(train_dir, '*.jpg'))
test_list = glob.glob(os.path.join(test_dir, '*.jpg'))
predictors = ['FamilySize', 'Title', 'Class*Age']
target = 'Survived'<prepare_x_and_y>
train_list,val_list = train_test_split(train_list,test_size=0.1 )
y_train = train_df['Survived'].values
train = train_df[predictors]
test = test_df[predictors]
x_train = train.values
x_test = test.values<predict_on_test>
class ImageTransform() :
def __init__(self, resize, mean, std):
self.data_transform = {
'train': transforms.Compose([
transforms.RandomResizedCrop(resize, scale=(0.5, 1.0)) ,
transforms.RandomHorizontalFlip() ,
transforms.ToTensor() ,
transforms.Normalize(mean, std)
 ]),
'val': transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(resize),
transforms.ToTensor() ,
transforms.Normalize(mean, std)
 ])
 }
def __call__(self, img, phase):
return self.data_transform[phase](img )
print("Start training")
 ada_oof_train, ada_oof_test = get_oof_predictions(ada, x_train, y_train, x_test)
 print("End AdaBoost")
 cat_oof_train, cat_oof_test = get_oof_predictions(cat, x_train, y_train, x_test)
 print("End CatBoost")
 ext_oof_train, ext_oof_test = get_oof_predictions(ext, x_train, y_train, x_test)
 print("End ExtraTrees")
 rfo_oof_train, rfo_oof_test = get_oof_predictions(rfo,x_train, y_train, x_test)
 print("End RandomForest")
 gbm_oof_train, gbm_oof_test = get_oof_predictions(gbm,x_train, y_train, x_test)
 print("End GradientBoost")
 svc_oof_train, svc_oof_test = get_oof_predictions(svc,x_train, y_train, x_test)
 print("End training" )<compute_train_metric>
class DogvsCatDataset(data.Dataset):
def __init__(self, file_list, transform=None, phase='train'):
self.file_list = file_list
self.transform = transform
self.phase = phase
def __len__(self):
return len(self.file_list)
 def __getitem__(self, idx):
img_path = self.file_list[idx]
img = Image.open(img_path)
 img_transformed = self.transform(img, self.phase)
 label = img_path.split('/')[-1].split('.')[0]
if label == 'dog':
label = 1
elif label == 'cat':
label = 0
return img_transformed, label
ada_feature_importance = ada.get_feature_importances(x_train,y_train)
 cat_feature_importance = cat.get_feature_importances(x_train,y_train)
 ext_feature_importance = ext.get_feature_importances(x_train,y_train)
 gbm_feature_importance = gbm.get_feature_importances(x_train,y_train)
 rfo_feature_importance = rfo.get_feature_importances(x_train,y_train )<create_dataframe>
size = 224
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 batch_size = 32
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
base_predictions_train = pd.DataFrame({
'AdaBoost': ada_oof_train.ravel() ,
'CatBoost': cat_oof_train.ravel() ,
'ExtraTrees': ext_oof_train.ravel() ,
'GradientBoost': gbm_oof_train.ravel() ,
'RandomForest': rfo_oof_train.ravel() ,
'SVM': svc_oof_train.ravel()
})
 base_predictions_train.head(10 )<concatenate>
train_dataset = DogvsCatDataset(train_list, transform=ImageTransform(size, mean, std), phase='train')
 val_dataset = DogvsCatDataset(val_list, transform=ImageTransform(size, mean, std), phase='val' )
x_train = np.concatenate(( ada_oof_train, cat_oof_train, ext_oof_train, gbm_oof_train, rfo_oof_train, svc_oof_train), axis=1)
 x_test = np.concatenate(( ada_oof_test, cat_oof_test, ext_oof_test, gbm_oof_test, rfo_oof_test, svc_oof_test), axis=1 )<choose_model_class>
train_dataloader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 val_dataloader = data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
 dataloader_dict = {'train': train_dataloader, 'val': val_dataloader}
print('Operation Check')
 batch_iterator = iter(train_dataloader)
 inputs, label = next(batch_iterator)
 print(inputs.size())
 print(label )
clf = xgb.XGBClassifier(
 learning_rate = 0.02,
n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
scale_pos_weight=1 )<train_model>
use_pretrained = True
net = models.resnet50(pretrained=use_pretrained)
 print(net )
xgbm = clf.fit(x_train, y_train )<predict_on_test>
net.fc = nn.Linear(in_features=2048, out_features=2)
 print('Done' )
predictions = xgbm.predict(x_test )<save_to_csv>
params_to_update = []
update_params_name = ['fc.weight', 'fc.bias']
for name, param in net.named_parameters() :
if name in update_params_name:
param.requires_grad = True
params_to_update.append(param)
 print(name)
 else:
param.requires_grad = False
submissionStacking = pd.DataFrame({ 'PassengerId': test_df["PassengerId"],'Survived': predictions })
 submissionStacking.to_csv("submission_ensamble.csv", index=False )<set_options>
def lr_schedule(epoch):
lr = 1e-3
if epoch > 95:
lr *= 0.5e-3
elif epoch > 80:
lr *= 1e-3
elif epoch > 50:
lr *= 1e-2
elif epoch > 20:
lr *= 1e-1
print('Learning rate: ', lr)
 return lr
%matplotlib inline 
<load_from_csv>
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(params=params_to_update,lr=lr_schedule(0))
test_df = pd.read_csv(".. /input/test.csv")
 train_df = pd.read_csv(".. /input/train.csv" )<sort_values>
def train_model(net, dataloader_dict, criterion, optimizer, num_epoch):
since = time.time()
best_model_wts = copy.deepcopy(net.state_dict())
 best_acc = 0.0
net = net.to(device)
 for epoch in range(num_epoch):
print('Epoch {}/{}'.format(epoch + 1, num_epoch))
print('-'*20)
 for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
for inputs, labels in tqdm(dataloader_dict[phase]):
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 _, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * inputs.size(0)
 epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloader_dict[phase].dataset)
 epoch_acc = epoch_corrects.double() / len(dataloader_dict[phase].dataset)
 print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(net.state_dict())
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
net.load_state_dict(best_model_wts)
 return net
total = train_df.isnull().sum().sort_values(ascending=False)
 percent_1 = train_df.isnull().sum() /train_df.isnull().count() *100
percent_2 =(round(percent_1, 1)).sort_values(ascending=False)
 missing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%'])
 missing_data.head(5 )<data_type_conversions>
num_epoch = 10
net = train_model(net, dataloader_dict, criterion, optimizer, num_epoch )
<count_values><EOS> 
id_list = []
pred_list = []
with torch.no_grad() :
for test_path in tqdm(test_list):
img = Image.open(test_path)
 _id = int(test_path.split('/')[-1].split('.')[0])
 transform = ImageTransform(size, mean, std)
 img = transform(img, phase='val')
 img = img.unsqueeze(0)
 img = img.to(device)
 net.eval()
outputs = net(img)
 preds = F.softmax(outputs, dim=1)[:, 1].tolist()
id_list.append(_id)
 pred_list.append(preds[0])
 res = pd.DataFrame({
'id': id_list,
'label': pred_list
})
 res.sort_values(by='id', inplace=True)
 res.reset_index(drop=True, inplace=True)
 res.to_csv('submission.csv', index=False )