Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<define_variables>
train_on_gpu = True
print("PyTorch Version: ",torch.__version__)
 print("Torchvision Version: ",torchvision.__version__)
 df_train = pd.read_csv(".. /input/train.csv")
 data_dir = ".. /input/train"
model_name = "densenet"
num_classes = 5005
batch_size = 32
num_epochs = 15
feature_extract = False
num_workers = 0
def prepare_labels(y):
values = np.array(y)
 label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
 onehot_encoder = OneHotEncoder(sparse=False)
 integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
 onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
 y = onehot_encoded
return y, label_encoder
y, le = prepare_labels(df_train['Id'])
 print(y)
 class WhaleDataset(Dataset):
def __init__(self, datafolder, datatype='train', df=None,
transform = transforms.Compose([transforms.ToTensor() ]), y=None):
self.datafolder = datafolder
self.datatype = datatype
self.y = y
if self.datatype == 'train':
self.df = df.values
self.image_files_list = [s for s in os.listdir(datafolder)]
self.transform = transform
def __len__(self):
return len(self.image_files_list)
 def __getitem__(self, idx):
if self.datatype == 'train':
img_name = os.path.join(self.datafolder, self.df[idx][0])
 label = self.y[idx]
elif self.datatype == 'test':
img_name = os.path.join(self.datafolder, self.image_files_list[idx])
 label = np.zeros(( 5005,))
image = Image.open(img_name ).convert('RGB')
 image = self.transform(image)
 if self.datatype == 'train':
return image, label
elif self.datatype == 'test':
return image, label, self.image_files_list[idx]
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
for param in model.parameters() :
param.requires_grad = False
def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
model_ft = None
input_size = 0
if model_name == "resnet":

 model_ft = models.resnet34(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
 input_size = 224
elif model_name == "alexnet":

 model_ft = models.alexnet(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
 input_size = 224
elif model_name == "vgg":

 model_ft = models.vgg11_bn(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
 input_size = 224
elif model_name == "squeezenet":

 model_ft = models.squeezenet1_0(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))
model_ft.num_classes = num_classes
input_size = 224
elif model_name == "densenet":

 model_ft = models.densenet121(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier.in_features
model_ft.classifier = nn.Linear(num_ftrs, num_classes)
 input_size = 224
elif model_name == "inception":

 model_ft = models.inception_v3(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.AuxLogits.fc.in_features
model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs,num_classes)
 input_size = 299
else:
print("Invalid model name, exiting...")
 exit()
return model_ft, input_size
model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)
 print(model_ft)
 data_transforms = transforms.Compose([
transforms.Resize(( 224,224)) ,
transforms.RandomHorizontalFlip(0.5),
transforms.RandomAffine(30),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 ])
 test_transforms = transforms.Compose([
transforms.Resize(( 224,224)) ,
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 ])
 def accuracy(outputs,labels):
preds = outputs.max(dim=1)[1]
return(preds==labels ).float().sum()
def train(batch_size = 32, epochs = 10, transform = None):
tfmd_dataset = WhaleDataset(datafolder = ".. /input/train", datatype='train', df = df_train,
transform = transform, y=y)
 dataloader = torch.utils.data.DataLoader(tfmd_dataset, batch_size=batch_size, shuffle=True,
num_workers=num_workers)
 criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model_ft.parameters() , lr=0.0001)
 if torch.cuda.is_available() :
model_ft.cuda()
nn.DataParallel(model_ft)
 print("GPU")
 epoch_loss_data = []
epoch_accuracy_data = []
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
 for epoch in range(1,epochs+1):
running_loss, running_loss_total,epoch_accuracy = 0.0, 0.0,0.0
for i, data in enumerate(dataloader, 0):
inputs, labels = data
labels = labels.double()
inputs, labels = inputs.to(device), labels.to(device)
 optimizer.zero_grad()
outputs = model_ft(inputs ).double()
loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
running_loss_total += loss.item()
if i%30 == 29:
print('[%d, %5d] loss: %.3f, epoch_accuracy:' %(epoch, i + 1,
running_loss))
running_loss = 0.0
if i%100 == 99:
torch.save(model_ft.state_dict() , 'checkpoint')
 torch.save(optimizer.state_dict() , 'optimizer_checkpoint')
 with open('Running_loss.p','wb')as f:
pickle.dump(running_loss_total,f)
 epoch_loss_data.append(running_loss_total)
categorical_features_indices = np.where(test.dtypes != np.float)[0]
print('Categorial Feature Indices: ', categorical_features_indices )<split>
train(transform=data_transforms,epochs=28,batch_size=32 )
%%time
def run_cv_model(categorical_indices, train, test, target, model_fn, params={}, eval_fn=None, label='model', n_folds=5):
kf = KFold(n_splits=n_folds)
 fold_splits = kf.split(train, target)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0]))
feature_importances = pd.DataFrame()
feature_importances['feature'] = test.columns
i = 1
for dev_index, val_index in fold_splits:
print('-------------------------------------------')
 print('Started ' + label + ' fold ' + str(i)+ f'/{n_folds}')
 dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target.iloc[dev_index], target.iloc[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, fi = model_fn(categorical_indices, dev_X, dev_y, val_X, val_y, test, params2)
 feature_importances[f'fold_{i}'] = fi
pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 print(label + ' cv score {}: {}'.format(i, cv_score), '
')
 i += 1
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std score : {}'.format(label, np.std(cv_scores)))
 pred_full_test = pred_full_test / n_folds
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'fi': feature_importances}
return results
def runCAT(categorical_indices, train_X, train_y, test_X, test_y, test_X2, params):
print('Pool Data')
 _train = Pool(train_X, label=train_y, cat_features = categorical_indices)
 _valid = Pool(test_X, label=test_y, cat_features = categorical_indices)
 print('Train CAT')
 model = CatBoostClassifier(**params)
 fit_model = model.fit(_train,
eval_set=_valid,
use_best_model=True,
verbose=1000,
plot=False)
 feature_im = fit_model.feature_importances_
print('Predict 1/2')
 pred_test_y = fit_model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = fit_model.predict_proba(test_X2)[:, 1]
return pred_test_y, pred_test_y2, feature_im
cat_params = {'loss_function': 'CrossEntropy',
'eval_metric': "AUC",
'task_type': "GPU",
'learning_rate': 0.01,
'iterations': 10000,
'random_seed': 42,
'od_type': "Iter",
'early_stopping_rounds': 500,
}
n_folds = 5
results = run_cv_model(categorical_features_indices, train, test, target, runCAT, cat_params, auc, 'cat', n_folds=n_folds )<save_to_csv>
sub = pd.read_csv(".. /input/sample_submission.csv")
 def test(batch_size, transform, model):
test_set = WhaleDataset(datafolder = ".. /input/test", datatype = "test", transform = transform)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=32, num_workers=num_workers)
 model.eval()
for(data, target, name)in test_loader:
data = data.cuda()
output = model(data)
 print(output)
 output = output.cpu().detach().numpy()
for i,(e, n)in enumerate(list(zip(output, name))):
sub.loc[sub['Image'] == n, 'Id'] = ' '.join(le.inverse_transform(e.argsort() [-5:][::-1]))
sub.to_csv('basic_model.csv', index=False)
<save_to_csv><EOS> 
test(32, test_transforms, model_ft )