Kaggle competition: PetFinder.my Adoption Prediction
train_df = pd.read_csv(str(Path(data_dir)/ 'train.csv'))
test_df = pd.read_csv(str(Path(data_dir)/ 'sample_submission.csv'))
train_df = train_df.sample(100 ).reset_index(drop=True)if debug else train_df<set_options>
def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic')
 def warn(*args, **kwargs):
pass
warnings.warn = warn
%matplotlib inline
pd.options.display.max_rows = 128
pd.options.display.max_columns = 128
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<categorify>
plt.rcParams['figure.figsize'] =(12, 9)
def feature_engineering(df, config=GlobalConfig):
df_ = df.copy()
df_ = pd.get_dummies(df_, columns=['label'])
 df_['label'] = df['label']
return df_<normalization>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
def train_transforms(config):
trans_params = config.trans_params
data_transform = A.Compose([
A.RandomResizedCrop(width=config.input_size[0], height=config.input_size[1],
scale=trans_params['random_resized_crop_scale'],
ratio=trans_params['random_resized_crop_ratio']),
A.Transpose(p=0.5),
A.HorizontalFlip(p=0.5),
A.ShiftScaleRotate(p=0.5),
A.HueSaturationValue(hue_shift_limit=trans_params['hue_shift_limit'],
sat_shift_limit=trans_params['sat_shift_limit'],
val_shift_limit=trans_params['val_shift_limit'], p=0.5),
A.RandomBrightnessContrast(brightness_limit=trans_params['brightness_limit'],
contrast_limit=trans_params['contrast_limit'], p=0.5),
A.Normalize(mean=trans_params['rgb_mean'], std=trans_params['rgb_std'], max_pixel_value=255.0),
A.CoarseDropout(p=0.5),
A.Cutout(p=0.5),
ToTensorV2() ,
])
 return data_transform
def valid_transforms(config):
trans_params = config.trans_params
data_transform = A.Compose([
A.Resize(width=config.scale_size[0], height=config.scale_size[1]),
A.CenterCrop(width=config.input_size[0], height=config.input_size[1]),
A.Normalize(mean=trans_params['rgb_mean'], std=trans_params['rgb_std'], max_pixel_value=255.0),
ToTensorV2() ,
])
 return data_transform
def test_transforms(config):
trans_params = config.trans_params
data_transform = A.Compose([
A.Resize(width=config.scale_size[0], height=config.scale_size[1]),
A.CenterCrop(width=config.input_size[0], height=config.input_size[1]),
A.Normalize(mean=trans_params['rgb_mean'], std=trans_params['rgb_std'], max_pixel_value=255.0),
ToTensorV2() ,
])
 return data_transform
def tta_transforms(config, num_augs=3, mode='normal'):
trans_params = config.trans_params
interpolation = getattr(Image, trans_params['interpolation'], 2)
 if mode == 'heavy':
main_transforms = RandomAugs(config=config, num_augs=num_augs)
 else:
main_transforms = CustomFourCrop(size=config.input_size)
 data_transform = transforms.Compose([
main_transforms,
transforms.Lambda(lambda augs: torch.stack([
transforms.ToTensor()(aug)for aug in augs])) ,
transforms.Lambda(lambda augs: torch.stack([
transforms.Normalize(trans_params['rgb_mean'], trans_params['rgb_std'] )(aug)for aug in augs])) ,
])
 return data_transform<data_type_conversions>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
class CassavaDataset(Dataset):
def __init__(self, config, df, phase, transforms, indices=None):
self.image_ids = df.image_id.values
self.data_transform = transforms
self.phase = phase
if phase in ['train', 'valid']:
self.targets = df[config.onehot_cols].values.astype(np.float32)
 images_dir = train_images_dir
else:
images_dir = test_images_dir
self.images_dir = images_dir
if indices is None:
indices = torch.from_numpy(df.index.values.astype(np.int))
self.indices = indices
def __len__(self):
return len(self.indices)
 def __getitem__(self, index):
index = self.indices[index]
image_id = self.image_ids[index]
image_path = str(Path(self.images_dir)/ image_id)
 image = cv2.imread(image_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.phase in ['train', 'valid']:
image = self.data_transform(image=image)['image']
target = self.targets[index]
return image_id, image, target
elif self.phase == 'tta':
image = transforms.ToPILImage()(image)
 tta_images = self.data_transform(image)
 return image_id, tta_images
else:
image = self.data_transform(image=image)['image']
return image_id, image<prepare_x_and_y>
def maek_features(src_df):
rescuer_count = src_df.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_CNT']
src_df = src_df.merge(rescuer_count, how='left', on='RescuerID')
 return src_df
def check_dataset(config=GlobalConfig, index=[0, 1, 2]):
train_df_ = train_df.copy()
train_df_ = feature_engineering(train_df_, config=config)
 train_dataset = CassavaDataset(config=DataConfig, df=train_df_, phase='train',
transforms=train_transforms(DataConfig))
for i in index:
image_id, image, target = train_dataset.__getitem__(index=i)
 image = np.clip(image.numpy().transpose(( 1, 2, 0)) , 0, 1)
 print(f"image_id: {image_id}, target: {target}")
 plt.imshow(image)
 plt.show()
check_dataset()<train_model>
train = maek_features(train)
 test = maek_features(test )
class DataLoaderMaker() :
def __init__(self, train_df, test_df, config=DataConfig):
self.config = config
self.batch_size = config.batch_size
self.num_workers = config.num_workers
self.train_df = train_df
self.test_df = test_df
def _train_dataset(self, indices):
return CassavaDataset(df=self.train_df, phase='train', transforms=train_transforms(self.config),
indices=indices, config=self.config)
 def train_dataloader(self, train_index):
return DataLoader(self._train_dataset(train_index), batch_size=self.batch_size,
num_workers=self.num_workers, drop_last=True, shuffle=True)
 def _valid_dataset(self, indices):
return CassavaDataset(df=self.train_df, phase='valid', transforms=valid_transforms(self.config),
indices=indices, config=self.config)
 def valid_dataloader(self, valid_index):
return DataLoader(self._valid_dataset(valid_index), batch_size=self.batch_size,
num_workers=self.num_workers, drop_last=False, shuffle=False)
 def _test_dataset(self):
return CassavaDataset(df=self.test_df, phase='test', transforms=test_transforms(self.config),
config=self.config)
 def test_dataloader(self):
return DataLoader(self._test_dataset() , batch_size=self.batch_size,
num_workers=self.num_workers, drop_last=False, shuffle=False)
 def _tta_dataset(self, num_augs):
return CassavaDataset(df=self.test_df, phase='tta', transforms=tta_transforms(self.config, num_augs),
config=self.config)
 def tta_dataloader(self, num_augs=3):
return DataLoader(self._tta_dataset(num_augs), batch_size=self.batch_size,
num_workers=self.num_workers, drop_last=False, shuffle=False )<choose_model_class>
train_img = pd.read_csv(".. /input/extractimagefeaturesfrompretrainednn/train_img_features.csv")
 test_img = pd.read_csv(".. /input/extractimagefeaturesfrompretrainednn/test_img_features.csv")
 train_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 test_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 train = pd.concat([train, train_img], axis=1)
 test = pd.concat([test, test_img], axis=1 )
class CassavaNet(nn.Module):
def __init__(self, config=ModelConfig):
super().__init__()
self.image_model = self._get_image_model(config)
 self.image_model.classifier = nn.Linear(in_features=self.image_model.classifier.in_features,
out_features=config.num_targets)
 def _get_image_model(self, config):
if kaggle:
image_model = timm.create_model(config.image_model_name, pretrained=False)
 image_model.load_state_dict(torch.load(config.path))
else:
image_model = timm.create_model(config.image_model_name, pretrained=True)
 return image_model
def forward(self, image):
x = self.image_model(image)
 return x<categorify>
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f )
def fix_model_state_dict(state_dict):
new_state_dict = OrderedDict()
for k, v in state_dict.items() :
name = k
if name.startswith('module.'):
name = name[7:]
new_state_dict[name] = v
return new_state_dict<train_model>
cat_ratings = ratings['cat_breeds']
dog_ratings = ratings['dog_breeds']
class Fitter(nn.Module):
def __init__(self, model, dataloader_maker, train_index=None, valid_index=None, config=FitterConfig):
super(Fitter, self ).__init__()
self.device = config.device
self.train_criterion = config.train_criterion.to(config.device)
 self.valid_criterion = config.valid_criterion.to(config.device)
 self.train_index = torch.from_numpy(train_index)
 self.valid_index = torch.from_numpy(valid_index)
 self.best_loss = np.Inf
self.best_acc = -np.Inf
self.train_dataloader = dataloader_maker.train_dataloader(self.train_index)
 self.valid_dataloader = dataloader_maker.valid_dataloader(self.valid_index)
 self.test_dataloader = dataloader_maker.test_dataloader()
self.tta_dataloader = dataloader_maker.tta_dataloader(config.num_augs_tta)
 if not kaggle and config.use_multi_gpus and torch.cuda.device_count() > 1:
model = nn.DataParallel(model, device_ids=config.device_ids)
 self.model = model.to(config.device)
 self.optimizer = config.optimizer(params=self.model.parameters())
 self.scheduler = config.scheduler(optimizer=self.optimizer)
 self.scaler = GradScaler()
self.num_epochs = config.num_epochs
self.batch_size = config.batch_size
self.early_stopping_start = config.early_stopping_start
self.early_stopping_rounds = config.early_stopping_rounds
self.num_targets = config.num_targets
self.step_scheduler = config.step_scheduler
self.valid_scheduler = config.valid_scheduler
self.scheduler_update_by_loss = config.scheduler_update_by_loss
self.iters_to_accumulate = config.iters_to_accumulate
self.onehot_cols = config.onehot_cols
self.change_device_config = config.change_device_config
self.use_tta = config.use_tta
self.weights_mode = config.weights_mode
def fit(self):
self.fold_logger = FoldLogger()
wait = 0
for epoch in range(self.num_epochs):
t = time.time()
loss, acc, roc_auc = self._train_one_epoch()
print(f"[RESULT]: Train.Epoch: {epoch}, acc: {acc.avg:.9f}, roc_auc: {roc_auc.avg:.9f}"
+ f", loss: {loss.avg:.9f}, time: {(time.time() - t):.5f}")
 self.fold_logger.update_train(train_loss=loss.avg, train_acc=acc.avg)
 t = time.time()
loss, acc, roc_auc, valid_preds_df = self._valid_one_epoch()
print(f"[RESULT]: Valid.Epoch: {epoch}, acc: {acc.avg:.9f}, roc_auc: {roc_auc.avg:.9f}"
+ f", loss: {loss.avg:.9f}, time: {(time.time() - t):.5f}")
 self.fold_logger.update_valid(valid_loss=loss.avg, valid_acc=acc.avg)
 if self.valid_scheduler:
if self.scheduler_update_by_loss:
self.scheduler.step(loss.avg)
 else:
self.scheduler.step()
current = acc.avg
if current > self.best_acc:
print(f'Validation acc increased({self.best_acc:.9f} --> {current:.9f} ).Saving model...')
 wait = 0
self.best_acc = current
loss_at_best_acc = loss.avg
roc_auc_at_best_acc = roc_auc.avg
self.weights_at_best_acc = copy.deepcopy(self.model.state_dict())
 if self.weights_mode == 'acc':
self.best_weights = self.weights_at_best_acc
current = loss.avg
if current < self.best_loss:
print(f'Validation loss decreased({self.best_loss:.9f} --> {current:.9f} ).Saving model...')
 wait = 0
self.best_loss = current
acc_at_best_loss = acc.avg
roc_auc_at_best_loss = roc_auc.avg
self.weights_at_best_loss = copy.deepcopy(self.model.state_dict())
 if self.weights_mode == 'loss':
self.best_weights = self.weights_at_best_loss
elif epoch > self.early_stopping_start:
wait += 1
print(f'EarlyStopping counter: {wait} out of {self.early_stopping_rounds}')
 if(self.early_stopping_rounds > 0)and(wait >= self.early_stopping_rounds):
print('Epoch %05d: early stopping' %(epoch))
break
if self.weights_mode == 'acc':
return loss_at_best_acc, self.best_acc, roc_auc_at_best_acc, valid_preds_df
else:
return self.best_loss, acc_at_best_loss, roc_auc_at_best_loss, valid_preds_df
def _train_one_epoch(self):
self.model.train()
epoch_loss = AverageMeter()
epoch_acc = AccMeter(n_class=self.num_targets)
 epoch_roc_auc = RocAucMeter(n_class=self.num_targets)
 for step,(image_ids, images, targets)in enumerate(tqdm(self.train_dataloader)) :
images = images.to(self.device, dtype=torch.float)
 targets = targets.to(self.device, dtype=torch.float)
 batch_size = len(image_ids)
 with autocast() :
preds = self.model(images)
 loss = self.train_criterion(preds, targets)
 self.scaler.scale(loss / self.iters_to_accumulate ).backward()
if(step + 1)% self.iters_to_accumulate == 0 or step + 1 == len(self.train_dataloader):
self.scaler.step(self.optimizer)
 self.scaler.update()
self.optimizer.zero_grad()
if self.step_scheduler:
self.scheduler.step()
epoch_loss.update(loss.item() , batch_size)
 epoch_acc.update(targets, preds)
 epoch_roc_auc.update(targets, preds)
 return epoch_loss, epoch_acc, epoch_roc_auc
def _valid_one_epoch(self):
self.model.eval()
epoch_loss = AverageMeter()
epoch_acc = AccMeter(n_class=self.num_targets)
 epoch_roc_auc = RocAucMeter(n_class=self.num_targets)
 image_ids_list = []
valid_preds = np.empty(( 0, self.num_targets), dtype=np.float)
 for image_ids, images, targets in tqdm(self.valid_dataloader):
images = images.to(self.device, dtype=torch.float)
 targets = targets.to(self.device, dtype=torch.float)
 batch_size = len(image_ids)
 with torch.no_grad() :
preds = self.model(images)
 loss = self.valid_criterion(preds, targets)
 epoch_loss.update(loss.item() , batch_size)
 epoch_acc.update(targets, preds)
 epoch_roc_auc.update(targets, preds)
 image_ids_list += list(image_ids)
 valid_preds = np.append(valid_preds, preds.data.cpu().numpy() , axis=0)
 valid_preds_df = pd.concat([pd.DataFrame(image_ids_list, columns=['image_id']),
pd.DataFrame(valid_preds, columns=self.onehot_cols)], axis=1)
 return epoch_loss, epoch_acc, epoch_roc_auc, valid_preds_df
def validation(self):
loss, acc, roc_auc, valid_preds_df = self._valid_one_epoch()
return loss.avg, acc.avg, roc_auc.avg, valid_preds_df
def _prediction(self):
self.set_model_weights(self.best_weights)
 self.model.eval()
image_ids_list = []
test_preds = np.empty(( 0, self.num_targets), dtype=np.float)
 for image_ids, images in tqdm(self.test_dataloader):
images = images.to(self.device, dtype=torch.float)
 batch_size = len(image_ids)
 with torch.no_grad() :
preds = self.model(images)
 image_ids_list += list(image_ids)
 test_preds = np.append(test_preds, preds.data.cpu().numpy() , axis=0)
 test_preds_df = pd.concat([pd.DataFrame(image_ids_list, columns=['image_id']),
pd.DataFrame(test_preds, columns=self.onehot_cols)], axis=1)
 return test_preds_df
def _prediction_with_tta(self):
self.set_model_weights(self.best_weights)
 self.model.eval()
image_ids_list = []
test_preds = np.empty(( 0, self.num_targets), dtype=np.float)
 for image_ids, tta_images in tqdm(self.tta_dataloader):
tta_images = tta_images.to(self.device, dtype=torch.float ).transpose(0, 1)
 n_augs, batch_size, ch, height, width = tta_images.size()
with torch.no_grad() :
preds_ = torch.zeros(batch_size, self.num_targets ).to(self.device)
 for i in range(n_augs):
preds = self.model(tta_images[i])
 preds_ += preds.to(self.device)
 preds_ /= n_augs
image_ids_list += list(image_ids)
 test_preds = np.append(test_preds, preds_.data.cpu().numpy() , axis=0)
 test_preds_df = pd.concat([pd.DataFrame(image_ids_list, columns=['image_id']),
pd.DataFrame(test_preds, columns=self.onehot_cols)], axis=1)
 return test_preds_df
def prediction(self):
if self.use_tta:
test_preds_df = self._prediction_with_tta()
else:
test_preds_df = self._prediction()
return test_preds_df
def get_best_model_weights(self):
return self.best_weights
def set_model_weights(self, weights):
self.model.load_state_dict(weights)
 def save_model_weights(self, path):
self.model.eval()
torch.save(self.model.state_dict() , path)
 def save_best_model_weights(self, path):
torch.save(self.best_weights, path)
 def load_model_weights(self, path):
if self.change_device_config:
self.best_weights = fix_model_state_dict(torch.load(path))
else:
self.best_weights = torch.load(path)
 self.model.load_state_dict(self.best_weights)
 print(f"weights: {path} loaded.")
 def get_fold_log(self, fold=None):
if fold is not None:
title = f'convergence check of fold {fold}'
fig_title = f'fold{fold}.png'
self.fold_logger.get_graph(title=title, fig_title=fig_title)
 else:
title = f'convergence check'
self.fold_logger.get_graph(title=title)
 def delete(self):
del self.model, self.optimizer, self.scheduler, self.scaler<compute_test_metric>
breed_id = {}
for id,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[id] = name
def get_loss(preds, targets, config=FitterConfig):
preds = torch.from_numpy(preds ).to(config.device ).float()
targets = torch.from_numpy(targets ).to(config.device ).float()
with torch.no_grad() :
loss = config.valid_criterion(preds, targets)
 return loss<create_dataframe>
breed_names_1 = [i for i in cat_ratings.keys() ]
breed_names_2 = [i for i in dog_ratings.keys() ]
def get_label(df, config=GlobalConfig):
return pd.concat([df, pd.DataFrame(np.argmax(df[config.onehot_cols].values, axis=-1), columns=['label'])], axis=1 )<categorify>
for id in train['Breed1']:
if id in breed_id.keys() :
name = breed_id[id]
if name in breed_names_1:
for key in cat_ratings[name].keys() :
train[key] = cat_ratings[name][key]
if name in breed_names_2:
for key in dog_ratings[name].keys() :
train[key] = dog_ratings[name][key]
def get_probability(df, config=GlobalConfig):
preds = torch.from_numpy(df[config.onehot_cols].values ).to(config.device)
 with torch.no_grad() :
probs = nn.functional.softmax(preds, dim=-1 ).data.cpu().numpy()
df_ = pd.concat([df.drop(columns=config.onehot_cols), pd.DataFrame(probs, columns=config.onehot_cols)], axis=1)
 return df_<compute_test_metric>
state_gdp = {
41336: 116.679,
41325: 40.596,
41367: 23.02,
41401: 190.075,
41415: 5.984,
41324: 37.274,
41332: 42.389,
41335: 52.452,
41330: 67.629,
41380: 5.642,
41327: 81.284,
41345: 80.167,
41342: 121.414,
41326: 280.698,
41361: 32.270
}
state_population = {
41336: 33.48283,
41325: 19.47651,
41367: 15.39601,
41401: 16.74621,
41415: 0.86908,
41324: 8.21110,
41332: 10.21064,
41335: 15.00817,
41330: 23.52743,
41380: 2.31541,
41327: 15.61383,
41345: 32.06742,
41342: 24.71140,
41326: 54.62141,
41361: 10.35977
}
train["state_gdp"] = train.State.map(state_gdp)
 train["state_population"] = train.State.map(state_population)
 test["state_gdp"] = test.State.map(state_gdp)
 test["state_population"] = test.State.map(state_population )
def get_confusion_matrix(preds_list, targets_list):
cm = ConfusionMatrix(targets_list, preds_list)
 cm.relabel(mapping={0: 'cbb', 1: 'cbsd', 2: 'cgm', 3: 'cmd', 4: 'healthy'})
 cm.save_obj(os.path.join(cache_dir, 'cm'))<choose_model_class>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )
def evaluate_model_with_cv(train_df, test_df, config=GlobalConfig):
seed_everything(config.seed)
 dataloader_maker = DataLoaderMaker(train_df, test_df, config=DataConfig)
 skf = StratifiedKFold(n_splits=config.num_folds, random_state=config.seed, shuffle=config.shuffle)
 oof_valid_preds_df_ = pd.DataFrame(index=[], columns=['image_id'] + config.onehot_cols)
 fold_best_losses = []
for fold,(train_index, valid_index)in enumerate(skf.split(train_df.values, train_df.label.values)) :
t = time.time()
print(f'----- fold {fold}/{config.num_folds - 1} Started.-----')
 model = CassavaNet()
fitter = Fitter(model, dataloader_maker, train_index, valid_index)
 if mode == 'training':
if config.load_weights:
file_path = Path(load_weights_dir)/ f'{config.image_model_name}_{str(fold)}.pth'
fitter.load_model_weights(str(file_path))
file_path = Path(save_weights_dir)/ f'{config.image_model_name}_{str(fold)}.pth'
fold_best_loss, fold_acc, fold_roc_auc, fold_valid_preds_df = fitter.fit()
fitter.save_best_model_weights(str(file_path))
fitter.get_fold_log(fold)
 else:
file_path = Path(load_weights_dir)/ f'{config.image_model_name}_{str(fold)}.pth'
fitter.load_model_weights(str(file_path))
fold_best_loss, fold_acc, fold_roc_auc, fold_valid_preds_df = fitter.validation()
print(f"[fold {fold} best]: Valid.acc: {fold_acc:.9f}, roc_auc: {fold_roc_auc:.9f}"
+ f", loss: {fold_best_loss:.9f}, time: {(time.time() - t):.5f}")
 fold_best_losses.append(fold_best_loss)
 oof_valid_preds_df_ = pd.concat([oof_valid_preds_df_, fold_valid_preds_df],
sort=False, ignore_index=True)
 fold_preds_df = fitter.prediction()
if fold == 0:
oof_preds_df = fold_preds_df.copy()
else:
oof_preds_df[config.onehot_cols] += fold_preds_df[config.onehot_cols]
fitter.delete()
del model, fitter
del fold_preds_df, fold_valid_preds_df
gc.collect()
oof_valid_preds_df = pd.merge(train_df.image_id, oof_valid_preds_df_, on='image_id', how='outer')
 oof_preds_df[config.onehot_cols] = oof_preds_df[config.onehot_cols] / config.num_folds
oof_labels = train_df.label.values
oof_valid_preds = np.argmax(oof_valid_preds_df[config.onehot_cols].values, axis=-1)
 oof_valid_accuracy = metrics.accuracy_score(oof_labels, oof_valid_preds, normalize=True)
 get_confusion_matrix(preds_list=oof_valid_preds.tolist() , targets_list=oof_labels.tolist())
 print(f"[OOF]: acc: {oof_valid_accuracy:.9f}, loss(CV): {np.mean(fold_best_losses):.9f}"
+ f", std of fold best: {np.std(fold_best_losses):.9f}")
 return oof_preds_df, oof_valid_preds_df<save_to_csv>
test_df_ids = test[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )
def run_everything() :
t = time.time()
print("device: {}".format(GlobalConfig.device))
train_df_ = train_df.copy()
test_df_ = test_df.copy()
train_df_ = feature_engineering(train_df_)
 oof_preds_df, oof_valid_preds_df = evaluate_model_with_cv(train_df_, test_df_)
 oof_valid_preds_df = get_label(oof_valid_preds_df)
 oof_preds_df = get_label(oof_preds_df)
 submission_df = oof_preds_df.drop(columns=GlobalConfig.onehot_cols ).copy()
oof_valid_preds_df.to_csv(str(Path(cache_dir)/ 'valid_preds.csv'), index=False)
 oof_preds_df.to_csv(str(Path(cache_dir)/ 'preds.csv'), index=False)
 submission_df.to_csv(str(Path(cache_dir)/ 'submission.csv'), index=False)
 oof_probs_df = get_probability(oof_preds_df)
 oof_probs_df.to_csv(str(Path(cache_dir)/ 'probs.csv'), index=False)
 print(f'All Completed.total time: {(time.time() - t):.5f}' )<load_from_csv>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
valid_preds_df = pd.read_csv(str(Path(cache_dir)/ 'valid_preds.csv'))
valid_preds_df<load_from_csv>
aggregates = ['mean', 'sum', 'var']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
cm = ConfusionMatrix(file=open(os.path.join(cache_dir, 'cm.obj'), 'r'))<load_from_csv>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
preds_df = pd.read_csv(str(Path(cache_dir)/ 'preds.csv'))
preds_df<load_from_csv>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
probs_df = pd.read_csv(str(Path(cache_dir)/ 'probs.csv'))
probs_df<load_from_csv>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))
submission_df = pd.read_csv(str(Path(cache_dir)/ 'submission.csv'))
submission_df<import_modules>
column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
print('\tinteger columns:
{}'.format(int_cols))
print('
\tfloat columns:
{}'.format(float_cols))
print('
\tto encode categorical columns:
{}'.format(cat_cols))
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
BATCH_SIZE = 64
ROW = 300
COL = 300
train_csv_loc = '.. /input/cassava-leaf-disease-classification/train.csv'
train_location = '.. /input/cassava-leaf-disease-classification/train_images/'
test_location = '.. /input/cassava-leaf-disease-classification/test_images'<normalization>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
def augmentations(file):
file = read_file(file)
 file = image.decode_jpeg(file, channels=3)
 transform = A.Compose([
A.RandomRotate90() ,
A.Flip() ,
A.Transpose() ,
A.OneOf([
A.IAAAdditiveGaussianNoise() ,
A.GaussNoise() ,
], p=0.2),
A.OneOf([
A.MotionBlur(p=.2),
A.MedianBlur(blur_limit=3, p=0.1),
A.Blur(blur_limit=3, p=0.1),
], p=0.2),
A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2),
A.OneOf([
A.OpticalDistortion(p=0.3),
A.GridDistortion(p=.1),
A.IAAPiecewiseAffine(p=0.3),
], p=0.2),
A.OneOf([
A.CLAHE(clip_limit=2),
A.IAASharpen() ,
A.IAAEmboss() ,
A.RandomBrightnessContrast() ,
A.HueSaturationValue(hue_shift_limit=30),
A.RandomFog(p=.3)
 ], p=0.7),
A.HueSaturationValue(p=0.3),
A.OneOf([
A.CoarseDropout(max_holes=16, max_height=16, max_width=16),
A.Cutout(num_holes=16, max_h_size=16, max_w_size=16),
A.ElasticTransform(alpha=1, sigma=25)
 ], p=0.4),
])
 file = transform(image=file.numpy())['image']
file = preprocess_input(file)
 file = image.resize(file, [ROW, COL])
 return file<normalization>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
def fetch_image_without_aug(filename , label):
image_file = read_file(filename)
 image_file = image.decode_jpeg(image_file, channels=3)
 image_file = preprocess_input(image_file)
 image_file = image.resize(image_file, [ROW, COL])
 return image_file, label<prepare_x_and_y>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
def fetch_image_with_aug(filename , label):
aug_img = numpy_function(func=augmentations, inp=[filename], Tout=float32)
 return aug_img, label<create_dataframe>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )
def getDatasetFromDataframe(train_files, train_labels, val_files, val_labels):
train_ds = Dataset.from_tensor_slices(( train_files, train_labels))
train_ds = train_ds.shuffle(len(train_files))
train_ds = train_ds.map(fetch_image_with_aug , num_parallel_calls=16)
 train_ds = train_ds.batch(BATCH_SIZE)
 train_ds = train_ds.prefetch(AUTOTUNE)
 val_ds = Dataset.from_tensor_slices(( val_files, val_labels))
val_ds = val_ds.shuffle(len(val_files))
val_ds = val_ds.map(fetch_image_without_aug , num_parallel_calls=16)
 val_ds = val_ds.batch(BATCH_SIZE)
 val_ds = val_ds.prefetch(AUTOTUNE)
 return train_ds , val_ds<choose_model_class>
n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
def create_model(training = True, weights = "imagenet"):
base_model = EfficientNetB3(weights=weights , include_top=False , input_shape=(ROW,COL,3))
x = Input(shape =(ROW,COL,3))
out_1 = base_model(x,training = training)
 out_1 = GlobalAveragePooling2D(name = 'encoding' )(out_1)
 out_1 = Dropout(0.5 )(out_1)
 output = Dense(5 , activation="softmax" )(out_1)
 final_model = Model(inputs = x , outputs = output)
 final_model.summary()
return final_model<feature_engineering>
categorical_features = ["Type", "Breed1", "Breed2", "Color1" ,"Color2", "Color3", "State"]
impact_coding_map = {}
for f in categorical_features:
print("Impact coding for {}".format(f))
X_train["impact_encoded_{}".format(f)], impact_coding_mapping, default_coding = impact_coding(X_train, f, target="AdoptionSpeed")
 impact_coding_map[f] =(impact_coding_mapping, default_coding)
 mapping, default_mean = impact_coding_map[f]
X_test["impact_encoded_{}".format(f)] = X_test.apply(lambda x: mapping[x[f]] if x[f] in mapping
else default_mean, axis=1)
 for cat in categorical_features:
X_train = frequency_encoding(X_train, cat)
 X_test = frequency_encoding(X_test, cat )
train_csv = pd.read_csv(train_csv_loc)
 train_csv['image_id'] = train_csv['image_id'].map(lambda x: train_location+x)
 Folder_name = "Exp_"
create_k_folds = StratifiedKFold(n_splits=5)
 fold_number = 1
for train_index, test_index in create_k_folds.split(train_csv['image_id'] , train_csv['label']):
print("Fold Number {} is starting it's training".format(fold_number))
print("Creating Callbacks...")
 Folder_name = Folder_name + str(fold_number)
 callbacks, histories = create_callbacks(Folder_name)
 print("Importing Datasets...")
 X_train, X_test = train_csv['image_id'].loc[train_index], train_csv['image_id'].loc[test_index]
y_train, y_test = train_csv['label'].loc[train_index], train_csv['label'].loc[test_index]
train_ds, val_ds = getDatasetFromDataframe(X_train, y_train, X_test, y_test)
 final_model = create_model()
final_model.compile(optimizer=RMSprop(learning_rate=1e-4),
loss=SparseCategoricalCrossentropy() , metrics=['accuracy'])
 try:
print("Starting training...")
 hist = final_model.fit(train_ds, epochs=15,
validation_data=val_ds, callbacks=callbacks, workers=16)
 except:
print("Error...")
 fold_number += 1<load_pretrained>
np.sum(pd.isnull(X_test))
checkpoints = glob.glob(os.path.join('.. /input/efficient-net-weights/Weights2' , 'Exp2_*.h5'))
Models = []
for checkpoint in checkpoints:
new_model = create_model(training = False, weights = None)
 new_model.load_weights(checkpoint)
 Models.append(new_model)
 print('Models loaded...')
 images = os.listdir(test_location)
 results = []
for single in images:
image = load_img(test_location+'/'+single , target_size=(ROW,COL))
image = img_to_array(image)
 image = preprocess_input(image)
 batch = np.array([image])
 res = []
for model in Models:
res.append(model.predict(batch))
res = np.mean(res , 0)
 results.append({"image_id" : single , "label" : np.argmax(res)} )<save_to_csv>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
submission = pd.DataFrame(results)
 submission.to_csv('submission.csv' , index = False )<import_modules>
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 10000
n_splits = 5
import pandas as pd
from fastai.vision.all import *
import os<set_options>
X_train = X_train.drop('img_Unnamed: 0',axis=1)
 X_test = X_test.drop('img_Unnamed: 0',axis=1 )
print(torch.cuda.is_available() , torch.backends.cudnn.enabled)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True<define_variables>
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train_lgb = np.zeros(( X_train.shape[0]))
oof_test_lgb = np.zeros(( X_test.shape[0], n_splits))
qwk_scores = []
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(val_pred, y_val)
 coefficients = optR.coefficients()
coefficients[0] = 1.645
coefficients[1] = 2.115
coefficients[3] = 2.84
pred_val_y_k = optR.predict(val_pred, coefficients)
 print("Valid Counts = ", Counter(y_val))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(y_val, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train_lgb[valid_index] = val_pred
oof_test_lgb[:, i] = test_pred
i += 1
print('{} cv mean QWK score : {}'.format('LGBM', np.mean(qwk_scores)) )
path = Path('.. /input/cassava-leaf-disease-classification' )<load_from_csv>
importance_type= "split"
idx_sort = np.argsort(model.feature_importance(importance_type=importance_type)) [::-1]
names_sorted = np.array(model.feature_name())[idx_sort]
imports_sorted = model.feature_importance(importance_type=importance_type)[idx_sort]
for n, im in zip(names_sorted, imports_sorted):
print(n, im )
sample_sub = pd.read_csv(path/'sample_submission.csv')
 sample_sub<count_values>
optR = OptimizedRounder()
optR.fit(oof_train_lgb, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train_lgb, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk )
train = pd.read_csv(path/'train.csv')
 print(train.shape)
 print(train.label.value_counts())
 print(train.label.value_counts(1))<filter>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions_lgb = optR.predict(oof_train_lgb, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions_lgb)))
 test_predictions_lgb = optR.predict(oof_test_lgb.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions_lgb)) )
train = train[~train['image_id'].isin(['1562043567.jpg', '3551135685.jpg', '2252529694.jpg'])]<prepare_x_and_y>
print("True Distribution:")
 print(pd.value_counts(X_train['AdoptionSpeed'], normalize=True ).sort_index())
 print("
Train Predicted Distribution:")
 print(pd.value_counts(train_predictions_lgb, normalize=True ).sort_index())
 print("
Test Predicted Distribution:")
 print(pd.value_counts(test_predictions_lgb, normalize=True ).sort_index() )
def get_x(r):
return path/'train_images'/r['image_id']
def get_y(r):
return r['label']<load_pretrained>
df = pd.concat([X_train, X_test], axis=0)
 df.head(2 )
dls = data.dataloaders(train, bs=bs)
 dls.show_batch(max_n=9 )<choose_model_class>
df_ = pd.concat([train,test],axis=0)
learn = cnn_learner(dls, resnet50, loss_func=LabelSmoothingCrossEntropy() , metrics=accuracy ).to_native_fp16()<find_best_params>
word_vec_size = 300
max_words = 100
max_word_features = 25000
def transform_text(text, tokenizer):
tokenizer.fit_on_texts(text)
 text_emb = tokenizer.texts_to_sequences(text)
 text_emb = sequence.pad_sequences(text_emb, maxlen=max_words)
 return text_emb
desc_tokenizer = text.Tokenizer(num_words=max_word_features)
 desc_embs = transform_text(df_["Description"].astype(str), desc_tokenizer )
learn.lr_find()<choose_model_class>
text_mode = "fasttext"
if text_mode == "fasttext":
embedding_file = ".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec"
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(embedding_file))
word_index = desc_tokenizer.word_index
print('Word index len:', len(word_index))
vocabulary_size = min(max_word_features, len(word_index)) + 1
text_embs = np.zeros(( vocabulary_size, word_vec_size))
for word, i in word_index.items() :
if i >= max_word_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: text_embs[i] = embedding_vector
del(embeddings_index)
 elif text_mode == "word2vec":
embedding_file = ".. /input/word2vec-google/GoogleNews-vectors-negative300.bin"
print("Loading word vectors...")
 word_vectors = KeyedVectors.load_word2vec_format(embedding_file, binary=True)
 print("Matching word vectors...")
 EMBEDDING_DIM=300
word_index = desc_tokenizer.word_index
vocabulary_size=min(len(word_index)+1,max_word_features)
 text_embs = np.zeros(( vocabulary_size, EMBEDDING_DIM))
for word, i in word_index.items() :
if i>=max_word_features:
continue
try:
embedding_vector = word_vectors[word]
text_embs[i] = embedding_vector
except KeyError:
text_embs[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)
 del(word_vectors )
cbs1 = [MixUp(alpha = 0.7)]
cbs2 = [MixUp(alpha = 0.3)]<train_model>
np.sum(pd.isnull(df))
learn.freeze()
learn.fit_one_cycle(25, slice(3e-2), cbs = cbs1 )<find_best_params>
cat_vars = ["Type", "Breed1", "Breed2", "Color1", "Color2", "Color3", "Gender", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized", "Health", "State"]
cont_vars = ["Fee", "PhotoAmt", "VideoAmt", "Age", "Quantity",'RescuerID_CNT']
learn.unfreeze()
learn.lr_find()<train_model>
def preproc(df):
global cont_vars
for var in cat_vars:
df[var] = LabelEncoder().fit_transform(df[var])
 for var in cont_vars:
df[var] = MinMaxScaler().fit_transform(df[var].values.reshape(-1,1))
return df
learn.fit_one_cycle(5, slice(2e-5/50), cbs = cbs2 )<load_from_csv>
df_scaled = preproc(df)
 train_df = df_scaled[:len(train)]
test_df = df_scaled[len(train):]
len(train_df), len(test_df )
sub = pd.read_csv(path/'sample_submission.csv')
 sub<prepare_output>
def get_keras_data(df, description_embeds):
X = {var: df[var].values for var in cont_vars+cat_vars}
X["description"] = description_embeds
for i in range(256): X[f"img_{i}"] = df[f"img_{i}"]
return X
sub2 = sub.copy()
sub2['image_id'] = sub2['image_id'].apply(lambda x: f'.. /test_images/{x}' )<categorify>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class QWKEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.history = []
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, batch_size=1000, verbose=0)
 y_pred = eval_predict(self.y_val, y_pred)
 score = quadratic_weighted_kappa(self.y_val, y_pred)
 print("QWK - epoch: %d - score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history): self.model.save('checkpoint.h5')
 def eval_predict(y=[], y_pred=[], coeffs=None, ret_coeffs=False):
optR = OptimizedRounder()
if not coeffs:
optR.fit(y_pred.reshape(-1,), y)
 coeffs = optR.coefficients()
if ret_coeffs: return optR.coefficients()
return optR.predict(y_pred, coeffs ).reshape(-1, )
learn = learn.to_native_fp32()<train_model>
def rmse(y, y_pred):
return K.sqrt(K.mean(K.square(y-y_pred), axis=-1))
def get_model(emb_n=10, dout=.25, batch_size=1000):
inps = []
embs = []
nums = []
for var in cat_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 embs.append(( Embedding(df[var].max() +1, emb_n )(inp)))
 for var in cont_vars:
inp = Input(shape=[1], name=var)
 inps.append(inp)
 nums.append(( inp))
desc_inp = Input(shape=(max_words,), name="description")
 inps.append(desc_inp)
 emb_desc = Embedding(vocabulary_size, word_vec_size, weights=[text_embs] )(desc_inp)
 emb_desc = SpatialDropout1D (.4 )(emb_desc)
 emb_desc = Reshape(( max_words, word_vec_size, 1))(emb_desc)
 filter_sizes=[1,3,1]
convs = []
for filter_size in filter_sizes:
conv = Conv2D(32, kernel_size=(filter_size, word_vec_size),
kernel_initializer="normal", activation="relu" )(emb_desc)
 convs.append(MaxPool2D(pool_size=(max_words-filter_size+1, 1))(conv))
img_fts = []
for i in range(256):
inp = Input(shape=[1], name=f"img_{i}")
 inps.append(inp)
 img_fts.append(( inp))
img_fts = concatenate(img_fts)
 img_fts = Dropout (.5 )(img_fts)
 img_fts = BatchNormalization()(img_fts)
 img_fts = Dense(64, activation="relu" )(img_fts)
 convs = concatenate(convs)
 convs = Flatten()(convs)
 convs = BatchNormalization()(convs)
 embs = Flatten()(concatenate(embs))
embs = Dropout(dout )(Dense(64, activation="relu", kernel_initializer="he_normal" )(embs))
nums = concatenate(nums)
 nums = Dense(32, activation="relu" )(nums)
 nums = Dropout(0.5 )(nums)
 x = concatenate([embs, nums, convs, img_fts])
 x = BatchNormalization()(x)
 dense_n = [256, 64]
for n in dense_n:
x = BatchNormalization()(x)
 x = Dense(n, activation="relu", kernel_initializer="he_normal" )(x)
 x = BatchNormalization()(x)
 x = Dropout (.2 )(x)
 out = Dense(1, activation="linear" )(x)
 model = Model(inputs=inps, outputs=out)
 opt = Adam()
model.compile(optimizer=opt, loss='mse',)
 return model
test_dl = learn.dls.test_dl(sub2 )<define_variables>
nfolds=5
folds = StratifiedKFold(n_splits=nfolds,shuffle=True, random_state=15)
 avg_train_kappa = 0
avg_valid_kappa = 0
batch_size=1000
coeffs=None
x_test = get_keras_data(test_df, desc_embs[len(train_df):])
 adoptions_keras = np.zeros(( len(test_df),))
oof_train_keras = np.zeros(( train_df.shape[0]))
i =0
for train_idx, valid_idx in folds.split(train_df[cat_vars+cont_vars], train_df["AdoptionSpeed"]):
x_train = get_keras_data(train_df.iloc[train_idx], desc_embs[train_idx])
 x_valid = get_keras_data(train_df.iloc[valid_idx], desc_embs[valid_idx])
 y_train, y_valid = train_df["AdoptionSpeed"][train_idx].values, train_df["AdoptionSpeed"][valid_idx].values
model = get_model()
clr_tri = CyclicLR(base_lr=2e-3, max_lr=4e-2, step_size=len(train_df)//batch_size, mode="triangular2")
 qwk_eval = QWKEvaluation(validation_data=(x_valid, y_valid), interval=1)
 history = model.fit(x_train, y_train, batch_size=batch_size, validation_data=(x_valid, y_valid),
epochs=20, callbacks=[clr_tri, qwk_eval])
 model.load_weights('checkpoint.h5')
 y_train_pred = eval_predict(y_train, model.predict(x_train, batch_size=1000))
y_valid_pred = eval_predict(y_valid, model.predict(x_valid, batch_size=1000))
avg_train_kappa += quadratic_weighted_kappa(y_train_pred, y_train)
 avg_valid_kappa += quadratic_weighted_kappa(y_valid_pred, y_valid)
 coeffs = eval_predict(y_valid, model.predict(x_valid, batch_size=1000), ret_coeffs=True)
 adoptions_keras += model.predict(x_test, batch_size=batch_size ).reshape(-1,)
 oof_train_keras[valid_idx] = y_valid_pred
print("
avg train kappa:", avg_train_kappa/nfolds,)
 print("
avg valid kappa:", avg_valid_kappa/nfolds, )
test_dl.show_batch()<feature_engineering>
Counter(X_train['AdoptionSpeed'])
preds, _ = learn.tta(dl=test_dl )<prepare_output>
Counter(X_train['AdoptionSpeed'])
sub['label'] = preds.argmax(dim=-1 ).numpy()
sub<save_to_csv>
coeffs[0] = 1.645
coeffs[1] = 2.115
coeffs[3] = 2.84
print('True Counter',Counter(X_train['AdoptionSpeed']))
train_pred_keras = eval_predict(y_pred=oof_train_keras, coeffs=list(coeffs)).astype(int)
 print('Train pred Counter',Counter(train_pred_keras))
test_pred_keras = eval_predict(y_pred=adoptions_keras/nfolds, coeffs=list(coeffs)).astype(int)
 print('Test pred Counter',Counter(test_pred_keras))
sub.to_csv('submission.csv',index=False )<define_variables>
train_stack = np.vstack([train_pred_keras, train_predictions_lgb] ).transpose()
test_stack = np.vstack([test_pred_keras, test_predictions_lgb] ).transpose()
OUTPUT_DIR = './'
MODEL_DIR = '.. /input/cassava-resnext50-32x4d-weights/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'<init_hyperparams>
from sklearn.linear_model import Ridge
class CFG:
debug=False
num_workers=8
model_name='resnext50_32x4d'
size=512
batch_size=32
seed=2020
target_size=5
target_col='label'
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
inference=True<load_from_csv>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(train_stack.shape[0])
 predictions = np.zeros(test_stack.shape[0])
 qwk_scores = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_stack, X_train['AdoptionSpeed'].values)) :
trn_data, trn_y = train_stack[trn_idx], X_train['AdoptionSpeed'].iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], X_train['AdoptionSpeed'].iloc[val_idx].values
clf = Ridge(alpha=1)
 clf.fit(trn_data, trn_y)
 val_pred = clf.predict(val_data)
 pred_val_y_k = eval_predict(y_pred=val_pred, coeffs=list(coeffs)).astype(int)
 print("Valid Counts = ", Counter(val_y))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(val_y, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 oof[val_idx] = clf.predict(val_data)
 predictions += clf.predict(test_stack)/ folds.n_splits
print('{} cv mean QWK score : {}'.format('Stacked_model_ridge', np.mean(qwk_scores)) )
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test.head()<normalization>
pred_final = eval_predict(y_pred=predictions, coeffs=list(coeffs)).astype(int )
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['image_id'].values
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform:
augmented = self.transform(image=image)
 image = augmented['image']
return image<categorify>
Counter(pred_final)
def get_transforms(*, data):
if data == 'valid':
return A.Compose([
A.Resize(CFG.size, CFG.size),
A.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
] )<choose_model_class>
Counter(test_pred_keras)
class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x<load_pretrained>
Counter(test_predictions_lgb)
<save_to_csv><EOS> 
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': pred_final.astype(np.int32)})
 submission.head()
submission.to_csv('submission.csv', index=False )