Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<define_variables>
%%time
! nvidia-smi
sub_files = [
'.. /input/ensemble/sub_832.csv',
'.. /input/ensemble/sub_824.csv',
]
sub_weight = [
0.832**2,
0.824**2
]
Hlabel = 'Image'
Htarget = 'Id'
npt = 5
place_weights = {}
for i in range(npt):
place_weights[i] =(1 /(i + 1))
print(place_weights)
 lg = len(sub_files)
 sub = [None]*lg
for i, file in enumerate(sub_files):
print("Reading {}: w={} - {}".format(i, sub_weight[i], file))
reader = csv.DictReader(open(file,"r"))
sub[i] = sorted(reader, key=lambda d: str(d[Hlabel]))
out = open("sub_ens.csv", "w", newline='')
 writer = csv.writer(out)
 writer.writerow([Hlabel,Htarget])
 for p, row in enumerate(sub[0]):
target_weight = {}
for s in range(lg):
row1 = sub[s][p]
for ind, trgt in enumerate(row1[Htarget].split(' ')) :
target_weight[trgt] = target_weight.get(trgt,0)+(place_weights[ind]*sub_weight[s])
 tops_trgt = sorted(target_weight, key=target_weight.get, reverse=True)[:npt]
writer.writerow([row1[Hlabel], " ".join(tops_trgt)])
 out.close()<import_modules>
warnings.filterwarnings("ignore" )
import pandas as pd<import_modules>
path_t=Path('.. /input/humpback-whale-identification/')
 path_b=Path('.. /input/')
 path1='.'
df = pd.read_csv(path_t/'train.csv');
fastai.__version__
import pandas as pd<import_modules>
exclude_list=['0b1e39ff.jpg',
'0c11fa0c.jpg',
'1b089ea6.jpg',
'2a2ecd4b.jpg',
'2c824757.jpg',
'3e550c8a.jpg',
'56893b19.jpg',
'613539b4.jpg',
'6530809b.jpg',
'6b753246.jpg',
'6b9f5632.jpg',
'75c94986.jpg',
'7f048f21.jpg',
'7f7702dc.jpg',
'806cf583.jpg',
'95226283.jpg',
'a3e9070d.jpg',
'ade8176b.jpg',
'b1cfda8a.jpg',
'b24c8170.jpg',
'b7ea8be4.jpg',
'b9315c19.jpg',
'b985ae1e.jpg',
'baf56258.jpg',
'c4ad67d8.jpg',
'c5da34e7.jpg',
'c5e3df74.jpg',
'ced4a25c.jpg',
'd14f0126.jpg',
'e0b00a14.jpg',
'e6ce415f.jpg',
'e9bd2e9c.jpg',
'f4063698.jpg',
'f9ba7040.jpg']
new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 trn_imgs=train_df.copy().reset_index(drop=True)
 cnter = Counter(trn_imgs.Id.values)
 trn_imgs['cnt']=trn_imgs['Id'].apply(lambda x: cnter[x])
 trn_imgs['target'] = 0
trn_imgs1 = trn_imgs.copy()
trn_imgs1['target'] = 1
target_col = 3
trn_imgs.head(1)
 trn_imgs=trn_imgs[~trn_imgs.Image.isin(exclude_list)]
from fastai import *
from fastai.vision import *
from fastai.datasets import *
from fastai.metrics import *
from fastai.vision.data import *
from fastai.vision.image import *<define_variables>
def read_img(fname,box_df,img,sz=224):
x0,y0,x1,y1 = tuple(box_df.loc[fname,['x0','y0','x1','y1']].tolist())
 l1,l0 = img.shape[1],img.shape[2]
b0,b1 = x1-x0, y1-y0
x0n,x1n = max(int(x0 - b0*0.05),0), min(int(x1 + b0*0.05),l0-1)
 y0n,y1n = max(int(y0 - b1*0.05),0), min(int(y1 + b1*0.05),l1-1)
 img=to_np(img)
 if not(x0 >= x1 or y0 >= y1):
None
img = img[:,y0n:y1n, x0n:x1n]
return Image(pil2tensor(img.astype(np.float)/255, np.float32 ).float() )
path = untar_data(URLs.DOGS )<import_modules>
from fastai.imports import *<set_options>
def open_4_channel2(fname):
fname = str(fname)
 bbox_df = pd.read_csv(path_b/'cropped-img'/'bounding_boxes.csv' ).set_index('Image')
 x0,y0,x1,y1=bbox_df.loc[fname[fname.rfind('/')+1:]]
area=(x0,y0,x1,y1)
 img = cv2.imread(fname)
 img=crop_loose_bbox(img,area)
 return Image(pil2tensor(img/255, np.float32 ).float() )
torch.cuda.is_available()<set_options>
trn_imgs=trn_imgs.append(trn_imgs.loc[trn_imgs.cnt==2],ignore_index=True)
 trn_imgs=trn_imgs.append(trn_imgs.loc[trn_imgs.cnt==1],ignore_index=True )
torch.backends.cudnn.enabled<define_variables>
val_idx=[]
for i in trn_imgs[trn_imgs.cnt>5].Id.unique() :
tmp=list(trn_imgs.loc[trn_imgs.Id==i].index.values)
 val_idx=val_idx+(random.sample(tmp,1))
len(val_idx)
 for i in trn_imgs[(trn_imgs.cnt<5)&(trn_imgs.cnt>2)].Id.unique() :
tmp=list(trn_imgs.loc[trn_imgs.Id==i].index.values)
 if len(val_idx)< 1300 :
val_idx=val_idx+(random.sample(tmp,1))
len(val_idx)
sz=224<choose_model_class>
val_idx=list(trn_imgs.iloc[val_idx].index.values)
 trn_idx=set(list(trn_imgs.index.values)) -set(val_idx)
 df_i=trn_imgs.iloc[list(trn_idx)].reset_index(drop=True)
arch=models.resnet34<load_pretrained>
data = ImageDataBunch.from_folder(path,test='test1',
ds_tfms=get_transforms() ,
size=sz,
num_workers=0 )<choose_model_class>
test_ids = list(sorted({fname for fname in os.listdir(path_t/'test')}))
test_fnames = [path_t/'test'/test_id for test_id in test_ids]
test_fnames[:3]
learn = create_cnn(data, models.resnet34, metrics=accuracy )<find_best_params>
src1.train.x.create_func = open_4_channel2
src1.train.x.open = open_4_channel2
src1.valid.x.create_func = open_4_channel2
src1.valid.x.open = open_4_channel2
src1.add_test(test_fnames);
src1.test.x.create_func = open_4_channel2
src1.test.x.open = open_4_channel2
trn_tfms,_ = get_transforms(do_flip=False, flip_vert=True, max_rotate=5., max_zoom=1.08,
max_lighting=0.15, max_warp=0.)
data1 =(src1.transform(( trn_tfms,trn_tfms), size=224,resize_method=ResizeMethod.SQUISH)
.databunch(bs=64,num_workers=0 ).normalize(imagenet_stats))
data2 =(src1.transform(( trn_tfms,trn_tfms), size=484,resize_method=ResizeMethod.SQUISH)
.databunch(bs=36,num_workers=0 ).normalize(imagenet_stats))
learn.lr_find()<train_model>
learn.fit(3 )<prepare_output>
from fastai.metrics import accuracy
submission = pd.DataFrame(os.listdir(".. /input/test"),columns = ['FileName'])
 submission['id']= submission['FileName'].apply(lambda s: s[:len(s)-4])
 submission['label']=0.5
submission.head()<define_variables>
def dense(pre):
model = m.densenet121(pretrained=pre)
 model.classifier =(nn.Linear(1024, 5004))
return model
def _densenet_split(m): return(m[0][0][6],m[1] )
test_path = PATH+'test/'
test_path<predict_on_test>
def acc(input:Tensor, targs:Tensor)->Rank0Tensor:
"Compute accuracy with `targs` when `input` is bs * n_classes."
n = targs.shape[0]
input = input.argmax(dim=-1 ).view(n,-1)
 targs = targs.view(n,-1)
 return(input==targs ).float().mean()
for i in range(10):
img = open_image(test_path+submission['FileName'][i])
 result = learn.predict(img)
 print(submission['FileName'][i],result[0],result[1] )<predict_on_test>
i=torch.rand(3,2)
 print(i)
for i in range(len(submission)) :
img = open_image(test_path+submission['FileName'][i])
 result = learn.predict(img)
 submission.loc[i,'label']=float(result[2][1] )<save_to_csv>
class ArcMarginProduct(nn.Module):
r
def __init__(self, in_features, out_features=5004):
super(ArcMarginProduct, self ).__init__()
self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
stdv = 1./ math.sqrt(self.weight.size(1))
self.weight.data.uniform_(-stdv, stdv)
 def forward(self, features):
cosine = F.linear(F.normalize(features), F.normalize(self.weight.cuda()))
return cosine
class Customhead(nn.Module):
r
def __init__(self, in_features, out_features=5004):
super(Customhead, self ).__init__()
body = create_body(m.resnet50, True, -2)
 nf = num_features_model(nn.Sequential(*body.children())) * 2
self.head=create_head(nf, 1024,[2048], ps=0.5, bn_final=False)
 self.arc_margin=ArcMarginProduct(in_features,out_features)
 def forward(self, features):
x=self.head(features)
 cosine = self.arc_margin(x)
 cosine = cosine.clamp(-1, 1)
 return cosine
submission[['id','label']].to_csv('submission.csv',index=False )<load_from_csv>
class CustomheadRes(nn.Module):
r
def __init__(self, in_features, out_features=5004):
super(CustomheadRes, self ).__init__()
body = create_body(m.densenet121, True, -1)
 nf = num_features_model(nn.Sequential(*body.children())) * 2
self.head=create_head(nf, 1024, ps=0.5, bn_final=False)
 self.arc_margin=ArcMarginProduct(in_features,out_features)
 def forward(self, features):
x=self.head(features)
 cosine = self.arc_margin(x)
 cosine = cosine.clamp(-1, 1)
 return cosine
%%time
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )<drop_column>
class ArcFaceLoss(nn.modules.Module):
def __init__(self,s=30.0,m=0.5):
super(ArcFaceLoss, self ).__init__()
self.classify_loss = nn.CrossEntropyLoss()
self.s = s
self.easy_margin = False
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.th = math.cos(math.pi - m)
 self.mm = math.sin(math.pi - m)* m
def forward(self, inputs, labels, epoch=0,reduction=None):
cosine = inputs
sine = torch.sqrt(1.0 - torch.pow(cosine, 2))
phi = cosine * self.cos_m - sine * self.sin_m
if self.easy_margin:
phi = torch.where(cosine > 0, phi, cosine)
 else:
phi = torch.where(cosine > self.th, phi, cosine - self.mm)
 one_hot = torch.zeros(cosine.size() , device='cuda')
 one_hot.scatter_(1, labels.view(-1, 1 ).long() , 1)
 output =(one_hot * phi)+(( 1.0 - one_hot)* cosine)
 output *= self.s
loss1 = self.classify_loss(output, labels)
 loss2 = self.classify_loss(cosine, labels)
 gamma=1
loss=(loss1+gamma*loss2)/(1+gamma)
 return loss.mean()
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<count_values>
f1_score = partial(fbeta, thresh=0.4, beta=1)
 acc_02 = partial(accuracy_thresh, thresh=0.2)
 custom_head=Customhead(1024,5004)
 custom_headres=CustomheadRes(1024,5004)
 ar=ArcFaceLoss().cuda()
gc.collect()
learn1 = create_cnn(
 data2,
resnet501,
cut=-2,
split_on=_resnet_split,
lin_ftrs=[1024],
custom_head=custom_head,
loss_func=ar,
path=path1,
metrics=[accuracy], callback_fns= partial(GradientClipping, clip=1))
learn1.callback_fns.append(partial(SaveModelCallback,monitor='val_loss',mode='min'))
learn1.callback_fns.append(partial(ReduceLROnPlateauCallback, min_delta=1e-5, patience=3))
learn2 = create_cnn(
 data2,
dense,
cut=-1,
split_on=_densenet_split,
lin_ftrs=[512],
custom_head=custom_headres,
loss_func=ar,
path=path1,
metrics=[accuracy], callback_fns= partial(GradientClipping, clip=1))
learn2.callback_fns.append(partial(ReduceLROnPlateauCallback, min_delta=1e-5, patience=3))
def count_transform(column):
return column.map(column.value_counts().to_dict() )<define_variables>
print('Train_loss',learn1.recorder.losses[-1])
 print('Val loss',learn1.recorder.val_losses)
 print('Accuracy',learn1.recorder.metrics)
rang = {"Grandmaster" : 4, "Master" : 3, "Expert" : 2, "Contributor" : 1, "Novice" : 0}
temperature = {"Freezing" : 0, "Cold": 1, "Warm" : 2, "Hot": 3, "Boiling Hot" : 4, "Lava Hot" : 5}<concatenate>
print('Train_loss',learn2.recorder.losses[-1])
 print('Val loss',learn2.recorder.val_losses)
 print('Accuracy',learn2.recorder.metrics )
traintest = pd.concat([train, test] )<feature_engineering>
learn1.model.eval()
learn1.load('resnet_ar_c424_2')
 preds1,y1 = learn1.TTA(ds_type=DatasetType.Test,beta=0.30,with_loss=False,scale=1.1)
%%time
traintest['ord_1_new'] = traintest['ord_1'].map(rang)
 traintest['ord_2_new'] = traintest['ord_2'].map(temperature)
 traintest['ord_3_new'] = traintest['ord_3'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_3'])) })
 traintest['ord_4_new'] = traintest['ord_4'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_4'])) })
 traintest['ord_5_new_1'] = traintest['ord_5'].apply(lambda x: x[0])
 traintest['ord_5_new_1'] = traintest['ord_5_new_1'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_5_new_1'])) })
 traintest['ord_5_new_2'] = traintest['ord_5'].apply(lambda x: x[1])
 traintest['ord_5_new_2'] = traintest['ord_5_new_2'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_5_new_2'])) })
 traintest['ord_5_new_2'] = traintest['ord_5_new_2'].map({val : idx for idx, val in enumerate(np.unique(traintest['ord_5_new_2'])) })
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True ).to_sparse().to_coo()
traintest_new = csr_matrix(traintest[list(filter(lambda x: 'new' in x, traintest.columns)) ] )<concatenate>
def apk(actual, predicted, k=10):
if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i,p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i+1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=10):
return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)])
 def sigmoid_np(x):
return 1.0/(1.0 + np.exp(-x))
preds_tv = sigmoid_np(predsv )
feature_df = hstack([dummies, traintest_new] ).tocsr()<split>
i=preds_tv[99,:].argsort(descending=True)
 probs = preds_tv[0,i]
probs[:5]
np.stack(top_5s_v ).shape
labels_list[2]
%%time
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = KFold(n_splits=5)
 fold_splits = kf.split(train, target)
 cv_scores = []
pred_full_test = 0
coefs = []
pred_train = np.zeros(( train.shape[0]))
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
trn_res = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_val_y, pred_test_y = trn_res['pred_val_y'], trn_res['pred_test_y']
pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 coefs.append(trn_res['coef'])
 print(label + ' cv score {}: {}'.format(i, cv_score))
i += 1
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std score : {}'.format(label, np.std(cv_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores,
'coefs' : coefs}
return results
def runLR(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = LogisticRegression(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = model.predict_proba(test_X2)[:, 1]
return {'pred_val_y' : pred_test_y, 'pred_test_y' : pred_test_y2, 'coef' : model.coef_}
def runRLR(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = Ridge(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2)
 return pred_test_y, pred_test_y2
rr_params = {'alpha' : 1, 'solver': 'lsqr', "fit_intercept" : False}
lr_params = {'solver': 'lbfgs', 'C': 0.1,'max_iter' : 1000}
results = run_cv_model(train_ohe, test_ohe, target, runLR, lr_params, auc, 'Ridge' )<save_to_csv>
%%time
submission = pd.DataFrame({'id': test_id, 'target': results['test']})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
def sigmoid_np(x):
return 1.0/(1.0 + np.exp(-x))
preds_t = sigmoid_np(preds1)
%%time
sns.set()
%matplotlib inline
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )<drop_column>
i=preds_t[99,:].argsort(descending=True)
 probs = preds_t[99,i]
probs[:5]
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<feature_engineering>
top_5_classes = []
for top_5 in top_5s:
top_5_classes.append(' '.join([t for t in top_5]))
sub = pd.DataFrame({'Image': [path.name for path in data1.test_ds.x.items]})
 sub['Id'] = top_5_classes
sub.to_csv('pred_res484.csv',index=False)
 FileLink('pred_res484.csv')
<categorify>