Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<categorify>
print(os.listdir(".. /input"))
def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )<categorify>
def get_cat_val(X, arr, cat):
cat_list = []
for ind in arr:
cat_list.append(X[ind, cat])
 return cat_list
def get_dog_val(X, arr, dog):
dog_list = []
for ind in arr:
dog_list.append(X[ind, dog])
 return dog_list
def weighted_average(item):
try:
return np.average(item, weights=np.arange(len(item), 0, -1))
except ZeroDivisionError:
return 0
def get_train_meta() :
x_train = pd.read_csv(".. /input/train/train.csv")
 label_annotations = collections.defaultdict(list)
 label_scores = collections.defaultdict(list)
 for file in os.listdir('.. /input/train_metadata/'):
tmp = file.split() [0].split('.json')[0]
key, val = tmp.split('-')
 try:
with open('.. /input/train_metadata/'+file)as f:
data = json.load(f)
 if data.get('labelAnnotations'):
for element in data['labelAnnotations']:
label = element['description']
score = element['score']
label_annotations[key].append(label)
 label_scores[key].append(score)
 else:
label_annotations[key].append('N/A')
 label_scores[key].append(-1)
 except FileNotFoundError:
print('Oopsie')
 x_train['label_annotation'] = x_train['PetID'].map(label_annotations)
 x_train['label_score'] = x_train['PetID'].map(label_scores)
 x_train['label_annotation'] = [[word_tokenize(word)for word in sentence] for sentence in x_train['label_annotation']]
flatten = lambda l: [item for sublist in l for item in sublist]
x_train['label_annotation'] = x_train['label_annotation'].map(flatten)
 x_train['label_annotation'] = [[stem(word)for word in sentence] for sentence in x_train['label_annotation']]
list_to_string = lambda l: ' '.join(l)
 x_train['label_annotation'] = x_train['label_annotation'].map(list_to_string)
 vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(x_train['label_annotation'])
 ind = x_train.index.values
cat_val = get_cat_val(X, ind, vectorizer.vocabulary_['cat'])
 dog_val = get_dog_val(X, ind, vectorizer.vocabulary_['dog'])
 x_train.loc[:, 'label_cat'] = cat_val
x_train.loc[:, 'label_dog'] = dog_val
x_train = x_train.drop(columns=['label_annotation'])
 x_train['label_score'] = x_train['label_score'].map(weighted_average)
 return x_train
def get_test_meta() :
x_test = pd.read_csv(".. /input/test/test.csv")
 label_annotations = collections.defaultdict(list)
 label_scores = collections.defaultdict(list)
 for file in os.listdir('.. /input/test_metadata/'):
tmp = file.split() [0].split('.json')[0]
key, val = tmp.split('-')
 try:
with open('.. /input/test_metadata/'+file)as f:
data = json.load(f)
 if data.get('labelAnnotations'):
for element in data['labelAnnotations']:
label = element['description']
score = element['score']
label_annotations[key].append(label)
 label_scores[key].append(score)
 else:
label_annotations[key].append('N/A')
 label_scores[key].append(-1)
 except FileNotFoundError:
print('Oopsie')
 x_test['label_annotation'] = x_test['PetID'].map(label_annotations)
 x_test['label_score'] = x_test['PetID'].map(label_scores)
 x_test['label_annotation'] = [[word_tokenize(word)for word in sentence] for sentence in x_test['label_annotation']]
flatten = lambda l: [item for sublist in l for item in sublist]
x_test['label_annotation'] = x_test['label_annotation'].map(flatten)
 x_test['label_annotation'] = [[stem(word)for word in sentence] for sentence in x_test['label_annotation']]
list_to_string = lambda l: ' '.join(l)
 x_test['label_annotation'] = x_test['label_annotation'].map(list_to_string)
 vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(x_test['label_annotation'])
 ind = x_test.index.values
cat_val = get_cat_val(X, ind, vectorizer.vocabulary_['cat'])
 dog_val = get_dog_val(X, ind, vectorizer.vocabulary_['dog'])
 x_test.loc[:, 'label_cat'] = cat_val
x_test.loc[:, 'label_dog'] = dog_val
x_test = x_test.drop(columns=['label_annotation'])
 x_test['label_score'] = x_test['label_score'].map(weighted_average)
 return x_test
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T<prepare_x_and_y>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0<create_dataframe>
train = get_train_meta()
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md<find_best_params>
test = get_test_meta()
cut,lr_cut = model_meta[arch]<choose_model_class>
def extract_text_features(text):
tp = type(text)is str
if not tp:
text = "fsdfl"
features = []
header = []
header.append("PetID")
 bag_of_words = nltk.word_tokenize(text)
 header.append("Amount of Words")
 features.append(len(bag_of_words))
sent_text = nltk.sent_tokenize(text)
 header.append("Amount of Sentences")
 features.append(len(sent_text))
if not tp:
features = list(np.zeros(len(header)-1))
return header, features
def get_train_text_and_name_features() :
training_df = pd.read_csv('.. /input/train/train.csv')
 text_data = training_df[["PetID","Description","Name","AdoptionSpeed"]]
text_features_table = []
name_features_table = []
counter = 0
for(PID,profile,name,a_s)in text_data.values:
header,features = extract_text_features(profile)
 text_features_table.append([PID]+features)
 text_features_pd = pd.DataFrame(text_features_table,columns=header)
 return text_features_pd
def get_test_text_and_name_features() :
test_df = pd.read_csv('.. /input/test/test.csv')
 text_data = test_df[["PetID","Description","Name"]]
text_features_table = []
name_features_table = []
counter = 0
for(PID,profile,name)in text_data.values:
header,features = extract_text_features(profile)
 text_features_table.append([PID]+features)
 text_features_pd = pd.DataFrame(text_features_table,columns=header)
 return text_features_pd
def get_base(pre=True):
layers = cut_model(arch(pre), cut)
 return nn.Sequential(*layers )<concatenate>
des_train = get_train_text_and_name_features()
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
class Unet34(nn.Module):
def __init__(self, rn):
super().__init__()
self.rn = rn
self.sfs = [SaveFeatures(rn[i])for i in [2,4,5,6]]
self.up1 = UnetBlock(512,256,256)
 self.up2 = UnetBlock(256,128,256)
 self.up3 = UnetBlock(256,64,256)
 self.up4 = UnetBlock(256,64,256)
 self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)
 def forward(self,x):
x = F.relu(self.rn(x))
x = self.up1(x, self.sfs[3].features)
 x = self.up2(x, self.sfs[2].features)
 x = self.up3(x, self.sfs[1].features)
 x = self.up4(x, self.sfs[0].features)
 x = self.up5(x)
 return x[:,0]
def close(self):
for sf in self.sfs: sf.remove()
class UnetModel() :
def __init__(self,model,name='Unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]<compute_test_metric>
train['PetID']=train['PetID'].astype(str)
 test['PetID']=test['PetID'].astype(str )
def IoU(pred, targs):
pred =(pred > 0.5 ).astype(float)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )<compute_test_metric>
des_test = get_test_text_and_name_features()
def get_score(pred, true):
n_th = 10
b = 4
thresholds = [0.5 + 0.05*i for i in range(n_th)]
n_masks = len(true)
 n_pred = len(pred)
 ious = []
score = 0
for mask in true:
buf = []
for p in pred: buf.append(IoU(p,mask))
ious.append(buf)
 for t in thresholds:
tp, fp, fn = 0, 0, 0
for i in range(n_masks):
match = False
for j in range(n_pred):
if ious[i][j] > t: match = True
if not match: fn += 1
for j in range(n_pred):
match = False
for i in range(n_masks):
if ious[i][j] > t: match = True
if match: tp += 1
else: fp += 1
score +=(( b+1)*tp)/(( b+1)*tp + b*fn + fp)
 return score/n_th<categorify>
AoW_train = pd.DataFrame(des_train['Amount of Words'])
 AoW_test = pd.DataFrame(des_test['Amount of Words'] )
def split_mask(mask):
threshold = 0.5
threshold_obj = 30
labled,n_objs = ndimage.label(mask > threshold)
 result = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj): result.append(obj)
 return result<categorify>
x_train = train.join(AoW_train)
 x_test = test.join(AoW_test )
def get_mask_ind(img_id, df, shape =(768,768)) :
masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return []
if(type(masks)== str): masks = [masks]
result = []
for mask in masks:
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
result.append(img.reshape(shape ).T)
 return result<compute_train_metric>
def extract_features(X):
X_features = X.drop(["Name","RescuerID","Description","PetID"],axis=1)
 return X_features
x_train = extract_features(x_train)
 x_testt = extract_features(x_test )
class Score_eval() :
def __init__(self):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 self.score, self.count = 0.0, 0
def put(self,pred,name):
true = get_mask_ind(name, self.segmentation_df)
 self.score += get_score(pred,true)
 self.count += 1
def evaluate(self):
return self.score/self.count<concatenate>
X_mean = x_train.Fee.mean(axis=0)
 X_std = x_train.Fee.std(axis=0)
 x_train.Fee =(x_train.Fee-X_mean)/X_std
x_testt.Fee =(x_train.Fee-X_mean)/X_std
def aug_unit(x,fwd=True,mask=False):
return x
def aug_flipV(x,fwd=True,mask=False):
return x.flip(2)if mask else x.flip(3)
 def aug_flipH(x,fwd=True,mask=False):
return x.flip(1)if mask else x.flip(2)
 def aug_T(x,fwd=True,mask=False):
return torch.transpose(x,1,2)if mask else torch.transpose(x,2,3)
 def aug_rot_2(x,fwd=True,mask=False):
return aug_flipV(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_4cr(x,fwd=True,mask=False):
return aug_flipV(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipV(x,fwd,mask),fwd,mask)
 def aug_rot_4cw(x,fwd=True,mask=False):
return aug_flipH(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_2T(x,fwd=True,mask=False):
return aug_rot_2(aug_T(x,fwd,mask),fwd,mask)
 trms_side_on = [aug_unit,aug_flipH]
trms_top_down = [aug_unit,aug_flipV]
trms_dihedral = [aug_unit,aug_flipH,aug_flipV,aug_T,aug_rot_2,aug_rot_2T,
aug_rot_4cw,aug_rot_4cr]<categorify>
y_train = train['AdoptionSpeed'].values
def enc_img(img):
return torch.transpose(torch.tensor(img),0,2 ).unsqueeze(0)
 def dec_img(img):
return to_np(torch.transpose(img.squeeze(0),0,2))
def display_augs(x,augs=aug_unit):
columns = 4
n = len(augs)
 rows = n//4 + 1
fig=plt.figure(figsize=(columns*4, rows*4))
img = enc_img(x)
 for i in range(rows):
for j in range(columns):
idx = j+i*columns
if idx >= n: break
fig.add_subplot(rows, columns, idx+1)
 plt.axis('off')
 plt.imshow(dec_img(augs[idx](img)))
 plt.show()
img = np.array(Image.open(os.path.join(TRAIN,'ce69faa4b.jpg')))
 display_augs(img,trms_dihedral )<predict_on_test>
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold
def model_pred(learner, dl, F_save):
learner.model.eval() ;
name_list = dl.dataset.fnames
num_batchs = len(dl)
 t = tqdm(iter(dl), leave=False, total=num_batchs)
 count = 0
for x,y in t:
py = to_np(torch.sigmoid(learn.model(V(x))))
batch_size = len(py)
 for i in range(batch_size):
F_save(py[i],to_np(y[i]),name_list[count])
 count += 1
def pred_aug(x,aug=[aug_unit]):
pred = []
for aug_cur in aug:
py = to_np(aug_cur(torch.sigmoid(learn.model(V(aug_cur(x)))) ,
fwd=False, mask=True))
pred.append(py)
 pred = np.stack(pred, axis=0 ).mean(axis=0)
 return pred
def model_pred_aug(learner, dl, F_save, aug=[aug_unit]):
learner.model.eval() ;
name_list = dl.dataset.fnames
num_batchs = len(dl)
 t = tqdm(iter(dl), leave=False, total=num_batchs)
 count = 0
for x,y in t:
pred = pred_aug(x,aug)
 batch_size = len(pred)
 for i in range(batch_size):
F_save(pred[i],to_np(y[i]),name_list[count])
 count += 1<choose_model_class>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'silent': 1,
}
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
m = to_gpu(Unet34(get_base(False)))
 models = UnetModel(m )<define_variables>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
sz = 768
bs = 8
md = get_data(sz,bs )<choose_model_class>
def qwk(estimator,X,y, additionals = None):
N = 5
W = np.zeros(( N,N))
for i in range(N):
for j in range(N):
W[i,j]=(( i-j)**2)/(( N-1)**2)
 actuals = y
if additionals is not None:
preds = estimator.predict(X,additionals)
 else:
preds = estimator.predict(X)
 O = confusion_matrix(actuals,preds)
 O = O/O.sum()
act_hist=np.zeros([N])
 for act in actuals:
act_hist[act]+=1
pred_hist=np.zeros([N])
 for pred in preds:
pred_hist[pred]+=1
E = np.outer(act_hist,pred_hist)
 E = E/E.sum()
num = 0
den = 0
for i in range(N):
for j in range(N):
num+=W[i,j]*O[i,j]
den+=W[i,j]*E[i,j]
k = 1-num/den
return k
learn = ConvLearner(md, models)
 learn.models_path = PRETRAINED_SEGMENTATION_PATH
learn.load('Unet34_768_1')
 learn.models_path = PATH<compute_train_metric>
model, oof_train, oof_test = run_xgb(xgb_params, x_train,x_testt)
 optR = OptimizedRounder()
optR.fit(oof_train, y_train)
 coefficients = optR.coefficients()
kappa = qwk(optR,oof_train,y_train, additionals=coefficients)
 print("QWK = ", kappa )
score = Score_eval()
process_pred = lambda yp, y, name : score.put(split_mask(yp),name)
 model_pred_aug(learn, md.val_dl, process_pred, trms_dihedral)
 print('
',score.evaluate() )<load_from_csv>
coefficients_ = coefficients.copy()
kappa = qwk(optR,oof_train,y_train, additionals=coefficients_)
 print("QWK = ", kappa )
ship_detection = pd.read_csv(DETECTION_TEST_PRED)
 ship_detection.head()<data_type_conversions>
test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8 )
<set_options><EOS> 
submission = pd.DataFrame({'PetID': x_test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False )