Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<compute_train_metric>
!mkdir fastai_lib;cd fastai_lib;git clone https://github.com/rpauli/fastai.git;ln -s fastai_lib/fastai/old/fastai/.. /
train_pred = xg_model.predict(X)
 train_pred = np.expm1(train_pred)
 np.sqrt(mean_squared_log_error(train['revenue'], train_pred))<save_to_csv>
%load_ext autoreload
%autoreload
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 sub['revenue'] = np.expm1(xg_pred)
 sub.to_csv("xgb.csv", index=False )<define_search_model>
warnings.filterwarnings("ignore" )
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold
from xgboost import XGBRegressor 
<compute_train_metric>
def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(path_base, path_train))
return sorted([f2cat(f)for f in files], key=str.lower )
xgb = XGBRegressor()
baseline = cross_val_score(xgb, X, y, scoring='neg_mean_squared_error' ).mean()
baseline<train_on_grid>
def calc_apk(pred,actual,k=3):

 pred=to_np(pred)
 actual=to_np(actual)
 top_idxs=top_k_preds(pred,k)
 idx_0=(np.where(actual==1)[1][:,None]-top_idxs)
 idx_correct=idx_0==0
weighted_scores=1/np.arange(1,k+1)
 score_per_data=weighted_scores[np.argmax(idx_correct,axis=1)]*np.max(idx_correct,axis=1)
 mean_score=np.mean(score_per_data)
 return mean_score
def top_k_preds(pred,k):
top_idxs=np.argsort(-to_np(pred),axis=1)[:,:k]
return top_idxs
def acc_metric(preds,actual):
return accuracy_score(np.where(actual==1)[1],np.argmax(to_np(preds),axis=1))
param_dist = {"learning_rate": uniform(0, 1),
"gamma": uniform(0, 5),
"max_depth": range(1,50),
"n_estimators": range(1,300),
"min_child_weight": range(1,10)}
rs = RandomizedSearchCV(xgb, param_distributions=param_dist,
scoring='neg_mean_squared_error', n_iter=25)
 rs.fit(X, y);<set_options>
class FocalLoss(nn.Module):
def __init__(self, alpha=1, gamma=1, reduce=True):
super(FocalLoss, self ).__init__()
self.alpha = alpha
self.gamma = gamma
self.reduce = reduce
def forward(self, inputs, targets):
BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)
 pt = torch.exp(-BCE_loss)
 F_loss = self.alpha *(1-pt)**self.gamma * BCE_loss
if self.reduce:
return torch.mean(F_loss)
 else:
return F_loss
assert sys.version_info >=(3, 5)
 assert sklearn.__version__ >= "0.20"
%matplotlib inline
warnings.filterwarnings(action="ignore")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 print(os.listdir(".. /input")) 
<load_from_csv>
def draw_cv2(raw_strokes, size=256, lw=6, time_color=True):
img = np.zeros(( BASE_SIZE, BASE_SIZE,3), np.float32)
 t_max=max(1,len(raw_strokes))
velos=[np.sqrt(np.sum(np.square(np.diff(stroke)) ,axis=0)).astype(int)for stroke in raw_strokes]
velo_min=np.min([np.min(velo)for velo in velos])
 velo_max=max(1,np.max([np.max(velo)for velo in velos]))
for t, stroke in enumerate(raw_strokes):
velo=velos[t]
velo=(velo)*0.9/(velo_max)+0.1
velo*=255
for i in range(len(stroke[0])- 1):
try:
color =(255 - int(t*255./t_max), int(t*255./t_max), int(velo[i])) if time_color else 255
except:
print(t,t_max,t*255./t_max,int(t*255./t_max),int(velo[i]))
_ = cv2.line(img,(stroke[0][i], stroke[1][i]), 
(stroke[0][i + 1], stroke[1][i + 1]), color, lw)
 img/=255
if size != BASE_SIZE:
return cv2.resize(img,(size, size))
else:
return img
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
class DoodleDataset(Dataset):

 def __init__(self, x, y, transform=None,sz=64):
self.x = x
self.y = y
self.transform = transform
self.n = self.get_n()
self.c = self.get_c()
self.sz = sz
@abstractmethod
def get_x(self, i):
return draw_cv2(json.loads(self.x[i]),self.sz)
 @abstractmethod
def get_y(self, i):
y= np.zeros(self.c,dtype=np.float)
 y[int(self.y[i])]=1
return y
@property
def is_multi(self): return True
@property
def is_reg(self):return False
def get1item(self, idx):
x,y = self.get_x(idx),self.get_y(idx)
 return self.get(self.transform, x, y)
 def __getitem__(self, idx):
if isinstance(idx,slice):
xs,ys = zip(*[self.get1item(i)for i in range(*idx.indices(self.n)) ])
 return np.stack(xs),ys
return self.get1item(idx)
 def __len__(self): return self.n
def get(self, tfm, x, y):
return(x,y)if tfm is None else tfm(x,y)
 @abstractmethod
def get_n(self):
return len(self.y)
 @abstractmethod
def get_c(self):
return len(np.unique(self.y))
@abstractmethod
def get_sz(self):
return self.sz
@property
def is_multi(self):

 return True
class DoodleImageData(ImageData):
def get_dl(self, ds, shuffle):
if ds is None: return None
return DataLoader(ds, batch_size=self.bs, shuffle=shuffle,
num_workers=self.num_workers, pin_memory=False)
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<categorify>
class DoodleConvnetBuilder(ConvnetBuilder):
def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, pretrained=True):
self.f,self.c,self.is_multi,self.is_reg,self.xtra_cut = f,c,is_multi,is_reg,xtra_cut
if xtra_fc is None: xtra_fc = [512]
if ps is None: ps = [0.25]*len(xtra_fc)+ [0.5]
self.ps,self.xtra_fc = ps,xtra_fc
if f in model_meta: cut,self.lr_cut = model_meta[f]
else: cut,self.lr_cut = 0,0
cut-=xtra_cut
layers = cut_model(f(pretrained), cut)
 layers[0] = nn.Conv2d(1,64,kernel_size=(7,7),stride=(2,2),padding=(3, 3), bias=False)
 self.nf = model_features[f] if f in model_features else(num_features(layers)*2)
 if not custom_head: layers += [AdaptiveConcatPool2d() , Flatten() ]
self.top_model = nn.Sequential(*layers)
 n_fc = len(self.xtra_fc)+1
if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc
if custom_head: fc_layers = [custom_head]
else: fc_layers = self.get_fc_layers()
self.n_fc = len(fc_layers)
 self.fc_model = to_gpu(nn.Sequential(*fc_layers))
if not custom_head: apply_init(self.fc_model, kaiming_normal)
 self.model = to_gpu(nn.Sequential(*(layers+fc_layers)))
 class DoodleConvLearner(ConvLearner):
@classmethod
def pretrained(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
pretrained=True, **kwargs):
models = DoodleConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=pretrained)
 return cls(data, models, precompute, **kwargs)
 @classmethod
def lsuv_learner(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=False, **kwargs):
models = DoodleConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=False)
 convlearn=cls(data, models, precompute, **kwargs)
 convlearn.lsuv_init()
return convlearn
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )<train_model>
def get_stats(X_trn,X_val,y_trn,y_val,N=100):
bs=340
num_workers=4
sz=64
aug_tfms = None
tfms = tfms_from_stats(( np.array([0.,0.,0.]), np.array([1.,1.,1.])) , sz=sz, aug_tfms=aug_tfms,tfm_y=TfmType.NO)
 datasets=DoodleImageData.get_ds(DoodleDataset,(X_trn,y_trn),(X_val, y_val),tfms=tfms,sz=sz)
 doodleimageloader=DoodleImageData('.',datasets,bs,num_workers,enc.classes_)
 mn,var=[],[]
for i,(x,y)in tqdm(enumerate(iter(doodleimageloader.trn_dl))):
batch=to_np(x ).swapaxes(1,3 ).reshape([-1,3])
 mn.append(np.mean(batch,axis=0))
var.append(np.var(batch,axis=0))
if i>N:
break
channel_mean,channel_var=np.mean(mn,axis=0),np.mean(var,axis=0)
 return channel_mean,channel_var
train['source']='train'
test['source']='test'
data = pd.concat([train, test],ignore_index=True)
 print(train.shape, test.shape, data.shape )<count_values>
def get_set(N_cat,N_rows,N_skip):
if N_cats < len(list_all_categories()):
df = pd.concat([pd.read_csv(f'{path_base}/{path_train}/{cat}.csv',usecols=['drawing','word','key_id','recognized'],nrows=N_rows,skiprows=np.arange(1,N_skip+1)) for cat in tqdm(np.random.choice(list_all_categories() ,N_cats,replace=False)) ])
 elif N_cats == len(list_all_categories()):
df = pd.concat([pd.read_csv(f'{path_base}/{path_train}/{cat}.csv',usecols=['drawing','word','key_id','recognized'],nrows=N_rows,skiprows=np.arange(1,N_skip+1)) for cat in tqdm(list_all_categories())])
 else:
print('Somethings wrong')
 df_cleaned=df[df.recognized==True].drop(['recognized'],axis=1)
 df_cleaned.word=df_cleaned.word.str.replace(' ','_')
 return df
print("Belongs to a collection or not:
%s" %(data['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()))<categorify>
path_base='.. /input'
path_train='train_simplified'
def dict_one_hot_code(dataset,name_column, name_new_column):
list_of_values = list(dataset[name_column].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_values for i in j] ).most_common(20)
 top_list_values = [m[0] for m in Counter([i for j in list_of_values for i in j] ).most_common(20)]
for g in top_list_values:
dataset[name_new_column + "_" + g] = dataset[name_column].apply(lambda x: 1 if g in str(x)else 0)
 def counting_number_of_values_dict(dataset, name_column):
return dataset[name_column].apply(lambda x: len(x)if x!= {} else 0 )<feature_engineering>
np.random.seed(0)
 N_cats=len(list_all_categories())
 N_rows_train=10000
N_rows_val=100
N_cats
data['collection_name'] = data['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else "None")
 data['is_part_of_collection'] = data['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 data.drop('belongs_to_collection', axis = 1, inplace=True )<count_values>
test_file='test_simplified.csv'
df_test=pd.read_csv(f'{path_base}/{test_file}')
 df_test.info()
data['genres'].apply(lambda x: x[0]['name'] if x != {} else 0 ).value_counts()<drop_column>
enc=LabelEncoder()
enc.fit(df_trn.word)
 df_trn=shuffle(df_trn)
 df_val=shuffle(df_val)
 y_trn=enc.transform(df_trn.word)
 y_val=enc.transform(df_val.word)
 X_trn=df_trn.drawing.values
X_val=df_val.drawing.values
data['genre'] = data['genres'].apply(lambda x: x[0]['name'] if x != {} else "None")
 data.drop('genres', axis=1, inplace=True )<count_unique_values>
bs=256
num_workers=4
BASE_SIZE=256
sz=128
arch=resnet18
list_of_cast_names = list(data['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(20 )<count_values>
channel_mean,channel_var=get_stats(X_trn,X_val,y_trn,y_val,50 )
data['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()<feature_engineering>
aug_tfms = transforms_side_on+[RandomRotate(10)]
tfms = tfms_from_stats(( channel_mean, channel_var), sz=sz, aug_tfms=aug_tfms )
data['number_cast'] = counting_number_of_values_dict(data,'cast' )<count_values>
datasets=DoodleImageData.get_ds(DoodleDataset,(X_trn,y_trn),(X_val, y_val),tfms=tfms,sz=sz,test=df_test.drawing.values)
 datasets_warmup=DoodleImageData.get_ds(DoodleDataset,(X_trn[:200*bs],y_trn[:200*bs]),(X_val, y_val),tfms=tfms,sz=sz,test=df_test.drawing.values )
list_of_cast_characters = list(data['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )<feature_engineering>
doodleplot=DoodleImageData('.',datasets_warmup,8,num_workers,enc.classes_)
 doodle_warmup=DoodleImageData('.',datasets_warmup,bs,num_workers,enc.classes_)
 doodleimageloader=DoodleImageData('.',datasets,bs,num_workers,enc.classes_ )
top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
data['cast_name_' + g] = data['cast'].apply(lambda x: 1 if g in str(x)else 0 )<feature_engineering>
learn=ConvLearner.pretrained(arch,doodle_warmup,metrics=[acc_metric,calc_apk])
 learn.crit=FocalLoss()
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
data['cast_character_' + g] = data['cast'].apply(lambda x: 1 if g in str(x)else 0 )<drop_column>
learn.lr_find(end_lr=1000,start_lr=0.0001 )
data.drop('cast', axis=1, inplace=True )<count_values>
lr=10
learn.fit(lr,1,cycle_len=1,use_clr_beta =(10,10,0.95,0.85))
counting_number_of_values_dict(data,'crew' ).value_counts()<feature_engineering>
learn.save('freeze' )
data['crew_total'] = counting_number_of_values_dict(data, 'crew' )<categorify>
learn.load('freeze' )
dict_one_hot_code(data,name_column='crew',name_new_column='crew_name' )<drop_column>
learn.lr_find(end_lr=10000,start_lr=0.0001)
 learn.sched.plot(n_skip_end=1 )
data.drop('crew', axis=1, inplace=True )<count_values>
lr=100
lrs=[lr/9,lr/3,lr]
learn.set_data(doodleimageloader)
 learn.load('freeze')
 learn.unfreeze()
learn.fit(lrs,1,cycle_len=1,use_clr_beta =(10,10,0.95,0.85))
counting_number_of_values_dict(data,'Keywords' ).value_counts()<feature_engineering>
learn.save('unfreeze')
data['keywords_num'] = counting_number_of_values_dict(data,'Keywords' )<define_variables>
learn.load('unfreeze' )
list_of_keywords = list(data['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )<feature_engineering>
log_preds_test,y_test=learn.TTA(is_test=True,n_aug=16 )
data['all_Keywords'] = data['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '' )<feature_engineering>
mean_test_preds=np.mean(np.exp(log_preds_test),0)
 max_test_preds=np.max(np.exp(log_preds_test),0)
top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
data['keyword_' + g] = data['all_Keywords'].apply(lambda x: 1 if g in x else 0 )<drop_column>
def prepare_submission(preds,fname):
preds_transformed=enc.inverse_transform(top_k_preds(preds,3))
preds_joined=[' '.join(words)for words in preds_transformed]
df_test['word']=preds_joined
df_test.loc[:,['key_id','word']].to_csv(fname,index=None )
data.drop(['Keywords','all_Keywords'], axis=1, inplace=True )<count_values>
prepare_submission(mean_test_preds,f'submission_mean.csv')
 prepare_submission(max_test_preds,f'submission_max.csv')
<count_values><EOS> 
!rm -r fastai_lib/