Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<feature_engineering>
%matplotlib inline
device = torch.device('cuda' if torch.cuda.is_available() else "cpu" )
vectorizer = TfidfVectorizer(
 analyzer = 'word',
stop_words = 'english',
ngram_range =(1, 2),
min_df = 10,
sublinear_tf = True)
 overview_transf = vectorizer.fit_transform(train['text_merged'])
 overview_transf<train_model>
NROWS = 10000
VAL_NROWS = 500
path = '.. /input/train_simplified/'
filenames = glob.glob(os.path.join(path, '*.csv'))
NUM_CLASSES = len(filenames )
linreg = LinearRegression()
linreg.fit(overview_transf, train['revenue'] )<define_variables>
encode_dict = {}
path = '.. /input/train_simplified/'
filenames = glob.glob(os.path.join(path, '*.csv'))
filenames = sorted(filenames)
 print(len(filenames))
for ix, filename in enumerate(filenames):
class_name = filename.split('/')[-1].split('.')[0].replace(' ', '_')
 encode_dict[class_name] = ix
decode_dict = {value:key for key, value in encode_dict.items() }
keywords_newPos = [lexicon[i] for i in poscorrTop]
keywords_newNeg = [lexicon[i] for i in negcorrTop]
keywords_new = keywords_newPos + keywords_newNeg
keywords_new[0:10]<feature_engineering>
class ClassDataset(Dataset):
def __init__(self, csv_file_path, mode='train', nrows=15000, skiprows=0, size=128):
super().__init__()
self.df = pd.read_csv(csv_file_path, usecols=['drawing'], nrows=nrows, skiprows=0)
 self.mode = mode
self.size = size
if self.mode == 'train':
self.class_name = csv_file_path.split('/')[-1].split('.')[0].replace(' ', '_')
 self.label = encode_dict[self.class_name]
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
raw_strokes = ast.literal_eval(self.df.drawing[index])
 image = draw_cv2(raw_strokes, size=self.size)
 image = image.transpose(2, 0, 1)
 if self.mode == 'train':
return(image/255 ).astype('float32'), self.label
else:
return(image/255 ).astype('float32')
for k in keywords_new:
train['txt_' + k] = train['text_merged'].apply(lambda x: 1 if k in x else 0)
 keys_txt = ['txt_' + s for s in keywords_new]<feature_engineering>
dset = ConcatDataset([ClassDataset(filename, nrows=NROWS)for filename in filenames])
 val_dset = ConcatDataset([ClassDataset(filename, nrows=VAL_NROWS, skiprows=NROWS)for filename in filenames] )
test['budget_yn'] = test['budget'].apply(lambda x: 0 if x == 0 else 1)
 test['budget_per_year'] = test['budget']/test['release_date_year']
test['n_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['n_production_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['n_production_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['n_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 test['n_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for l in top_languages:
test['language_' + l] = test['original_language'].\
apply(lambda x: 1 if l in x else 0)
 test['genres_collapsed'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in genres:
test['genre_' + g] = test['genres_collapsed'].apply(lambda x: 1 if g in x else 0)
 test['production_companies_collapsed'] = test['production_companies'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for comp in top_prodCompanies:
test['production_company_' + comp] = test['production_companies_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 test['production_countries_collapsed'] = test['production_countries'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for comp in top_prodCountries:
test['production_country_' + comp] = test['production_countries_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 test['cast_collapsed'] = test['cast'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for lead in top_lead:
test['cast_' + lead] = test['cast_collapsed'].apply(lambda x: 1 if lead in x else 0)
 test['directors_collapsed'] = test['crew'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x if i['job'] == 'Director'])) if x != {} else '')
 for d in top_directors:
test['director_' + d] = test['directors_collapsed'].apply(lambda x: 1 if d in x else 0)
 for k in keywords_new:
test['txt_' + k] = test['text_merged'].apply(lambda x: 1 if k in x else 0 )<drop_column>
dloader = DataLoader(dset, batch_size=128, shuffle=True, num_workers=2)
 val_dloader = DataLoader(val_dset, batch_size=128, num_workers=2, shuffle=False )
cols2drop = ['genres', 'production_companies', 'production_countries', 'genres_collapsed',
'production_companies_collapsed', 'production_countries_collapsed', 'cast_collapsed',
'directors_collapsed', 'n_cast', 'n_crew']
train = train.drop(cols2drop, axis=1)
 test = test.drop(cols2drop, axis=1 )<compute_test_metric>
class Conv(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):
super().__init__()
self.layers = nn.Sequential(
 nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, stride=stride, padding=padding, bias=False),
nn.BatchNorm2d(out_channel),
nn.ReLU(inplace=True)
)
 def forward(self, input):
return self.layers(input)
 class Conv_dw_Conv(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):
super().__init__()
self.layers = nn.Sequential(
 nn.Conv2d(in_channel, in_channel, kernel_size, stride, padding, bias=False, groups=in_channel),
nn.BatchNorm2d(in_channel),
nn.ReLU(inplace=True),
Conv(in_channel, out_channel, kernel_size=1, stride=1, padding=0)
)
 def forward(self, input):
return self.layers(input)
 class MobileNet(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.num_classes = num_classes
self.model = nn.Sequential(
 Conv(3, 32, stride=2),
Conv_dw_Conv(32, 64, kernel_size=3, stride=1),
Conv_dw_Conv(64, 128, kernel_size=3, stride=2),
Conv_dw_Conv(128, 128, kernel_size=3, stride=1),
Conv_dw_Conv(128, 256, kernel_size=3, stride=2),
Conv_dw_Conv(256, 256, kernel_size=3, stride=1),
Conv_dw_Conv(256, 512, kernel_size=3, stride=2),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 512, kernel_size=3, stride=1),
Conv_dw_Conv(512, 1024, kernel_size=3, stride=2),
Conv_dw_Conv(1024, 1024, kernel_size=3, stride=1)
)
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(1024, num_classes)
 def forward(self, input):
x = self.model(input)
 x = self.avg_pool(x)
 x = x.view(-1, 1024)
 out = self.fc(x)
 return out
def RMSE(y_obs, y_pred):
n = len(y_obs)
 rmse = np.sqrt(1/n*np.sum(( y_pred-y_obs)**2))
return rmse
def RMSLE(y_obs, y_pred):
n = len(y_obs)
 rmsle = np.sqrt(1/n*np.sum(( np.log(y_pred)-np.log(y_obs)) **2))
return rmsle
def LinearRegression(X, y):
intercept_term = np.ones(shape = y.shape)
 X = np.concatenate(( intercept_term, X), 1)
 coeffs = np.linalg.inv(X.T.dot(X)).dot(X.T ).dot(y)
 return coeffs<prepare_x_and_y>
def apk(actual, predicted, k=3):

 if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
for i,p in enumerate(predicted):
if p == actual:
score = 1 /(i+1.0)
 return score
def mapk(actual, predicted, k=3):

 return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)] )
x = train['budget'].values
X = np.reshape(x,(len(x),1))
y = train['revenue'].values
y = np.reshape(y,(len(y),1))
np.shape(X), np.shape(y )<compute_test_metric>
def validation_score(model, val_data_loader):
model.eval()
sum_score = 0
count = 0
for images, labels in val_data_loader:
images = images.to(device)
 labels = labels.to(device)
 batch_size = images.size(0)
 output = model(images)
 topk = output.detach().topk(3, dim=1)[1]
sum_score += mapk(labels.cpu().numpy() , topk.cpu().numpy())* batch_size
count += batch_size
model.train()
return sum_score / count
model_baseline_coeffs = LinearRegression(X, y)
 model_baseline_pred_train = model_baseline_coeffs[0] + model_baseline_coeffs[1]*X
model_baseline_coeffs<compute_test_metric>
model = MobileNet(NUM_CLASSES ).to(device)
 criterion = nn.CrossEntropyLoss().to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=0.002)
 scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[60000, 130000, 160000, 190000], gamma=0.5 )
RMSLE(y, model_baseline_pred_train )<drop_column>
%%time
print("Start training...")
 epochs = 8
print_every = 1000
validate_every = 10000
model.train()
best_val_score = 0
curr_iter = 0
avg_loss = 0
avg_score = 0
for epoch in range(epochs):
for ix,(images, labels)in enumerate(dloader):
images = images.to(device)
 labels = labels.to(device)
 model.zero_grad()
output = model(images)
 loss = criterion(output, labels)
 loss.backward()
optimizer.step()
avg_loss += loss.item()
topk = output.detach().topk(3, dim=1)[1]
avg_score += mapk(labels.cpu().numpy() , topk.cpu().numpy())
 curr_iter += 1
if(curr_iter)% print_every == 0:
print('Epoch {}, Iteration {} - Train Loss: {:.4f}, MAP@3: {:.3f}'.format(epoch + 1, curr_iter, avg_loss/print_every, avg_score/print_every))
avg_loss = 0
avg_score = 0
if curr_iter % validate_every == 0:
val_score = validation_score(model, val_dloader)
 print('Epoch {}, Iteration {}: validation map@3: {}'.format(epoch + 1, curr_iter, val_score))
if val_score > best_val_score:
print('New best validation score: {}, saving model...'.format(val_score))
best_val_score = val_score
torch.save(model.state_dict() , 'model_checkpoint_best_val.ckpt')
train_selected = train[['budget', 'popularity', 'runtime',
'release_date_year', 'release_date_weekday', 'release_date_month',
'budget_yn', 'belongs2coll_yn', 'homepage_yn',
'n_genres', 'n_production_companies', 'n_production_countries',
'revenue']+keys_genre+keys_cast+keys_director+keys_production_company+
keys_production_country+keys_language]
train_selected = train_selected.dropna(axis = 0)
 X = train_selected.drop(['revenue'], axis=1)
 y = train_selected['revenue']<find_best_model_class>
torch.save(model.state_dict() , 'model_checkpoint.ckpt' )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
 model_RF = RandomForestRegressor(n_estimators = 100,
max_depth = 20)
 model_RF.fit(X_train, y_train)
 model_RF_pred_valset = model_RF.predict(X_test)
 RMSLE(y_test, model_RF_pred_valset )<predict_on_test>
test_dset = ClassDataset('.. /input/test_simplified.csv', mode='test', nrows=None)
 test_dloader = DataLoader(test_dset, batch_size=128, shuffle=False, num_workers=0)
 model.eval()
labels = []
for images in tqdm.tqdm(test_dloader):
images = images.to(device)
 output = model(images)
 _, pred = output.topk(3, 1)
 for i in range(len(images)) :
labels.append(' '.join([decode_dict[pred[i][j].item() ]for j in range(3)]))
model_RF.fit(X, y)
 model_RF_pred_train = model_RF.predict(X)
 RMSLE(y, model_RF_pred_train )<split>
%%time
submission = pd.read_csv('.. /input/test_simplified.csv', index_col='key_id' ,usecols=['key_id'])
 print(len(submission))
submission['word'] = labels
X_train, X_test, y_train, y_test = train_test_split(X, np.log1p(y), test_size=0.4)
 model_CatBoost = CatBoostRegressor(silent=True)
 model_CatBoost.fit(X_train, y_train)
 model_CatBoost_pred_valset = model_CatBoost.predict(X_test)
 RMSE(y_test, model_CatBoost_pred_valset )<predict_on_test>
submission.to_csv('submission_final.csv' )
<train_model><EOS> 
model.load_state_dict(torch.load('model_checkpoint_best_val.ckpt'))
model.eval()
labels = []
for images in tqdm.tqdm(test_dloader):
images = images.to(device)
 output = model(images)
 _, pred = output.topk(3, 1)
 for i in range(len(images)) :
labels.append(' '.join([decode_dict[pred[i][j].item() ]for j in range(3)]))
print(len(labels))
submission = pd.read_csv('.. /input/test_simplified.csv', index_col='key_id' ,usecols=['key_id'])
 print(len(submission))
submission['word'] = labels
submission.to_csv('submission_best_val.csv' )