Kaggle competition: Dog Breed Identification
<SOS> metric: MulticlassLoss Kaggle data source: dog-breed-identification<import_modules>
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow import keras
from keras.preprocessing.image import ImageDataGenerator as Imgen
from keras.models import Sequential,load_model
from keras.layers import Conv2D,MaxPooling2D,Flatten,Dense,GlobalAveragePooling2D,Dropout
from keras.preprocessing import image
import cv2
import pickle
from sklearn.metrics import mean_squared_error<define_variables>
labels = pd.read_csv(".. /input/dog-breed-identification/labels.csv")
 sample_sub = pd.read_csv(".. /input/dog-breed-identification/sample_submission.csv")
 labels.head()
allzeros = 0.3032<compute_test_metric>
def addjpg(id):
return id+".jpg"
tr['allzero'] = 0
np.sqrt(mean_squared_error(tr.deal_probability, tr['allzero']))<feature_engineering>
labels['id'] = labels['id'].apply(addjpg)
 sample_sub['id'] = sample_sub['id'].apply(addjpg )
allzeros / 0.2949544182370951<set_options>
data = Imgen(preprocessing_function=keras.applications.nasnet.preprocess_input,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
validation_split=0.1
)
sub.deal_probability *= 1.03<save_to_csv>
train_ds = data.flow_from_dataframe(
 labels,
directory = '.. /input/dog-breed-identification/train',
x_col = 'id',
y_col = 'breed',
subset="training",
color_mode="rgb",
target_size =(331,331),
class_mode="categorical",
batch_size=32,
shuffle=True,
seed=123, 
)
 val_ds = data.flow_from_dataframe(
 labels,
directory = '.. /input/dog-breed-identification/train',
x_col = 'id',
y_col = 'breed',
subset="validation",
color_mode="rgb",
target_size =(331,331),
class_mode="categorical",
batch_size=32,
shuffle=True,
seed=123,
)
sub.to_csv('beat-the-best-kernel.csv', index=False )<define_variables>
x,y = next(train_ds)
 x.shape
%%time
start_time = time.time()
SUBMIT_MODE = True
def rmse(predicted, actual):
return np.sqrt(((predicted - actual)** 2 ).mean() )<categorify>
from keras.applications.nasnet import NASNetLarge
from keras.applications.inception_resnet_v2 import InceptionResNetV2
%%time
class TargetEncoder:
def __repr__(self):
return 'TargetEncoder'
def __init__(self, cols, smoothing=1, min_samples_leaf=1, noise_level=0, keep_original=False):
self.cols = cols
self.smoothing = smoothing
self.min_samples_leaf = min_samples_leaf
self.noise_level = noise_level
self.keep_original = keep_original
@staticmethod
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def encode(self, train, test, target):
for col in self.cols:
if self.keep_original:
train[col + '_te'], test[col + '_te'] = self.encode_column(train[col], test[col], target)
 else:
train[col], test[col] = self.encode_column(train[col], test[col], target)
 return train, test
def encode_column(self, trn_series, tst_series, target):
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - self.min_samples_leaf)/ self.smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(['mean', 'count'], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return self.add_noise(ft_trn_series, self.noise_level), self.add_noise(ft_tst_series, self.noise_level )<string_transform>
base_model = InceptionResNetV2(include_top=False,
weights='imagenet',
input_shape=(331,331,3)
)
 base_model.trainable = False
%%time
stopwords = {x: 1 for x in stopwords.words('russian')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 non_alphanumpunct = re.compile(u'[^A-Za-z0-9\.?!,; \(\)\[\]'"\$]+')
 RE_PUNCTUATION = '|'.join([re.escape(x)for x in string.punctuation] )<load_from_csv>
model = Sequential([
base_model,
GlobalAveragePooling2D() ,
Dense(256,activation = 'relu'),
Dropout(0.5),
Dense(120,activation='softmax')
 ])
 model.summary()
%%time
train = pd.read_csv('.. /input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
 test = pd.read_csv('.. /input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
 print('[{}] Finished load data'.format(time.time() - start_time))<string_transform>
model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'] )
%%time
def normalize_text(text):
text = text.lower().strip()
for s in string.punctuation:
text = text.replace(s, ' ')
 text = text.strip().split(' ')
 return u' '.join(x for x in text if len(x)> 1 and x not in stopwords)
 print(train.description[0])
 print(normalize_text(train.description[0]))<prepare_x_and_y>
my_calls = [keras.callbacks.EarlyStopping(monitor='val_accuracy',patience=2),
keras.callbacks.ModelCheckpoint("Model.h5",verbose=1,save_best_only=True)]
%%time
train['is_train'] = 1
test['is_train'] = 0
print('[{}] Compiled train / test'.format(time.time() - start_time))
print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 y = train.deal_probability.copy()
nrow_train = train.shape[0]
merge = pd.concat([train, test])
 submission = pd.DataFrame(test.index)
 print('[{}] Compiled merge'.format(time.time() - start_time))
print('Merge shape: ', merge.shape)
 del train
del test
gc.collect()
print('[{}] Garbage collection'.format(time.time() - start_time))<feature_engineering>
hist = model.fit(train_ds,epochs=25,validation_data=val_ds,callbacks=my_calls, verbose=2, batch_size=20 )
%%time
print("Feature Engineering - Part 1")
 merge["price"] = np.log(merge["price"]+0.001)
 merge["price"].fillna(-999,inplace=True)
 merge["image_top_1"].fillna(-999,inplace=True)
 print("
Create Time Variables")
 merge["activation_weekday"] = merge['activation_date'].dt.weekday
print(merge.head(5))
gc.collect()<data_type_conversions>
model = load_model("./Model.h5" )
%%time
training_index = merge.loc[merge.activation_date<=pd.to_datetime('2017-04-07')].index
validation_index = merge.loc[merge.activation_date>=pd.to_datetime('2017-04-08')].index
merge.drop(["activation_date","image"],axis=1,inplace=True)
 merge.drop(["user_id"], axis=1,inplace=True )<data_type_conversions>
testgen = Imgen(preprocessing_function=keras.applications.inception_resnet_v2.preprocess_input )
%%time
print("
Text Features")
 textfeats = ["description", "title"]
for cols in textfeats:
merge[cols] = merge[cols].astype(str)
 merge[cols] = merge[cols].astype(str ).fillna('missing')
 merge[cols] = merge[cols].str.lower()
merge[cols + '_num_stopwords'] = merge[cols].apply(lambda x: len([w for w in x.split() if w in stopwords]))
merge[cols + '_num_punctuations'] = merge[cols].apply(lambda comment:(comment.count(RE_PUNCTUATION)))
 merge[cols + '_num_alphabets'] = merge[cols].apply(lambda comment: len([c for c in comment if c.isupper() ]))
merge[cols + '_num_digits'] = merge[cols].apply(lambda comment:(comment.count('[0-9]')))
 merge[cols + '_num_letters'] = merge[cols].apply(lambda comment: len(comment))
merge[cols + '_num_words'] = merge[cols].apply(lambda comment: len(comment.split()))
merge[cols + '_num_unique_words'] = merge[cols].apply(lambda comment: len(set(w for w in comment.split())))
 merge[cols + '_words_vs_unique'] = merge[cols+'_num_unique_words'] / merge[cols+'_num_words']
merge[cols + '_letters_per_word'] = merge[cols+'_num_letters'] / merge[cols+'_num_words']
merge[cols + '_punctuations_by_letters'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_letters']
merge[cols + '_punctuations_by_words'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_words']
merge[cols + '_digits_by_letters'] = merge[cols+'_num_digits'] / merge[cols+'_num_letters']
merge[cols + '_alphabets_by_letters'] = merge[cols+'_num_alphabets'] / merge[cols+'_num_letters']
merge[cols + '_stopwords_by_words'] = merge[cols+'_num_stopwords'] / merge[cols+'_num_words']
merge[cols + '_mean'] = merge[cols].apply(lambda x: 0 if len(x)== 0 else float(len(x.split())) / len(x)) * 10
merge['title_desc_len_ratio'] = merge['title_num_letters']/merge['description_num_letters']<split>
test_ds = testgen.flow_from_dataframe(
 sample_sub,
directory = '.. /input/dog-breed-identification/test',
x_col = 'id',
y_col = None,
target_size =(331,331),
class_mode= None,
batch_size=32,
shuffle=False
)
%%time
df_test = merge.loc[merge['is_train'] == 0]
df_train = merge.loc[merge['is_train'] == 1]
del merge
gc.collect()
df_test = df_test.drop(['is_train'], axis=1)
 df_train = df_train.drop(['is_train'], axis=1)
 print(df_train.shape)
 print(y.shape)
 if SUBMIT_MODE:
y_train = y
del y
gc.collect()
else:
df_train, df_test, y_train, y_test = train_test_split(df_train, y, test_size=0.2, random_state=144)
 print('[{}] Splitting completed.'.format(time.time() - start_time))<choose_model_class>
predictions = model.predict(test_ds,verbose=1 )
%%time
wb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29,
"norm": None,
"tf": 'binary',
"idf": None,
}), procs=8)
 wb.dictionary_freeze = True
X_name_train = wb.fit_transform(df_train['title'])
 print(X_name_train.shape)
 X_name_test = wb.transform(df_test['title'])
 print(X_name_test.shape)
 del(wb)
 gc.collect()<train_model>
pred = [np.argmax(i)for i in predictions]
%%time
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_name_train, y_train,
test_size = 0.5,
shuffle = False)
 print('[{}] Finished splitting'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=42, alpha=5)
 model.fit(X_train_1, y_train_1)
 print('[{}] Finished to train name ridge(1)'.format(time.time() - start_time))
name_ridge_preds1 = model.predict(X_train_2)
 name_ridge_preds1f = model.predict(X_name_test)
 print('[{}] Finished to predict name ridge(1)'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=42, alpha=5)
 model.fit(X_train_2, y_train_2)
 print('[{}] Finished to train name ridge(2)'.format(time.time() - start_time))
name_ridge_preds2 = model.predict(X_train_1)
 name_ridge_preds2f = model.predict(X_name_test)
 print('[{}] Finished to predict name ridge(2)'.format(time.time() - start_time))
name_ridge_preds_oof = np.concatenate(( name_ridge_preds2, name_ridge_preds1), axis=0)
 name_ridge_preds_test =(name_ridge_preds1f + name_ridge_preds2f)/ 2.0
print('RMSLE OOF: {}'.format(rmse(name_ridge_preds_oof, y_train)))
 if not SUBMIT_MODE:
print('RMSLE TEST: {}'.format(rmse(name_ridge_preds_test, y_test)))
 gc.collect()<choose_model_class>
file_list = test_ds.filenames
id_list = []
for name in file_list:
m = re.sub('test/', '', name)
 m = re.sub('.jpg', '', m)
 id_list.append(m )
%%time
wb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28,
"norm": "l2",
"tf": 1.0,
"idf": None}), procs=8)
 wb.dictionary_freeze = True
X_description_train = wb.fit_transform(df_train['description'].fillna(''))
print(X_description_train.shape)
 X_description_test = wb.transform(df_test['description'].fillna(''))
print(X_description_test.shape)
 print('-')
 del(wb)
 gc.collect()<split>
submission = pd.read_csv('.. /input/dog-breed-identification/sample_submission.csv' )
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_description_train, y_train,
test_size = 0.5,
shuffle = False)
 print('[{}] Finished splitting'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_1, y_train_1)
 print('[{}] Finished to train desc ridge(1)'.format(time.time() - start_time))
desc_ridge_preds1 = model.predict(X_train_2)
 desc_ridge_preds1f = model.predict(X_description_test)
 print('[{}] Finished to predict desc ridge(1)'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_2, y_train_2)
 print('[{}] Finished to train desc ridge(2)'.format(time.time() - start_time))
desc_ridge_preds2 = model.predict(X_train_1)
 desc_ridge_preds2f = model.predict(X_description_test)
 print('[{}] Finished to predict desc ridge(2)'.format(time.time() - start_time))
desc_ridge_preds_oof = np.concatenate(( desc_ridge_preds2, desc_ridge_preds1), axis=0)
 desc_ridge_preds_test =(desc_ridge_preds1f + desc_ridge_preds2f)/ 2.0
print('RMSLE OOF: {}'.format(rmse(desc_ridge_preds_oof, y_train)))
 if not SUBMIT_MODE:
print('RMSLE TEST: {}'.format(rmse(desc_ridge_preds_test, y_test)))
 gc.collect()<drop_column>
submission['id'] = id_list
submission.iloc[:,1:] =predictions
submission.head()
del X_train_1
del X_train_2
del y_train_1
del y_train_2
del name_ridge_preds1
del name_ridge_preds1f
del name_ridge_preds2
del name_ridge_preds2f
del desc_ridge_preds1
del desc_ridge_preds1f
del desc_ridge_preds2
del desc_ridge_preds2f
gc.collect()
print('[{}] Finished garbage collection'.format(time.time() - start_time))<drop_column>
final_df = submission.set_index('id')
 final_df.to_csv('Submission.csv' )