Kaggle competition: Dog Breed Identification
%%time
df_train.drop(['deal_probability', 'title', 'description'], axis=1, inplace=True)
 df_test.drop(['title', 'description'], axis=1, inplace=True)
 print('Remerged')
 dummy_cols = ['parent_category_name', 'category_name', 'user_type', 'image_top_1',
'region', 'city', 'param_1', 'param_2', 'param_3', 'activation_weekday']
numeric_cols = list(set(df_train.columns.values)- set(dummy_cols))
print(numeric_cols )<normalization>
%%time
%matplotlib inline
%%time
class Scaler(BaseEstimator, TransformerMixin):
def __init__(self, columns, copy=True, with_mean=True, with_std=True):
self.scaler = StandardScaler(copy, with_mean, with_std)
 self.columns = columns
def fit(self, X, y=None):
self.scaler.fit(X[self.columns], y)
 return self
def transform(self, X, y=None, copy=None):
init_col_order = X.columns
X_scaled = pd.DataFrame(self.scaler.transform(X[self.columns]), columns=self.columns, index=X.index)
 X_not_scaled = X[list(set(init_col_order)- set(self.columns)) ]
return pd.concat([X_not_scaled, X_scaled], axis=1)[init_col_order]
print('Scaler')
 scaler = Scaler(columns=numeric_cols)
 df_train = scaler.fit_transform(df_train)
 df_test = scaler.transform(df_test )<concatenate>
%%time
train_dir = '.. /input/dog-breed-identification/train'
test_dir ='.. /input/dog-breed-identification/test'
%%time
sparse_merge_train = hstack(( X_name_train, X_description_train)).tocsr()
sparse_merge_test = hstack(( X_name_test, X_description_test)).tocsr()
print(sparse_merge_train.shape)
 for col in dummy_cols:
print(col)
 lb = LabelBinarizer(sparse_output=True)
 sparse_merge_train = hstack(( sparse_merge_train, lb.fit_transform(df_train[[col]].fillna('')))).tocsr()
print(sparse_merge_train.shape)
 sparse_merge_test = hstack(( sparse_merge_test, lb.transform(df_test[[col]].fillna('')))).tocsr()<drop_column>
%%time
def append_ext(fn):
return fn+".jpg"
traindf = pd.read_csv('.. /input/dog-breed-identification/labels.csv',dtype=str)
 testdf = pd.read_csv('.. /input/dog-breed-identification/sample_submission.csv',dtype=str)
 traindf["id"] = traindf["id"].apply(append_ext)
 testdf["id"] = testdf["id"].apply(append_ext)
del X_description_test, X_name_test
del X_description_train, X_name_train
del lb, mask
gc.collect()<train_model>
%%time
train_datagen=ImageDataGenerator(rescale=1./255.,
horizontal_flip = True,
validation_split=0.02
)
%%time
print("
FM_FTRL Starting...........")
 if SUBMIT_MODE:
iters = 1
else:
iters = 1
rounds = 1
model = FM_FTRL(alpha=0.035, beta=0.001, L1=0.00001, L2=0.15, D=sparse_merge_train.shape[1],
alpha_fm=0.05, L2_fm=0.0, init_fm=0.01,
D_fm=100, e_noise=0, iters=iters, inv_link="identity", threads=4)
 if SUBMIT_MODE:
model.fit(sparse_merge_train, y_train)
 print('[{}] Train FM completed'.format(time.time() - start_time))
predsFM = model.predict(sparse_merge_test)
 print('[{}] Predict FM completed'.format(time.time() - start_time))
else:
for i in range(rounds):
model.fit(sparse_merge_train, y_train)
 predsFM = model.predict(sparse_merge_test)
 print('[{}] Iteration {}/{} -- RMSLE: {}'.format(time.time() - start_time, i + 1, rounds, rmse(predsFM, y_test)))
 del model
gc.collect()
if not SUBMIT_MODE:
print("FM_FTRL dev RMSLE:", rmse(predsFM, y_test)) 
<train_model>
BATCH_SIZE = 32
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(sparse_merge_train, y_train,
test_size = 0.5,
shuffle = False)
 print('[{}] Finished splitting'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_1, y_train_1)
 print('[{}] Finished to train ridge(1)'.format(time.time() - start_time))
ridge_preds1 = model.predict(X_train_2)
 ridge_preds1f = model.predict(sparse_merge_test)
 print('[{}] Finished to predict ridge(1)'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_2, y_train_2)
 print('[{}] Finished to train ridge(2)'.format(time.time() - start_time))
ridge_preds2 = model.predict(X_train_1)
 ridge_preds2f = model.predict(sparse_merge_test)
 print('[{}] Finished to predict ridge(2)'.format(time.time() - start_time))
ridge_preds_oof = np.concatenate(( ridge_preds2, ridge_preds1), axis=0)
 ridge_preds_test =(ridge_preds1f + ridge_preds2f)/ 2.0
print('RMSLE OOF: {}'.format(rmse(ridge_preds_oof, y_train)))
 if not SUBMIT_MODE:
print('RMSLE TEST: {}'.format(rmse(ridge_preds_test, y_test)) )<normalization>
image_size=(299,299 )
fselect = SelectKBest(f_regression, k=48000)
 train_features = fselect.fit_transform(sparse_merge_train, y_train)
 test_features = fselect.transform(sparse_merge_test)
 print('[{}] Select best completed'.format(time.time() - start_time))
del sparse_merge_train
del sparse_merge_test
gc.collect()
print('[{}] Garbage collection'.format(time.time() - start_time))<drop_column>
train_generator=train_datagen.flow_from_dataframe(
 dataframe=traindf,
directory=train_dir,
x_col="id",
y_col="breed",
subset="training",
batch_size=BATCH_SIZE,
seed=42,
shuffle=True,
class_mode="categorical",
target_size=image_size,
color_mode="rgb"
)
del ridge_preds1
del ridge_preds1f
del ridge_preds2
del ridge_preds2f
del X_train_1
del X_train_2
del y_train_1
del y_train_2
del model
gc.collect()
print('[{}] Finished garbage collection'.format(time.time() - start_time))<feature_engineering>
x,y = next(train_generator )
df_train['ridge'] = ridge_preds_oof
df_train['name_ridge'] = name_ridge_preds_oof
df_train['desc_ridge'] = desc_ridge_preds_oof
df_test['ridge'] = ridge_preds_test
df_test['name_ridge'] = name_ridge_preds_test
df_test['desc_ridge'] = desc_ridge_preds_test
print('[{}] Finished adding submodels'.format(time.time() - start_time))<categorify>
valid_generator=train_datagen.flow_from_dataframe(
 dataframe=traindf,
directory=train_dir,
x_col="id",
y_col="breed",
subset="validation",
batch_size=BATCH_SIZE,
seed=42,
shuffle=True,
class_mode="categorical",
target_size=image_size,
color_mode="rgb")
f_cats = ["region","city","parent_category_name","category_name","user_type","image_top_1"]
target_encode = TargetEncoder(min_samples_leaf=100, smoothing=10, noise_level=0.01,
keep_original=True, cols=f_cats)
 df_train, df_test = target_encode.encode(df_train, df_test, y_train)
 print('[{}] Finished target encoding'.format(time.time() - start_time))<set_options>
test_datagen=ImageDataGenerator(rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=testdf,
directory=test_dir,
x_col="id",
y_col=None,
batch_size=BATCH_SIZE,
seed=42,
shuffle=False,
class_mode=None,
target_size=image_size,
color_mode="rgb" )
del ridge_preds_oof
del ridge_preds_test
gc.collect()
print('[{}] Finished garbage collection'.format(time.time() - start_time))<categorify>
pretrained_model = tf.keras.applications.InceptionV3(
 weights='imagenet',
include_top=False ,
input_shape=shape 
)
 pretrained_model.trainable = False
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(120, activation='softmax')
 ] )
cols = ['region_te', 'city_te', 'parent_category_name_te', 'category_name_te',
'user_type_te', 'image_top_1_te', 'desc_ridge', 'name_ridge', 'ridge']
train_dummies = csr_matrix(df_train[cols].values)
 print('[{}] Finished dummyizing model 1/5'.format(time.time() - start_time))
test_dummies = csr_matrix(df_test[cols].values)
 print('[{}] Finished dummyizing model 2/5'.format(time.time() - start_time))
del df_train
del df_test
gc.collect()
print('[{}] Finished dummyizing model 3/5'.format(time.time() - start_time))
train_features = hstack(( train_features, train_dummies)).tocsr()
print('[{}] Finished dummyizing model 4/5'.format(time.time() - start_time))
test_features = hstack(( test_features, test_dummies)).tocsr()
print('[{}] Finished dummyizing model 5/5'.format(time.time() - start_time))<create_dataframe>
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 opt=tf.keras.optimizers.SGD(lr=1e-3, momentum=0.9)
 model.compile(optimizer = opt ,
loss="categorical_crossentropy",
metrics=["accuracy"])
 model.summary()
d_train = lgb.Dataset(train_features, label=y_train)
 del train_features
gc.collect()
if SUBMIT_MODE:
watchlist = [d_train]
else:
d_valid = lgb.Dataset(test_features, label=y_test)
 watchlist = [d_train, d_valid]
params = {
'learning_rate': 0.15,
'application': 'regression',
'max_depth': 13,
'num_leaves': 400,
'verbosity': -1,
'metric': 'RMSE',
'data_random_seed': 1,
'bagging_fraction': 0.8,
'feature_fraction': 0.6,
'nthread': 4,
'lambda_l1': 10,
'lambda_l2': 10
}
print('[{}] Finished compiling LGB'.format(time.time() - start_time))<train_model>
reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',factor=0.2,patience=5, min_lr=0.001)
 early = tf.keras.callbacks.EarlyStopping(patience=2,
min_delta=0.001,
restore_best_weights=True )
%%time
modelL = lgb.train(params,
train_set=d_train,
num_boost_round=400,
valid_sets=watchlist,
verbose_eval=50)
 predsL = modelL.predict(test_features)
 predsL[predsL < 0] = 0
predsL[predsL > 1] = 1
if not SUBMIT_MODE:
print("LGB RMSLE:", rmse(predsL, y_test))<drop_column>
STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size
STEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size
STEP_SIZE_TEST = test_generator.n//test_generator.batch_size
history = model.fit(train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=10,
callbacks=[early], )
del d_train
del modelL
if not SUBMIT_MODE:
del d_valid
gc.collect()<compute_test_metric>
score = model.evaluate(valid_generator,batch_size=32)
 print("Accuracy: {:.2f}%".format(score[1] * 100))
print("Loss: ",score[0])
preds_final = predsFM * 0.1 + predsL * 0.9
if not SUBMIT_MODE:
print('Final RMSE: ', rmse(preds_final, y_test))<save_to_csv>
from sklearn.metrics import f1_score
if SUBMIT_MODE:
submission['deal_probability'] = preds_final
submission['deal_probability'] = submission['deal_probability'].clip(0.0, 1.0)
 submission.to_csv('lgb_and_fm_separate_train_test.csv', index=False)
 print('[{}] Writing submission done'.format(time.time() - start_time))
print(submission.head(5))<set_options>
Y_pred = model.predict(valid_generator)
 y_pred = np.argmax(Y_pred, axis=1 )
sns.set()
%matplotlib inline<load_from_csv>
f_score = f1_score(valid_generator.classes,y_pred,average='macro')
 print('F1 score:',f_score)
train = pd.read_csv('.. /input/avito-demand-prediction/train.csv')
 test = pd.read_csv('.. /input/avito-demand-prediction/test.csv')
 train_sorted = train.sort_values(by = ["image"])
 test_sorted = test.sort_values(by = ["image"])
 tr_1 = pd.read_csv(".. /input/train-data-image-blurrness/1.csv")
 tr_2 = pd.read_csv(".. /input/train-data-image-blurrness/2.csv")
 tr_3 = pd.read_csv(".. /input/train-data-image-blurrness/3.csv")
 tr_4 = pd.read_csv(".. /input/train-data-image-blurrness/4.csv")
 tr_5 = pd.read_csv(".. /input/train-data-image-blurrness/5.csv")
 tr_6 = pd.read_csv(".. /input/train-data-image-blurrness/6.csv")
 tr_7 = pd.read_csv(".. /input/train-data-image-blurrness/7.csv")
 tr_8 = pd.read_csv(".. /input/train-data-image-blurrness/8.csv")
 tr_9 = pd.read_csv(".. /input/train-data-image-blurrness/9.csv")
 tr_10 = pd.read_csv(".. /input/train-data-image-blurrness/10.csv")
 tr_11 = pd.read_csv(".. /input/train-data-image-blurrness/11(12_13.5 ).csv")
 tr_12 = pd.read_csv(".. /input/train-data-image-blurrness/last.csv")
 frames = [tr_1, tr_2, tr_3, tr_4, tr_5, tr_6, tr_7, tr_8, tr_9, tr_10, tr_11, tr_12]
new = pd.concat(frames)
 new["File"] = new["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new = new.sort_values(by = ["File"])
 scores = list(new["Score"].values)+ [-1] *(len(train)-len(new))
train_sorted["image_blurrness_score"] = scores
train = train_sorted.sort_index()
te_1 = pd.read_csv(".. /input/image-blurrness-test/test_1.csv")
 te_2 = pd.read_csv(".. /input/image-blurrness-test/test_2.csv")
 te_3 = pd.read_csv(".. /input/image-blurrness-test/test_3.csv")
 te_4 = pd.read_csv(".. /input/image-blurrness-test/test_4.csv")
 te_5 = pd.read_csv(".. /input/image-blurrness-test/test_5.csv")
 frames_te = [te_1, te_2, te_3, te_4, te_5]
new_te = pd.concat(frames_te)
 new_te["File"] = new_te["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new_te = new_te.sort_values(by = ["File"])
 scores_te = list(new_te["Score"].values)+ [-1] *(len(test)-len(new_te))
test_sorted["image_blurrness_score"] = scores_te
test = test_sorted.sort_index() 
<load_from_csv>
pred=model.predict(test_generator )
gp = pd.read_csv(".. /input/aggregate/aggregated_features.csv")
 train = train.merge(gp, on='user_id', how='left')
 test = test.merge(gp, on='user_id', how='left')
 agg_cols = list(gp.columns)[1:]
del gp
gc.collect() 
<feature_engineering>
df_submission = pd.read_csv('/kaggle/input/dog-breed-identification/sample_submission.csv')
 df_submission.head()
count = lambda l1,l2: sum([1 for x in l1 if x in l2])
 for df in [train, test]:
df["price"] = np.log(df["price"]+0.001)
 df["image_blurrness_score"] = np.log(df["image_blurrness_score"]+0.001)
 df['description'].fillna('unknowndescription', inplace=True)
 df['title'].fillna('unknowntitle', inplace=True)
 df['weekday'] = pd.to_datetime(df['activation_date'] ).dt.day
for col in ['description', 'title']:
df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))
df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100
df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100
df['city'] = df['region'] + '_' + df['city']
df['num_desc_punct'] = df['description'].apply(lambda x: count(x, set(string.punctuation)))
 for col in agg_cols:
df[col].fillna(-1, inplace=True)
 for df in [train, test]:
df.price.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 df.image_blurrness_score.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True )<feature_engineering>
file_list = test_generator.filenames
id_list = []
for name in file_list:
m = re.sub('test/', '', name)
 m = re.sub('.jpg', '', m)
 id_list.append(m )
count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=25)
 title_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))
train_title_counts = title_counts[:len(train)]
test_title_counts = title_counts[len(train):]
count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'),
lowercase=True, ngram_range=(1, 2),
max_features=15000)
 desc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))
train_desc_counts = desc_counts[:len(train)]
test_desc_counts = desc_counts[len(train):]
train_title_counts.shape, train_desc_counts.shape
train<define_variables>
df_submission['id'] = id_list
df_submission.iloc[:,1:] = pred
df_submission.head()
target = 'deal_probability'
predictors = [
'num_desc_punct',
'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',
'avg_times_up_user', 'avg_days_up_user', 'n_user_items',
'price', 'item_seq_number', "image_blurrness_score"
]
categorical = [
'image_top_1', 'param_1', 'param_2', 'param_3',
'city', 'region', 'category_name', 'parent_category_name', 'user_type'
]
predictors = predictors + categorical<categorify>
breeds=['id','beagle','chihuahua','doberman','french_bulldog', 'golden_retriever', 'malamute','pug','saint_bernard','scottish_deerhound','tibetan_mastiff']
for feature in categorical:
print(f'Transforming {feature}...')
 encoder = LabelEncoder()
encoder.fit(train[feature].append(test[feature] ).astype(str))
train[feature] = encoder.transform(train[feature].astype(str))
test[feature] = encoder.transform(test[feature].astype(str))
train<train_model>
selected_breeds = df_submission.loc[:,breeds]
df = pd.concat([train[predictors], test[predictors]], axis = 0)
 NFOLDS = 5
SEED = 42
VALID = False
traindex = train.index
testdex = test.index
ntrain = train.shape[0]
ntest = test.shape[0]
kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)
 y = train.deal_probability.copy()
class SklearnWrapper(object):
def __init__(self, clf, seed=0, params=None, seed_bool = True):
if(seed_bool == True):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def get_oof(clf, x_train, y, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(kf):
print('
Fold {}'.format(i))
x_tr = x_train[train_index]
y_tr = y[train_index]
x_te = x_train[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)
 ridge_params = {'alpha':20.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}
ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
 ridge_oof_train, ridge_oof_test = get_oof(ridge, np.array(df[:ntrain]), y, np.array(df[ntrain:]))
def rmse(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(( y - y0), 2)))
 rms = sqrt(mean_squared_error(y, ridge_oof_train))
print('Ridge OOF RMSE: {}'.format(rms))
ridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])
 df['ridge_preds'] = ridge_preds
train = df[:ntrain]
train["deal_probability"] = y
test = df[ntrain:]
predictors.append("ridge_preds" )<prepare_x_and_y>
final_sub = df_submission.set_index('id')
 final_sub.to_csv('Submission.csv' )