Kaggle competition: Dog Breed Identification
train_index, valid_index = train_test_split(np.arange(len(train)) , test_size=0.1, random_state=42)
 x_train = scipy.sparse.hstack([
train_desc_counts[train_index],
train_title_counts[train_index],
train.loc[train_index, predictors]
], format='csr')
 y_train = train.loc[train_index, target]
x_valid = scipy.sparse.hstack([
train_desc_counts[valid_index],
train_title_counts[valid_index],
train.loc[valid_index, predictors]
], format='csr')
 y_valid = train.loc[valid_index, target]
x_test = scipy.sparse.hstack([
test_desc_counts,
test_title_counts,
test.loc[:, predictors]
], format='csr')
 dtrain = lgb.Dataset(x_train, label=y_train,
feature_name=list(feature_names),
categorical_feature=categorical)
 dvalid = lgb.Dataset(x_valid, label=y_valid,
feature_name=list(feature_names),
categorical_feature=categorical )<train_model>
%%time
%matplotlib inline
rounds = 16000
early_stop_rounds = 500
params = {
'objective' : 'regression',
'metric' : 'rmse',
'num_leaves' : 32,
'max_depth': 15,
'learning_rate' : 0.02,
'feature_fraction' : 0.6,
'verbosity' : -1
}
evals_result = {}
model = lgb.train(params, dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train', 'valid'],
num_boost_round=rounds,
early_stopping_rounds=early_stop_rounds,
verbose_eval=500 )<save_to_csv>
%%time
train_dir = '.. /input/dog-breed-identification/train'
test_dir ='.. /input/dog-breed-identification/test'
subm = pd.read_csv('.. /input/avito-demand-prediction/sample_submission.csv')
 subm['deal_probability'] = np.clip(model.predict(x_test), 0, 1)
 subm.to_csv('Aggregate_Ridge.csv', index=False )<set_options>
%%time
def append_ext(fn):
return fn+".jpg"
traindf = pd.read_csv('.. /input/dog-breed-identification/labels.csv',dtype=str)
 testdf = pd.read_csv('.. /input/dog-breed-identification/sample_submission.csv',dtype=str)
 traindf["id"] = traindf["id"].apply(append_ext)
 testdf["id"] = testdf["id"].apply(append_ext)
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
print(os.listdir(".. /input")) 
<load_from_csv>
%%time
train_datagen=ImageDataGenerator(rescale=1./255.,
zoom_range = [0.7,1],
horizontal_flip = True,
validation_split=0.05
)
train_df = pd.read_csv(".. /input/train.csv", parse_dates=["activation_date"])
 test_df = pd.read_csv(".. /input/test.csv", parse_dates=["activation_date"])
 print("Train file rows and columns are : ", train_df.shape)
 print("Test file rows and columns are : ", test_df.shape )<define_variables>
BATCH_SIZE = 32
temp_data = StringIO()
region_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, region_df, how="left", on="region")
 test_df = pd.merge(test_df, region_df, how="left", on="region" )<merge>
train_generator=train_datagen.flow_from_dataframe(
 dataframe=traindf,
directory=train_dir,
x_col="id",
y_col="breed",
subset="training",
batch_size=BATCH_SIZE,
seed=42,
shuffle=True,
class_mode="categorical",
target_size=(224,224),
color_mode="rgb"
)
temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="parent_category_name", how="left")
 test_df = pd.merge(test_df, temp_df, how="left", on="parent_category_name")
 temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="category_name", how="left")
 test_df = pd.merge(test_df, temp_df, on="category_name", how="left")
 train_df["title_nwords"] = train_df["title"].apply(lambda x: len(x.split()))
test_df["title_nwords"] = test_df["title"].apply(lambda x: len(x.split()))<feature_engineering>
x,y = next(train_generator )
train_df["description"].fillna("NA", inplace=True)
 train_df["desc_nwords"] = train_df["description"].apply(lambda x: len(x.split()))
test_df["description"].fillna("NA", inplace=True)
 test_df["desc_nwords"] = test_df["description"].apply(lambda x: len(x.split()))<categorify>
valid_generator=train_datagen.flow_from_dataframe(
 dataframe=traindf,
directory=train_dir,
x_col="id",
y_col="breed",
subset="validation",
batch_size=BATCH_SIZE,
seed=42,
shuffle=True,
class_mode="categorical",
target_size=(224,224),
color_mode="rgb")
tfidf_vec = TfidfVectorizer(ngram_range=(1,1))
full_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['title'].values.tolist() )<concatenate>
test_datagen=ImageDataGenerator(rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=testdf,
directory=test_dir,
x_col="id",
y_col=None,
batch_size=BATCH_SIZE,
seed=42,
shuffle=False,
class_mode=None,
target_size=(224,224),
color_mode="rgb" )
n_comp = 5
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<categorify>
pretrained_model = tf.keras.applications.ResNet50V2(
 weights='imagenet',
include_top=False ,
input_shape=[224,224,3] 
)
 pretrained_model.trainable = False
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(120, activation='softmax')
 ] )
tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)
 full_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())
 n_comp = 5
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<categorify>
opt=tf.keras.optimizers.SGD(lr=1e-4, momentum=0.9)
 model.compile(optimizer = opt ,
loss="categorical_crossentropy",
metrics=["accuracy"])
 model.summary()
train_df['param123'] = train_df['param_1'].fillna('')+ " " + train_df['param_2'].fillna('')+ " " + train_df['param_3'].fillna('')
 test_df['param123'] = test_df['param_1'].fillna('')+ " " + test_df['param_2'].fillna('')+ " " + test_df['param_3'].fillna('' )<categorify>
early = tf.keras.callbacks.EarlyStopping(patience=2,
min_delta=0.001,
restore_best_weights=True )
tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)
 full_tfidf = tfidf_vec.fit_transform(train_df['param123'].values.tolist() + test_df['param123'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['param123'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['param123'].values.tolist())
 n_comp = 5
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_params_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_params_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<feature_engineering>
STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size
STEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size
STEP_SIZE_TEST = test_generator.n//test_generator.batch_size
history = model.fit(train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=25,
callbacks=[early], )
train_df["activation_weekday"] = train_df["activation_date"].dt.weekday
test_df["activation_weekday"] = test_df["activation_date"].dt.weekday
train_df.head()<categorify>
score = model.evaluate(valid_generator,batch_size=32)
 print("Accuracy: {:.2f}%".format(score[1] * 100))
print("Loss: ",score[0])
train_df["price_new"] = train_df["price"].values
train_df["price_new"].fillna(np.nanmedian(train_df["price"].values), inplace=True)
 test_df["price_new"] = test_df["price"].values
test_df["price_new"].fillna(np.nanmedian(train_df["price"].values), inplace=True)
 cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))
 cols_to_drop = ["item_id", "user_id", "title", "description", "activation_date", "image", "param_1", "param_2", "param_3", "param123", "price_new"]
train_X = train_df.drop(cols_to_drop + ["region_en", "parent_category_name_en", "category_name_en", "deal_probability"], axis=1)
 test_X = test_df.drop(cols_to_drop + ["region_en", "parent_category_name_en", "category_name_en"], axis=1)
 train_y = train_df["deal_probability"].values
test_id = test_df["item_id"].values<train_model>
from sklearn.metrics import f1_score, confusion_matrix
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 200,
"learning_rate" : 0.01,
"bagging_fraction" : 0.8,
"feature_fraction" : 0.8,
"bagging_freq" : 5,
"bagging_seed" : 2018,
"verbosity" : -1,
"min_child_samples":20
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=20, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 return pred_test_y, model, evals_result<train_model>
target_names = []
for key in train_generator.class_indices:
target_names.append(key )
dev_X = train_X.iloc[:-100000,:]
val_X = train_X.iloc[-100000:,:]
dev_y = train_y[:-100000]
val_y = train_y[-100000:]
print(dev_X.shape, val_X.shape, test_X.shape )<train_model>
Y_pred = model.predict(valid_generator)
 y_pred = np.argmax(Y_pred, axis=1 )
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X )<save_to_csv>
cm = confusion_matrix(valid_generator.classes,y_pred)
pred_test[pred_test>1] = 1
pred_test[pred_test<0] = 0
sub_df = pd.DataFrame({"item_id":test_id})
 sub_df["deal_probability"] = pred_test
sub_df.to_csv("baseline_lgb.csv", index=False )<import_modules>
f_score = f1_score(valid_generator.classes,y_pred,average='macro')
 print('F1 score:',f_score)
import pandas as pd
import numpy as np
import re
import gc
from keras.preprocessing import text, sequence
import math
from xgboost.sklearn import XGBRegressor
import xgboost as xgb
from sklearn import metrics
import matplotlib.pyplot as plt<load_from_csv>
pred=model.predict(test_generator )
train = pd.read_csv('.. /input/readyforuse/trainforuse_mix.csv')
 test = pd.read_csv('.. /input/readyforuse/testforuse_mix.csv')
 submission = pd.read_csv('.. /input/avito-demand-prediction/sample_submission.csv')
 train = train.drop(columns = ['item_id'])
 train['date'] = pd.to_datetime(train['activation_date'] ).dt.weekday.astype('int')
 train = train.drop(columns = ['activation_date','image'])
 test = test.drop(columns = ['item_id'])
 test['date'] = pd.to_datetime(test['activation_date'] ).dt.weekday.astype('int')
 test = test.drop(columns = ['activation_date','image'] )<define_variables>
df_submission = pd.read_csv('/kaggle/input/dog-breed-identification/sample_submission.csv')
 df_submission.head()
encoding_predictors = ['user_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'item_seq_number', 'user_type', 'date']<count_missing_values>
file_list = test_generator.filenames
id_list = []
for name in file_list:
m = re.sub('test/', '', name)
 m = re.sub('.jpg', '', m)
 id_list.append(m )
for i in encoding_predictors:
print(str(i)+': ' + str(len(set(train[i]))))
print('Na contains? ' + str(train[i].isnull().values.any()))<count_missing_values>
df_submission['id'] = id_list
df_submission.iloc[:,1:] = pred
df_submission.head()
for i in encoding_predictors:
print(str(i)+': ' + str(len(set(test[i]))))
print('Na contains? ' + str(test[i].isnull().values.any()))<define_search_space>
breeds=['id','beagle','chihuahua','doberman','french_bulldog', 'golden_retriever', 'malamute','pug','saint_bernard','scottish_deerhound','tibetan_mastiff']
l_region = dict(zip(list(set(train['region'])) ,range(1,29)))
 l_parent_category_name = dict(zip(list(set(train['parent_category_name'])) ,range(1,10)))
 l_category_name = dict(zip(list(set(train['category_name'])) ,range(1,48)))
 l_user_type = dict(zip(list(set(train['user_type'])) ,range(1,4)))
 l_date = dict(zip(list(set(train['date'])) ,range(1,8)))
 l_param1 = dict(zip(list(set(train['param_1'])) ,range(1,373)))
 l_param2 = dict(zip(list(set(train['param_2'])) ,range(1,273)))
 l_city = dict(zip(list(set(train['city'])) ,range(1,1734)))
 l_param3 = dict(zip(list(set(train['param_3'])) ,range(1,1221)) )<categorify>
selected_breeds = df_submission.loc[:,breeds]
<categorify><EOS> 
final_sub = df_submission.set_index('id')
 final_sub.to_csv('Submission.csv' )