Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<define_variables>
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'
temp_data = StringIO()
region_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, region_df, how="left", on="region" )<count_values>
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
temp_series = train_df['region_en'].value_counts() 
(temp_series[:5] )<define_variables>
from collections import OrderedDict
sizes =(np.array(( temp_series / temp_series.sum())*100))<count_values>
map={'yes':1,"no":0}
for df in [train,test]:
df['dependency']=df['dependency'].replace(map ).astype(np.float64)
 df['edjefe']=df['edjefe'].replace(map ).astype(np.float64)
 df['edjefa']=df['edjefa'].replace(map ).astype(np.float64)
 train[['dependency','edjefe','edjefa']].describe()
cnt_srs = train_df['city'].value_counts().head(20)
 type(cnt_srs )<merge>
test['Target']=np.nan
data=train.append(test,ignore_index=True )
temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="parent_category_name", how="left" )<define_variables>
X = [[0],[1],[2]]
Y = [ 1, 2, 1 ]
dtc = DTC(max_depth=1 )
temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="category_name", how="left" )<define_variables>
dtc.fit(X,Y)
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
cnt_srs = train_df['activation_date']<feature_engineering>
dtc.fit(X,Y,sample_weight=[1,2,3])
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
train_df["title_nwords"] = train_df["title"].apply(lambda x: len(x.split()))
test_df["title_nwords"] = test_df["title"].apply(lambda x: len(x.split()))<categorify>
data['Target'].value_counts()
tfidf_vec = TfidfVectorizer(ngram_range=(1,1))
full_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['title'].values.tolist())
<choose_model_class>
heads=data.loc[data['parentesco1'] == 1].copy()
train_labels=data.loc[(data['Target'].notnull())&(data['parentesco1']==1),['Target','idhogar']]
n_comp = 3
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
<create_dataframe>
type_counts=train_labels['Target'].value_counts()
svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))<categorify>
type_counts=pd.DataFrame(type_counts )
svd_obj.transform(train_tfidf )<create_dataframe>
type_counts['level']=type_counts.index
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))<rename_columns>
type_counts['level']=type_counts['level'].replace(poverty_mapping)
train_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]<create_dataframe>
type_counts['Count']=type_counts['Target']
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<correct_missing_values>
all_equal=train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() ==1 )
test_df["description"].fillna("NA", inplace=True )<feature_engineering>
not_equal= all_equal[all_equal!=True]
train_df["desc_nwords"] = train_df["description"].apply(lambda x: len(x.split()))
train_df["desc_nwords"]<count_values>
example=train.loc[train['idhogar']==not_equal.index[0],['idhogar', 'parentesco1', 'Target']]
test_df["desc_nwords"] = test_df["description"].apply(lambda x: len(x.split()))
cnt_srs = train_df['desc_nwords'].value_counts()
cnt_srs<categorify>
households_leader=train.groupby('idhogar')['parentesco1'].sum()
tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)
 full_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())
 n_comp = 3
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<prepare_x_and_y>
households_leader_flase=train.loc[train['idhogar'].isin(households_leader
[households_leader !=1].index),:]
train_y = train_df["deal_probability"].values
test_id = test_df["item_id"].values<feature_engineering>
for household in not_equal.index:
true_target= int(train[(train['idhogar']==household)& 
(train['parentesco1']==1.0)]['Target'])
 train.loc[train['idhogar']==household, 'Target']=true_target
train_df["activation_weekday"] = train_df["activation_date"].dt.weekday
test_df["activation_weekday"] = test_df["activation_date"].dt.weekday 
<count_unique_values>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns={0:'total'})
 missing['Percent']=missing['total']/len(data)
 missing.sort_values('Percent',ascending=False ).head(10 )
np.unique(train_df[col].values ).shape<categorify>
heads.groupby('v18q')['v18q1'].apply(lambda x:x.isnull().sum() )
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1", "param_2", "param_3"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')) )<categorify>
data['v18q1']=data['v18q1'].fillna(0 )
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1", "param_2", "param_3"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))
 cols_to_drop = ["item_id", "user_id", "title", "description", "activation_date", "image"]
train_X = train_df.drop(cols_to_drop + ["region_en", "parent_category_name_en", "category_name_en", "price_new", "deal_probability"], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1 )<find_best_params>
own_variables=[x for x in data if x.startswith('tipo')]
model.best_iteration<create_dataframe>
owns=pd.DataFrame(data.loc[data['tipovivi3']==1,'Target'] )
def run_lgb(train_X, train_y, val_X, val_y,alpha):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 30,
"learning_rate" : alpha,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=10000, verbose_eval=20, evals_result=evals_result)
 return model, evals_result<train_model>
owns['Target'].value_counts()
start_time = time.time()
dev_X = train_X.iloc[:-200000,:]
val_X = train_X.iloc[-200000:,:]
dev_y = train_y[:-200000]
val_y = train_y[-200000:]
print(dev_X.shape, val_X.shape, test_X.shape)
 learningList = [0.001,0.003,0.009,0.027,0.081,0.243,0.729,2.178,6.514,19.51]
train_error = list()
test_error = list()
for i in learningList:
print("the learning_rate 0f ",i)
 model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, i)
 pred_train = model.predict(dev_X, num_iteration=model.best_iteration)
 pred_val= model.predict(val_X,num_iteration = model.best_iteration)
 train_error.append(metrics.mean_squared_error(dev_y,pred_train))
test_error.append(metrics.mean_squared_error(val_y,pred_val))
del model, evals_result
plt.plot(learningList,train_error)
 plt.plot(learningList,test_error)
 plt.show()
print("--- %s seconds ---" %(time.time() - start_time))<set_options>
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
print(os.listdir(".. /input")) 
<load_from_csv>
data['v2a1-missing'] = data['v2a1'].isnull()
train_df = pd.read_csv(".. /input/train.csv", parse_dates=["activation_date"])
 test_df = pd.read_csv(".. /input/test.csv", parse_dates=["activation_date"])
 print("Train file rows and columns are : ", train_df.shape)
 print("Test file rows and columns are : ", test_df.shape)
 train_prd = pd.read_csv(".. /input/periods_train.csv", parse_dates=["activation_date","date_from", "date_to"])
 test_prd = pd.read_csv(".. /input/periods_test.csv", parse_dates=["activation_date","date_from", "date_to"])
 print("Period Train file rows and columns are : ", train_prd.shape)
 print("Period Test file rows and columns are : ", test_prd.shape )<merge>
data['v2a1-missing'].value_counts()
train_prd['days'] =(train_prd['date_to'] - train_prd['date_from'] ).dt.days
test_prd['days'] =(test_prd['date_to'] - test_prd['date_from'] ).dt.days
enc = train_prd.groupby('item_id')['days'].agg('sum' ).astype(np.float32 ).reset_index()
enc.head(5)
 train_df = pd.merge(train_df, enc, how='left', on='item_id')
 test_df = pd.merge(test_df, enc, how='left', on='item_id' )<feature_engineering>
data['v2a1'] = data['v2a1'].fillna(value=data['tipovivi3'] )
train_df["activation_weekday"] = train_df["activation_date"].dt.weekday
test_df["activation_weekday"] = test_df["activation_date"].dt.weekday
train_df["activation_month"] = train_df["activation_date"].dt.month
test_df["activation_month"] = test_df["activation_date"].dt.month
train_df["title_nwords"] = train_df["title"].apply(lambda x: len(x.split()))
test_df["title_nwords"] = test_df["title"].apply(lambda x: len(x.split()))
train_df["description"].fillna("NA", inplace=True)
 test_df["description"].fillna("NA", inplace=True)
 train_df["desc_nwords"] = train_df["description"].apply(lambda x: len(x.split()))
test_df["desc_nwords"] = test_df["description"].apply(lambda x: len(x.split()))
train_df['param123'] = train_df['param_1'].fillna('')+ " " + train_df['param_2'].fillna('')+ " " + train_df['param_3'].fillna('')
 test_df['param123'] = test_df['param_1'].fillna('')+ " " + test_df['param_2'].fillna('')+ " " + test_df['param_3'].fillna('')
 enc = train_df.groupby('category_name')['image_top_1'].agg(lambda x:x.value_counts().index[0] ).astype(np.float32 ).reset_index()
enc.columns = ['category_name' ,'image_top_1_impute']
train_df = pd.merge(train_df, enc, how='left', on='category_name')
 test_df = pd.merge(test_df, enc, how='left', on='category_name')
 train_df['image_top_1'].fillna(train_df['image_top_1_impute'], inplace=True)
 test_df['image_top_1'].fillna(test_df['image_top_1_impute'], inplace=True)
 enc = train_df.groupby('category_name')['days'].agg('median' ).astype(np.float32 ).reset_index()
enc.columns = ['category_name' ,'days_impute']
train_df = pd.merge(train_df, enc, how='left', on='category_name')
 test_df = pd.merge(test_df, enc, how='left', on='category_name')
 train_df['days'].fillna(train_df['days_impute'], inplace=True)
 test_df['days'].fillna(test_df['days_impute'], inplace=True)
 test_df['image'] = test_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 train_df['image'] = train_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 train_df['city'] = train_df['city'] + "_" + train_df['region']
test_df['city'] = test_df['city'] + "_" + test_df['region']
train_df['price'].fillna(0, inplace=True)
 test_df['price'].fillna(0, inplace=True)
 train_df['price'] = np.log1p(train_df['price'])
 test_df['price'] = np.log1p(test_df['price'])
 price_mean = train_df['price'].mean()
price_std = train_df['price'].std()
train_df['price'] =(train_df['price'] - price_mean)/ price_std
test_df['price'] =(test_df['price'] - price_mean)/ price_std
cat_cols = ['category_name', 'image_top_1']
num_cols = ['price', 'deal_probability']
for c in cat_cols:
for c2 in num_cols:
enc = train_df.groupby(c)[c2].agg(['median'] ).astype(np.float32 ).reset_index()
enc.columns = ['_'.join([str(c), str(c2), str(c3)])if c3 != c else c for c3 in enc.columns]
train_df = pd.merge(train_df, enc, how='left', on=c)
 test_df = pd.merge(test_df, enc, how='left', on=c )<categorify>
data.loc[(( data['age']>19)|(data['age']<7)) &(data['rez_esc'].isnull()), 'rez_esc']=0
tfidf_vec = TfidfVectorizer(ngram_range=(1,1))
full_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())
<categorify>
data['rez_esc-missing'] = data['rez_esc'].isnull()
train_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['title'].values.tolist())
 n_comp = 5
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<categorify>
data['rez_esc']=data['rez_esc'].fillna(0 )
tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)
 full_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())
 n_comp = 5
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<categorify>
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5
tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)
 full_tfidf = tfidf_vec.fit_transform(train_df['param123'].values.tolist() + test_df['param123'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['param123'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['param123'].values.tolist())
 n_comp = 5
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_params_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_params_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<categorify>
id_ = ['Id', 'idhogar', 'Target']
train = train_df
test = test_df
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))
 cols_to_drop = ["item_id", "user_id", "title", "description", "activation_date", "image", "param_2", "param_3"
, "param123", "image_top_1_impute", "days_impute"]
train_X = train_df.drop(cols_to_drop + ["deal_probability"], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1)
 train_y = train_df["deal_probability"].values
test_id = test_df["item_id"].values<train_model>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
dev_X = train_X.iloc[:-100000,:]
val_X = train_X.iloc[-100000:,:]
dev_y = train_y[:-100000]
val_y = train_y[-100000:]
print(dev_X.shape, val_X.shape, test_X.shape )<train_model>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 1000,
"learning_rate" : 0.02,
"bagging_fraction" : 0.75,
"feature_fraction" : 0.6,
"bagging_freq" : 5,
"bagging_seed" : 2018,
"verbosity" : -1,
"max_depth": 18,
"min_child_samples":100
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 2500, valid_sets=[lgval], early_stopping_rounds=50, verbose_eval=50, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 return pred_test_y, model, evals_result<train_model>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X )<save_to_csv>
heads = data.loc[data['parentesco1'] == 1, :]
heads = heads[id_ + hh_bool + hh_cont + hh_ordered]
heads.shape
pred_test[pred_test>1] = 1
pred_test[pred_test<0] = 0
sub_df = pd.DataFrame({"item_id":test_id})
 sub_df["deal_probability"] = pred_test
sub_df.to_csv("baseline_lgb.csv", index=False )<import_modules>
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
import pandas as pd
import numpy as np
import time
import gc<set_options>
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True )
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
cores = 4
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_true - y_pred)) )<data_type_conversions>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
def preprocess_dataset(dataset):
t1 = time.time()
print("Filling Missing Values.....")
 dataset['price'] = dataset['price'].fillna(0 ).astype('float32')
 dataset['param_1'].fillna(value='missing', inplace=True)
 dataset['param_2'].fillna(value='missing', inplace=True)
 dataset['param_3'].fillna(value='missing', inplace=True)
 dataset['param_1'] = dataset['param_1'].astype(str)
 dataset['param_2'] = dataset['param_2'].astype(str)
 dataset['param_3'] = dataset['param_3'].astype(str)
 print("Casting data types to type Category.......")
 dataset['category_name'] = dataset['category_name'].astype('category')
 dataset['parent_category_name'] = dataset['parent_category_name'].astype('category')
 dataset['region'] = dataset['region'].astype('category')
 dataset['city'] = dataset['city'].astype('category')
 dataset['image_top_1'] = dataset['image_top_1'].fillna('missing')
 dataset['image_code'] = dataset['image_top_1'].astype('str')
 del dataset['image_top_1']
gc.collect()
del dataset['activation_date']
gc.collect()
print("Creating New Feature.....")
 dataset['param123'] =(dataset['param_1']+'_'+dataset['param_2']+'_'+dataset['param_3'] ).astype(str)
 del dataset['param_2'], dataset['param_3']
gc.collect()
print("PreProcessing Function completed.")
 return dataset
def keras_fit(train):
t1 = time.time()
train['title_description']=(train['title']+" "+train['description'] ).astype(str)
 del train['description'], train['title']
gc.collect()
print("Start Tokenization.....")
 tokenizer = text.Tokenizer(num_words = max_words_title_description)
 all_text = np.hstack([train['title_description'].str.lower() ])
 tokenizer.fit_on_texts(all_text)
 del all_text
gc.collect()
print("Loading Test for Label Encoding on Train + Test")
 use_cols_test = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'image_top_1', 'activation_date']
test = pd.read_csv(".. /input/avito-demand-prediction/test.csv", usecols = use_cols_test)
 test['image_top_1'] = test['image_top_1'].fillna('missing')
 test['image_code'] = test['image_top_1'].astype('str')
 del test['image_top_1']
gc.collect()
del test['activation_date']
gc.collect()
test['param_1'].fillna(value='missing', inplace=True)
 test['param_1'] = test['param_1'].astype(str)
 test['param_2'].fillna(value='missing', inplace=True)
 test['param_2'] = test['param_2'].astype(str)
 test['param_3'].fillna(value='missing', inplace=True)
 test['param_3'] = test['param_3'].astype(str)
 print("Creating New Feature.....")
 test['param123'] =(test['param_1']+'_'+test['param_2']+'_'+test['param_3'] ).astype(str)
 del test['param_2'], test['param_3']
gc.collect()
ntrain = train.shape[0]
DF = pd.concat([train, test], axis = 0)
 del train, test
gc.collect()
print(DF.shape)
 print("Start Label Encoding process.... ")
 le_region = LabelEncoder()
le_region.fit(DF.region)
 le_city = LabelEncoder()
le_city.fit(DF.city)
 le_category_name = LabelEncoder()
le_category_name.fit(DF.category_name)
 le_parent_category_name = LabelEncoder()
le_parent_category_name.fit(DF.parent_category_name)
 le_param_1 = LabelEncoder()
le_param_1.fit(DF.param_1)
 le_param123 = LabelEncoder()
le_param123.fit(DF.param123)
 le_image_code = LabelEncoder()
le_image_code.fit(DF.image_code)
 train = DF[0:ntrain]
del DF
gc.collect()
train['price'] = np.log1p(train['price'])
 train['avg_days_up_user'] = np.log1p(train['avg_days_up_user'])
 train['avg_times_up_user'] = np.log1p(train['avg_times_up_user'])
 train['n_user_items'] = np.log1p(train['n_user_items'])
 train['item_seq_number'] = np.log(train['item_seq_number'])
 print("Fit on Train Function completed.")
 return train, tokenizer, le_region, le_city, le_category_name, le_parent_category_name, le_param_1, le_param123, le_image_code
def keras_train_transform(dataset):
t1 = time.time()
dataset['seq_title_description']= tokenizer.texts_to_sequences(dataset.title_description.str.lower())
 print("Transform done for test")
 print("Time taken for Sequence Tokens is"+str(time.time() -t1))
del train['title_description']
gc.collect()
dataset['region'] = le_region.transform(dataset['region'])
 dataset['city'] = le_city.transform(dataset['city'])
 dataset['category_name'] = le_category_name.transform(dataset['category_name'])
 dataset['parent_category_name'] = le_parent_category_name.transform(dataset['parent_category_name'])
 dataset['param_1'] = le_param_1.transform(dataset['param_1'])
 dataset['param123'] = le_param123.transform(dataset['param123'])
 dataset['image_code'] = le_image_code.transform(dataset['image_code'])
 print("Transform on test function completed.")
 return dataset
def keras_test_transform(dataset):
t1 = time.time()
dataset['title_description']=(dataset['title']+" "+dataset['description'] ).astype(str)
 del dataset['description'], dataset['title']
gc.collect()
dataset['seq_title_description']= tokenizer.texts_to_sequences(dataset.title_description.str.lower())
 print("Transform done for test")
 print("Time taken for Sequence Tokens is"+str(time.time() -t1))
del dataset['title_description']
gc.collect()
dataset['region'] = le_region.transform(dataset['region'])
 dataset['city'] = le_city.transform(dataset['city'])
 dataset['category_name'] = le_category_name.transform(dataset['category_name'])
 dataset['parent_category_name'] = le_parent_category_name.transform(dataset['parent_category_name'])
 dataset['param_1'] = le_param_1.transform(dataset['param_1'])
 dataset['param123'] = le_param123.transform(dataset['param123'])
 dataset['image_code'] = le_image_code.transform(dataset['image_code'])
 dataset['price'] = np.log1p(dataset['price'])
 dataset['item_seq_number'] = np.log(dataset['item_seq_number'])
 dataset['avg_days_up_user'] = np.log1p(dataset['avg_days_up_user'])
 dataset['avg_times_up_user'] = np.log1p(dataset['avg_times_up_user'])
 dataset['n_user_items'] = np.log1p(dataset['n_user_items'])
 print("Transform on test function completed.")
 return dataset
def get_keras_data(dataset):
X = {
'seq_title_description': pad_sequences(dataset.seq_title_description, maxlen=max_seq_title_description_length)
 ,'region': np.array(dataset.region)
 ,'city': np.array(dataset.city)
 ,'category_name': np.array(dataset.category_name)
 ,'parent_category_name': np.array(dataset.parent_category_name)
 ,'param_1': np.array(dataset.param_1)
 ,'param123': np.array(dataset.param123)
 ,'image_code':np.array(dataset.image_code)
 ,'avg_ad_days':np.array(dataset.avg_days_up_user)
 ,'avg_ad_times':np.array(dataset.avg_times_up_user)
 ,'n_user_items':np.array(dataset.n_user_items)
 ,'price': np.array(dataset[["price"]])
 ,'item_seq_number': np.array(dataset[["item_seq_number"]])
 }
print("Data ready for Vectorization")
 return X 
<load_from_csv>
heads['walls+roof+floor'] = heads['walls'] + heads['roof'] + heads['floor']
dtypes_train = {
'price': 'float32',
'deal probability': 'float32',
'item_seq_number': 'uint32'
}
use_cols = ['item_id', 'user_id', 'image_top_1', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'title', 'description', 'price', 'item_seq_number', 'activation_date', 'deal_probability']
train = pd.read_csv(".. /input/avito-demand-prediction/train.csv", parse_dates=["activation_date"], usecols = use_cols, dtype = dtypes_train)
 train_features = pd.read_csv('.. /input/avito-additional-features/aggregated_features.csv')
 train = train.merge(train_features, on = ['user_id'], how = 'left')
 del train_features
gc.collect()
train['avg_days_up_user'] = train['avg_days_up_user'].fillna(0 ).astype('uint32')
 train['avg_times_up_user'] = train['avg_times_up_user'].fillna(0 ).astype('uint32')
 train['n_user_items'] = train['n_user_items'].fillna(0 ).astype('uint32')
 y_train = np.array(train['deal_probability'])
 del train['deal_probability']
gc.collect()
max_seq_title_description_length = 100
max_words_title_description = 200000
train = preprocess_dataset(train)
 train, tokenizer, le_region, le_city, le_category_name, le_parent_category_name, le_param_1, le_param123, le_image_code = keras_fit(train)
 train = keras_train_transform(train)
 print("Tokenization done and TRAIN READY FOR Validation splitting")
 max_region = np.max(train.region.max())+2
max_city= np.max(train.city.max())+2
max_category_name = np.max(train.category_name.max())+2
max_parent_category_name = np.max(train.parent_category_name.max())+2
max_param_1 = np.max(train.param_1.max())+2
max_param123 = np.max(train.param123.max())+2
max_image_code = np.max(train.image_code.max())+2
del train['item_id'], train['user_id']
gc.collect() 
<prepare_x_and_y>
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))
EMBEDDING_DIM1 = 300
EMBEDDING_FILE1 = '.. /input/fasttest-common-crawl-russian/cc.ru.300.vec'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index1 = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE1))
vocab_size = len(tokenizer.word_index)+2
EMBEDDING_DIM1 = 300
embedding_matrix1 = np.zeros(( vocab_size, EMBEDDING_DIM1))
print(embedding_matrix1.shape)
 c = 0
c1 = 0
w_Y = []
w_No = []
for word, i in tokenizer.word_index.items() :
if word in embeddings_index1:
c +=1
embedding_vector = embeddings_index1[word]
w_Y.append(word)
 else:
embedding_vector = None
w_No.append(word)
 c1 +=1
if embedding_vector is not None:
embedding_matrix1[i] = embedding_vector
print(c,c1, len(w_No), len(w_Y))
print(embedding_matrix1.shape)
 del embeddings_index1
gc.collect()
print(" FAST TEXT DONE")
<define_variables>
heads['bonus'] = 1 *(heads['refrig'] +
heads['computer'] + 
(heads['v18q1'] > 0)+
heads['television'])
def RNN_model() :
seq_title_description = Input(shape=[100], name="seq_title_description")
 region = Input(shape=[1], name="region")
 city = Input(shape=[1], name="city")
 category_name = Input(shape=[1], name="category_name")
 parent_category_name = Input(shape=[1], name="parent_category_name")
 param_1 = Input(shape=[1], name="param_1")
 param123 = Input(shape=[1], name="param123")
 image_code = Input(shape=[1], name="image_code")
 price = Input(shape=[1], name="price")
 item_seq_number = Input(shape = [1], name = 'item_seq_number')
 avg_ad_days = Input(shape=[1], name="avg_ad_days")
 avg_ad_times = Input(shape=[1], name="avg_ad_times")
 n_user_items = Input(shape=[1], name="n_user_items")
 emb_seq_title_description = Embedding(vocab_size, EMBEDDING_DIM1, weights = [embedding_matrix1], trainable = False )(seq_title_description)
 emb_region = Embedding(vocab_size, 10 )(region)
 emb_city = Embedding(vocab_size, 10 )(city)
 emb_category_name = Embedding(vocab_size, 10 )(category_name)
 emb_parent_category_name = Embedding(vocab_size, 10 )(parent_category_name)
 emb_param_1 = Embedding(vocab_size, 10 )(param_1)
 emb_param123 = Embedding(vocab_size, 10 )(param123)
 emb_image_code = Embedding(vocab_size, 10 )(image_code)
 rnn_layer1 = GRU(50 )(emb_seq_title_description)
 main_l = concatenate([
rnn_layer1
, Flatten()(emb_region)
 , Flatten()(emb_city)
 , Flatten()(emb_category_name)
 , Flatten()(emb_parent_category_name)
 , Flatten()(emb_param_1)
 , Flatten()(emb_param123)
 , Flatten()(emb_image_code)
 , avg_ad_days
, avg_ad_times
, n_user_items
, price
, item_seq_number
])
 main_l = Dropout(0.1 )(Dense(512,activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(64,activation='relu' )(main_l))
output = Dense(1,activation="sigmoid" )(main_l)
 model = Model([seq_title_description, region, city, category_name, parent_category_name, param_1, param123, price, item_seq_number, image_code, avg_ad_days, avg_ad_times, n_user_items], output)
 model.compile(optimizer = 'adam',
loss= root_mean_squared_error,
metrics = [root_mean_squared_error])
 return model
def rmse(y, y_pred):
Rsum = np.sum(( y - y_pred)**2)
 n = y.shape[0]
RMSE = np.sqrt(Rsum/n)
 return RMSE
def eval_model(model, X_test1):
val_preds = model.predict(X_test1)
 y_pred = val_preds[:, 0]
y_true = np.array(y_test1)
 yt = pd.DataFrame(y_true)
 yp = pd.DataFrame(y_pred)
 print(yt.isnull().any())
 print(yp.isnull().any())
 v_rmse = rmse(y_true, y_pred)
 print(" RMSE for VALIDATION SET: "+str(v_rmse))
return v_rmse
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1<load_from_csv>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
def predictions(model):
t1 = time.time()
def load_test() :
for df in pd.read_csv('.. /input/avito-demand-prediction/test.csv', chunksize= 250000):
yield df
item_ids = np.array([], dtype=np.int32)
 preds= np.array([], dtype=np.float32)
 i = 0
for df in load_test() :
i +=1
print(df.dtypes)
 item_id = df['item_id']
print(" Chunk number is "+str(i))
test = preprocess_dataset(df)
 train_features = pd.read_csv('.. /input/avito-additional-features/aggregated_features.csv')
 test = test.merge(train_features, on = ['user_id'], how = 'left')
 del train_features
gc.collect()
print(test.dtypes)
 test['avg_days_up_user'] = test['avg_days_up_user'].fillna(0 ).astype('uint32')
 test['avg_times_up_user'] = test['avg_times_up_user'].fillna(0 ).astype('uint32')
 test['n_user_items'] = test['n_user_items'].fillna(0 ).astype('uint32')
 test = keras_test_transform(test)
 del df
gc.collect()
print(test.dtypes)
 X_test = get_keras_data(test)
 del test
gc.collect()
Batch_Size = 512*3
preds1 = modelRNN.predict(X_test, batch_size = Batch_Size, verbose = 1)
 print(preds1.shape)
 del X_test
gc.collect()
print("RNN Prediction is done")
 preds1 = preds1.reshape(-1,1)
 preds1 = np.clip(preds1, 0, 1)
 print(preds1.shape)
 item_ids = np.append(item_ids, item_id)
 print(item_ids.shape)
 preds = np.append(preds, preds1)
 print(preds.shape)
 print("All chunks done")
 t2 = time.time()
print("Total time for Parallel Batch Prediction is "+str(t2-t1))
return preds<feature_engineering>
heads.loc[(heads.v14a == 1)&(heads.sanitario1 == 1)&(heads.abastaguano == 0), "v14a"] = 0
heads.loc[(heads.v14a == 1)&(heads.sanitario1 == 1)&(heads.abastaguano == 0), "sanitario1"] = 0
train1 = np.array(train.values)
 del train
gc.collect()
def get_data_frame(dataset):
DF = pd.DataFrame()
DF['avg_days_up_user'] = np.array(dataset[:,0])
 DF['avg_times_up_user'] = np.array(dataset[:,1])
 DF['category_name'] = np.array(dataset[:,2])
 DF['city'] = np.array(dataset[:,3])
 DF['image_code'] = np.array(dataset[:,4])
 DF['item_seq_number'] = np.array(dataset[:,5])
 DF['n_user_items'] = np.array(dataset[:,6])
 DF['param123'] = np.array(dataset[:,7])
 DF['param_1'] = np.array(dataset[:,8])
 DF['parent_category_name'] = np.array(dataset[:,9])
 DF['price'] = np.array(dataset[:,10])
 DF['region'] = np.array(dataset[:,11])
 DF['seq_title_description'] = np.array(dataset[:,12])
 return DF<split>
range_ = lambda x: x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()
skf = KFold(n_splits = 3)
 Kfold_preds_final = []
k = 0
RMSE = []
for train_idx, test_idx in skf.split(train1, y_train):
print("Number of Folds.. "+str(k+1))
epochs = 1
batch_size = 512 * 3
steps =(int(train1.shape[0]/batch_size)) *epochs
lr_init, lr_fin = 0.009, 0.0045
lr_decay = exp_decay(lr_init, lr_fin, steps)
 modelRNN = RNN_model()
K.set_value(modelRNN.optimizer.lr, lr_init)
 K.set_value(modelRNN.optimizer.decay, lr_decay)
 X_train1, X_test1 = train1[train_idx], train1[test_idx]
print(X_train1.shape, X_test1.shape)
 y_train1, y_test1 = y_train[train_idx], y_train[test_idx]
print(y_train1.shape, y_test1.shape)
 gc.collect()
print(type(X_train1))
print(X_train1.shape)
 print(type(X_train1[:,12]))
X_train_final = get_data_frame(X_train1)
 X_test_final = get_data_frame(X_test1)
 del X_train1, X_test1
gc.collect()
X_train_f = get_keras_data(X_train_final)
 X_test_f = get_keras_data(X_test_final)
 del X_train_final, X_test_final
gc.collect()
for i in range(3):
hist = modelRNN.fit(X_train_f, y_train1, batch_size=batch_size+(batch_size*(2*i)) , epochs=epochs, validation_data=(X_test_f, y_test1), verbose=1)
 del X_train_f
gc.collect()
v_rmse = eval_model(modelRNN, X_test_f)
 RMSE.append(v_rmse)
 del X_test_f
del y_train1, y_test1
gc.collect()
preds = predictions(modelRNN)
 del modelRNN
gc.collect()
print("Predictions done for Fold "+str(k))
print(preds.shape)
 Kfold_preds_final.append(preds)
 del preds
gc.collect()
print("Number of folds completed.... "+str(len(Kfold_preds_final)))
 print(Kfold_preds_final[k][0:10])
 print("All Folds completed"+str(k+1))
print("RNN FOLD MODEL Done" )<normalization>
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()
pred_final1 = np.average(Kfold_preds_final, axis =0)
 print(pred_final1.shape)
 min_value = min(RMSE)
 RMSE_idx = RMSE.index(min_value)
 print(RMSE_idx)
 pred_final2 = Kfold_preds_final[RMSE_idx]
print(pred_final2.shape)
 gc.collect()<save_to_csv>
ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )
test_cols = ['item_id']
test = pd.read_csv('.. /input/avito-demand-prediction/test.csv', usecols = test_cols)
 submission1 = pd.DataFrame(columns = ['item_id', 'deal_probability'])
 submission1['item_id'] = test['item_id']
submission1['deal_probability'] = pred_final1
print("Check Submission NOW!!!!!!!!@")
 submission1.to_csv("Avito_Shanth_RNN_AVERAGE.csv", index=False)
<save_to_csv>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
submission2 = pd.DataFrame(columns = ['item_id', 'deal_probability'])
 submission2['item_id'] = test['item_id']
submission2['deal_probability'] = pred_final2
print("Check Submission NOW!!!!!!!!@")
 submission2.to_csv("Avito_Shanth_RNN_MIN.csv", index=False )<load_from_csv>
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()
print(os.listdir(".. /input"))
df = pd.read_csv(".. /input/avito-demand-prediction/train.csv")
 test = pd.read_csv(".. /input/avito-demand-prediction/test.csv")
 df_sorted = df.sort_values(by = ["image"])
 test_sorted = test.sort_values(by = ["image"] )<load_from_csv>
train_set['adult'] = train_set['hogar_adul'] - train_set['hogar_mayor']
train_set['dependency_count'] = train_set['hogar_nin'] + train_set['hogar_mayor']
train_set['dependency'] = train_set['dependency_count'] / train_set['adult']
train_set['child_percent'] = train_set['hogar_nin']/train_set['hogar_total']
train_set['elder_percent'] = train_set['hogar_mayor']/train_set['hogar_total']
train_set['adult_percent'] = train_set['hogar_adul']/train_set['hogar_total']
test_set['adult'] = test_set['hogar_adul'] - test_set['hogar_mayor']
test_set['dependency_count'] = test_set['hogar_nin'] + test_set['hogar_mayor']
test_set['dependency'] = test_set['dependency_count'] / test_set['adult']
test_set['child_percent'] = test_set['hogar_nin']/test_set['hogar_total']
test_set['elder_percent'] = test_set['hogar_mayor']/test_set['hogar_total']
test_set['adult_percent'] = test_set['hogar_adul']/test_set['hogar_total']
train_set['rent_per_adult'] = train_set['v2a1']/train_set['hogar_adul']
train_set['rent_per_person'] = train_set['v2a1']/train_set['hhsize']
test_set['rent_per_adult'] = test_set['v2a1']/test_set['hogar_adul']
test_set['rent_per_person'] = test_set['v2a1']/test_set['hhsize']
train_set['overcrowding_room_and_bedroom'] =(train_set['hacdor'] + train_set['hacapo'])/2
test_set['overcrowding_room_and_bedroom'] =(test_set['hacdor'] + test_set['hacapo'])/2
tr_1 = pd.read_csv(".. /input/train-data-image-blurrness/1.csv")
 tr_2 = pd.read_csv(".. /input/train-data-image-blurrness/2.csv")
 tr_3 = pd.read_csv(".. /input/train-data-image-blurrness/3.csv")
 tr_4 = pd.read_csv(".. /input/train-data-image-blurrness/4.csv")
 tr_5 = pd.read_csv(".. /input/train-data-image-blurrness/5.csv")
 tr_6 = pd.read_csv(".. /input/train-data-image-blurrness/6.csv")
 tr_7 = pd.read_csv(".. /input/train-data-image-blurrness/7.csv")
 tr_8 = pd.read_csv(".. /input/train-data-image-blurrness/8.csv")
 tr_9 = pd.read_csv(".. /input/train-data-image-blurrness/9.csv")
 tr_10 = pd.read_csv(".. /input/train-data-image-blurrness/10.csv")
 tr_11 = pd.read_csv(".. /input/train-data-image-blurrness/11(12_13.5 ).csv")
 tr_12 = pd.read_csv(".. /input/train-data-image-blurrness/last.csv")
 frames = [tr_1, tr_2, tr_3, tr_4, tr_5, tr_6, tr_7, tr_8, tr_9, tr_10, tr_11, tr_12]
new = pd.concat(frames)
 new["File"] = new["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new = new.sort_values(by = ["File"])
 scores = list(new["Score"].values)+ [-1] *(len(df)-len(new))
df_sorted["image_blurrness_score"] = scores
df = df_sorted.sort_index()<load_from_csv>
train_set['r4h1_percent_in_male'] = train_set['r4h1'] / train_set['r4h3']
train_set['r4m1_percent_in_female'] = train_set['r4m1'] / train_set['r4m3']
train_set['r4h1_percent_in_total'] = train_set['r4h1'] / train_set['hhsize']
train_set['r4m1_percent_in_total'] = train_set['r4m1'] / train_set['hhsize']
train_set['r4t1_percent_in_total'] = train_set['r4t1'] / train_set['hhsize']
test_set['r4h1_percent_in_male'] = test_set['r4h1'] / test_set['r4h3']
test_set['r4m1_percent_in_female'] = test_set['r4m1'] / test_set['r4m3']
test_set['r4h1_percent_in_total'] = test_set['r4h1'] / test_set['hhsize']
test_set['r4m1_percent_in_total'] = test_set['r4m1'] / test_set['hhsize']
test_set['r4t1_percent_in_total'] = test_set['r4t1'] / test_set['hhsize']
te_1 = pd.read_csv(".. /input/image-blurrness-test/test_1.csv")
 te_2 = pd.read_csv(".. /input/image-blurrness-test/test_2.csv")
 te_3 = pd.read_csv(".. /input/image-blurrness-test/test_3.csv")
 te_4 = pd.read_csv(".. /input/image-blurrness-test/test_4.csv")
 te_5 = pd.read_csv(".. /input/image-blurrness-test/test_5.csv")
 frames_te = [te_1, te_2, te_3, te_4, te_5]
new_te = pd.concat(frames_te)
 new_te["File"] = new_te["File"].apply(lambda x : x.split("/")[-1].split(".")[0])
 new_te = new_te.sort_values(by = ["File"])
 scores_te = list(new_te["Score"].values)+ [-1] *(len(test)-len(new_te))
test_sorted["image_blurrness_score"] = scores_te
test = test_sorted.sort_index()<drop_column>
train_set['rent_per_room'] = train_set['v2a1']/train_set['rooms']
train_set['bedroom_per_room'] = train_set['bedrooms']/train_set['rooms']
train_set['elder_per_room'] = train_set['hogar_mayor']/train_set['rooms']
train_set['adults_per_room'] = train_set['adult']/train_set['rooms']
train_set['child_per_room'] = train_set['hogar_nin']/train_set['rooms']
train_set['male_per_room'] = train_set['r4h3']/train_set['rooms']
train_set['female_per_room'] = train_set['r4m3']/train_set['rooms']
train_set['room_per_person_household'] = train_set['hhsize']/train_set['rooms']
test_set['rent_per_room'] = test_set['v2a1']/test_set['rooms']
test_set['bedroom_per_room'] = test_set['bedrooms']/test_set['rooms']
test_set['elder_per_room'] = test_set['hogar_mayor']/test_set['rooms']
test_set['adults_per_room'] = test_set['adult']/test_set['rooms']
test_set['child_per_room'] = test_set['hogar_nin']/test_set['rooms']
test_set['male_per_room'] = test_set['r4h3']/test_set['rooms']
test_set['female_per_room'] = test_set['r4m3']/test_set['rooms']
test_set['room_per_person_household'] = test_set['hhsize']/test_set['rooms']
train_set['rent_per_bedroom'] = train_set['v2a1']/train_set['bedrooms']
train_set['edler_per_bedroom'] = train_set['hogar_mayor']/train_set['bedrooms']
train_set['adults_per_bedroom'] = train_set['adult']/train_set['bedrooms']
train_set['child_per_bedroom'] = train_set['hogar_nin']/train_set['bedrooms']
train_set['male_per_bedroom'] = train_set['r4h3']/train_set['bedrooms']
train_set['female_per_bedroom'] = train_set['r4m3']/train_set['bedrooms']
train_set['bedrooms_per_person_household'] = train_set['hhsize']/train_set['bedrooms']
test_set['rent_per_bedroom'] = test_set['v2a1']/test_set['bedrooms']
test_set['edler_per_bedroom'] = test_set['hogar_mayor']/test_set['bedrooms']
test_set['adults_per_bedroom'] = test_set['adult']/test_set['bedrooms']
test_set['child_per_bedroom'] = test_set['hogar_nin']/test_set['bedrooms']
test_set['male_per_bedroom'] = test_set['r4h3']/test_set['bedrooms']
test_set['female_per_bedroom'] = test_set['r4m3']/test_set['bedrooms']
test_set['bedrooms_per_person_household'] = test_set['hhsize']/test_set['bedrooms']
train_set['tablet_per_person_household'] = train_set['v18q1']/train_set['hhsize']
train_set['phone_per_person_household'] = train_set['qmobilephone']/train_set['hhsize']
test_set['tablet_per_person_household'] = test_set['v18q1']/test_set['hhsize']
test_set['phone_per_person_household'] = test_set['qmobilephone']/test_set['hhsize']
train_set['age_12_19'] = train_set['hogar_nin'] - train_set['r4t1']
test_set['age_12_19'] = test_set['hogar_nin'] - test_set['r4t1']
df_train= df
df_test = test<define_variables>
train_set['num_over_18'] = 0
train_set['num_over_18'] = train_set[train.age >= 18].groupby('idhogar' ).transform("count")
 train_set['num_over_18'] = train_set.groupby("idhogar")["num_over_18"].transform("max")
 train_set['num_over_18'] = train_set['num_over_18'].fillna(0)
 test_set['num_over_18'] = 0
test_set['num_over_18'] = test_set[test.age >= 18].groupby('idhogar' ).transform("count")
 test_set['num_over_18'] = test_set.groupby("idhogar")["num_over_18"].transform("max")
 test_set['num_over_18'] = test_set['num_over_18'].fillna(0 )
parent_category_name_map = {"Личные вещи" : "Personal belongings",
"Для дома и дачи" : "For the home and garden",
"Бытовая электроника" : "Consumer electronics",
"Недвижимость" : "Real estate",
"Хобби и отдых" : "Hobbies & leisure",
"Транспорт" : "Transport",
"Услуги" : "Services",
"Животные" : "Animals",
"Для бизнеса" : "For business"} 
<define_variables>
train_set=train_set.drop(columns='idhogar')
 test_set=test_set.drop(columns='idhogar' )
region_map = {"Свердловская область" : "Sverdlovsk oblast",
"Самарская область" : "Samara oblast",
"Ростовская область" : "Rostov oblast",
"Татарстан" : "Tatarstan",
"Волгоградская область" : "Volgograd oblast",
"Нижегородская область" : "Nizhny Novgorod oblast",
"Пермский край" : "Perm Krai",
"Оренбургская область" : "Orenburg oblast",
"Ханты-Мансийский АО" : "Khanty-Mansi Autonomous Okrug",
"Тюменская область" : "Tyumen oblast",
"Башкортостан" : "Bashkortostan",
"Краснодарский край" : "Krasnodar Krai",
"Новосибирская область" : "Novosibirsk oblast",
"Омская область" : "Omsk oblast",
"Белгородская область" : "Belgorod oblast",
"Челябинская область" : "Chelyabinsk oblast",
"Воронежская область" : "Voronezh oblast",
"Кемеровская область" : "Kemerovo oblast",
"Саратовская область" : "Saratov oblast",
"Владимирская область" : "Vladimir oblast",
"Калининградская область" : "Kaliningrad oblast",
"Красноярский край" : "Krasnoyarsk Krai",
"Ярославская область" : "Yaroslavl oblast",
"Удмуртия" : "Udmurtia",
"Алтайский край" : "Altai Krai",
"Иркутская область" : "Irkutsk oblast",
"Ставропольский край" : "Stavropol Krai",
"Тульская область" : "Tula oblast"} 
<define_variables>
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v14a']
train_set = train_set.drop(needless_cols, axis=1)
 test_set = test_set.drop(needless_cols, axis=1 )
category_map = {"Одежда, обувь, аксессуары":"Clothing, shoes, accessories",
"Детская одежда и обувь":"Children's clothing and shoes",
"Товары для детей и игрушки":"Children's products and toys",
"Квартиры":"Apartments",
"Телефоны":"Phones",
"Мебель и интерьер":"Furniture and interior",
"Предложение услуг":"Offer services",
"Автомобили":"Cars",
"Ремонт и строительство":"Repair and construction",
"Бытовая техника":"Appliances",
"Товары для компьютера":"Products for computer",
"Дома, дачи, коттеджи":"Houses, villas, cottages",
"Красота и здоровье":"Health and beauty",
"Аудио и видео":"Audio and video",
"Спорт и отдых":"Sports and recreation",
"Коллекционирование":"Collecting",
"Оборудование для бизнеса":"Equipment for business",
"Земельные участки":"Land",
"Часы и украшения":"Watches and jewelry",
"Книги и журналы":"Books and magazines",
"Собаки":"Dogs",
"Игры, приставки и программы":"Games, consoles and software",
"Другие животные":"Other animals",
"Велосипеды":"Bikes",
"Ноутбуки":"Laptops",
"Кошки":"Cats",
"Грузовики и спецтехника":"Trucks and buses",
"Посуда и товары для кухни":"Tableware and goods for kitchen",
"Растения":"Plants",
"Планшеты и электронные книги":"Tablets and e-books",
"Товары для животных":"Pet products",
"Комнаты":"Room",
"Фототехника":"Photo",
"Коммерческая недвижимость":"Commercial property",
"Гаражи и машиноместа":"Garages and Parking spaces",
"Музыкальные инструменты":"Musical instruments",
"Оргтехника и расходники":"Office equipment and consumables",
"Птицы":"Birds",
"Продукты питания":"Food",
"Мотоциклы и мототехника":"Motorcycles and bikes",
"Настольные компьютеры":"Desktop computers",
"Аквариум":"Aquarium",
"Охота и рыбалка":"Hunting and fishing",
"Билеты и путешествия":"Tickets and travel",
"Водный транспорт":"Water transport",
"Готовый бизнес":"Ready business",
"Недвижимость за рубежом":"Property abroad"} 
<define_variables>
train_set= train_set.fillna(0)
 test_set= test_set.fillna(0 )
params_top35_map = {'Женская одежда':"Women's clothing",
'Для девочек':'For girls',
'Для мальчиков':'For boys',
'Продам':'Selling',
'С пробегом':'With mileage',
'Аксессуары':'Accessories',
'Мужская одежда':"Men's Clothing",
'Другое':'Other','Игрушки':'Toys',
'Детские коляски':'Baby carriages',
'Сдам':'Rent',
'Ремонт, строительство':'Repair, construction',
'Стройматериалы':'Building materials',
'iPhone':'iPhone',
'Кровати, диваны и кресла':'Beds, sofas and armchairs',
'Инструменты':'Instruments',
'Для кухни':'For kitchen',
'Комплектующие':'Accessories',
'Детская мебель':"Children's furniture",
'Шкафы и комоды':'Cabinets and chests of drawers',
'Приборы и аксессуары':'Devices and accessories',
'Для дома':'For home',
'Транспорт, перевозки':'Transport, transportation',
'Товары для кормления':'Feeding products',
'Samsung':'Samsung',
'Сниму':'Hire',
'Книги':'Books',
'Телевизоры и проекторы':'Televisions and projectors',
'Велосипеды и самокаты':'Bicycles and scooters',
'Предметы интерьера, искусство':'Interior items, art',
'Другая':'Other','Косметика':'Cosmetics',
'Постельные принадлежности':'Bed dress',
'С/х животные' :'Farm animals','Столы и стулья':'Tables and chairs'} 
<feature_engineering>
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'] )
df_train['region_en'] = df_train['region'].apply(lambda x : region_map[x])
 df_train['parent_category_name_en'] = df_train['parent_category_name'].apply(lambda x : parent_category_name_map[x])
 df_train['category_name_en'] = df_train['category_name'].apply(lambda x : category_map[x])
 del df_train['region']
del df_train['parent_category_name']
del df_train['category_name']
df_train
df_test['region_en'] = df_test['region'].apply(lambda x : region_map[x])
 df_test['parent_category_name_en'] = df_test['parent_category_name'].apply(lambda x : parent_category_name_map[x])
 df_test['category_name_en'] = df_test['category_name'].apply(lambda x : category_map[x])
 del df_test['region']
del df_test['parent_category_name']
del df_test['category_name']
df_test
df_train<drop_column>
train_set = pd.DataFrame(train_set, columns = features)
 corr_matrix = train_set.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 1)]
to_drop
new_df_train = df_train.copy()
new_df_test = df_test.copy()
del new_df_test["image"]
del new_df_train["image"]
del new_df_test["activation_date"]
del new_df_train["activation_date"] 
<categorify>
test_set = pd.DataFrame(test_set, columns = features)
 train_set, test_set = train_set.align(test_set, axis = 1, join = 'inner')
 features = list(train_set.columns )
stopWords_en = stopwords.words('english')
 stopWords_ru = stopwords.words('russian')
 tr =new_df_train.copy()
te =new_df_test.copy()
tri=tr.shape[0]
y = tr.deal_probability.copy()
lb=LabelEncoder()
List_Var=['item_id', 'description']
def Concat_Text(df,Columns,Name):
df=df.copy()
df.loc[:,Columns].fillna(" ",inplace=True)
 df[Name]=df[Columns[0]].astype('str')
 for col in Columns[1:]:
df[Name]=df[Name]+' '+df[col].astype('str')
 return df
def Ratio_Words(df):
df=df.copy()
df['description']=df['description'].astype('str')
 df['num_words_description']=df['description'].apply(lambda x:len(x.split()))
Unique_Words=df['description'].apply(lambda x: len(set(x.split())))
 df['Ratio_Words_description']=Unique_Words/df['num_words_description']
return df
def Lenght_Columns(df,Columns):
df=df.copy()
Columns_Len=['len_'+s for s in Columns]
for col in Columns:
df[col]=df[col].astype('str')
 for x,y in zip(Columns,Columns_Len):
df[y]=df[x].apply(len)
 return df
tr_te=tr[tr.columns.difference(["deal_probability"])].append(te)\ 
.pipe(Concat_Text,['city','param_1'],'txt1')\ 
.pipe(Concat_Text,['title','description'],'txt2' ).pipe(Ratio_Words ).pipe(Lenght_Columns,['title','description','param_1'] ).assign(category_name_en=lambda x: pd.Categorical(x['category_name_en'] ).codes,
parent_category_name_en=lambda x:pd.Categorical(x['parent_category_name_en'] ).codes,
region_en=lambda x:pd.Categorical(x['region_en'] ).codes,
user_type=lambda x:pd.Categorical(x['user_type'] ).codes, image_top_1=lambda x:pd.Categorical(x['image_top_1'] ).codes,
param_1=lambda x:lb.fit_transform(x['param_1'].fillna('-1' ).astype('str')) ,
param_2=lambda x:lb.fit_transform(x['param_2'].fillna('-1' ).astype('str')) , param_3=lambda x:lb.fit_transform(x['param_3'].fillna('-1' ).astype('str')) ,
user_id=lambda x:lb.fit_transform(x['user_id'].astype('str')) ,
city=lambda x:lb.fit_transform(x['city'].astype('str')) ,
price=lambda x: np.log1p(x['price'].fillna(0)) ,
image_blurrness_score=lambda x: np.log1p(x['image_blurrness_score'].fillna(0)) ,
title=lambda x: x['title'].astype('str')).drop(labels=List_Var,axis=1)
 tr_te.price.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te.image_blurrness_score.replace(to_replace=[np.inf, -np.inf,np.nan], value=-1,inplace=True)
 tr_te
del tr,te
gc.collect()
tr_te.loc[:,'txt2']=tr_te.txt2.apply(lambda x:x.lower().replace("[^[:alpha:]]"," " ).replace("\\s+", " "))
print("Processing Text")
 print("Text 1")
 vec1=CountVectorizer(ngram_range=(1,2),dtype=np.uint8,min_df=5, binary=True,max_features=3000)
 m_tfidf1=vec1.fit_transform(tr_te.txt1)
 tr_te.drop(labels=['txt1'],inplace=True,axis=1)
 print("Text 2")
 vec2=TfidfVectorizer(ngram_range=(1,2),stop_words=stopWords_ru,min_df=3,max_df=0.4,sublinear_tf=True,norm='l2',max_features=5500,dtype=np.uint8)
 m_tfidf2=vec2.fit_transform(tr_te.txt2)
 tr_te.drop(labels=['txt2'],inplace=True,axis=1)
 print("Title")
 vec3=CountVectorizer(ngram_range=(3,6),analyzer='char_wb',dtype=np.uint8,min_df=5, binary=True,max_features=2000)
 m_tfidf3=vec3.fit_transform(tr_te.title)
 tr_te.drop(labels=['title'],inplace=True,axis=1)
 data = hstack(( tr_te.values,m_tfidf1,m_tfidf2,m_tfidf3)).tocsr()
print(data.shape)
 del tr_te,m_tfidf1,m_tfidf2,m_tfidf3
gc.collect()
dtest=data[tri:]
X=data[:tri]
del data
gc.collect()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.15, random_state=23)
<create_dataframe>
estimator = RandomForestClassifier(random_state = 10, n_estimators = 100, n_jobs = -1)
 selector = RFECV(estimator, step = 1, cv = 3, scoring= scorer, n_jobs = -1 )
dtrain =lgb.Dataset(data = X_train, label = y_train)
 dval =lgb.Dataset(data = X_valid, label = y_valid)
 Dparam = {'objective' : 'regression',
'boosting_type': 'gbdt',
'metric' : 'rmse',
'nthread' : 4,
'shrinkage_rate':0.03,
'max_depth':18,
'min_child_weight': 8,
'bagging_fraction':0.75,
'feature_fraction':0.75,
'lambda_l1':0,
'lambda_l2':0,
'num_leaves':31,
'verbosity' : -1}
print("Training Model")
 m_lgb=lgb.train(params=Dparam,train_set=dtrain,num_boost_round=30000, early_stopping_rounds=50, valid_sets=[dtrain,dval], verbose_eval=50,valid_names=['train','valid'])
 fig, ax = plt.subplots(figsize=(10, 14))
lgb.plot_importance(m_lgb, max_num_features=50, ax=ax)
 plt.title("Light GBM Feature Importance")
 Pred=m_lgb.predict(dtest)
 Pred[Pred<0.0005]=0
Pred[Pred>1]=1
print("Output Model")
 LGB_text=pd.read_csv(".. /input/avito-demand-prediction/sample_submission.csv")
 LGB_text['deal_probability']=Pred
LGB_text.to_csv("Blurrness_nans_0.03_15000.csv", index=False )<import_modules>
selector.fit(train_set, train_labels )
import pandas as pd
import numpy as np
import time
import gc<set_options>
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
cores = 4
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_true - y_pred)) )<data_type_conversions>
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected, columns = selected_features)
 test_selected = pd.DataFrame(test_selected, columns = selected_features )
def preprocess_dataset(dataset):
t1 = time.time()
print("Filling Missing Values.....")
 dataset['price'] = dataset['price'].fillna(0 ).astype('float32')
 dataset['param_1'].fillna(value='missing', inplace=True)
 dataset['param_2'].fillna(value='missing', inplace=True)
 dataset['param_3'].fillna(value='missing', inplace=True)
 dataset['param_1'] = dataset['param_1'].astype(str)
 dataset['param_2'] = dataset['param_2'].astype(str)
 dataset['param_3'] = dataset['param_3'].astype(str)
 print("Casting data types to type Category.......")
 dataset['category_name'] = dataset['category_name'].astype('category')
 dataset['parent_category_name'] = dataset['parent_category_name'].astype('category')
 dataset['region'] = dataset['region'].astype('category')
 dataset['city'] = dataset['city'].astype('category')
 print("Creating New Feature.....")
 dataset['param123'] =(dataset['param_1']+'_'+dataset['param_2']+'_'+dataset['param_3'] ).astype(str)
 del dataset['param_2'], dataset['param_3']
gc.collect()
print("PreProcessing Function completed.")
 return dataset
def keras_fit(train):
t1 = time.time()
train['title_description']=(train['title']+" "+train['description'] ).astype(str)
 del train['description'], train['title']
gc.collect()
print("Start Tokenization.....")
 tokenizer = text.Tokenizer(num_words = max_words_title_description)
 all_text = np.hstack([train['title_description'].str.lower() ])
 tokenizer.fit_on_texts(all_text)
 del all_text
del train['activation_date']
gc.collect()
print("Loading Test for Label Encoding on Train + Test")
 use_cols_test = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3']
test = pd.read_csv(".. /input/avito-demand-prediction/test.csv", usecols = use_cols_test)
 test['param_1'].fillna(value='missing', inplace=True)
 test['param_1'] = test['param_1'].astype(str)
 test['param_2'].fillna(value='missing', inplace=True)
 test['param_2'] = test['param_2'].astype(str)
 test['param_3'].fillna(value='missing', inplace=True)
 test['param_3'] = test['param_3'].astype(str)
 print("Creating New Feature.....")
 test['param123'] =(test['param_1']+'_'+test['param_2']+'_'+test['param_3'] ).astype(str)
 del test['param_2'], test['param_3']
gc.collect()
ntrain = train.shape[0]
DF = pd.concat([train, test], axis = 0)
 del train, test
gc.collect()
print(DF.shape)
 print("Start Label Encoding process.... ")
 le_region = LabelEncoder()
le_region.fit(DF.region)
 le_city = LabelEncoder()
le_city.fit(DF.city)
 le_category_name = LabelEncoder()
le_category_name.fit(DF.category_name)
 le_parent_category_name = LabelEncoder()
le_parent_category_name.fit(DF.parent_category_name)
 le_param_1 = LabelEncoder()
le_param_1.fit(DF.param_1)
 le_param123 = LabelEncoder()
le_param123.fit(DF.param123)
 train = DF[0:ntrain]
test = DF[ntrain:]
del DF
gc.collect()
train['price'] = np.log1p(train['price'])
 train['item_seq_number'] = np.log(train['item_seq_number'])
 print("Fit on Train Function completed.")
 return train, tokenizer, le_region, le_city, le_category_name, le_parent_category_name, le_param_1, le_param123
def keras_train_transform(dataset):
t1 = time.time()
dataset['seq_title_description']= tokenizer.texts_to_sequences(dataset.title_description.str.lower())
 print("Transform done for test")
 print("Time taken for Sequence Tokens is"+str(time.time() -t1))
del train['title_description']
gc.collect()
dataset['region'] = le_region.transform(dataset['region'])
 dataset['city'] = le_city.transform(dataset['city'])
 dataset['category_name'] = le_category_name.transform(dataset['category_name'])
 dataset['parent_category_name'] = le_parent_category_name.transform(dataset['parent_category_name'])
 dataset['param_1'] = le_param_1.transform(dataset['param_1'])
 dataset['param123'] = le_param123.transform(dataset['param123'])
 print("Transform on test function completed.")
 return dataset
def keras_test_transform(dataset):
t1 = time.time()
dataset['title_description']=(dataset['title']+" "+dataset['description'] ).astype(str)
 del dataset['description'], dataset['title']
gc.collect()
dataset['seq_title_description']= tokenizer.texts_to_sequences(dataset.title_description.str.lower())
 print("Transform done for test")
 print("Time taken for Sequence Tokens is"+str(time.time() -t1))
del dataset['activation_date'], dataset['title_description']
gc.collect()
dataset['region'] = le_region.transform(dataset['region'])
 dataset['city'] = le_city.transform(dataset['city'])
 dataset['category_name'] = le_category_name.transform(dataset['category_name'])
 dataset['parent_category_name'] = le_parent_category_name.transform(dataset['parent_category_name'])
 dataset['param_1'] = le_param_1.transform(dataset['param_1'])
 dataset['param123'] = le_param123.transform(dataset['param123'])
 dataset['price'] = np.log1p(dataset['price'])
 dataset['item_seq_number'] = np.log(dataset['item_seq_number'])
 print("Transform on test function completed.")
 return dataset
def get_keras_data(dataset):
X = {
'seq_title_description': pad_sequences(dataset.seq_title_description, maxlen=max_seq_title_description_length)
 ,'region': np.array(dataset.region)
 ,'city': np.array(dataset.city)
 ,'category_name': np.array(dataset.category_name)
 ,'parent_category_name': np.array(dataset.parent_category_name)
 ,'param_1': np.array(dataset.param_1)
 ,'param123': np.array(dataset.param123)
 ,'price': np.array(dataset[["price"]])
 ,'item_seq_number': np.array(dataset[["item_seq_number"]])
 }
print("Data ready for Vectorization")
 return X<load_from_csv>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
dtypes_train = {
'price': 'float32',
'deal probability': 'float32',
'item_seq_number': 'uint32'
}
use_cols = ['item_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'title', 'description', 'price', 'item_seq_number', 'activation_date', 'deal_probability']
train = pd.read_csv(".. /input/avito-demand-prediction/train.csv", parse_dates=["activation_date"], usecols = use_cols, dtype = dtypes_train)
 y_train = train['deal_probability']
del train['deal_probability']
gc.collect()
max_seq_title_description_length = 100
max_words_title_description = 200000
train = preprocess_dataset(train)
 train, tokenizer, le_region, le_city, le_category_name, le_parent_category_name, le_param_1, le_param123 = keras_fit(train)
 train = keras_train_transform(train)
 print("Tokenization done and TRAIN READY FOR Validation splitting")
 max_region = np.max(train.region.max())+1
max_city= np.max(train.city.max())+1
max_category_name = np.max(train.category_name.max())+1
max_parent_category_name = np.max(train.parent_category_name.max())+1
max_param_1 = np.max(train.param_1.max())+1
max_param123 = np.max(train.param123.max())+1
print("Train Test Split")
 x_train_f, x_valid_f, y_train_f, y_valid_f = train_test_split(train, y_train, train_size=0.95, random_state=233)
 print(x_train_f.shape, x_valid_f.shape)
 print(y_train_f.shape, y_valid_f.shape)
 del train, y_train
gc.collect()
X_train = get_keras_data(x_train_f)
 X_valid = get_keras_data(x_valid_f)
 del x_train_f, x_valid_f
gc.collect() 
<prepare_x_and_y>
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
EMBEDDING_DIM1 = 300
EMBEDDING_FILE1 = '.. /input/fasttest-common-crawl-russian/cc.ru.300.vec'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index1 = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE1))
vocab_size = len(tokenizer.word_index)+1
EMBEDDING_DIM1 = 300
embedding_matrix1 = np.zeros(( vocab_size, EMBEDDING_DIM1))
print(embedding_matrix1.shape)
 c = 0
c1 = 0
w_Y = []
w_No = []
for word, i in tokenizer.word_index.items() :
if word in embeddings_index1:
c +=1
embedding_vector = embeddings_index1[word]
w_Y.append(word)
 else:
embedding_vector = None
w_No.append(word)
 c1 +=1
if embedding_vector is not None:
embedding_matrix1[i] = embedding_vector
print(c,c1, len(w_No), len(w_Y))
print(embedding_matrix1.shape)
 del embeddings_index1
gc.collect()
print(" FAST TEXT DONE")
<choose_model_class>
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None,random_state_int=101,early_stopping=300):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = 4, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True, random_state= random_state_int)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds =early_stopping,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
callbacks=[lgb.reset_parameter(learning_rate=learning_rate_power_0997)])
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
def RNN_model() :
seq_title_description = Input(shape=[X_train["seq_title_description"].shape[1]], name="seq_title_description")
 region = Input(shape=[1], name="region")
 city = Input(shape=[1], name="city")
 category_name = Input(shape=[1], name="category_name")
 parent_category_name = Input(shape=[1], name="parent_category_name")
 param_1 = Input(shape=[1], name="param_1")
 param123 = Input(shape=[1], name="param123")
 price = Input(shape=[1], name="price")
 item_seq_number = Input(shape = [1], name = 'item_seq_number')
 emb_seq_title_description = Embedding(vocab_size, EMBEDDING_DIM1, weights = [embedding_matrix1], trainable = True )(seq_title_description)
 emb_region = Embedding(max_region, 10 )(region)
 emb_city = Embedding(max_city, 10 )(city)
 emb_category_name = Embedding(max_category_name, 10 )(category_name)
 emb_parent_category_name = Embedding(max_parent_category_name, 10 )(parent_category_name)
 emb_param_1 = Embedding(max_param_1, 10 )(param_1)
 emb_param123 = Embedding(max_param123, 10 )(param123)
 rnn_layer1 = GRU(25 )(emb_seq_title_description)
 main_l = concatenate([
rnn_layer1
, Flatten()(emb_region)
 , Flatten()(emb_city)
 , Flatten()(emb_category_name)
 , Flatten()(emb_parent_category_name)
 , Flatten()(emb_param_1)
 , Flatten()(emb_param123)
 , price
, item_seq_number
])
 main_l = Dropout(0.1 )(Dense(512,activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(64,activation='relu' )(main_l))
output = Dense(1,activation="sigmoid" )(main_l)
 model = Model([seq_title_description, region, city, category_name, parent_category_name, param_1, param123, price, item_seq_number ], output)
 model.compile(optimizer = 'adam',
loss= root_mean_squared_error,
metrics = [root_mean_squared_error])
 return model
def rmse(y, y_pred):
Rsum = np.sum(( y - y_pred)**2)
 n = y.shape[0]
RMSE = np.sqrt(Rsum/n)
 return RMSE
def eval_model(model):
val_preds = model.predict(X_valid)
 y_pred = val_preds[:, 0]
y_true = np.array(y_valid_f)
 yt = pd.DataFrame(y_true)
 yp = pd.DataFrame(y_pred)
 print(yt.isnull().any())
 print(yp.isnull().any())
 v_rmse = rmse(y_true, y_pred)
 print(" RMSE for VALIDATION SET: "+str(v_rmse))
return v_rmse
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1<train_model>
hyp1 = {
'colsample_bytree': 0.88,
'learning_rate': 0.08,
'min_child_samples': 90, 'num_leaves': 34, 'subsample': 0.94, 'reg_lambda': 0.5,
'class_weight': 'balanced'}
hyp2 = {
'colsample_bytree': 0.88,
'learning_rate': 0.08,
'min_child_samples': 90, 'num_leaves': 14, 'subsample': 0.94, 'reg_lambda': 0.5,
'class_weight': 'balanced'}
hyp3 = {
'colsample_bytree': 0.78,
'learning_rate': 0.08,
'min_child_samples': 45, 'num_leaves': 14, 'subsample': 0.64, 'reg_lambda': 0.1,
'class_weight': 'balanced'}
hyp4 = {
'colsample_bytree': 0.72,
'learning_rate': 0.08,
'min_child_samples': 30, 'num_leaves': 18, 'subsample': 0.64, 'reg_lambda': 0.1,
'class_weight': 'balanced'}
epochs = 1
BATCH_SIZE = 512 * 3
steps = int(len(X_train['seq_title_description'])/BATCH_SIZE)* epochs
lr_init, lr_fin = 0.009, 0.0045
lr_decay = exp_decay(lr_init, lr_fin, steps)
 modelRNN = RNN_model()
K.set_value(modelRNN.optimizer.lr, lr_init)
 K.set_value(modelRNN.optimizer.decay, lr_decay)
 del embedding_matrix1
gc.collect()
for i in range(3):
history = modelRNN.fit(X_train, y_train_f
, epochs=epochs
, batch_size=(BATCH_SIZE+(BATCH_SIZE*(i)))
 , validation_data =(X_valid, y_valid_f)
 , verbose=1 
)
 v_rmse = eval_model(modelRNN)
 print("Finished Fitting the model" )<drop_column>
%%capture --no-display
submission1, gbm_fi_selected, valid_scores_selected = model_gbm(train_set, train_labels, test_set, test_ids,hyp = hyp1,random_state_int=103,early_stopping=300)
 submission2, gbm_fi_selected, valid_scores_selected = model_gbm(train_set, train_labels, test_set, test_ids,hyp = hyp1,random_state_int=103,early_stopping=300)
 submission3, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids)
del X_train, y_train_f, X_valid, y_valid_f
gc.collect()<set_options>
def model_gbm_2(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None,random_state_int=101,early_stopping=300):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = 4, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True, random_state= random_state_int)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds =early_stopping,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
gc.collect()<load_from_csv>
%%capture --no-display
submission4, gbm_fi_selected, valid_scores_selected = model_gbm_2(train_set, train_labels, test_set, test_ids,hyp = hyp3,random_state_int=103,early_stopping=300 )
t1 = time.time()
def load_test() :
for df in pd.read_csv('.. /input/avito-demand-prediction/test.csv', chunksize= 250000):
yield df
item_ids = np.array([], dtype=np.int32)
 preds= np.array([], dtype=np.float32)
 i = 0
for df in load_test() :
i +=1
print(df.dtypes)
 item_id = df['item_id']
print(" Chunk number is "+str(i))
test = preprocess_dataset(df)
 test = keras_test_transform(df)
 del df
gc.collect()
X_test = get_keras_data(test)
 del test
gc.collect()
preds1 = modelRNN.predict(X_test, batch_size = BATCH_SIZE, verbose = 1)
 print(preds1.shape)
 del X_test
gc.collect()
print("RNN Prediction is done")
 preds1 = preds1.reshape(-1,1)
 preds1 = np.clip(preds1, 0, 1)
 print(preds1.shape)
 item_ids = np.append(item_ids, item_id)
 print(item_ids.shape)
 preds = np.append(preds, preds1)
 print(preds.shape)
 print("All chunks done")
 t2 = time.time()
print("Total time for Parallel Batch Prediction is "+str(t2-t1))<save_to_csv>
suball=submission1.merge(submission2, on='Id')
 suball=suball.merge(submission4, on='Id')
 suball=suball.merge(submission3, on='Id')
 suball_2=suball.merge(submission4, on='Id' )
submission = pd.DataFrame(columns = ['item_id', 'deal_probability'])
 submission['item_id'] = item_ids
submission['deal_probability'] = preds
print("Check Submission NOW!!!!!!!!@")
 submission.to_csv("Avito_Shanth_RNN.csv", index=False )<set_options>
suball_2['index']=suball_2.index
sns.set()
%matplotlib inline<load_from_csv>
modle_13= pd.DataFrame(suball_2.mode(axis=1)[0] )
used_cols = ['item_id', 'user_id']
train = pd.read_csv('.. /input/train.csv', usecols=used_cols)
 train_active = pd.read_csv('.. /input/train_active.csv', usecols=used_cols)
 test = pd.read_csv('.. /input/test.csv', usecols=used_cols)
 test_active = pd.read_csv('.. /input/test_active.csv', usecols=used_cols)
 train_periods = pd.read_csv('.. /input/periods_train.csv', parse_dates=['date_from', 'date_to'])
 test_periods = pd.read_csv('.. /input/periods_test.csv', parse_dates=['date_from', 'date_to'])
 train.head()<remove_duplicates>
modle_13['index']=modle_13.index
all_samples = pd.concat([
train,
train_active,
test,
test_active
] ).reset_index(drop=True)
 all_samples.drop_duplicates(['item_id'], inplace=True)
 del train_active
del test_active
gc.collect()<concatenate>
final = suball_2.merge(modle_13, on ='index' )
all_periods = pd.concat([
train_periods,
test_periods
])
 del train_periods
del test_periods
gc.collect()
all_periods.head()<feature_engineering>
final_res=final[['Id',0]]
all_periods['days_up'] = all_periods['date_to'].dt.dayofyear - all_periods['date_from'].dt.dayofyear<feature_engineering>
final_res.columns=['Id','Target']
<merge><EOS> 
final_res.to_csv('submission.csv',index=False )