Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<categorify>
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import RandomizedSearchCV,cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import make_scorer
from itertools import product
import re
import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords
i = 0
for item in lbl_ls:
trdf = train_n[item]
cols = trdf.columns.to_series().values
if i==0:
lbl_trdf = pd.DataFrame(np.repeat(cols[None, :], len(trdf), 0)[trdf.astype(bool ).values], trdf.index[trdf.any(1)], columns = [lbl_strls[i]])
 else:
lbl_trdf = pd.concat([lbl_trdf, pd.DataFrame(np.repeat(cols[None, :], len(trdf), 0)[trdf.astype(bool ).values],
trdf.index[trdf.any(1)], columns = [lbl_strls[i]])], axis = 1)
 i+=1
j = 0
for item in lbl_ls:
tedf = test_n[item]
cols = tedf.columns.to_series().values
if j==0:
lbl_tedf = pd.DataFrame(np.repeat(cols[None, :], len(tedf), 0)[tedf.astype(bool ).values],
tedf.index[tedf.any(1)], columns = [lbl_strls[j]])
 else:
lbl_tedf = pd.concat([lbl_tedf, pd.DataFrame(np.repeat(cols[None, :], len(tedf), 0)[tedf.astype(bool ).values],
tedf.index[tedf.any(1)], columns = [lbl_strls[j]])], axis = 1)
 j+=1
print(lbl_trdf.shape, lbl_tedf.shape)
 pd.concat([lbl_trdf.head() , lbl_tedf.head() ], sort=False )<categorify>
!pip install pymorphy2
!pip install pymorphy2-dicts
!pip install DAWG-Python
for i in range(0, len(lbl_strls)) :
if i%10==0:
print(i)
 tr_ls = lbl_trdf.iloc[:,i].tolist()
lbl_trdf.iloc[:,i] = LabelEncoder().fit_transform(tr_ls)
 te_ls = lbl_tedf.iloc[:,i].tolist()
lbl_tedf.iloc[:,i] = LabelEncoder().fit_transform(te_ls )<concatenate>
import pymorphy2
pd.concat([lbl_trdf.head() , lbl_tedf.head() ], sort=False )<concatenate>
sample_submissions=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv")
 sales_train_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 items_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 shops_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv")
 item_categories_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv" )
train_n = pd.concat([train_n, lbl_trdf], axis = 1)
 test_n = pd.concat([test_n, lbl_tedf], axis = 1 )<drop_column>
print(sales_train_df["shop_id"].nunique())
 print(test_df["shop_id"].nunique() )
for item in lbl_ls:
train_n = train_n.drop(labels=item, axis=1)
 test_n = test_n.drop(labels=item, axis=1 )<concatenate>
print(sales_train_df["item_id"].nunique())
 print(test_df["item_id"].nunique() )
pd.concat([train_n.head() , test_n.head() ], sort = False )<concatenate>
nltk.download("stopwords" )
train_n = shift_cols_last(train_n, col_index)
 train_n.head()<drop_column>
def text_preprocessing(text):
txt=""
morph=pymorphy2.MorphAnalyzer()
stop_words_ru=stopwords.words("russian")
 stop_words_eng=stopwords.words("english")
 stroka=re.sub('[^A-Za-zА-Яа-я]+',' ',text)
 strs=stroka.split(" ")
 filtered_strs=[morph.parse(w.lower())[0].normal_form for w in strs if(w not in stop_words_ru and w not in stop_words_eng)]
txt=" ".join(filtered_strs)
 return txt
def shift_cols_front(df, col_index):

 cols = df.columns.tolist()
i=0
for item in col_index:
if i==0:
new_cols = [cols[0]] + [cols[item]] + cols[1:item] + cols[item+1:]
elif item < col_index[i-1]:
new_cols = new_cols[0:i+1] + [new_cols[item+1]] + new_cols[1+i:item+1] + new_cols[item+2:]
elif item > col_index[i-1]:
new_cols = new_cols[0:i+1] + [new_cols[item]] + new_cols[1+i:item] + new_cols[item+1:]
i+=1
return df[new_cols]<concatenate>
train_n = shift_cols_front(train_n, col_index)
 test_n = shift_cols_front(test_n, col_index )<sort_values>
sales_train_df=sales_train_df.join(shops_df.set_index("shop_id"),on="shop_id",how="left" ).join(
 items_df.set_index("item_id"),on="item_id",how="left" ).join(item_categories_df.set_index("item_category_id")
 ,on="item_category_id",how="left")
 sales_train_df.head().T
tr_corr = df.corr()
sorted_cor = tr_corr['Target'][tr_corr['Target'].isna() ==False].sort_values(ascending=False)
 print(f'Most Positive Correlations:
{sorted_cor.head(20)}
Most Negative Correlations:
{sorted_cor.tail(20)}' )<concatenate>
test_df=test_df.join(shops_df.set_index("shop_id"),on="shop_id",how="left" ).join(
 items_df.set_index("item_id"),on="item_id",how="left" ).join(item_categories_df.set_index("item_category_id")
 ,on="item_category_id",how="left")
 test_df.head().T
train_class1_3 = train_n[train_n.Target!=4]
train_n.drop(index = train_class1_3.index, inplace=True)
 train_class1_3 = pd.concat([train_class1_3, train_class1_3, train_class1_3], sort=False, ignore_index = True)
 train_n = pd.concat([train_n, train_class1_3], sort=False, ignore_index = True)
 print(len(train_n))<prepare_x_and_y>
sales_train_df[sales_train_df.item_cnt_day>2000]
X_arr = train_n.iloc[:,2:-1].values
y = train_n.iloc[:,-1].values
te_X = test_n.iloc[:,2:].values<normalization>
sales_train_df[sales_train_df.item_category_id==9]["item_cnt_day"].mean()
scaler = MinMaxScaler(feature_range=(0, 1))
X_arr = scaler.fit_transform(X_arr)
 te_X = scaler.fit_transform(te_X )<train_model>
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==11373)]["item_cnt_day"].mean()
input_dim = X_arr.shape[1]
print(f'Train features
{input_dim}' )<count_missing_values>
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>2000])]
assert not np.any(np.isnan(X_arr))<train_on_grid>
sales_train_df[sales_train_df.item_price>50000]
model3 = LogisticRegression(penalty='l2', solver = 'newton-cg', multi_class='multinomial', verbose = 1)
 scores3 = cross_validate(model3, X_arr, y, cv=10, n_jobs = 4, return_train_score = True, verbose=1, scoring = 'f1_macro')
 model3.fit(X_arr,y )<train_model>
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_price>50000])]
scores3<predict_on_test>
sales_train_df[sales_train_df.item_cnt_day>800]
pred3 = model3.predict(te_X )<save_to_csv>
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==20949)]["item_cnt_day"].mean()
pred3s = pd.DataFrame(pred3, columns = ['Target'])
 pred3s = pred3s.assign(Id=test_n.Id)
 pred3s = pred3s[['Id','Target']]
pred3s.Target = pd.to_numeric(pred3s.Target, downcast='signed')
 pred3s.to_csv('log_reg.csv', header = True, index=False)
 pred3s.head()<create_dataframe>
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>800])]
pd.DataFrame.from_dict(model3_dict, orient = 'Index', columns=['Value'] )<create_dataframe>
test_unique_items=test_df[~test_df["item_id"].isin(item_id_uniq)]
test_unique_items.head()
pd.DataFrame.from_dict(classification_report(y, model3.predict(X_arr), output_dict=True), orient='index' )<train_on_grid>
test_unique_items["item_id"].nunique()
model7 = BaggingClassifier()
scores7 = cross_validate(model7, X_arr, y, cv=10, verbose=1, return_train_score = True, n_jobs=4, scoring = 'f1_macro')
 model7.fit(X_arr,y )<predict_on_test>
%%time
train=[]
col=["date_block_num","shop_id","item_id"]
for i in range(34):
sales=sales_train_df[sales_train_df.date_block_num==i]
train.append(np.array(list(product([i],sales.shop_id.unique() ,sales.item_id.unique())) ,dtype="int16"))
train=pd.DataFrame(np.vstack(train),columns=col)
 train["date_block_num"]=train["date_block_num"].astype(np.int8)
 train["shop_id"]=train["shop_id"].astype(np.int8)
 train["item_id"]=train["item_id"].astype(np.int16)
 print(train.head() )
pred7 = model7.predict(te_X )<save_to_csv>
month_df=sales_train_df[["date_block_num","shop_id","item_id","item_cnt_day"]].groupby(["date_block_num","shop_id","item_id"] ).sum()
month_df.head()
pred7s = pd.DataFrame(pred7, columns = ['Target'])
 pred7s = pred7s.assign(Id=test_n.Id)
 pred7s = pred7s[['Id','Target']]
pred7s.Target = pd.to_numeric(pred7s.Target, downcast='signed')
 pred7s.to_csv('bagging.csv', header = True, index=False)
 pred7s.head()<train_on_grid>
date_block_total["year"]=date_block_total.date_block_num.apply(lambda x:(( x//12)+2013))
date_block_total["month"]=date_block_total.date_block_num.apply(lambda x:(x%12))
date_block_total.head()
model10 = GradientBoostingClassifier()
scores10 = cross_validate(model10, X_arr, y, cv=10, verbose=1, return_train_score = True, n_jobs=4, scoring = 'f1_macro')
 model10.fit(X_arr,y )<predict_on_test>
train["year"]=train.date_block_num.apply(lambda x:(( x//12)+2013))
train["month"]=train.date_block_num.apply(lambda x:(x%12))
train["year"]=train["year"].astype(np.int16)
 train["month"]=train["month"].astype(np.int8)
 train.head()
pred10 = model10.predict(te_X )<save_to_csv>
test_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 test_df["date_block_num"]=34
test_df["year"]=(( 34//12)+2013)
 test_df["month"]=(34%12)
 test_df["date_block_num"]=test_df["date_block_num"].astype(np.int8)
 test_df["shop_id"]=test_df["shop_id"].astype(np.int8)
 test_df["item_id"]=test_df["item_id"].astype(np.int16)
 test_df["year"]=test_df["year"].astype(np.int16)
 test_df["month"]=test_df["month"].astype(np.int8)
 test_df.head()
pred10s = pd.DataFrame(pred10, columns = ['Target'])
 pred10s = pred10s.assign(Id=test_n.Id)
 pred10s = pred10s[['Id','Target']]
pred10s.Target = pd.to_numeric(pred10s.Target, downcast='signed')
 pred10s.to_csv('grad_boosting.csv', header = True, index=False)
 pred10s.head()<categorify>
categorical=["year","month"]
final_pred = np.array([])
 for i in range(0,len(te_X)) :
final_pred = np.append(final_pred, mode([pred3[i], pred7[i], pred10[i]])[0] )<save_to_csv>
train=train.join(month_df,on=["date_block_num","shop_id","item_id"],how="left")
 train["item_cnt_day"]=train["item_cnt_day"].fillna(0 ).clip(0,20 ).astype(np.float16)
 train.head()
pred = pd.DataFrame(final_pred, columns = ['Target'])
 pred = pred.assign(Id=test_n.Id)
 pred = pred[['Id','Target']]
pred.Target = pd.to_numeric(pred.Target, downcast='signed')
 pred.to_csv('prediction.csv', header = True, index=False)
 pred.head()<set_options>
train=pd.concat([train,test_df],ignore_index=True,sort=False,keys=col)
 train.fillna(0,inplace=True)
 train.head()
init_notebook_mode(connected=True )<load_from_csv>
def create_lag_features(df,feature):
numeric_features=[]
for i in range(1,7):
lagged=df.copy()
lagged=lagged[["date_block_num","shop_id","item_id",feature]]
lagged.columns=["date_block_num","shop_id","item_id",feature+"_lag_"+str(i)]
numeric_features.append(feature+"_lag_"+str(i))
lagged["date_block_num"]+=i
df=df.join(lagged.set_index(["date_block_num","shop_id","item_id"]),on=["date_block_num","shop_id","item_id"],how="left")
 return df,numeric_features
data=pd.read_csv(".. /input/costa-rican-household-poverty-prediction/train.csv")
 test=pd.read_csv(".. /input/costa-rican-household-poverty-prediction/test.csv")

label_dict={1:"extreme poverty",2:"moderate poverty",3:"vulnerable households",4:"non-vulnerable",}

 features=data.columns
info={}
x=open(".. /input/description/description.txt")
 for x,line in enumerate(x):
info[line.split(",",1)[0]]= line.split(",",1)[1].replace("
","")
 info 
<drop_column>
%%time
train,n_col=create_lag_features(train,"item_cnt_day")
 train=train.fillna(0)
 numeric=n_col
data=data.drop(columns=["area2","female"])
 data=data.drop(columns=["estadocivil1","estadocivil2","estadocivil3","estadocivil4","estadocivil5"])
 test=test.drop(columns=["area2","female"])
 test=test.drop(columns=["estadocivil1","estadocivil2","estadocivil3","estadocivil4","estadocivil5"] )<feature_engineering>
total_cnt_for_items=sales_train_df[["date_block_num","item_id","item_cnt_day"]].groupby(["date_block_num","item_id"] ).sum()
total_cnt_for_items.columns=["total_cnt_for_items"]
total_cnt_for_items.head()
data["depend"] =(data["hogar_nin"]+data["hogar_mayor"]+1)/(1+data["hogar_adul"])
 test["depend"] =(test["hogar_nin"]+test["hogar_mayor"]+1)/(1+test["hogar_adul"])
 set(data["depend"] )<feature_engineering>
train=train.join(total_cnt_for_items,on=["date_block_num","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_items"]=train["total_cnt_for_items"].astype(np.float16)
 train.head().T
data[["depend","Target"]].corr(method="spearman")
 info["depend"]=info["dependency"]<drop_column>
%%time
train,n_col=create_lag_features(train,"total_cnt_for_items")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_items"],axis=1)
 numeric=numeric+n_col
data=data.drop(columns="dependency")
 test=test.drop(columns="dependency" )<define_variables>
total_cnt_for_shops=sales_train_df[["date_block_num","shop_id","item_cnt_day"]].groupby(["date_block_num","shop_id"] ).sum()
total_cnt_for_shops.columns=["total_cnt_for_shops"]
total_cnt_for_shops.head()
id_ = ['Id', 'idhogar', 'Target']
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc']
ind_ordered = ['rez_esc', 'escolari', 'age']
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<count_unique_values>
sum_cnt_shops=total_cnt_for_shops.reset_index()
cnt_shops=sum_cnt_shops[["shop_id","total_cnt_for_shops"]].groupby("shop_id",as_index=False ).mean()
cnt_shops=cnt_shops.join(shops_df.set_index(["shop_id"]),on="shop_id",how="left")
 cnt_shops.head()
all_equal = data.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<groupby>
cnt_shops.loc[cnt_shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
cnt_shops['shop_city'] = cnt_shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 cnt_shops.loc[cnt_shops.shop_city == '!Якутск', 'shop_city'] = 'Якутск'
cnt_shops['shop_type'] = cnt_shops['shop_name'].str.split(' ' ).map(lambda x: x[1])
 cnt_shops.head()
households_leader = data.groupby('idhogar')['parentesco1'].sum()
households_no_head = households_leader[households_leader==0]
print('There are {} households without a head.'.format(len(households_leader[households_leader==0])) )<feature_engineering>
train=train.join(cnt_shops[["shop_id","shop_city","shop_type"]].set_index(["shop_id"]),on="shop_id",how="left")
 train.fillna(0,inplace=True)
 train["shop_city"]=train["shop_city"].astype(str)
 train["shop_type"]=train["shop_type"].astype(str)
 train.head().T
for household in not_equal.index:
true_target = int(data[(data['idhogar'] == household)&(data['parentesco1'] == 1.0)]['Target'])
 data.loc[data['idhogar'] == household, 'Target'] = true_target
all_equal = data.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<drop_column>
categorical=categorical+["shop_city","shop_type"]
data=data.drop(columns=[i for i in data.columns if i.startswith("SQ")==True])
 data=data.drop(columns=["agesq"] )<drop_column>
train=train.join(total_cnt_for_shops,on=["date_block_num","shop_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_shops"]=train["total_cnt_for_shops"].astype(np.float16)
 train.head().T
test=test.drop(columns=[i for i in test.columns if i.startswith("SQ")==True])
 test=test.drop(columns=["agesq"] )<count_missing_values>
%%time
train,n_col=create_lag_features(train,"total_cnt_for_shops")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_shops"],axis=1)
 numeric=numeric+n_col
null=data.isnull().any()
null[null==True]<data_type_conversions>
total_cnt_for_item_categories=sales_train_df[["date_block_num","item_category_id","item_cnt_day"]].groupby(["date_block_num","item_category_id"] ).sum()
total_cnt_for_item_categories.columns=["total_cnt_for_item_categories"]
total_cnt_for_item_categories.head()
data["v2a1"]=data["v2a1"].fillna(0)
 data["v18q1"]=data["v18q1"].fillna(0)
 test["v2a1"]=test["v2a1"].fillna(0)
 test["v18q1"]=test["v18q1"].fillna(0 )<filter>
sum_cnt_for_item_category["category"]=sum_cnt_for_item_category.item_category_name.str.split("-" ).str[0]
sum_cnt_for_item_category["subcategory"]=sum_cnt_for_item_category.item_category_name.str.split("-" ).str[1]
sum_cnt_for_item_category.head()
data["Target"][data["meaneduc"].isnull() ==True] 
<feature_engineering>
train=train.join(items_df.set_index(["item_id"]),on=["item_id"],how="left")
 train=train.join(sum_cnt_for_item_category[["item_category_id","category","subcategory"]].set_index(["item_category_id"]),on=["item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["category"]=train["category"].astype(str)
 train["subcategory"]=train["subcategory"].astype(str)
 train.tail().T
mode_edu=statistics.mode(data["meaneduc"][data["Target"]==4][data["meaneduc"].isnull() ==False])
 data["meaneduc"]=data["meaneduc"].fillna(mode_edu)
 test["meaneduc"]=test["meaneduc"].fillna(mode_edu)
<drop_column>
categorical=categorical+["category","subcategory"]
set(data["rez_esc"].fillna(0))
data=data.drop(columns=["rez_esc"])
 test=test.drop(columns=["rez_esc"] )<define_variables>
train=train.join(total_cnt_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["total_cnt_for_item_categories"]=train["total_cnt_for_item_categories"].astype(np.float16)
 train.head().T
features=["escolari","meaneduc","instlevel1","instlevel2","instlevel3","instlevel4","instlevel5","instlevel6",
"instlevel7","instlevel8","instlevel9"]<categorify>
%%time
train,n_col=create_lag_features(train,"total_cnt_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["total_cnt_for_item_categories"],axis=1)
 numeric=numeric+n_col
encoding={i:int(i[-1])for i in [j for j in features if "inst" in j]}
insti=["instlevel1","instlevel2","instlevel3","instlevel4","instlevel5","instlevel6",
"instlevel7","instlevel8","instlevel9"]
for i in range(len(data)) :
for j in insti:
if data.loc[i,j]==1:
data.loc[i,j]=encoding[j]
for i in range(len(test)) :
for j in insti:
if test.loc[i,j]==1:
test.loc[i,j]=encoding[j]
data["qualified"]=data[insti].sum(axis=1)
 test["qualified"]=test[insti].sum(axis=1)
 info["qualified"]="sum of all instlevels"
data=data.drop(columns=insti)
 test=test.drop(columns=insti )<data_type_conversions>
mean_price=sales_train_df[["date_block_num","shop_id","item_id","item_price"]].groupby(["date_block_num","shop_id","item_id"] ).mean()
mean_price.columns=["mean_price"]
mean_price.head()
avg_qual=data.groupby("idhogar")["qualified"].apply(lambda x: x.mean())
 avg_qual_test=test.groupby("idhogar")["qualified"].apply(lambda x: x.mean())
 info["avg_qual"]="average academic qualification level of a household"
data["avg_qual"]=""
for i in range(len(data)) :
data.loc[i,"avg_qual"]=avg_qual[data.loc[i,"idhogar"]]
data["avg_qual"]=data["avg_qual"].astype(float)
 for i in range(len(test)) :
test.loc[i,"avg_qual"]=avg_qual_test[test.loc[i,"idhogar"]]
test["avg_qual"]=test["avg_qual"].astype(float )<drop_column>
train=train.join(mean_price,on=["date_block_num","shop_id","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price"]=train["mean_price"].astype(np.float16)
 train.head().T
data=data.drop(columns=["escolari","meaneduc"])
 test=test.drop(columns=["escolari","meaneduc"] )<categorify>
%%time
train,n_col=create_lag_features(train,"mean_price")
 train=train.fillna(0)
 train=train.drop(["mean_price"],axis=1)
 numeric=numeric+n_col
walls={"epared1":1,"epared2":2,"epared3":3}
roof={"etecho1":1,"etecho2":2,"etecho3":3}
floor={"eviv1":1,"eviv2":2,"eviv3":3}
data["walls"]=""
data["roof"]=""
data["floor"]=""
test["walls"]=""
test["roof"]=""
test["floor"]=""
for i in range(len(data)) :
for j in walls.keys() :
if data.loc[i,j]==1:
data.loc[i,"walls"]=walls[j]
for i in range(len(data)) :
for j in roof.keys() :
if data.loc[i,j]==1:
data.loc[i,"roof"]=roof[j]
for i in range(len(data)) :
for j in floor.keys() :
if data.loc[i,j]==1:
data.loc[i,"floor"]=floor[j]
for i in range(len(test)) :
for j in walls.keys() :
if test.loc[i,j]==1:
test.loc[i,"walls"]=walls[j]
for i in range(len(test)) :
for j in roof.keys() :
if test.loc[i,j]==1:
test.loc[i,"roof"]=roof[j]
for i in range(len(test)) :
for j in floor.keys() :
if test.loc[i,j]==1:
test.loc[i,"floor"]=floor[j] 
<feature_engineering>
mean_price_for_items=sales_train_df[["date_block_num","item_id","item_price"]].groupby(["date_block_num","item_id"] ).mean()
mean_price_for_items.columns=["mean_price_for_items"]
mean_price_for_items.head()
data["housing"]=data["walls"]+data["floor"]+data["roof"]
test["housing"]=test["walls"]+test["floor"]+test["roof"]<drop_column>
train=train.join(mean_price_for_items,on=["date_block_num","item_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price_for_items"]=train["mean_price_for_items"].astype(np.float16)
 train.head().T
data=data.drop(columns=["walls","floor","roof"])
 test=test.drop(columns=["walls","floor","roof"] )<define_variables>
%%time
train,n_col=create_lag_features(train,"mean_price_for_items")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_items"],axis=1)
 numeric=numeric+n_col
features1=["mobilephone","computer","television","refrig","v18q"]<feature_engineering>
mean_price_for_item_categories=sales_train_df[["date_block_num","item_category_id","item_price"]].groupby(["date_block_num","item_category_id"] ).mean()
mean_price_for_item_categories.columns=["mean_price_for_item_categories"]
mean_price_for_item_categories.head()
data["equipped"] = data[features1].sum(axis=1)
 test["equipped"] = test[features1].sum(axis=1)
 info["equipped"]="how tech savvy a family is" 
<drop_column>
train=train.join(mean_price_for_item_categories,on=["date_block_num","item_category_id"],how="left")
 train.fillna(0,inplace=True)
 train["mean_price_for_item_categories"]=train["mean_price_for_item_categories"].astype(np.float16)
 train.head().T
data=data.drop(columns=features1)
 test=test.drop(columns=features1 )<feature_engineering>
%%time
train,n_col=create_lag_features(train,"mean_price_for_item_categories")
 train=train.fillna(0)
 train=train.drop(["mean_price_for_item_categories"],axis=1)
 numeric=numeric+n_col
data["tab_cap"]=(data["v18q1"]/data["hogar_total"] ).fillna(0)
 data["phone_cap"]=(data["qmobilephone"]/data["hogar_total"] ).fillna(0)
 data["tech_cap"]=data["tab_cap"]+data["phone_cap"]
test["tab_cap"]=(test["v18q1"]/test["hogar_total"] ).fillna(0)
 test["phone_cap"]=(test["qmobilephone"]/test["hogar_total"] ).fillna(0)
 test["tech_cap"]=test["tab_cap"]+test["phone_cap"]<drop_column>
sales_train_df["revenue"]=sales_train_df["item_price"]*sales_train_df["item_cnt_day"]
total_revenue=sales_train_df[["date_block_num","shop_id","revenue"]].groupby(["date_block_num","shop_id"] ).sum()
total_revenue.columns=["total_shop_revenue"]
total_revenue.head()
data=data.drop(columns=["tab_cap","phone_cap","qmobilephone","v18q1"])
 test=test.drop(columns=["tab_cap","phone_cap","qmobilephone","v18q1"] )<drop_column>
train=train.join(total_revenue,on=["date_block_num","shop_id"],how="left")
 train["total_shop_revenue"]=train["total_shop_revenue"].astype(np.float32)
 train.fillna(0,inplace=True)
 train.head().T
data=data.drop(columns=["tamhog","tamviv","hhsize"])
 test=test.drop(columns=["tamhog","tamviv","hhsize"] )<define_variables>
%%time
train,n_col=create_lag_features(train,"total_shop_revenue")
 train=train.fillna(0.0)
 train=train.drop(["total_shop_revenue"],axis=1)
 numeric=numeric+n_col
features=["bedrooms","overcrowding","hacdor","rooms","hacapo"]<drop_column>
groups = train.groupby(train.date_block_num ).groups
sorted_groups = [value for(key, value)in sorted(groups.items())]
cv=[(np.concatenate(sorted_groups[:8]),np.concatenate(sorted_groups[8:])) , 
(np.concatenate(sorted_groups[:16]),np.concatenate(sorted_groups[16:])) , 
(np.concatenate(sorted_groups[:24]),np.concatenate(sorted_groups[24:])) ]
data=data.drop(columns=["hacapo","hacdor","bedrooms"])
 test=test.drop(columns=["hacapo","hacdor","bedrooms"] )<define_variables>
y_train=train["item_cnt_day"]
features=['tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4',"tipovivi5","v2a1"]<drop_column>
X_train_categorical=train[categorical]
X_test_categorical=test_df[categorical]
X_train_categorical["subcategory"]=X_train_categorical["subcategory"].astype(str)
 X_test_categorical["subcategory"]=X_test_categorical["subcategory"].astype(str)
 X_test_categorical["year"]=X_test_categorical["year"].astype(np.int16)
 X_test_categorical["month"]=X_test_categorical["month"].astype(np.int8)
 X_test_categorical.loc[X_test_categorical.category=="PC ","category"]="Игры PC "
X_test_categorical.loc[X_test_categorical.subcategory==" Гарнитуры/Наушники","subcategory"]=" Аксессуары для игр"
data=data.drop(columns="v2a1")
 test=test.drop(columns="v2a1" )<drop_column>
for feature in categorical:
le=LabelEncoder()
print(feature)
 X_train_categorical[feature]=le.fit_transform(X_train_categorical[feature])
 X_test_categorical[feature]=le.transform(X_test_categorical[feature] )
data=data.drop(columns=["r4h1","r4h2","r4m1","r4m2","r4m3","r4t2","r4t3"])
 test=test.drop(columns=["r4h1","r4h2","r4m1","r4m2","r4m3","r4t2","r4t3"] )<drop_column>
X_train_numeric=train[numeric]
X_test_numeric=test_df[numeric]
data=data.drop(columns=["lugar1","lugar2","lugar3","lugar4","lugar5","lugar6"])
 test=test.drop(columns=["lugar1","lugar2","lugar3","lugar4","lugar5","lugar6"] )<filter>
label_cat_not_num_train=pd.concat([X_train_categorical,X_train_numeric],axis=1)
 label_cat_not_num_train.head().T
data=data[data["parentesco1"]==1]<import_modules>
label_cat_not_num_test=pd.concat([X_test_categorical,X_test_numeric],axis=1)
 label_cat_not_num_test.head().T
from imblearn.over_sampling import RandomOverSampler<prepare_x_and_y>
def downcast_type(df):
for feature in categorical:
df[feature]=df[feature].astype(np.int8 )
y=data["Target"]
x=data.drop(columns=["Target","Id"])
 test_labels=test["Id"]
test=test.drop(columns=["Id"])
 x=x.drop(columns=["edjefe","edjefa"])
 test=test.drop(columns=["edjefe","edjefa"])
 x=x.drop(columns=["hogar_nin","hogar_adul","hogar_mayor"])
 test=test.drop(columns=["hogar_nin","hogar_adul","hogar_mayor"])
 x=x.drop(columns=["idhogar"])
 test=test.drop(columns=["idhogar"])
<train_model>
def RMSE(y,predictions):
return np.sqrt(mean_squared_error(y,predictions))
scorer=make_scorer(RMSE,False )
ros = RandomOverSampler(random_state=0)
 x_res, y_res = ros.fit_resample(x, y)
 Counter(y_res )<import_modules>
%%time
baseline = -cross_val_score(
 XGBRegressor(max_depth=10,
subsample=0.8,
colsample_bytree=0.9,
colsample_bylevel=0.7,
min_child_weight=200,
n_estimators=1000,
learning_rate=0.025,
objective="reg:squarederror",
tree_method="hist"), label_cat_not_num_train, y_train, scoring=scorer,cv=cv
).mean()
print(baseline )
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error, accuracy_score
import xgboost as xgb
import lightgbm as lgb<import_modules>
%%time
estimator=XGBRegressor(max_depth=10,
subsample=0.8,
colsample_bytree=0.9,
colsample_bylevel=0.7,
min_child_weight=200,
n_estimators=1000,
learning_rate=0.025,
objective="reg:squarederror",
tree_method="hist")
 estimator.fit(label_cat_not_num_train,y_train)
 importances=estimator.feature_importances_
predictions=estimator.predict(label_cat_not_num_test )
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score, make_scorer,accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import Pipeline<compute_test_metric>
sample_submissions["item_cnt_month"]=predictions
sample_submissions.head()
<choose_model_class><EOS> 
sample_submissions.to_csv("xgboost_lagged_features_6.csv",index=False )