Kaggle competition: Predict Future Sales
train["year"] = train["created"].dt.year
train["month"] = train["created"].dt.month
train["day"] = train["created"].dt.dayofweek
train["hour"] = train["created"].dt.hour<concatenate>
ts = time.time()
target_features_df=[]
col_labels=['item_avg_sales','item_avg_sales_over_sold']
target_features_df.append(moving_statistics(train_1,item_encoding,['month_id','item_id'],col_labels,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,item_encoding,['month_id','item_id'],col_labels,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,item_encoding,['month_id','item_id'],col_labels,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,item_encoding,['month_id','item_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,item_encoding,['month_id','item_id'],col_labels,[1]))
target_features_df.append(lag_features(train_1,item_encoding,['month_id','item_id'],col_labels,[12]))
target_features_df.append(time_mapping(train_1,item_encoding,['item_id'],col_labels,'item_month_id_of_last_sale',suffix='_last_sale'))
target_features_df.append(moving_statistics(train_1,item_encoding,['month_id','item_id'],['item_max_quantity'],34,'max',suffix='_absolute_max'))
col_labels_seniority_1=['item_avg_sales_seniority_1']
target_features_df.append(moving_statistics(train_1,item_encoding_seniority_1,['month_id','item_id'],col_labels_seniority_1,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,item_encoding_seniority_1,['month_id','item_id'],col_labels_seniority_1,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,item_encoding_seniority_1,['month_id','item_id'],col_labels_seniority_1,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,item_encoding_seniority_1,['month_id','item_id'],col_labels_seniority_1,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,item_encoding_seniority_1,['month_id','item_id'],col_labels_seniority_1,[1]))
target_features_df.append(lag_features(train_1,item_encoding_seniority_1,['month_id','item_id'],col_labels_seniority_1,[12]))
target_features_df.append(time_mapping(train_1,item_encoding_seniority_1,['item_id'],col_labels_seniority_1,'item_month_id_of_last_sale',suffix='_last_sale'))
print('time : ' +str(time.time() - ts))
col_labels=['shop_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(moving_statistics(train_1,shop_encoding,['month_id','shop_id'],col_labels,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,shop_encoding,['month_id','shop_id'],col_labels,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,shop_encoding,['month_id','shop_id'],col_labels,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,shop_encoding,['month_id','shop_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,shop_encoding,['month_id','shop_id'],col_labels,[1]))
target_features_df.append(lag_features(train_1,shop_encoding,['month_id','shop_id'],col_labels,[12]))
target_features_df.append(moving_statistics(train_1,shop_encoding_seniority_1,['month_id','shop_id'],col_labels_seniority_1,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,shop_encoding_seniority_1,['month_id','shop_id'],col_labels_seniority_1,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,shop_encoding_seniority_1,['month_id','shop_id'],col_labels_seniority_1,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,shop_encoding_seniority_1,['month_id','shop_id'],col_labels_seniority_1,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,shop_encoding_seniority_1,['month_id','shop_id'],col_labels_seniority_1,[1]))
target_features_df.append(lag_features(train_1,shop_encoding_seniority_1,['month_id','shop_id'],col_labels_seniority_1,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['category_months_since_release_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(moving_statistics(train_1,category_months_since_release_encoding_seniority_1,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_1,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,category_months_since_release_encoding_seniority_1,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_1,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,category_months_since_release_encoding_seniority_1,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_1,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,category_months_since_release_encoding_seniority_1,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_1,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,category_months_since_release_encoding_seniority_1,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_1,[1]))
target_features_df.append(lag_features(train_1,category_months_since_release_encoding_seniority_1,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_1,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_months_since_release_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(moving_statistics(train_1,shop_category_months_since_release_encoding_seniority_1,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_1,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,shop_category_months_since_release_encoding_seniority_1,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_1,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,shop_category_months_since_release_encoding_seniority_1,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_1,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,shop_category_months_since_release_encoding_seniority_1,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_1,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,shop_category_months_since_release_encoding_seniority_1,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_1,[1]))
target_features_df.append(lag_features(train_1,shop_category_months_since_release_encoding_seniority_1,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_1,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['supercategory_months_since_release_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(moving_statistics(train_1,supercategory_months_since_release_encoding_seniority_1,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,supercategory_months_since_release_encoding_seniority_1,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,supercategory_months_since_release_encoding_seniority_1,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,supercategory_months_since_release_encoding_seniority_1,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,supercategory_months_since_release_encoding_seniority_1,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,[1]))
target_features_df.append(lag_features(train_1,supercategory_months_since_release_encoding_seniority_1,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_supercategory_months_since_release_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(moving_statistics(train_1,shop_supercategory_months_since_release_encoding_seniority_1,['month_id','shop_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,shop_supercategory_months_since_release_encoding_seniority_1,['month_id','shop_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,shop_supercategory_months_since_release_encoding_seniority_1,['month_id','shop_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,shop_supercategory_months_since_release_encoding_seniority_1,['month_id','shop_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,shop_supercategory_months_since_release_encoding_seniority_1,['month_id','shop_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,[1]))
target_features_df.append(lag_features(train_1,shop_supercategory_months_since_release_encoding_seniority_1,['month_id','shop_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_1,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_avg_sales_compared_to_category']
target_features_df.append(moving_statistics(train_1,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_1,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_1,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_1,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,[1]))
target_features_df.append(lag_features(train_1,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(rational_fraction(train_1,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_1,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_1,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(rational_fraction(train_1,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_1,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_1,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['supercategory_avg_sales']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(rational_fraction(train_1,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_1,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_1,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales_compared_to_supercategory']
col_labels_seniority_1=[col+'_seniority_1' for col in col_labels]
target_features_df.append(moving_statistics(train_1,category_encoding,['month_id','item_category_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_1,category_encoding,['month_id','item_category_id'],col_labels,[1]))
target_features_df.append(lag_features(train_1,category_encoding,['month_id','item_category_id'],col_labels,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['category_month_of_release_avg_sales_compared_to_category']
target_features_df.append(moving_statistics(train_1,category_month_of_release_encoding,['month_id','item_category_id','item_month_of_release'],col_labels,34,'mean',suffix='_absolute_mean'))
col_labels=['supercategory_month_of_release_avg_sales_compared_to_supercategory']
target_features_df.append(moving_statistics(train_1,supercategory_month_of_release_encoding,['month_id','item_supercategory_id','item_month_of_release'],col_labels,34,'mean',suffix='_absolute_mean'))
print('time : ' +str(time.time() - ts))
train_1=pd.concat([train_1]+target_features_df,axis=1,sort=False)
 del target_features_df
gc.collect()
print('time : ' +str(time.time() - ts))
print(train_1.info(null_counts=True,verbose=True))
train_1
train_numeric = train._get_numeric_data()
train_numeric = pd.concat([train_numeric,train["interest_level"],train["train_test"]],axis=1)
 train_numeric.head()<drop_column>
for df in dfs:
exec("del "+df)
 exec("del "+df+"_seniority_1")
 gc.collect()
train_numeric = train_numeric.drop(['year','train_test'],axis=1);<categorify>
train_1.drop(features_to_discard,axis=1,inplace=True)
 train_1.info(null_counts=True,verbose=True )
le = preprocessing.LabelEncoder()
train_numeric["interest_level"] = le.fit_transform(train_numeric["interest_level"].fillna('0'))
np.unique(train_numeric["interest_level"] )<filter>
create_directory(os.path.join(DATA_FOLDER, 'training'))
train_1.to_pickle(os.path.join(DATA_FOLDER,'training/train_1_pred.pkl'))
train_numeric = train_numeric.query('interest_level>0' )<choose_model_class>
del train_1
gc.collect()
vectorizer = TfidfVectorizer()<feature_engineering>
reset_variable_space
tfidfeats = vectorizer.fit_transform(train["description"] )<import_modules>
train_2=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_2.pkl'))
print(train_2.info(null_counts=True))
from sklearn.decomposition import TruncatedSVD<choose_model_class>
split_col='month_id'
target_col='item_quantity'
raw_features=[]
label_encoding=['shop_id',
'item_id',
'item_category_id',
'item_supercategory_id',
'item_category_console_id',
'item_category_is_digital'
]
mapping_columns=['item_month_id_of_last_sale',
'item_month_id_of_release',
'item_month_of_release',
'item_month_id_of_last_sale_in_shop',
'item_month_id_of_first_sale_in_shop'
]
features_to_discard=['item_id',
'shop_id',
'item_supercategory_id',
'item_category_console_id',
'item_category_is_digital'
]
features_to_discard+=mapping_columns
raw_features+=['shop_months_since_opening']
raw_features+=['item_freq_in_seniority']
raw_features+=['item_category_freq']
raw_features+=['item_supercategory_freq']
raw_features+=['item_category_console_freq']
raw_features+=['item_category_digital_freq']
raw_features+=['item_category_freq_in_seniority']
raw_features+=['item_supercategory_freq_in_seniority']
raw_features+=['item_category_console_freq_in_seniority']
raw_features+=['item_category_digital_freq_in_seniority']
raw_features+=['category_semiannual_avg_recent_rsd']
raw_features+=['category_semiannual_avg_annual_rsd']
raw_features+=['item_months_since_release']
raw_features+=['item_months_since_first_sale_in_shop']
raw_features+=['item_months_since_last_sale_in_shop']
train_2=train_2[[split_col]+label_encoding+mapping_columns+[target_col]+raw_features]
print(train_2.info(null_counts=True))
gc.collect()
svd = TruncatedSVD(n_components=100 )<categorify>
item_prices=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/price_features/item_prices.pkl'))
category_prices=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/price_features/category_prices.pkl'))
lsa_feats = svd.fit_transform(tfidfeats )<create_dataframe>
ts = time.time()
price_features_df=[]
col_labels=['item_price_median']
col_labels+=['item_price_median_compared_to_category_price_median']
price_features_df.append(time_mapping(train_2,item_prices,['item_id'],col_labels,'item_month_id_of_last_sale',suffix='_last_sale'))
price_features_df.append(moving_statistics(train_2,item_prices,['month_id','item_id'],col_labels,34,'mean',suffix='_absolute_mean'))
time_window=34
price_features_df.append(moving_statistics(train_2,item_prices,['month_id','item_id'],['item_price_min'],34,'min',suffix='_absolute_min'))
price_features_df.append(moving_statistics(train_2,item_prices,['month_id','item_id'],['item_price_max'],34,'max',suffix='_absolute_max'))
print('time : ' +str(time.time() - ts))
price_features_df.append(moving_statistics(train_2,category_prices,['month_id','item_category_id'],['category_price_median'],3,'mean',suffix='_recent_mean'))
print('time : ' +str(time.time() - ts))
train_2=pd.concat([train_2]+price_features_df,axis=1,sort=False)
 del price_features_df
gc.collect()
print('time : ' +str(time.time() - ts))
train_2=downcast_dtypes(train_2)
 print('time : ' +str(time.time() - ts))
print(train_2.info(null_counts=True))
train_2
lsa_feats = pd.DataFrame(lsa_feats)
 lsa_feats_colnames = ["lsa_" + str(i+1)for i in range(100)]
lsa_feats.columns = lsa_feats_colnames
train = pd.concat([train,lsa_feats],axis=1 )<data_type_conversions>
del item_prices
del category_prices
gc.collect()
train["features"] = train["features"].astype("str" )<feature_engineering>
dfs=['shop_encoding',
'item_encoding',
'category_encoding',
'supercategory_encoding',
'shop_category_encoding',
'shop_supercategory_encoding',
'category_months_since_release_encoding',
'supercategory_months_since_release_encoding',
'category_month_of_release_encoding',
'supercategory_month_of_release_encoding',
'category_months_since_first_sale_in_shop_encoding',
'supercategory_months_since_first_sale_in_shop_encoding',
'shop_category_months_since_release_encoding',
'shop_supercategory_months_since_release_encoding',
'shop_category_months_since_first_sale_in_shop_encoding',
'shop_supercategory_months_since_first_sale_in_shop_encoding'
]
for df in dfs:
exec(df+"=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+".pkl')) ")
 exec(df+"_seniority_2=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+"_seniority_2.pkl')) ")
 shop_item_encoding=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/shop_item_encoding.pkl'))
tfidfeats2 = vectorizer.fit_transform(train["features"] )<choose_model_class>
feature_correlation_analysis(train_2,item_encoding,item_encoding_seniority_2,'item',['month_id','item_id'],2)
 compare_features(item_encoding,item_encoding_seniority_2,'item',['month_id','item_id'],2,20,20,20 )
svd2 = TruncatedSVD(n_components=20 )<feature_engineering>
feature_correlation_analysis(train_2,supercategory_encoding,supercategory_encoding_seniority_2,'supercategory',['month_id','item_supercategory_id'],2)
 compare_features(supercategory_encoding,supercategory_encoding_seniority_2,'supercategory',['month_id','item_supercategory_id'],2,20,20,20 )
lsa_feats2 = svd2.fit_transform(tfidfeats2 )<create_dataframe>
feature_correlation_analysis(train_2,shop_supercategory_encoding,shop_supercategory_encoding_seniority_2,'shop_supercategory',['month_id','shop_id','item_supercategory_id'],2)
 compare_features(shop_supercategory_encoding,shop_supercategory_encoding_seniority_2,'shop_supercategory',['month_id','shop_id','item_supercategory_id'],2,20,20,20 )
lsa_feats2 = pd.DataFrame(lsa_feats2)
 lsa_feats2_colnames = ["lsa2_" + str(i+1)for i in range(20)]
lsa_feats2.columns = lsa_feats2_colnames
train = pd.concat([train,lsa_feats2],axis=1 )<import_modules>
feature_correlation_analysis(train_2,category_months_since_release_encoding,category_months_since_release_encoding_seniority_2,'category_months_since_release',['month_id','item_category_id','item_months_since_release'],2)
 compare_features(category_months_since_release_encoding,category_months_since_release_encoding_seniority_2,'category_months_since_release',['month_id','item_category_id','item_months_since_release'],2,20,20,20 )
from textblob import TextBlob<feature_engineering>
feature_correlation_analysis(train_2,category_months_since_first_sale_in_shop_encoding,category_months_since_first_sale_in_shop_encoding_seniority_2,'category_months_since_first_sale_in_shop',['month_id','item_category_id','item_months_since_first_sale_in_shop'],2)
 compare_features(category_months_since_first_sale_in_shop_encoding,category_months_since_first_sale_in_shop_encoding_seniority_2,'category_months_since_first_sale_in_shop',['month_id','item_category_id','item_months_since_first_sale_in_shop'],2,20,20,20 )
description_polarity, description_subjectivity = [],[]
for desc in train["description"]:
desc_blob= TextBlob(desc)
 description_polarity.append(desc_blob.sentiment.polarity)
 description_subjectivity.append(desc_blob.sentiment.subjectivity)
 train["description_polarity"] = description_polarity
train["description_subjectivity"] = description_subjectivity 
<feature_engineering>
tmp=train_2[['month_id','shop_id','item_id','item_category_id','item_months_since_release','item_months_since_first_sale_in_shop']]
tmp2=category_months_since_release_encoding_seniority_2.loc[(category_months_since_release_encoding_seniority_2['month_id']>12)&(category_months_since_release_encoding_seniority_2['month_id']<34),['month_id','item_category_id','item_months_since_release','category_months_since_release_avg_sales_seniority_2','category_months_since_release_avg_sales_over_sold_seniority_2']]
tmp3=category_months_since_release_encoding.loc[(category_months_since_release_encoding['month_id']>12)&(category_months_since_release_encoding['month_id']<34),['month_id','item_category_id','item_months_since_release','category_months_since_release_avg_sales','category_months_since_release_avg_sales_over_sold']]
tmp=tmp.join(tmp2.set_index(['month_id','item_category_id','item_months_since_release']),on=['month_id','item_category_id','item_months_since_release'])
 tmp=tmp.join(tmp3.set_index(['month_id','item_category_id','item_months_since_release']),on=['month_id','item_category_id','item_months_since_release'])
 tmp4=category_months_since_first_sale_in_shop_encoding.loc[(category_months_since_first_sale_in_shop_encoding['month_id']>12)&(category_months_since_first_sale_in_shop_encoding['month_id']<34),['month_id','item_category_id','item_months_since_first_sale_in_shop','category_months_since_first_sale_in_shop_avg_sales','category_months_since_first_sale_in_shop_avg_sales_over_sold']]
tmp=tmp.join(tmp4.set_index(['month_id','item_category_id','item_months_since_first_sale_in_shop']),on=['month_id','item_category_id','item_months_since_first_sale_in_shop'])
 print(tmp[['item_months_since_release','item_months_since_first_sale_in_shop']].corr().loc['item_months_since_release',:])
 sns.heatmap(tmp.drop(['month_id','shop_id','item_id','item_category_id','item_months_since_release','item_months_since_first_sale_in_shop'],axis=1 ).corr() ,annot=True)
 del tmp, tmp2, tmp3, tmp4
train["photos_count"] = train.photos.apply(len )<import_modules>
feature_correlation_analysis(train_2,shop_category_months_since_release_encoding,shop_category_months_since_release_encoding_seniority_2,'shop_category_months_since_release',['month_id','shop_id','item_category_id','item_months_since_release'],2 )
import lightgbm as lgb
from sklearn.model_selection import GridSearchCV<data_type_conversions>
ts = time.time()
target_features_df=[]
target_features_df.append(moving_statistics(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],[1]))
target_features_df.append(lag_features(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],[12]))
target_features_df.append(linear_combination(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],weights=extralin2,suffix='_extralin2'))
target_features_df.append(linear_combination(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],weights=extralin3,suffix='_extralin3'))
target_features_df.append(moving_statistics(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],34,'min',suffix='_absolute_min'))
target_features_df.append(moving_statistics(train_2,shop_item_encoding,['month_id','shop_id','item_id'],['item_quantity'],34,'max',suffix='_absolute_max'))
print('time : ' +str(time.time() - ts))
col_labels=['item_avg_sales','item_avg_sales_over_sold']
target_features_df.append(moving_statistics(train_2,item_encoding,['month_id','item_id'],col_labels,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,item_encoding,['month_id','item_id'],col_labels,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,item_encoding,['month_id','item_id'],col_labels,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,item_encoding,['month_id','item_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,item_encoding,['month_id','item_id'],col_labels,[1]))
target_features_df.append(lag_features(train_2,item_encoding,['month_id','item_id'],col_labels,[12]))
target_features_df.append(time_mapping(train_2,item_encoding,['item_id'],col_labels,'item_month_id_of_last_sale',suffix='_last_sale'))
col_labels_seniority_2=['item_avg_sales_seniority_2']
target_features_df.append(moving_statistics(train_2,item_encoding_seniority_2,['month_id','item_id'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,item_encoding_seniority_2,['month_id','item_id'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,item_encoding_seniority_2,['month_id','item_id'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,item_encoding_seniority_2,['month_id','item_id'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,item_encoding_seniority_2,['month_id','item_id'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,item_encoding_seniority_2,['month_id','item_id'],col_labels_seniority_2,[12]))
target_features_df.append(time_mapping(train_2,item_encoding_seniority_2,['item_id'],col_labels_seniority_2,'item_month_id_of_last_sale',suffix='_last_sale'))
print('time : ' +str(time.time() - ts))
col_labels=['shop_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['supercategory_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_supercategory_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,shop_supercategory_encoding_seniority_2,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,shop_supercategory_encoding_seniority_2,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,shop_supercategory_encoding_seniority_2,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,shop_supercategory_encoding_seniority_2,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,shop_supercategory_encoding_seniority_2,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,shop_supercategory_encoding_seniority_2,['month_id','shop_id','item_supercategory_id'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['category_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,shop_category_encoding_seniority_2,['month_id','shop_id','item_category_id'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_encoding_seniority_2,['month_id','shop_id','item_category_id'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_encoding_seniority_2,['month_id','shop_id','item_category_id'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_encoding_seniority_2,['month_id','shop_id','item_category_id'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,shop_category_encoding_seniority_2,['month_id','shop_id','item_category_id'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,shop_category_encoding_seniority_2,['month_id','shop_id','item_category_id'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['category_months_since_release_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,category_months_since_release_encoding_seniority_2,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,category_months_since_release_encoding_seniority_2,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,category_months_since_release_encoding_seniority_2,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,category_months_since_release_encoding_seniority_2,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,category_months_since_release_encoding_seniority_2,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,category_months_since_release_encoding_seniority_2,['month_id','item_category_id','item_months_since_release'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_months_since_release_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,shop_category_months_since_release_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_months_since_release_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_months_since_release_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_months_since_release_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,shop_category_months_since_release_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,shop_category_months_since_release_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_release'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['supercategory_months_since_release_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,supercategory_months_since_release_encoding_seniority_2,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,supercategory_months_since_release_encoding_seniority_2,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,supercategory_months_since_release_encoding_seniority_2,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,supercategory_months_since_release_encoding_seniority_2,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,supercategory_months_since_release_encoding_seniority_2,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,supercategory_months_since_release_encoding_seniority_2,['month_id','item_supercategory_id','item_months_since_release'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['category_months_since_first_sale_in_shop_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_months_since_first_sale_in_shop_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,shop_category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,shop_category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,[1]))
target_features_df.append(lag_features(train_2,shop_category_months_since_first_sale_in_shop_encoding_seniority_2,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],col_labels_seniority_2,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_category_avg_sales_compared_to_category']
target_features_df.append(moving_statistics(train_2,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,34,'mean',suffix='_absolute_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,12,'mean',suffix='_annual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,6,'mean',suffix='_semiannual_mean'))
target_features_df.append(moving_statistics(train_2,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,[1]))
target_features_df.append(lag_features(train_2,shop_category_encoding,['month_id','shop_id','item_category_id'],col_labels,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(rational_fraction(train_2,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,shop_encoding,['month_id','shop_id'],col_labels,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(rational_fraction(train_2,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,category_encoding,['month_id','item_category_id'],['category_avg_sales'],weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['supercategory_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(rational_fraction(train_2,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,supercategory_encoding,['month_id','item_supercategory_id'],col_labels,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales_compared_to_supercategory']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(moving_statistics(train_2,category_encoding,['month_id','item_category_id'],col_labels,3,'mean',suffix='_recent_mean'))
target_features_df.append(lag_features(train_2,category_encoding,['month_id','item_category_id'],col_labels,[1]))
target_features_df.append(lag_features(train_2,category_encoding,['month_id','item_category_id'],col_labels,[12]))
print('time : ' +str(time.time() - ts))
col_labels=['shop_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(rational_fraction(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,shop_encoding_seniority_2,['month_id','shop_id'],col_labels_seniority_2,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['supercategory_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(rational_fraction(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,supercategory_encoding_seniority_2,['month_id','item_supercategory_id'],col_labels_seniority_2,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
col_labels=['category_avg_sales']
col_labels_seniority_2=[col+'_seniority_2' for col in col_labels]
target_features_df.append(rational_fraction(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,weights_num=[0,0,0,0,0,0,0,0,0,0,0,12],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag12_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,weights_num=[12,0,0,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_lag1_to_annual_mean'))
target_features_df.append(rational_fraction(train_2,category_encoding_seniority_2,['month_id','item_category_id'],col_labels_seniority_2,weights_num=[4,4,4,0,0,0,0,0,0,0,0,0],weights_denom=[1,1,1,1,1,1,1,1,1,1,1,1],suffix='_compare_recent_mean_to_annual_mean'))
print('time : ' +str(time.time() - ts))
col_labels=['category_month_of_release_avg_sales_compared_to_category']
target_features_df.append(moving_statistics(train_2,category_month_of_release_encoding,['month_id','item_category_id','item_month_of_release'],col_labels,34,'mean',suffix='_absolute_mean'))
col_labels=['supercategory_month_of_release_avg_sales_compared_to_supercategory']
target_features_df.append(moving_statistics(train_2,supercategory_month_of_release_encoding,['month_id','item_supercategory_id','item_month_of_release'],col_labels,34,'mean',suffix='_absolute_mean'))
print('time : ' +str(time.time() - ts))
train_2=pd.concat([train_2]+target_features_df,axis=1,sort=False)
 del target_features_df
gc.collect()
print('time : ' +str(time.time() - ts))
print(train_2.info(null_counts=True,verbose=True))
train_2
train = train.drop(['building_id','created','description','display_address',
'features','manager_id','photos','street_address'],axis=1)
 train['display_address_count'] = train['display_address_count'].astype("float")
 train['manager_id_count'] = train['manager_id_count'].astype("float")
<drop_column>
for df in dfs:
exec("del "+df)
 exec("del "+df+"_seniority_2")
 del shop_item_encoding
gc.collect()
test = train[train.train_test == "test"]
train = train[train.train_test == "train"]
train.pop("train_test")
 train_target = train.pop("interest_level")
 test = test.drop(['interest_level','train_test'],axis=1)
 train.head()<choose_model_class>
train_2.drop(features_to_discard,axis=1,inplace=True)
 train_2.info(null_counts=True,verbose=True )
gradient_booster = lgb.LGBMClassifier()<define_search_space>
create_directory(os.path.join(DATA_FOLDER, 'training'))
train_2.to_pickle(os.path.join(DATA_FOLDER,'training/train_2_pred.pkl'))
grid_parameters = {'n_estimators': [100,500,1000],'num_leaves': [10,30,50],
'reg_alpha' : [0.2,0.4,0.6],'learning_rate' : [0.05,0.1]}<choose_model_class>
del train_2
gc.collect()
gs = GridSearchCV(estimator=gradient_booster, param_grid = grid_parameters,
cv= 4, scoring= 'neg_log_loss' )<train_model>
reset_variable_space
gs.fit(train,train_target )<find_best_params>
def datasplit_train_val(train_df,n_months_val=1,month_id_first=0):
if month_id_first==0:
month_id_first=train_df['month_id'].min()
month_id_split=train_df['month_id'].max() -(n_months_val-1)
 X_train=train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<month_id_split),:].astype(np.float32)
 Y_train=X_train['item_quantity'].astype(np.float32)
 X_train.drop(['month_id','item_quantity'],axis=1,inplace=True)
 print(X_train.shape, Y_train.shape)
 X_val=train_df.loc[train_df['month_id']>=month_id_split,:].astype(np.float32)
 Y_val=X_val['item_quantity'].astype(np.float32)
 X_val.drop(['month_id','item_quantity'],axis=1,inplace=True)
 print(X_val.shape , Y_val.shape)
 return(X_train,Y_train,X_val,Y_val)
 def datasplit_train_test(train_df,month_id_first=0):
if month_id_first==0:
month_id_first=train_df['month_id'].min()
X_train=train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<34),:].astype(np.float32)
 Y_train=X_train['item_quantity'].astype(np.float32)
 X_train.drop(['month_id','item_quantity'],axis=1,inplace=True)
 print(X_train.shape, Y_train.shape)
 X_test=train_df.loc[train_df['month_id']==34,:].astype(np.float32)
 X_test.drop(['month_id','item_quantity'],axis=1,inplace=True)
 print(X_test.shape)
 return(X_train,Y_train,X_test)
 def datasplit_evalset(train_df,month_id_first=0):
if month_id_first==0:
month_id_first=train_df['month_id'].min()
eval_set=[]
for mid in train_df['month_id'].unique() :
if(mid>=month_id_first)&(mid<34):
X_val=train_df.loc[(train_df['month_id']==mid),:].astype(np.float32)
 Y_val=X_val['item_quantity'].astype(np.float32)
 X_val.drop(['month_id','item_quantity'],axis=1,inplace=True)
 eval_set.append(( X_val,Y_val))
return eval_set
best_log_loss = -1 * np.max(gs.cv_results_['mean_test_score'])
 print("La log-loss de validation croisée du meilleur parametrage est de " + str(best_log_loss))<find_best_params>
def target_analysis_val(train_df,n_months_val=1,month_id_first=0): 
(X_train,Y_train,X_val,Y_val)= datasplit_train_val(train_df,n_months_val,month_id_first)
 evalset = datasplit_evalset(train_df,month_id_first)
 base_score=Y_train.mean()
Y_guess_train=0*Y_train
rmse_guess_train=np.sqrt(mean_squared_error(Y_train, Y_guess_train))
Y_base_train=0*Y_train+base_score
rmse_base_train=np.sqrt(mean_squared_error(Y_train, Y_base_train))
print('
print('mean target value: '+str(base_score))
print('RMSE(guess,base): '+str(rmse_guess_train)+' , '+str(rmse_base_train))
base_score_val=Y_val.mean()
Y_guess_val=0*Y_val
rmse_guess_val=np.sqrt(mean_squared_error(Y_val, Y_guess_val))
Y_base_val=0*Y_val+base_score_val
rmse_base_val=np.sqrt(mean_squared_error(Y_val, Y_base_val))
print('
print('mean target value: '+str(base_score_val))
print('RMSE(guess,base): '+str(rmse_guess_val)+' , '+str(rmse_base_val))
base_score_evalset=[Y.mean() for(_,Y)in evalset]
rmse_guess_evalset=[]
rmse_base_evalset=[]
for(_,Y)in evalset:
Y_guess=0*Y
rmse_guess_evalset.append(np.sqrt(mean_squared_error(Y, Y_guess)))
 Y_base=0*Y+Y.mean()
rmse_base_evalset.append(np.sqrt(mean_squared_error(Y, Y_base)))
 print('
print('mean target value: '+str([round(b*1e4)/1e4 for b in base_score_evalset]))
print('RMSE(guess): '+str([round(b*1e2)/1e2 for b in rmse_guess_evalset]))
print('RMSE(base): '+str([round(b*1e2)/1e2 for b in rmse_base_evalset]))
fig,axes=plt.subplots(1,2,figsize=(15,5))
axes[0].plot(train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<34),'month_id'].unique() ,base_score_evalset,'-o')
 axes[0].set_xlabel('month_id')
 axes[0].set_ylabel('mean of target value')
 axes[0].grid(True)
 axes[0].set_ylim(bottom=0)
 axes[1].plot(train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<34),'month_id'].unique() ,rmse_guess_evalset,'-o')
 axes[1].plot(train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<34),'month_id'].unique() ,rmse_base_evalset,'-o')
 axes[1].set_xlabel('month_id')
 axes[1].set_ylabel('rmse')
 axes[1].legend(['guess = 0','mean value'])
 axes[1].grid(True)
 axes[1].set_ylim(bottom=0)
 return rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset
def target_analysis_test(train_df,month_id_first=0): 
(X_train,Y_train,_)= datasplit_train_test(train_df,month_id_first)
 evalset = datasplit_evalset(train_df,month_id_first)
 base_score=Y_train.mean()
base_score_evalset=[Y.mean() for(_,Y)in evalset]
Y_guess_train=0*Y_train
rmse_guess_train=np.sqrt(mean_squared_error(Y_train, Y_guess_train))
Y_base_train=0*Y_train+base_score
rmse_base_train=np.sqrt(mean_squared_error(Y_train, Y_base_train))
print('
print('mean target value: '+str(base_score))
print('RMSE(guess,base): '+str(rmse_guess_train)+' , '+str(rmse_base_train))
rmse_guess_evalset=[]
rmse_base_evalset=[]
for(_,Y)in evalset:
Y_guess=0*Y
rmse_guess_evalset.append(np.sqrt(mean_squared_error(Y, Y_guess)))
 Y_base=0*Y+Y.mean()
rmse_base_evalset.append(np.sqrt(mean_squared_error(Y, Y_base)))
 print('
print('mean target value: '+str([round(b*1e4)/1e4 for b in base_score_evalset]))
print('RMSE(guess): '+str([round(b*1e2)/1e2 for b in rmse_guess_evalset]))
print('RMSE(base): '+str([round(b*1e2)/1e2 for b in rmse_base_evalset]))
fig,axes=plt.subplots(1,2,figsize=(15,5))
axes[0].plot(train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<34),'month_id'].unique() ,base_score_evalset,'-o')
 axes[0].set_xlabel('month_id')
 axes[0].set_ylabel('mean of target value')
 axes[0].grid(True)
 axes[0].set_ylim(bottom=0)
 axes[1].plot(train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<34),'month_id'].unique() ,rmse_guess_evalset,'-o')
 axes[1].plot(train_df.loc[(train_df['month_id']>=month_id_first)&(train_df['month_id']<34),'month_id'].unique() ,rmse_base_evalset,'-o')
 axes[1].set_xlabel('month_id')
 axes[1].set_ylabel('rmse')
 axes[1].legend(['guess = 0','mean value'])
 axes[1].grid(True)
 axes[1].set_ylim(bottom=0)
 return rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset
gs.best_estimator_<find_best_params>
def performance_analysis_val(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_val,rmse_base_val,rmse_guess_evalset,rmse_base_evalset):
rmse_evalset_last=[]
rmse_evalset_min=[]
for evset in xgbreg.evals_result().values() :
rmse_evalset_last.append(evset['rmse'][-1])
 rmse_evalset_min.append(min(evset['rmse']))
print('
print('RMSE(last, min, base, guess): '+str(rmse_evalset_last[0])+' , '+str(rmse_evalset_min[0])+' , '+str(rmse_base_train)+' , '+str(rmse_guess_train))
print()
print('
print('RMSE(last, min, base, guess): '+str(rmse_evalset_last[1])+' , '+str(rmse_evalset_min[1])+' , '+str(rmse_base_val)+' , '+str(rmse_guess_val))
print()
print('
print('RMSE(last): '+str([round(b*1e4)/1e4 for b in rmse_evalset_last[2:]]))
print('RMSE(min): '+str([round(b*1e4)/1e4 for b in rmse_evalset_min[2:]]))
print('RMSE(guess): '+str([round(b*1e4)/1e4 for b in rmse_guess_evalset]))
print('RMSE(base): '+str([round(b*1e4)/1e4 for b in rmse_base_evalset]))
fig,axes=plt.subplots(1,2,figsize=(15,5))
for i in range(2,len(xgbreg.evals_result())) :
evset=xgbreg.evals_result() ['validation_'+str(i)]['rmse']
axes[0].plot(evset,'-o')
 axes[0].legend(range(0,len(eval_set)))
 axes[0].plot(xgbreg.evals_result() ['validation_0']['rmse'],'k',linewidth=5)
 axes[0].plot(xgbreg.evals_result() ['validation_1']['rmse'],'k--',linewidth=5)
 axes[1].plot(rmse_evalset_last[2:],'-o')
 axes[1].plot(0*np.array(rmse_evalset_last[2:])+rmse_evalset_last[0],'k')
 axes[1].plot(0*np.array(rmse_evalset_last[2:])+rmse_evalset_last[1],'k--')
 axes[1].plot(rmse_evalset_min[2:],'-o')
 axes[1].plot(0*np.array(rmse_evalset_min[2:])+rmse_evalset_last[0],'r')
 axes[1].plot(0*np.array(rmse_evalset_min[2:])+rmse_evalset_last[1],'r--')
 axes[1].plot(rmse_guess_evalset,'k-s')
 axes[1].set_xlabel('month_id')
 axes[1].set_ylabel('rmse(last,min)')
 axes[1].grid(True)
 axes[1].set_ylim(bottom=0)
 def performance_analysis_test(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_evalset,rmse_base_evalset):
rmse_evalset_last=[]
rmse_evalset_min=[]
for evset in xgbreg.evals_result().values() :
rmse_evalset_last.append(evset['rmse'][-1])
 rmse_evalset_min.append(min(evset['rmse']))
print('
print('RMSE(last, min, base, guess): '+str(rmse_evalset_last[0])+' , '+str(rmse_evalset_min[0])+' , '+str(rmse_base_train)+' , '+str(rmse_guess_train))
print()
print('
print('RMSE(last): '+str([round(b*1e4)/1e4 for b in rmse_evalset_last[1:]]))
print('RMSE(min): '+str([round(b*1e4)/1e4 for b in rmse_evalset_min[1:]]))
print('RMSE(base): '+str([round(b*1e4)/1e4 for b in rmse_base_evalset]))
print('RMSE(guess): '+str([round(b*1e4)/1e4 for b in rmse_guess_evalset]))
fig,axes=plt.subplots(1,2,figsize=(15,5))
for i in range(1,len(xgbreg.evals_result())) :
evset=xgbreg.evals_result() ['validation_'+str(i)]['rmse']
axes[0].plot(evset,'-o')
 axes[0].legend(range(0,len(eval_set)))
 axes[0].plot(xgbreg.evals_result() ['validation_0']['rmse'],'k',linewidth=5)
 axes[1].plot(rmse_evalset_last[1:],'-o')
 axes[1].plot(0*np.array(rmse_evalset_last[1:])+rmse_evalset_last[0],'k')
 axes[1].plot(rmse_evalset_min[1:],'-o')
 axes[1].plot(0*np.array(rmse_evalset_min[1:])+rmse_evalset_last[0],'r')
 axes[1].plot(rmse_guess_evalset,'k-s')
 axes[1].set_xlabel('month_id')
 axes[1].set_ylabel('rmse(last,min)')
 axes[1].grid(True)
 axes[1].set_ylim(bottom=0 )
gs.best_estimator_.classes_<prepare_output>
loaded=%who_ls
loaded.append('loaded' )
predictions = gs.predict_proba(test)
 predictions = pd.DataFrame(predictions)
 predictions.columns = ['high','low','medium']<prepare_output>
dataset_name='train_0_pred'
n_months_val=2
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
train_X.drop(train_X.loc[train_X['month_id']==34].index,axis=0,inplace=True)
 print(train_X.info(null_counts=True,verbose=True))
submission['high'] = predictions['high']
submission['medium'] = predictions['medium']
submission['low'] = predictions['low']
submission.head()<save_to_csv>
rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset = target_analysis_val(train_X,n_months_val=2,month_id_first=18 )
submission.to_csv('projet ML.csv', index=False);<load_pretrained>
features_to_keep_0=[]
features_to_keep_0+=['item_category_id']
features_to_keep_0+=['item_category_freq']
features_to_keep_0+=['item_supercategory_freq']
features_to_keep_0+=['item_category_console_freq']
features_to_keep_0+=['item_category_freq_in_seniority']
features_to_keep_0+=['item_supercategory_freq_in_seniority']
features_to_keep_0+=['item_category_console_freq_in_seniority']
features_to_keep_0+=['shop_category_price_median_absolute_mean']
features_to_keep_0+=['shop_category_price_median_lag_12']
features_to_keep_0+=['category_price_median_recent_mean']
features_to_keep_0+=['category_price_median_lag_12']
features_to_keep_0+=['supercategory_price_median_lag_12']
features_to_keep_0+=['category_price_min_absolute_min']
features_to_keep_0+=['category_price_max_absolute_max']
features_to_keep_0+=['category_price_min_annual_min']
features_to_keep_0+=['category_price_max_annual_max']
features_to_keep_0+=['category_price_median_compared_to_supercategory_price_median_absolute_mean']
features_to_keep_0+=['shop_avg_sales_seniority_0_semiannual_mean']
features_to_keep_0+=['supercategory_avg_sales_seniority_0_semiannual_mean']
features_to_keep_0+=['supercategory_avg_sales_seniority_0_lag_12']
features_to_keep_0+=['shop_supercategory_avg_sales_seniority_0_annual_mean']
features_to_keep_0+=['shop_supercategory_avg_sales_seniority_0_lag_12']
features_to_keep_0+=['category_avg_sales_seniority_0_annual_mean']
features_to_keep_0+=['category_avg_sales_seniority_0_semiannual_mean']
features_to_keep_0+=['shop_category_avg_sales_seniority_0_annual_mean']
features_to_keep_0+=['shop_category_avg_sales_seniority_0_semiannual_mean']
features_to_keep_0+=['shop_category_avg_sales_seniority_0_recent_mean']
features_to_keep_0+=['shop_category_avg_sales_seniority_0_lag_1']
features_to_keep_0+=['shop_category_avg_sales_seniority_0_lag_12']
features_to_keep_0+=['shop_category_avg_sales_compared_to_category_annual_mean']
features_to_keep_0+=['shop_category_avg_sales_compared_to_category_recent_mean']
features_to_keep_0+=['shop_category_avg_sales_compared_to_category_lag_12']
features_to_keep_0+=['shop_supercategory_avg_sales_compared_to_supercategory_recent_mean']
features_to_keep_0+=['supercategory_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_0+=['supercategory_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_0+=['category_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_0+=['category_avg_sales_compared_to_supercategory_recent_mean']
features_to_keep_0+=['supercategory_avg_sales_seniority_0_compare_lag12_to_annual_mean']
def load_json_from_path(path):
try:
with open(path, "r")as f:
return json.load(f)
 except OSError as error:
if error.errno != errno.ENOENT:
raise error
return None
def json_to_pandas_dataframe(dictionary):
columns = list(dictionary.keys())
 rows = sorted(list(set(itertools.chain.from_iterable([
list(dictionary[k].keys())
 for k in columns
]))), key=lambda x: int(x))
data = list(map(list, zip(*([rows] + [
[
dictionary[column][r] if r in dictionary[column] else None
for r in rows
]
for column in columns
]))))
df = pd.DataFrame(data, columns=['id'] + columns)
 df.set_index('id')
 return df
def load_training_test_data(training_data_path, test_data_path):
return(
 json_to_pandas_dataframe(load_json_from_path(training_data_path)) ,
json_to_pandas_dataframe(load_json_from_path(test_data_path)) 
)

def string_to_category_name(string):
return string.lower().replace(" ", "_")
 def categories_from_column(data_frame, column):
return list(set(list(itertools.chain.from_iterable(
 data_frame[column].tolist() 
))))
def normalize_whitespace(string):
return re.sub(r"\s+", " ", string)
 def normalize_category(category):
return normalize_whitespace(re.sub(r"[\*\-\!\&]", " ", category.lower())).strip()
def normalize_categories(categories):
return [
normalize_category(c)for c in categories
]
def sliding_window(sequence, n):

 it = iter(sequence)
 window = deque(( next(it, None)for _ in range(n)) , maxlen=n)
 yield list(window)
 for element in it:
window.append(element)
 yield list(window)
 def create_ngrams(content, n):
for ngram in sliding_window(content.split() , n):
yield " ".join(ngram)
 def create_ngrams_up_to_n(content, n):
for i in range(n):
yield from create_ngrams(content, i)
 def count_ngrams_up_to_n(content, n):
return Counter(list(create_ngrams_up_to_n(content, n)))
 def remove_small_or_stopwords_from_ranking(ranking, nlp, min_len):
for word, rank in ranking:
if nlp.vocab[word].is_stop or len(word)< min_len:
continue
yield word, rank
def column_list_to_category_flags(data_frame, column, grams):
categories = [
"{}_{}".format(column, string_to_category_name(n))
for n in grams
]
row_cleaned_categories = [
normalize_category(" ".join(r))
for r in data_frame[column].tolist()
]
category_flags = pd.DataFrame.from_records([
[1 if gram in r else 0 for gram in grams]
for r in row_cleaned_categories
], columns=categories)
 return pd.concat(( data_frame, category_flags), axis=1)
 def remap_column(data_frame, column, new_column, mapping):
data_frame[new_column] = data_frame[column].transform(mapping)
 return data_frame
def remap_date_column_to_days_before(data_frame,
column,
new_column,
reference_date):
data_frame[new_column] = data_frame[column].transform(
 lambda x:(reference_date - datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")).days 
)
 return data_frame
def map_categorical_column_to_category_ids(train_data_frame,
test_data_frame,
column,
new_column,
min_freq=1):
category_counts = Counter(train_data_frame[column])+ Counter(test_data_frame[column])
 category_to_unknown_mapping = {
category: category if count >= min_freq else "Unknown"
for category, count in category_counts.items()
}
category_to_id_map = {
category: i
for i, category in enumerate(sorted([
category_to_unknown_mapping[c] for c in 
(set(train_data_frame[column])| set(test_data_frame[column]))
]))
}
id_to_category_map = {
i: category
for category, i in category_to_id_map.items()
}
return(
 category_to_unknown_mapping,
category_to_id_map,
id_to_category_map,
remap_column(train_data_frame,
column,
new_column,
lambda x: category_to_id_map[category_to_unknown_mapping[x]]),
remap_column(test_data_frame,
column,
new_column,
lambda x: category_to_id_map[category_to_unknown_mapping[x]])
)
 def remap_columns_with_transform(train_data_frame,
test_data_frame,
column,
new_column,
transform):

 return(
 remap_column(train_data_frame,
column,
new_column,
transform),
remap_column(test_data_frame,
column,
new_column,
transform)
)
 def normalize_description(description):

 description = description.lower()
description = re.sub(r"<[^<]+?(>|$)", " ", description)
 description = re.sub(r"[0-9\-]+", " ", description)
 description = re.sub(r"[a-z0-9]@[a-z0-9]\.[a-z]", " ", description)
 description = re.sub(r"[\!]+", "! ", description)
 description = re.sub(r"[\-\:]", " ", description)
 description = re.sub("\*", " ", description)
 return re.sub(r"\s+", " ", description)
 def add_epsilon(array):
return np.array([a + 10e-10 if a == 0 else a for a in array])
 def numerical_feature_engineering_on_dataframe(dataframe,
numerical_columns):

 for lhs_column, rhs_column in itertools.combinations(numerical_columns, 2):
dataframe['{}_add_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] + dataframe[rhs_column]
dataframe['{}_sub_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] - dataframe[rhs_column]
dataframe['{}_mul_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] * dataframe[rhs_column]
dataframe['{}_div_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] / add_epsilon(dataframe[rhs_column])
 return dataframe
def numerical_feature_engineering(train_data_frame,
test_data_frame,
numerical_columns):

 return(
 numerical_feature_engineering_on_dataframe(train_data_frame,
numerical_columns),
numerical_feature_engineering_on_dataframe(test_data_frame,
numerical_columns), 
)
 def normalize_eastwest(eastwest):
eastwest = eastwest.lower().strip()
if not eastwest:
return ""
if eastwest[0] == "e":
return "e"
elif eastwest[0] == "w":
return "w"
else:
return ""
STREET_MAPPING = {
"st": "street",
"ave": "avenue",
"pl": "place",
"rd": "road"
}
def normalize_name(name):
m = re.match(r"(?P<address>[\w\s]+ )(?P<st>st|street|ave|avenue|place|pl|road|rd ).*",
name.lower().strip())
 if not m:
return name.lower().strip()
return "{address} {street}".format(
 address=m.groupdict() ["address"].strip() ,
street=STREET_MAPPING.get(m.groupdict() ["st"], m.groupdict() ["st"])
)
 def normalize_address(address_dict):
return "{eastwest} {name}".format(
 eastwest=normalize_eastwest(address_dict["eastwest"] or ""),
name=normalize_name(address_dict["name"] or "")
)
 def parse_address_components_from_address(address):
m = re.match(r"(?P<number>[0-9]*\s+)?\s*(?P<eastwest>East|West|E\s|W\s)?\s*(?P<name>[A-Za-z0-9\.\-\s]*)",
normalize_whitespace(address),
flags=re.IGNORECASE)
 return {
"normalized": normalize_address(m.groupdict())if m is not None else address
}
def parse_address_components_for_column(dataframe, column):
return pd.concat(( dataframe, pd.DataFrame.from_records([
{
"{}_{}".format(column, key): value for key, value in
parse_address_components_from_address(cell ).items()
}
for cell in dataframe[column]
])) , axis=1)
 def parse_address_components(train_data_frame,
test_data_frame,
columns):
return(
 functools.reduce(lambda df, c: parse_address_components_for_column(df,
c),
columns,
train_data_frame),
functools.reduce(lambda df, c: parse_address_components_for_column(df,
c),
columns,
test_data_frame)
)
 def count_json_column(dataframe, column):
return pd.DataFrame([
len(c)for c in dataframe[column]
])
 def count_json(train_data_frame,
test_data_frame,
column):
train_data_frame["{}_count".format(column)] = count_json_column(train_data_frame,
column)
 test_data_frame["{}_count".format(column)] = count_json_column(test_data_frame,
column)
 return train_data_frame, test_data_frame

 def maybe_cuda(tensor):

 if torch.cuda.is_available() :
return tensor.cuda()
return tensor.cpu()
class Doc2Vec(nn.Module):

 def __init__(self,
embedding_size,
hidden_layer_size,
vocab_size,
output_size,
batch_size):
super().__init__()
self.hidden_dim = hidden_layer_size
self.embedding = nn.Embedding(vocab_size, embedding_size)
 self.batch_size = batch_size
self.embedding_size = embedding_size
self.hidden =(maybe_cuda(torch.randn(2, batch_size, self.hidden_dim)) ,
maybe_cuda(torch.randn(2, batch_size, self.hidden_dim)))
 self.lstm = nn.LSTM(embedding_size,
self.hidden_dim,
num_layers=1,
bidirectional=True)
 self.linear = nn.Linear(self.hidden_dim * 2, output_size)
 def sentence_embedding(self, sentence):
self.hidden =(maybe_cuda(torch.randn(2, self.batch_size, self.hidden_dim)) ,
maybe_cuda(torch.randn(2, self.batch_size, self.hidden_dim)))
 embeddings = self.embedding(sentence)
 out, self.hidden = self.lstm(embeddings.view(-1, self.batch_size, self.embedding_size),
self.hidden)
 added = self.hidden[0] + self.hidden[1]
return added / torch.norm(added)
 def forward(self, sentence):
embedding = self.sentence_embedding(sentence)
 lin = self.linear(embedding.view(-1, self.hidden_dim * 2))
return F.softmax(lin, dim=1)
 def train_model(model, optimizer, epochs, sentence_tensors, label_tensors):
for epoch in range(epochs):
total_loss = 0
loss_criterion = nn.CrossEntropyLoss()
shuffled_sentence_tensors, shuffled_label_tensors = shuffle(
 sentence_tensors, label_tensors 
)
 progressable_tensors = tqdm(
 zip(shuffled_sentence_tensors, shuffled_label_tensors),
total=len(shuffled_label_tensors),
desc="Processing sentence vectors" 
)
 for sentence_tensor, label_tensor in progressable_tensors:
optimizer.zero_grad()
preds = model(sentence_tensor)
 loss = loss_criterion(preds,
label_tensor)
 loss.backward()
optimizer.step()
total_loss += loss.item()
progressable_tensors.set_postfix(loss=loss.item())
 print('Epoch', epoch, 'total loss', total_loss)
 with torch.no_grad() :
print("Model Accuracy:",
compute_model_accuracy(model,
shuffled_sentence_tensors[0],
shuffled_label_tensors[0]))
return model
def characters_to_one_hot_lookups(all_characters):
set_characters = set(all_characters)
 character_to_one_hot = {
c: i for i, c in enumerate(sorted(set_characters))
}
one_hot_to_character = {
c: i for i, c in enumerate(sorted(set_characters))
}
return character_to_one_hot, one_hot_to_character
def character_sequence_to_matrix(sentence, character_to_one_hot):
return np.array([character_to_one_hot[c] for c in sentence])
 def compute_model_accuracy(model, sentence_tensors, label_tensors):

 predictions = np.argmax(model(maybe_cuda(sentence_tensors)).detach().cpu().numpy() , axis=1 ).flatten()
labels = label_tensors.detach().cpu().numpy().flatten()
return len([p for p in(predictions == labels)if p == True])/ len(predictions)
 def pad_sentence(sentence, padding):
truncated = sentence[:padding]
return truncated +(" " *(padding - len(truncated)))
 def to_batches(sentences, batch_size, pad_value):
for i in range(math.ceil(len(sentences)/ batch_size)) :
batch = [
sentences[i * batch_size + j]
for j in range(min(( batch_size, len(sentences[i * batch_size:]))))
]
padding = [
pad_value
for k in range(max(0, batch_size - len(sentences[i * batch_size:])))
 ]
yield torch.stack(batch + padding, dim=0)
 def documents_to_vectors_model(train_documents,
test_documents,
labels,
epochs,
parameters,
learning_rate,
load=None,
save=None,
sentence_length=1000,
batch_size=200):

 character_to_one_hot, one_hot_to_character = characters_to_one_hot_lookups(
 "".join(train_documents)+ "".join(test_documents)
)
 train_sentence_tensors = list(to_batches([
maybe_cuda(torch.tensor(character_sequence_to_matrix(pad_sentence(sentence, sentence_length),
character_to_one_hot), dtype=torch.long))
for sentence in train_documents
], batch_size, maybe_cuda(torch.tensor([character_to_one_hot[" "] for i in range(sentence_length)]))))
label_tensors = list(to_batches([maybe_cuda(torch.tensor(i)) for i in labels],
batch_size,
maybe_cuda(torch.tensor(0))))
model = maybe_cuda(Doc2Vec(parameters,
parameters * 2,
len(character_to_one_hot.keys()), max(labels)+ 1,
batch_size))
if not load:
optimizer = optim.Adam(model.parameters() , lr=learning_rate)
 try:
train_model(model,
optimizer,
epochs,
train_sentence_tensors,
label_tensors)
 except KeyboardInterrupt:
print("Interrupted, saving current state now")
 else:
model.load_state_dict(torch.load(load))
if save:
torch.save(model.state_dict() , save)
 return character_to_one_hot, one_hot_to_character, model
def generate_document_vector_embeddings_from_model(model,
character_to_one_hot,
sentences,
sentence_length,
batch_size):
sentence_tensors = list(to_batches([
maybe_cuda(torch.tensor(character_sequence_to_matrix(pad_sentence(sentence, sentence_length),
character_to_one_hot), dtype=torch.long))
for sentence in sentences
], batch_size, maybe_cuda(torch.tensor([character_to_one_hot[" "] for i in range(sentence_length)]))))
with torch.no_grad() :
return np.row_stack([
model.sentence_embedding(sentence_tensor ).detach().cpu().numpy()
for sentence_tensor in sentence_tensors
] ).reshape(-1, model.hidden_dim * 2)[:len(sentences)]
def column_to_doc_vectors(train_data_frame,
test_data_frame,
description_column,
target_column,
document_vector_column,
epochs=100,
parameters=40,
learning_rate=0.01,
load=None,
save=None,
sentence_length=1000,
batch_size=200):

 train_descriptions = list(train_data_frame[description_column])
 test_descriptions = list(test_data_frame[description_column])
 labels = list(train_data_frame[target_column])
 character_to_one_hot, one_hot_to_character, model = documents_to_vectors_model(
 train_descriptions,
test_descriptions,
labels,
epochs,
parameters,
learning_rate,
load=load,
save=save,
sentence_length=sentence_length,
batch_size=batch_size 
)
 train_description_vectors = pd.DataFrame(
 generate_document_vector_embeddings_from_model(
 model,
character_to_one_hot,
train_descriptions,
sentence_length,
batch_size 
)
)
 test_description_vectors = pd.DataFrame(
 generate_document_vector_embeddings_from_model(
 model,
character_to_one_hot,
test_descriptions,
sentence_length,
batch_size 
)
)
 return(
 pd.concat(( train_data_frame, train_description_vectors), axis=1),
pd.concat(( test_data_frame, test_description_vectors), axis=1)
)

def sklearn_pipeline_steps(categorical_columns, verbose=False):
return(
('one_hot',
OneHotEncoder(cols=categorical_columns, verbose=verbose)) , 
('scaling', StandardScaler())
)
 def basic_logistic_regression_pipeline(categorical_columns,
verbose=False):
return Pipeline((
*sklearn_pipeline_steps(categorical_columns, verbose=verbose), 
('xgb', xgb.XGBClassifier(
 n_estimators=1000,
seed=42,
objective='multi:softprob',
subsample=0.8,
colsample_bytree=0.8, 
)) 
))
def calculate_statistics(statistics, test_labels, predictions):
return {
k: s(test_labels, predictions)
 for k, s in statistics.items()
}
def format_statistics(calculated_statistics):
return ", ".join([
"{0}: {1:.2f}".format(k, s)
 for k, s in calculated_statistics.items()
])
 def prediction_accuracy(labels, predictions):
return(
 len([a for a, b in zip(labels, predictions)if a == b])/ len(predictions)
)
 def fit_one_split(model, features, labels, statistics, train_index, test_index):
train_data, train_labels = features.iloc[train_index], labels[train_index]
test_data, test_labels = features.iloc[test_index], labels[test_index]
model.fit(train_data, train_labels)
 predictions = model.predict(test_data)
 return(
 test_labels,
predictions,
calculate_statistics(statistics, test_labels, predictions)
)
 def test_model_with_k_fold_cross_validation(model,
features,
labels,
statistics,
n_splits=5,
random_state=42):
kf = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 test_labels, predictions = [], []
for i,(train_index, test_index)in enumerate(kf.split(features, labels)) :
fold_test_labels, fold_predictions, calculated_statistics = fit_one_split(
 model,
features,
labels,
statistics,
train_index,
test_index 
)
 print('Fold', i, format_statistics(calculated_statistics))
test_labels.extend(fold_test_labels)
 predictions.extend(fold_predictions)
 return(
 calculate_statistics(statistics, test_labels, predictions),
test_labels,
predictions,
model 
)
 def get_prediction_probabilities_with_columns(model,
test_dataframe,
keep_columns):
return pd.concat(( test_dataframe[keep_columns],
pd.DataFrame(model.predict_proba(test_dataframe.drop(keep_columns, axis=1)))) ,
axis=1)
<load_pretrained>
features_to_discard_0=list(set(list(train_X.columns)) - set(['month_id','item_quantity']+features_to_keep_0))
train_X.drop(features_to_discard_0,axis=1,inplace=True)
 train_X.fillna(-1,inplace=True)
(X_train,Y_train,X_val,Y_val)= datasplit_train_val(train_X,n_months_val,month_id_first)
 eval_set = [(X_train,Y_train),(X_val,Y_val)]+datasplit_evalset(train_X,month_id_first)
 print('number of features to keep : '+str(len(features_to_keep_0)))
 print('number of features kept : '+str(len(X_train.columns)))
 print(X_train.info(null_counts=True,verbose=True))
del train_X
gc.collect()
nlp = spacy.load("en" )<set_options>
xgb_params_0={'objective': 'reg:squarederror',
'max_depth': 4,
'min_child_weight': 1e3,
'subsample': 0.8,
'colsample_bytree': 0.8,
'learning_rate': 0.01,
'n_estimators': 300,
'base_score': 0.6,
'n_jobs': 4,
'random_state': 2
}
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_0)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )
torch.cuda.is_available()<load_pretrained>
del X_train, Y_train, X_val, Y_val
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset
del xgbreg
gc.collect()
( TRAIN_DATAFRAME, TEST_DATAFRAME)= \
load_training_test_data(os.path.join('.. ', 'input', 'train.json'),
os.path.join('.. ', 'input', 'test.json'))<sort_values>
reset_variable_space
normalized_categories = sorted(normalize_categories(categories_from_column(TRAIN_DATAFRAME, 'features')))
 normalized_categories[:50]<sort_values>
dataset_name='train_1_pred'
n_months_val=2
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
train_X.drop(train_X.loc[train_X['month_id']==34].index,axis=0,inplace=True)
 print(train_X.info(null_counts=True,verbose=True))
most_common_ngrams = sorted(count_ngrams_up_to_n(" ".join(normalized_categories), 3 ).most_common() ,
key=lambda x:(-x[1], x[0]))
most_common_ngrams[:50]<sort_values>
rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset = target_analysis_val(train_X,n_months_val=2,month_id_first=18 )
most_common_ngrams = sorted(list(remove_small_or_stopwords_from_ranking(most_common_ngrams, nlp, 3)) ,
key=lambda x:(-x[1], x[0]))
most_common_ngrams[:50]<feature_engineering>
features_to_keep_1=[]
features_to_keep_1+=['item_category_id']
features_to_keep_1+=['item_category_freq']
features_to_keep_1+=['item_supercategory_freq']
features_to_keep_1+=['item_category_console_freq']
features_to_keep_1+=['item_category_digital_freq']
features_to_keep_1+=['shop_months_since_opening']
features_to_keep_1+=['shop_opening']
features_to_keep_1+=['item_freq_in_seniority']
features_to_keep_1+=['item_months_since_release']
features_to_keep_1+=['item_price_median_absolute_mean']
features_to_keep_1+=['item_price_median_last_sale']
features_to_keep_1+=['item_price_min_absolute_min']
features_to_keep_1+=['item_price_max_absolute_max']
features_to_keep_1+=['category_price_median_recent_mean']
features_to_keep_1+=['item_max_quantity_absolute_max']
features_to_keep_1+=['item_avg_sales_recent_mean']
features_to_keep_1+=['item_avg_sales_lag_1']
features_to_keep_1+=['item_avg_sales_last_sale']
features_to_keep_1+=['item_avg_sales_over_sold_recent_mean']
features_to_keep_1+=['item_avg_sales_over_sold_lag_1']
features_to_keep_1+=['item_avg_sales_over_sold_last_sale']
features_to_keep_1+=['item_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['item_avg_sales_seniority_1_lag_1']
features_to_keep_1+=['item_avg_sales_seniority_1_last_sale']
features_to_keep_1+=['shop_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_avg_sales_lag_12']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_annual_mean']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_recent_mean']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_lag_1']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_lag_12']
features_to_keep_1+=['shop_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_1+=['supercategory_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_1+=['supercategory_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_1+=['category_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_1+=['category_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_1+=['category_avg_sales_compared_to_supercategory_recent_mean']
features_to_keep_1+=['category_avg_sales_compared_to_supercategory_lag_12']
features_to_keep_1+=['supercategory_month_of_release_avg_sales_compared_to_supercategory_absolute_mean']
features_to_keep_1+=['category_month_of_release_avg_sales_compared_to_category_absolute_mean']
TRAIN_DATAFRAME = column_list_to_category_flags(TRAIN_DATAFRAME, 'features', list(map(operator.itemgetter(0), most_common_ngrams[:100])))
 TEST_DATAFRAME = column_list_to_category_flags(TEST_DATAFRAME, 'features', list(map(operator.itemgetter(0), most_common_ngrams[:100])) )<categorify>
features_to_discard_1=list(set(list(train_X.columns)) - set(['month_id','item_quantity']+features_to_keep_1))
train_X.drop(features_to_discard_1,axis=1,inplace=True)
 train_X.fillna(-1,inplace=True)
(X_train,Y_train,X_val,Y_val)= datasplit_train_val(train_X,n_months_val,month_id_first)
 eval_set = [(X_train,Y_train),(X_val,Y_val)]+datasplit_evalset(train_X,month_id_first)
 print('number of features to keep : '+str(len(features_to_keep_1)))
 print('number of features kept : '+str(len(X_train.columns)))
 print(X_train.info(null_counts=True,verbose=True))
del train_X
gc.collect()
TRAIN_DATAFRAME = remap_date_column_to_days_before(TRAIN_DATAFRAME, "created", "created_days_ago", datetime.datetime(2017, 1, 1))
TEST_DATAFRAME = remap_date_column_to_days_before(TEST_DATAFRAME, "created", "created_days_ago", datetime.datetime(2017, 1, 1))<categorify>
xgb_params_1={'objective': 'reg:squarederror',
'max_depth': 5,
'min_child_weight': 1e3,
'subsample': 0.8,
'colsample_bytree': 0.8,
'learning_rate': 0.05,
'n_estimators': 500,
'base_score': 0.05,
'n_jobs': 4,
'random_state': 2
}
Y_train.clip(0,6,inplace=True)
 ts=time.time()
xgbreg=XGBRegressor(**xgb_params_1)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )
TRAIN_DATAFRAME = remap_column(TRAIN_DATAFRAME, "interest_level", "label_interest_level", lambda x: {
"high": 0,
"medium": 1,
"low": 2
}[x])
 TEST_DATAFRAME["label_interest_level"] = 0<categorify>
del X_train, Y_train, X_val, Y_val
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset
del xgbreg
gc.collect()
( BUILDING_ID_UNKNOWN_REMAPPING,
BUILDING_ID_TO_BUILDING_CATEGORY,
BUILDING_CATEGORY_TO_BUILDING_ID,
TRAIN_DATAFRAME,
TEST_DATAFRAME)= map_categorical_column_to_category_ids(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'building_id',
'building_id_category',
min_freq=40
)<categorify>
reset_variable_space
( MANAGER_ID_UNKNOWN_REMAPPING,
MANAGER_ID_TO_MANAGER_CATEGORY,
MANAGER_CATEGORY_TO_MANAGER_ID,
TRAIN_DATAFRAME,
TEST_DATAFRAME)= map_categorical_column_to_category_ids(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'manager_id',
'manager_id_category',
min_freq=40
)<string_transform>
dataset_name='train_2_pred'
n_months_val=2
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
train_X.drop(train_X.loc[train_X['month_id']==34].index,axis=0,inplace=True)
 print(train_X.info(null_counts=True,verbose=True))
TRAIN_DATAFRAME, TEST_DATAFRAME = parse_address_components(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
[
"display_address",
"street_address"
]
)<categorify>
rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset = target_analysis_val(train_X,n_months_val=2,month_id_first=18 )
( DISP_ADDR_ID_UNKNOWN_REMAPPING,
DISP_ADDR_TO_DISP_ADDR_CATEGORY,
DISP_ADDR_CATEGORY_TO_DISP_ADDR_ID,
TRAIN_DATAFRAME,
TEST_DATAFRAME)= map_categorical_column_to_category_ids(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'display_address_normalized',
'display_address_category',
min_freq=10
)<count_values>
features_to_keep_2=[]
features_to_keep_2+=['item_category_id']
features_to_keep_2+=['item_category_freq']
features_to_keep_2+=['item_supercategory_freq']
features_to_keep_2+=['item_category_console_freq']
features_to_keep_2+=['category_semiannual_avg_recent_rsd']
features_to_keep_2+=['category_semiannual_avg_annual_rsd']
features_to_keep_2+=['shop_months_since_opening']
features_to_keep_2+=['item_months_since_release']
features_to_keep_2+=['item_months_since_first_sale_in_shop']
features_to_keep_2+=['item_months_since_last_sale_in_shop']
features_to_keep_2+=['item_price_median_absolute_mean']
features_to_keep_2+=['item_price_median_last_sale']
features_to_keep_2+=['item_price_min_absolute_min']
features_to_keep_2+=['item_price_max_absolute_max']
features_to_keep_2+=['item_price_median_compared_to_category_price_median_absolute_mean']
features_to_keep_2+=['item_price_median_compared_to_category_price_median_last_sale']
features_to_keep_2+=['category_price_median_recent_mean']
features_to_keep_2+=['item_quantity_annual_mean']
features_to_keep_2+=['item_quantity_semiannual_mean']
features_to_keep_2+=['item_quantity_recent_mean']
features_to_keep_2+=['item_quantity_lag_1']
features_to_keep_2+=['item_quantity_lag_12']
features_to_keep_2+=['item_quantity_extralin2']
features_to_keep_2+=['item_quantity_extralin3']
features_to_keep_2+=['item_quantity_absolute_min']
features_to_keep_2+=['item_quantity_absolute_max']
features_to_keep_2+=['item_avg_sales_recent_mean']
features_to_keep_2+=['item_avg_sales_lag_1']
features_to_keep_2+=['item_avg_sales_last_sale']
features_to_keep_2+=['item_avg_sales_over_sold_recent_mean']
features_to_keep_2+=['item_avg_sales_over_sold_lag_1']
features_to_keep_2+=['item_avg_sales_over_sold_lag_12']
features_to_keep_2+=['item_avg_sales_over_sold_last_sale']
features_to_keep_2+=['shop_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_lag_1']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['shop_supercategory_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['category_avg_sales_seniority_2_annual_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_semiannual_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_lag_1']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_annual_mean']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_semiannual_mean']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_lag_1']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['category_months_since_release_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['shop_category_months_since_release_avg_sales_seniority_2_absolute_mean']
features_to_keep_2+=['shop_category_months_since_release_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['supercategory_months_since_release_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['category_months_since_first_sale_in_shop_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['shop_category_months_since_first_sale_in_shop_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['shop_category_avg_sales_compared_to_category_annual_mean']
features_to_keep_2+=['shop_category_avg_sales_compared_to_category_recent_mean']
features_to_keep_2+=['shop_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_2+=['supercategory_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_2+=['supercategory_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_2+=['category_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_2+=['category_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_2+=['category_avg_sales_compared_to_supercategory_recent_mean']
features_to_keep_2+=['category_avg_sales_compared_to_supercategory_lag_12']
features_to_keep_2+=['shop_avg_sales_seniority_2_compare_lag12_to_annual_mean']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_compare_lag12_to_annual_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_compare_lag12_to_annual_mean']
features_to_keep_2+=['supercategory_month_of_release_avg_sales_compared_to_supercategory_absolute_mean']
features_to_keep_2+=['category_month_of_release_avg_sales_compared_to_category_absolute_mean']
TRAIN_DATAFRAME, TEST_DATAFRAME = count_json(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
"photos"
)<feature_engineering>
features_to_discard_2=list(set(list(train_X.columns)) - set(['month_id','item_quantity']+features_to_keep_2))
train_X.drop(features_to_discard_2,axis=1,inplace=True)
 train_X['item_quantity_extralin2'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X['item_quantity_extralin3'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X.fillna(-1,inplace=True)
(X_train,Y_train,X_val,Y_val)= datasplit_train_val(train_X,n_months_val,month_id_first)
 eval_set = [(X_train,Y_train),(X_val,Y_val)]+datasplit_evalset(train_X,month_id_first)
 print('number of features to keep : '+str(len(features_to_keep_2)))
 print('number of features kept : '+str(len(X_train.columns)))
 print(X_train.info(null_counts=True,verbose=True))
del train_X
gc.collect()
NUMERICAL_COLUMNS = [
'bathrooms',
'bedrooms',
'price',
'latitude',
'longitude',
'photos_count'
]
TRAIN_DATAFRAME, TEST_DATAFRAME = numerical_feature_engineering(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
NUMERICAL_COLUMNS
)<categorify>
xgb_params_2={'objective': 'reg:squarederror',
'max_depth': 6,
'min_child_weight': 300,
'subsample': 0.8,
'colsample_bytree': 0.8,
'learning_rate': 0.02,
'n_estimators': 2000,
'base_score': 0.4,
'n_jobs': 4,
'random_state': 2
}
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_2)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )
TRAIN_DATAFRAME, TEST_DATAFRAME = remap_columns_with_transform(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'description',
'clean_description',
normalize_description
)<drop_column>
del X_train, Y_train, X_val, Y_val
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset
del xgbreg
gc.collect()
DROP_COLUMNS = [
'id',
'created',
'building_id',
'clean_description',
'description',
'features',
'display_address',
'display_address_normalized',
'manager_id',
'photos',
'street_address',
'street_address_normalized',
'interest_level',
]
TRAIN_DATAFRAME = TRAIN_DATAFRAME.drop(DROP_COLUMNS, axis=1)
 TEST_DATAFRAME = TEST_DATAFRAME.drop(DROP_COLUMNS[:-1], axis=1 )<drop_column>
reset_variable_space
[(i, x)for i, x in enumerate(np.all(np.isfinite(TRAIN_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1)) , axis=0)) if not x]<drop_column>
create_directory(os.path.join(DATA_FOLDER, 'predictions'))
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/'))
TRAIN_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1 ).columns[119]<prepare_x_and_y>
dataset_name='train_0_pred'
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
print(train_X.info(null_counts=True,verbose=True))
np.argwhere(~np.isfinite(TEST_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1 ).as_matrix()))<define_variables>
rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset = target_analysis_test(train_X,month_id_first=18 )
CATEGORICAL_COLUMNS =('building_id_category', 'manager_id_category' )<data_type_conversions>
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_0)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )
( LINEAR_MODEL_STATISTICS,
LINEAR_MODEL_LABELS,
LINEAR_MODEL_PREDICTIONS,
LINEAR_MODEL)= test_model_with_k_fold_cross_validation(
 basic_logistic_regression_pipeline(CATEGORICAL_COLUMNS),
TRAIN_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1 ).astype(float),
TRAIN_DATAFRAME['label_interest_level'],
{
"mse_loss": mean_squared_error,
"accuracy": prediction_accuracy
},
n_splits=2 
)
 print('Linear Model', format_statistics(LINEAR_MODEL_STATISTICS))<set_options>
performance_analysis_test(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_evalset,rmse_base_evalset )
pd.set_option("display.max_columns",200)
 pd.set_option("display.max_rows",500)
 TRAIN_DATAFRAME[TRAIN_DATAFRAME.isnull().any(axis=1)]<create_dataframe>
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/xgbreg_seniority0'))
pickle.dump(xgbreg, open(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority0/model.pickle'), 'wb'))
X_test.to_pickle(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority0/X_test.pkl'))
table = get_prediction_probabilities_with_columns(LINEAR_MODEL,
TEST_DATAFRAME.drop('label_interest_level', axis=1),
['listing_id'])
 table.columns = ['listing_id', 'high', 'medium', 'low']<save_to_csv>
del X_train, Y_train, X_test
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset
del xgbreg
del features_to_keep_0, features_to_discard_0
del xgb_params_0
loaded.remove('features_to_keep_0')
 loaded.remove('features_to_discard_0')
 loaded.remove('xgb_params_0')
 gc.collect()
table.to_csv('submission.csv', columns=['listing_id', 'high', 'medium', 'low'], index=False )<import_modules>
reset_variable_space
import numpy as np
import pandas as pd
import datetime
import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss<feature_engineering>
dataset_name='train_1_pred'
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
print(train_X.info(null_counts=True,verbose=True))
def PrepareFeatures(TestOrTrain):
source = f'.. /input/{TestOrTrain}.json'
data = pd.read_json(source)
 ulimit = np.percentile(data.price.values, 99)
 data['price'][data['price']>ulimit] = ulimit
data['hasDesc'] = data['description'].apply(lambda x: len(x.strip())!=0)
 data["nFeatures"] = data["features"].apply(len)
 data["nDescWords"] = data["description"].apply(lambda x: len(x.split(" ")))
 data['nPhotos'] = data['photos'].apply(lambda x: min(10, len(x)))
 data['created'] = pd.to_datetime(data['created'])
 data['month'] = data['created'].dt.month
data['weekday'] = data['created'].apply(lambda x: x.weekday())
 if TestOrTrain == 'train':
interest_level_map = {'low': 0, 'medium': 1, 'high': 2}
data['interest_level'] = data['interest_level'].apply(lambda x: interest_level_map[x])
 return data
def CreateCategFeat(data, features_list):
f_dict = {'hasParking':['parking', 'garage'], 'hasGym':['gym', 'fitness', 'health club'],
'hasPool':['swimming pool', 'pool'], 'noFee':['no fee', "no broker's fees"],
'hasElevator':['elevator'], 'hasGarden':['garden', 'patio', 'outdoor space'],
'isFurnished': ['furnished', 'fully equipped'],
'reducedFee':['reduced fee', 'low fee'],
'hasAC':['air conditioning', 'central a/c', 'a/c', 'central air', 'central ac'],
'hasRoof':['roof', 'sundeck', 'private deck', 'deck'],
'petFriendly':['pets allowed', 'pet friendly', 'dogs allowed', 'cats allowed'],
'shareable':['shares ok'], 'freeMonth':['month free'],
'utilIncluded':['utilities included']}
for feature in features_list:
data[feature] = False
for ind, row in data.iterrows() :
for f in row['features']:
f = f.lower().replace('-', '')
 if any(e in f for e in f_dict[feature]):
data.at[ind, feature]= True<prepare_x_and_y>
rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset = target_analysis_test(train_X,month_id_first=18 )
data = PrepareFeatures('train')
 cat_features = ['hasParking', 'hasGym', 'hasPool', 'noFee', 'hasElevator',
'hasGarden', 'isFurnished', 'reducedFee', 'hasAC', 'hasRoof',
'petFriendly', 'shareable', 'freeMonth', 'utilIncluded']
CreateCategFeat(data, cat_features)
 features = ["bathrooms", "bedrooms", "latitude", "longitude", "price",
"nPhotos", "hasDesc", 'nFeatures', 'nDescWords', "month", 'weekday']
features.extend(cat_features)
 X = data[features]
y = data["interest_level"]<train_model>
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_1)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.05)
 param = {'eval_metric':"mlogloss", 'eta':0.02, 'objective':'multi:softprob', 'silent':1,
'max_depth':10, 'num_class':3, 'subsample':0.7, 'colsample_bytree':0.7}
num_rounds = 500
xgtrain = xgb.DMatrix(X_train, label=y_train)
 model = xgb.train(param, xgtrain, num_rounds )<predict_on_test>
performance_analysis_test(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_evalset,rmse_base_evalset )
xgval = xgb.DMatrix(X_val)
 y_val_pred = model.predict(xgval)
 log_loss(y_val, y_val_pred )<predict_on_test>
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/xgbreg_seniority1'))
pickle.dump(xgbreg, open(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority1/model.pickle'), 'wb'))
X_test.to_pickle(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority1/X_test.pkl'))
test = PrepareFeatures('test')
 CreateCategFeat(test, cat_features)
 X_test = test[features]
xgtest = xgb.DMatrix(X_test)
 y = model.predict(xgtest )<save_to_csv>
del X_train, Y_train, X_test
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset
del xgbreg
del features_to_keep_1, features_to_discard_1
del xgb_params_1
loaded.remove('features_to_keep_1')
 loaded.remove('features_to_discard_1')
 loaded.remove('xgb_params_1')
 gc.collect()
labels2idx = {'low': 0, 'medium': 1, 'high': 2}
sub = pd.DataFrame()
sub["listing_id"] = test["listing_id"]
for label in labels2idx.keys() :
sub[label] = y[:, labels2idx[label]]
sub.to_csv("submission_rf.csv", index=False )<import_modules>
reset_variable_space
import os
import sys
import operator
import numpy as np
import pandas as pd
from scipy import sparse
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer<train_model>
dataset_name='train_2_pred'
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
print(train_X.info(null_counts=True,verbose=True))
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 0.7
param['subsample'] = 0.8
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<load_from_disk>
rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset = target_analysis_test(train_X,month_id_first=18 )
data_path = ".. /input/"
train_file = data_path + "train.json"
test_file = data_path + "test.json"
train_df = pd.read_json(train_file)
 test_df = pd.read_json(test_file)
 print(train_df.shape)
 print(test_df.shape )<define_variables>
train_X.drop(features_to_discard_2,axis=1,inplace=True)
 train_X['item_quantity_extralin2'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X['item_quantity_extralin3'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X.fillna(-1,inplace=True)
(X_train,Y_train,X_test)= datasplit_train_test(train_X,month_id_first)
 eval_set = [(X_train,Y_train)]+datasplit_evalset(train_X,month_id_first)
 print('number of features to keep : '+str(len(features_to_keep_2)))
 print('number of features kept : '+str(len(X_train.columns)))
 print(X_train.info(null_counts=True,verbose=True))
del train_X
gc.collect()
features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"]<feature_engineering>
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_2)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
train_df['total_room'] = train_df['bedrooms'] + train_df['bathrooms']
train_df['price_per_room'] = train_df['price'] / train_df['total_room']
test_df['total_room'] = test_df['bedrooms'] + test_df['bathrooms']
test_df['price_per_room'] = test_df['price'] / test_df['total_room']
features_to_use.extend(["num_photos", "num_features", "num_description_words","created_year", "created_month", "created_day", "listing_id", "created_hour", 'price_per_room'] )<categorify>
performance_analysis_test(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_evalset,rmse_base_evalset )
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )<feature_engineering>
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/xgbreg_seniority2'))
pickle.dump(xgbreg, open(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority2/model.pickle'), 'wb'))
X_test.to_pickle(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority2/X_test.pkl'))
target_num_map = {'high':0, 'medium':1, 'low':2}
train_df['label'] = train_df['interest_level'].apply(lambda x: target_num_map[x])
 manager_score = train_df['label'].groupby(train_df['manager_id'] ).mean()
manager_score = pd.DataFrame(manager_score)
 manager_score.columns = { 'manager_score' }
manager_score['manager_id'] = manager_score.index
manager_score.head()<merge>
del X_train, Y_train, X_test
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset
del xgbreg
del features_to_keep_2, features_to_discard_2
del xgb_params_2
loaded.remove('features_to_keep_2')
 loaded.remove('features_to_discard_2')
 loaded.remove('xgb_params_2')
 gc.collect()
train_df = pd.merge(train_df, manager_score, on='manager_id')
 train_df.head()<merge>
reset_variable_space
test_df = pd.merge(test_df, manager_score, on='manager_id', how='left')
 test_df.head()
test_df.shape<concatenate>
def predict_seniority(seniority,ntree=0):
xgbreg = pickle.load(open(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority'+str(seniority)+'/model.pickle'), 'rb'))
X_test = pd.read_pickle(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority'+str(seniority)+'/X_test.pkl'))
Y_pred_test=xgbreg.predict(X_test,ntree_limit=ntree ).clip(0,20)
 Y_pred=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_'+str(seniority)+'.pkl'))
Y_pred=Y_pred.loc[Y_pred['month_id']==34,['shop_id','item_id']]
Y_pred['prediction']=Y_pred_test
return Y_pred
features_to_use.append('manager_score' )<feature_engineering>
ntrees=[200,400,2000]
Y_0=predict_seniority(seniority=0,ntree=ntrees[0])
 Y_1=predict_seniority(seniority=1,ntree=ntrees[1])
 Y_2=predict_seniority(seniority=2,ntree=ntrees[2])
 Y_pred=pd.concat([Y_0,Y_1,Y_2],axis=0,sort=False)
 test_df = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'test.csv'))
test_df=test_df.join(Y_pred.set_index(['shop_id','item_id']),on=['shop_id','item_id'])
 print(test_df.info(null_counts=True))
test_df
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"] )<prepare_x_and_y>
submission = pd.DataFrame({
"ID": test_df['ID'],
"item_cnt_month": test_df['prediction']
})
 submission.to_csv('xgb_prediction.csv', index=False )
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<compute_train_metric>
del test_df
del Y_0, Y_1, Y_2, Y_pred
del submission
gc.collect()
cv_scores = []
kf = model_selection.KFold(n_splits=10, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<save_to_csv>
reset_variable_space