Kaggle competition: Santander Customer Transaction Prediction
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Number of rows and columns in train set : ",train_df.shape)
 print("Number of rows and columns in test set : ",test_df.shape )
data = pd.read_pickle('data.pkl' )<drop_column>
corr_df.loc[corr_df.target>0.05].index[1:]
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
train_df_out_rem=train_df.iloc[:,1:]
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
train_df_out_rem=train_df_out_rem.reset_index(drop=True )
del data
gc.collect() ;<train_model>
from sklearn.ensemble import RandomForestClassifier
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
clf = RandomForestClassifier(n_estimators=100, max_depth=8,random_state=0 )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<import_modules>
clf.fit(train_df_out_rem.iloc[:,1:], train_df_out_rem.iloc[:,0] )
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import RandomizedSearchCV,cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import make_scorer
from sklearn.ensemble import RandomForestRegressor
from itertools import product
import re
import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords 
<install_modules>
importance_dict=dict(zip(list(train_df_out_rem.columns[1:]),list(clf.feature_importances_)) )
!pip install pymorphy2
!pip install pymorphy2-dicts
!pip install DAWG-Python<import_modules>
from sklearn.preprocessing import PolynomialFeatures
import pymorphy2<load_from_csv>
poly = PolynomialFeatures(interaction_only=True )
sample_submissions=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv")
 sales_train_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 items_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 shops_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv")
 item_categories_df=pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv" )<count_unique_values>
poly_features=poly.fit_transform(train_df_out_rem[features_names_upd] )
print(sales_train_df["shop_id"].nunique())
 print(test_df["shop_id"].nunique() )<count_unique_values>
poly_features_df=pd.DataFrame(poly_features)
 poly_features_df.columns=['Imp_feature_'+str(i)for i in range(poly_features.shape[1])]
poly_features_df.head()
print(sales_train_df["item_id"].nunique())
 print(test_df["item_id"].nunique() )<load_pretrained>
other_cols=train_df_out_rem.loc[:, ~train_df_out_rem.columns.isin(features_names_upd)]
nltk.download("stopwords" )<string_transform>
train_final_df=pd.concat([poly_features_df,other_cols],axis=1 )
def text_preprocessing(text):
txt=""
morph=pymorphy2.MorphAnalyzer()
stop_words_ru=stopwords.words("russian")
 stop_words_eng=stopwords.words("english")
 stroka=re.sub('[^A-Za-zА-Яа-я]+',' ',text)
 strs=stroka.split(" ")
 filtered_strs=[morph.parse(w.lower())[0].normal_form for w in strs if(w not in stop_words_ru and w not in stop_words_eng)]
txt=" ".join(filtered_strs)
 return txt<feature_engineering>
train_final_df.target.value_counts(dropna=False )
<merge>
train_x=train_final_df.loc[:, train_final_df.columns != 'target']
train_y=train_final_df.target
sales_train_df=sales_train_df.join(shops_df.set_index("shop_id"),on="shop_id",how="left" ).join(
 items_df.set_index("item_id"),on="item_id",how="left" ).join(item_categories_df.set_index("item_category_id")
 ,on="item_category_id",how="left")
 sales_train_df.head().T<merge>
poly_features_test=poly.transform(test_df[features_names_upd] )
test_df=test_df.join(shops_df.set_index("shop_id"),on="shop_id",how="left" ).join(
 items_df.set_index("item_id"),on="item_id",how="left" ).join(item_categories_df.set_index("item_category_id")
 ,on="item_category_id",how="left")
 test_df.head().T<filter>
poly_features_test_df=pd.DataFrame(poly_features_test)
 poly_features_test_df.columns=['Imp_feature_'+str(i)for i in range(poly_features_test.shape[1])]
poly_features_test_df.head()
sales_train_df[sales_train_df.item_cnt_day>2000]<filter>
other_cols_test=test_df.loc[:, ~test_df.columns.isin(features_names_upd)]
sales_train_df[sales_train_df.item_category_id==9]["item_cnt_day"].mean()<filter>
id_code=other_cols_test.ID_code
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==11373)]["item_cnt_day"].mean()<filter>
other_cols_test=other_cols_test.drop(columns=['ID_code'] )
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>2000])]<filter>
final_test_df=pd.concat([poly_features_test_df,other_cols_test],axis=1 )
sales_train_df[sales_train_df.item_price>50000]<filter>
train_x=reduce_mem_usage(train_x)
 final_test_df=reduce_mem_usage(final_test_df )
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_price>50000])]<filter>
gc.collect()
del train_df,test_df,train_df_out_rem
sales_train_df[sales_train_df.item_cnt_day>800]<filter>
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold, StratifiedKFold
import lightgbm as lgb
sales_train_df[(sales_train_df.shop_id==12)&(sales_train_df.item_id==20949)]["item_cnt_day"].mean()<filter>
def kfold_lightgbm(train_df,target,test_df, num_folds, stratified = True, debug= False):
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=326)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=326)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df,target)) :
train_x, train_y = train_df.iloc[train_idx], target[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], target[valid_idx]
lgb_train = lgb.Dataset(train_x,
label=train_y,
free_raw_data=False)
 lgb_test = lgb.Dataset(valid_x,
label=valid_y,
free_raw_data=False)
 params = {
'num_leaves': 20,
'max_bin': 60,
'min_data_in_leaf': 17,
'learning_rate': 0.07,
'min_sum_hessian_in_leaf': 0.000446,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'lambda_l1': 4,
'lambda_l2': 1,
'min_gain_to_split': 0.15,
'max_depth': 18,
'save_binary': True,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
}
reg = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_test],
valid_names=[ 'test'],
num_boost_round=10000,
early_stopping_rounds= 200,
verbose_eval=100 
)
 oof_preds[valid_idx] = reg.predict(valid_x, num_iteration=reg.best_iteration)
 sub_preds += reg.predict(test_df, num_iteration=reg.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = list(test_df.columns)
 fold_importance_df["importance"] = np.log1p(reg.feature_importance(importance_type='gain', iteration=reg.best_iteration))
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del reg, train_x, train_y, valid_x, valid_y
gc.collect()
return sub_preds,oof_preds
sales_train_df=sales_train_df[~sales_train_df.isin(sales_train_df[sales_train_df.item_cnt_day>800])]<filter>
test_preds,train_preds=kfold_lightgbm(train_x,train_y.values,final_test_df, 10 )
test_unique_items=test_df[~test_df["item_id"].isin(item_id_uniq)]
test_unique_items.head()<count_unique_values>
roc_auc_score(train_y.values,train_preds )
test_unique_items["item_id"].nunique()<data_type_conversions>
subm=pd.read_csv('.. /input/sample_submission.csv' )
<groupby><EOS> 
subm.target=test_preds
subm.to_csv('submission.csv',index=False )