Kaggle competition: Predict Future Sales
<save_to_csv><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: rmse Kaggle data source: predict-future-sales<count_values>
import pandas as pd
import numpy as np
df_test['Target'].value_counts()<define_variables>
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' )
prefix = ".. /input/costa-rican-household-poverty-prediction/"<set_options>
train.columns = ['date', 'date_block_num', 'shop_id', 'item_id', 'item_price', 'item_cnt_day']
train
warnings.filterwarnings('ignore' )<set_options>
train[train.date_block_num==0].sort_values(['shop_id','item_id'], ascending=True )
init_notebook_mode(connected=True)
 cf.set_config_file(offline=True, world_readable=True, theme='ggplot')
 %matplotlib inline 
<train_on_grid>
train['date_block_num'] += 1
train['date_block_num']
def search_optimised_k(n_clusters_list, random_state, X):
silhouette_score_list = []
result_dict = {}
for n_clusters in n_clusters_list:
km_predict = KMeans(random_state=random_state, n_jobs=-1, n_clusters=n_clusters ).fit_predict(X)
 silhouette_score_list.append(silhouette_score(X=X, labels=km_predict, random_state=16446054))
result_dict["silhouette_score"] = silhouette_score_list
result_dict["n_clusters"] = n_clusters_list
return result_dict 
<train_model>
shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv');
print(shops.shape)
 shops.head()
class knn_agent:
def __init__(self):
self.uniform_neighbours_num = None
self.distance_neighbours_num = None
self.uniform_knn = None
self.distance_knn = None
self.f1_df = None
def optimise_params(self, X_train, y_train, X_dev, y_dev, max_k_num):
uniform_f1_test_list = []
distance_f1_test_list = []
for i in range(1, max_k_num):
uniform_knn = KNeighborsClassifier(n_neighbors=i, weights="uniform")
 distance_knn = KNeighborsClassifier(n_neighbors=i, weights="distance")
 uniform_prediction = uniform_knn.fit(X_train, y_train ).predict(X_dev)
 distance_prediction = distance_knn.fit(X_train, y_train ).predict(X_dev)
 uniform_f1_test_list.append(f1_score(y_true=y_dev, y_pred=uniform_prediction, average='macro'))
distance_f1_test_list.append(f1_score(y_true=y_dev, y_pred=distance_prediction, average='macro'))
self.f1_df = pd.DataFrame(data={"F1(uniform)": uniform_f1_test_list,
"F1(distance)": distance_f1_test_list},
index=range(1, max_k_num))
self.uniform_neighbours_num = self.f1_df.sort_values("F1(uniform)", ascending=False ).head(1 ).index.values.item(0)
 self.distance_neighbours_num = self.f1_df.sort_values("F1(distance)", ascending=False ).head(1 ).index.values.item(0)
 self.uniform_knn = KNeighborsClassifier(n_neighbors=self.uniform_neighbours_num, weights="uniform")
 self.distance_knn = KNeighborsClassifier(n_neighbors=self.distance_neighbours_num, weights="distance")
 self.plot_validation_result()
def plot_validation_result(self):
self.f1_df.iplot(kind='scatter', yTitle="F1 Score", xTitle="K number", title="Optimised K Number Searching")
 def predict(self, X_train, y_train, X_predict):
return(self.uniform_knn.fit(X_train, y_train ).predict(X_predict), self.distance_knn.fit(X_train, y_train ).predict(X_predict))
def evaluate(self, X_train, y_train, X_test, y_test):
uniform_prediction = self.uniform_knn.fit(X_train, y_train ).predict(X_test)
 distance_prediction = self.distance_knn.fit(X_train, y_train ).predict(X_test)
 return(f1_score(y_true=y_test, y_pred=uniform_prediction, average="macro"),
f1_score(y_true=y_test, y_pred=distance_prediction, average="macro"))<predict_on_test>
items= pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv');
print(items.shape)
 items.head()
def generate_final_predict_df(input_prediction):
input_prediction = input_prediction.astype(int)
 X_predict["Target"] = input_prediction
tmp_household_predict_df = X_predict.set_index("idhogar")
 tmp_X_predict = all_predict_df.loc[all_predict_df["parentesco1"] == 1, :].set_index("idhogar")
 tmp_X_predict.loc[tmp_household_predict_df.index, "Target"] = tmp_household_predict_df.Target
tmp_X_predict = tmp_X_predict.set_index("Id")
 tmp_all_predict_df = all_predict_df.set_index("Id")
 tmp_all_predict_df.loc[tmp_X_predict.index, "Target"] = tmp_X_predict.Target
tmp_all_predict_df["Target"] = tmp_all_predict_df["Target"].fillna(1 ).astype(int)
 tmp_all_predict_df
tmp_all_predict_df["Target"] = tmp_all_predict_df["Target"].fillna(1 ).astype(int)
 final_predict_df = tmp_all_predict_df.reset_index() [["Id", "Target"]]
return final_predict_df 
<compute_test_metric>
cats= pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv');
print(cats.shape)
 cats.head()
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f}(std: {1:.3f})".format(
 results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("" )<predict_on_test>
sample_submission= pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv');
print(sample_submission.shape)
 sample_submission.head()
def ada_report(ada_models, X_train, y_train, X_test, y_test):
ada_tree_real, ada_tree_discrete = ada_models
real_test_f1 = []
real_train_f1 = []
discrete_test_f1 = []
discrete_train_f1 = []
trees_discrete_num = len(ada_tree_discrete)
 trees_real_num = len(ada_tree_real)
 for real_test_predict, real_train_predict, discrete_test_predict, discrete_train_predict in zip(
 ada_tree_real.staged_predict(X_test),
ada_tree_real.staged_predict(X_train),
ada_tree_discrete.staged_predict(X_test),
ada_tree_discrete.staged_predict(X_train)
):
real_test_f1.append(
 f1_score(y_pred=real_test_predict, y_true=y_test, average='macro')
)
 real_train_f1.append(
 f1_score(y_pred=real_train_predict, y_true=y_train, average='macro')
)
 discrete_test_f1.append(
 f1_score(y_pred=discrete_test_predict, y_true=y_test, average='macro')
)
 discrete_train_f1.append(
 f1_score(y_pred=discrete_train_predict, y_true=y_train, average='macro')
)
 f1s = [
real_test_f1,
real_train_f1,
discrete_test_f1,
discrete_train_f1,
]
labels = ["SAMME.R: Test", "SAMME.R: Train", "SAMME: Test", "SAMME: Train"]
colours = ['rgb(22, 96, 167)', 'rgb(22, 96, 167)', 'rgb(205, 12, 24)', 'rgb(205, 12, 24)']
styles = [None, "dash", None, "dash"]
fig = tools.make_subplots(cols=1, rows=3, print_grid=False, shared_xaxes=True)
 for f1, label, colour, style in zip(f1s, labels, colours, styles):
trace = go.Scatter(
 x=np.arange(1, trees_real_num, 1),
y=f1,
name= label,
line=dict(
 color=colour,
dash=style 
)
)
 fig.append_trace(trace, 1, 1)
 fig["layout"]["yaxis1"].update(title="F1 Macro Score")
 trace_real_error = go.Scatter(
 x=np.arange(1, trees_real_num, 1),
y=ada_tree_real.estimator_errors_,
name="SAMME.R: Error", 
)
 trace_discrete_error = go.Scatter(
 x=np.arange(1, trees_real_num, 1),
y=ada_tree_discrete.estimator_errors_,
name="SAMME: Error", 
)
 fig.append_trace(trace_real_error, 2, 1)
 fig.append_trace(trace_discrete_error, 2, 1)
 fig["layout"]["yaxis2"].update(title="Error")
 trace_real_weights = go.Scatter(
 x=np.arange(1, trees_real_num, 1),
y=ada_tree_real.estimator_weights_,
name="SAMME.R: Weights", 
)
 trace_discrete_weights = go.Scatter(
 x=np.arange(1, trees_real_num, 1),
y=ada_tree_discrete.estimator_weights_,
name="SAMME: Weights", 
)
 fig.append_trace(trace_real_weights, 3, 1)
 fig.append_trace(trace_discrete_weights, 3, 1)
 fig["layout"]["yaxis3"].update(title="Weights")
 fig["layout"].update(title="Adaboost Discrete and Real Algorithm Comparison")
 iplot(fig )<define_variables>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]
poverty_map = {
1:"extreme poverty",
2 : "moderate poverty" ,
3 : "vulnerable households",
4 : "non vulnerable households"
}<load_from_csv>
train[train.item_price<0]
raw_train_df = pd.read_csv(prefix + "train.csv" )<load_from_csv>
median = train[(train.date_block_num==4)&(train.shop_id==32)&(train.item_id==2973)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
train[train.item_price<0]
raw_predict_df = pd.read_csv(prefix + "test.csv" )<load_from_csv>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
example_predict_df = pd.read_csv(prefix + "sample_submission.csv" )<concatenate>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
raw_predict_df = pd.concat([raw_predict_df, pd.Series(np.nan, name='Target')], axis=1 )<concatenate>
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
raw_all_df = pd.concat([raw_train_df, raw_predict_df], ignore_index=True )<data_type_conversions>
items.drop(['item_name'], axis=1, inplace=True )
raw_all_df["edjefe"] = raw_all_df["edjefe"].apply(lambda x: 1 if x == "yes" else x ).apply(lambda x: 0 if x == "no" else x)
 raw_all_df["edjefa"] = raw_all_df["edjefa"].apply(lambda x: 1 if x == "yes" else x ).apply(lambda x: 0 if x == "no" else x)
 raw_all_df["dependency"] = raw_all_df["dependency"].apply(lambda x: 1 if x == "yes" else x ).apply(lambda x: 0 if x == "no" else x)
 raw_all_df["edjefe"] = pd.to_numeric(raw_all_df["edjefe"])
 raw_all_df["edjefa"] = pd.to_numeric(raw_all_df["edjefa"])
 raw_all_df["dependency"] = pd.to_numeric(raw_all_df["dependency"] )<count_missing_values>
ts = time.time()
matrix = []
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
cols = ['date_block_num','shop_id','item_id']
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
raw_all_df.isnull().sum() [raw_all_df.isnull().sum() > 0]<drop_column>
train['revenue'] = train['item_price'] * train['item_cnt_day']
train.head()
clean_all_df = raw_all_df.drop(["v2a1", "rez_esc"], axis=1)
 clean_all_df["v18q1"].fillna(0, inplace=True )<count_missing_values>
matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
matrix.shape
clean_all_df.isnull().sum() [clean_all_df.isnull().sum() > 0]<define_variables>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 test.head()
id_col = ['Id', 'idhogar', 'Target']<define_variables>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
matrix[matrix.date_block_num==34]
ind_bool_col = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone',]
ind_ordered_col = ['escolari', 'age']<define_variables>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
matrix
hh_bool_col = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area2' , 'area1']
hh_ordered_col = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'v18q1', 'hogar_nin', 'tamhog','tamviv','hhsize', 'r4t3', 'hogar_total',
'hogar_adul','hogar_mayor', 'bedrooms', 'qmobilephone']
hh_cont_col = ['dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
sqr_col = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<drop_column>
ts = time.time()
matrix = lag_feature(matrix, [1,2,3,6,12], 'item_cnt_month')
 time.time() - ts
clean_all_df.drop(sqr_col, axis=1, inplace=True )<define_variables>
cols_list = id_col + ind_bool_col + ind_ordered_col + hh_bool_col + hh_ordered_col + hh_cont_col<drop_column>