Kaggle competition: Predict Future Sales
train.fillna(0, inplace = True)
 train.replace(-np.inf, 0, inplace = True)
 train.replace(np.inf, 0, inplace = True)
 test_df.fillna(0, inplace = True)
 test_df.replace(-np.inf, 0, inplace = True)
 test_df.replace(np.inf, 0, inplace = True )<choose_model_class>
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID' )
skf = StratifiedKFold(n_splits=50, shuffle=True, random_state=2019 )<train_model>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
preds = np.zeros(( test_df.shape[0],9))
measured= np.zeros(( train.shape[0]))
total_score = 0
for n_fold,(train_idx, valid_idx)in enumerate(skf.split(train.values, y['surface'].values)) :
print(n_fold)
 model = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 model.fit(train.iloc[train_idx],y['surface'][train_idx])
 measured[valid_idx] = model.predict(train.iloc[valid_idx])
 preds += model.predict_proba(test_df)/ 50
total_score += model.score(train.iloc[valid_idx],y['surface'][valid_idx])
 print("Fold: {} score: {}".format(n_fold,model.score(train.iloc[valid_idx],y['surface'][valid_idx])))
 print('Avg Accuracy', total_score / 50)
<feature_engineering>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
sub['surface'] = le.inverse_transform(preds.argmax(axis=1))
sub<save_to_csv>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )
sub.to_csv('submission.csv', index=False )<save_to_csv>
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )
sub.to_csv('submission.csv', index=False )<load_from_csv>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
data = pd.read_csv('.. /input/X_train.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 test = pd.read_csv('.. /input/X_test.csv' )<create_dataframe>
train['revenue'] = train['item_price'] * train['item_cnt_day']
data = data[['series_id', 'measurement_number', 'orientation_X',
'orientation_Y', 'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z']]<merge>
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts
f_data = pd.merge(data, target, how='inner')
 full_data = f_data[['series_id', 'measurement_number', 'orientation_X',
'orientation_Y', 'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z', 'surface']]<import_modules>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
from sklearn.ensemble import BaggingClassifier, RandomForestClassifier<groupby>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
colum = ['series_id','orientation_X', 'orientation_Y',
'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z']
tr_dat = full_data[colum+['surface']].groupby(['series_id'] ).max()
tr_dat_1 = full_data[colum].groupby(['series_id'] ).min()
tr_dat_2 = full_data[colum].groupby(['series_id'] ).median()
tr_dat_3 = full_data[colum].groupby(['series_id'] ).mean()
tr_dat_4 = full_data[colum].groupby(['series_id'] ).std()
tr_dat_5 = full_data[colum].groupby(['series_id'] ).quantile(0.25)
 tr_dat_6 = full_data[colum].groupby(['series_id'] ).quantile(0.5)
 tr_dat_7 = full_data[colum].groupby(['series_id'] ).quantile(0.75)
 tr_dat_8 = full_data[colum].groupby(['series_id'] ).max() /full_data[colum].groupby(['series_id'] ).min()<groupby>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
tr_dat_5 = full_data[colum].groupby(['series_id'] ).quantile(0.25)
 tr_dat_5.head()<merge>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
ss = [tr_dat,tr_dat_1,tr_dat_2,tr_dat_3,tr_dat_4,tr_dat_5,tr_dat_6,tr_dat_7,tr_dat_8]
df_final = reduce(lambda left,right: pd.merge(left,right,on='series_id'), ss)
 df_final.head()<groupby>
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
ts_dat = test[colum].groupby(['series_id'] ).max()
ts_dat_1 = test[colum].groupby(['series_id'] ).min()
ts_dat_2 = test[colum].groupby(['series_id'] ).median()
ts_dat_3 = test[colum].groupby(['series_id'] ).mean()
ts_dat_4 = test[colum].groupby(['series_id'] ).std()
ts_dat_5 = test[colum].groupby(['series_id'] ).quantile(0.25)
 ts_dat_6 = test[colum].groupby(['series_id'] ).quantile(0.5)
 ts_dat_7 = test[colum].groupby(['series_id'] ).quantile(0.75)
 ts_dat_8 = test[colum].groupby(['series_id'] ).max() /test[colum].groupby(['series_id'] ).min()
ts_ss = [ts_dat,ts_dat_1,ts_dat_2,ts_dat_3,ts_dat_4,ts_dat_5,ts_dat_6,ts_dat_7,ts_dat_8]
ts_final = reduce(lambda left,right: pd.merge(left,right,on='series_id'), ts_ss)
 cols = ['orientation_X_x', 'orientation_Y_x', 'orientation_Z_x',
'orientation_W_x', 'angular_velocity_X_x', 'angular_velocity_Y_x',
'angular_velocity_Z_x', 'linear_acceleration_X_x',
'linear_acceleration_Y_x', 'linear_acceleration_Z_x',
'orientation_X_y', 'orientation_Y_y', 'orientation_Z_y',
'orientation_W_y', 'angular_velocity_X_y', 'angular_velocity_Y_y',
'angular_velocity_Z_y', 'linear_acceleration_X_y',
'linear_acceleration_Y_y', 'linear_acceleration_Z_y', 'orientation_X_x',
'orientation_Y_x', 'orientation_Z_x', 'orientation_W_x',
'angular_velocity_X_x', 'angular_velocity_Y_x', 'angular_velocity_Z_x',
'linear_acceleration_X_x', 'linear_acceleration_Y_x',
'linear_acceleration_Z_x', 'orientation_X_y', 'orientation_Y_y',
'orientation_Z_y', 'orientation_W_y', 'angular_velocity_X_y',
'angular_velocity_Y_y', 'angular_velocity_Z_y',
'linear_acceleration_X_y', 'linear_acceleration_Y_y',
'linear_acceleration_Z_y', 'orientation_X_x', 'orientation_Y_x',
'orientation_Z_x', 'orientation_W_x', 'angular_velocity_X_x',
'angular_velocity_Y_x', 'angular_velocity_Z_x',
'linear_acceleration_X_x', 'linear_acceleration_Y_x',
'linear_acceleration_Z_x', 'angular_velocity_X_y',
'angular_velocity_Y_y', 'angular_velocity_Z_y',
'linear_acceleration_X_y', 'linear_acceleration_Y_y',
'linear_acceleration_Z_y', 'orientation_W_y', 'orientation_X_y',
'orientation_Y_y', 'orientation_Z_y', 'angular_velocity_X_x',
'angular_velocity_Y_x', 'angular_velocity_Z_x',
'linear_acceleration_X_x', 'linear_acceleration_Y_x',
'linear_acceleration_Z_x', 'orientation_W_x', 'orientation_X_x',
'orientation_Y_x', 'orientation_Z_x', 'angular_velocity_X_y',
'angular_velocity_Y_y', 'angular_velocity_Z_y',
'linear_acceleration_X_y', 'linear_acceleration_Y_y',
'linear_acceleration_Z_y', 'orientation_W_y', 'orientation_X_y',
'orientation_Y_y', 'orientation_Z_y', 'orientation_X', 'orientation_Y',
'orientation_Z', 'orientation_W', 'angular_velocity_X',
'angular_velocity_Y', 'angular_velocity_Z', 'linear_acceleration_X',
'linear_acceleration_Y', 'linear_acceleration_Z']<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
trains = df_final[cols]
targs = df_final['surface']
tests = ts_final[cols]<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
new_feat_x = []
new_feat_y = []
new_feat_z = []
new_feat_w = []
skm = LinearRegression()
for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_X'].index))
skm.fit(x, qq['orientation_X'].values)
 new_feat_x.append([skm.intercept_, skm.coef_[0]])
 for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Y'].index))
skm.fit(x, qq['orientation_Y'].values)
 new_feat_y.append([skm.intercept_, skm.coef_[0]])
 for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Z'].index))
skm.fit(x, qq['orientation_Z'].values)
 new_feat_z.append([skm.intercept_, skm.coef_[0]])
 for i in range(3810):
qq = data[data.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_W'].index))
skm.fit(x, qq['orientation_W'].values)
 new_feat_w.append([skm.intercept_, skm.coef_[0]])
 new_feat_x = np.array(new_feat_x)
 new_feat_y = np.array(new_feat_y)
 new_feat_z = np.array(new_feat_z)
 new_feat_w = np.array(new_feat_w)
<feature_engineering>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
trains['intercept_x'] = new_feat_x[:,0]
trains['coef_x'] = new_feat_x[:,1]
trains['intercept_y'] = new_feat_y[:,0]
trains['coef_y'] = new_feat_y[:,1]
trains['intercept_z'] = new_feat_z[:,0]
trains['coef_z'] = new_feat_z[:,1]
trains['intercept_w'] = new_feat_w[:,0]
trains['coef_w'] = new_feat_w[:,1]<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
new_feat_x = []
new_feat_y = []
new_feat_z = []
new_feat_w = []
new_feat_ax = []
new_feat_ay = []
new_feat_az = []
for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_X'].index))
skm.fit(x, qq['orientation_X'].values)
 new_feat_x.append([skm.intercept_, skm.coef_[0]])
 for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Y'].index))
skm.fit(x, qq['orientation_Y'].values)
 new_feat_y.append([skm.intercept_, skm.coef_[0]])
 for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_Z'].index))
skm.fit(x, qq['orientation_Z'].values)
 new_feat_z.append([skm.intercept_, skm.coef_[0]])
 for i in range(3816):
qq = test[test.series_id==i]
x = np.transpose(np.atleast_2d(qq['orientation_W'].index))
skm.fit(x, qq['orientation_Z'].values)
 new_feat_w.append([skm.intercept_, skm.coef_[0]])
 new_feat_x = np.array(new_feat_x)
 new_feat_y = np.array(new_feat_y)
 new_feat_z = np.array(new_feat_z)
 new_feat_w = np.array(new_feat_w)
 tests['intercept_x'] = new_feat_x[:,0]
tests['coef_x'] = new_feat_x[:,1]
tests['intercept_y'] = new_feat_y[:,0]
tests['coef_y'] = new_feat_y[:,1]
tests['intercept_z'] = new_feat_z[:,0]
tests['coef_z'] = new_feat_z[:,1]
tests['intercept_w'] = new_feat_w[:,0]
tests['coef_w'] = new_feat_w[:,1]<predict_on_test>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
clf = RandomForestClassifier(n_estimators = 2000,
max_depth=20,
min_samples_split=5,
class_weight='balanced')
 clf.fit(trains, targs)
 ress = clf.predict(tests )<save_to_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
submission = pd.read_csv(".. /input/sample_submission.csv")
 print(len(submission))
print(test.shape)
 submission['surface'] = ress
submission.to_csv("NEW_submission.csv", index = False)
 submission.head()<load_from_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
train=pd.read_csv(".. /input/X_train.csv")
 test=pd.read_csv(".. /input/X_test.csv")
 label=pd.read_csv(".. /input/y_train.csv" )<feature_engineering>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
def feature_extraction(raw_frame):
frame = pd.DataFrame()
raw_frame['orientation'] = raw_frame['orientation_X'] + raw_frame['orientation_Y'] + raw_frame['orientation_Z']+ raw_frame['orientation_W']
raw_frame['angular_velocity'] = raw_frame['angular_velocity_X'] + raw_frame['angular_velocity_Y'] + raw_frame['angular_velocity_Z']
raw_frame['linear_acceleration'] = raw_frame['linear_acceleration_X'] + raw_frame['linear_acceleration_Y'] + raw_frame['linear_acceleration_Y']
raw_frame['velocity_to_acceleration'] = raw_frame['angular_velocity'] / raw_frame['linear_acceleration']
raw_frame['velocity_linear_acceleration'] = raw_frame['linear_acceleration'] * raw_frame['angular_velocity']
for col in raw_frame.columns[3:]:
frame[col + '_mean'] = raw_frame.groupby(['series_id'])[col].mean()
frame[col + '_std'] = raw_frame.groupby(['series_id'])[col].std()
frame[col + '_max'] = raw_frame.groupby(['series_id'])[col].max()
frame[col + '_min'] = raw_frame.groupby(['series_id'])[col].min()
frame[col + '_max_to_min'] = frame[col + '_max'] / frame[col + '_min']
frame[col + '_mean_abs_change'] = raw_frame.groupby('series_id')[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
frame[col + '_abs_max'] = raw_frame.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 return frame<feature_engineering>
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
train = feature_extraction(train)
 test = feature_extraction(test)
 train.head()<prepare_output>
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
label=label["surface"]<categorify>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
label_encoder = preprocessing.LabelEncoder()
label_encoder.fit(label)
 label=label_encoder.transform(label )<normalization>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
Y_train = label
X_train = train
test=test
X_train.shape, Y_train.shape
scaler = StandardScaler()
scaler.fit(X_train)
 train_img = scaler.transform(X_train)
 test_img = scaler.transform(test )<split>
matrix['month'] = matrix['date_block_num'] % 12
kfold = StratifiedKFold(n_splits=5)
 train_x,val_x,train_y,val_y = train_test_split(X_train, Y_train, test_size = 0.10, random_state=14)
 train_x.shape,val_x.shape,train_y.shape,val_y.shape<create_dataframe>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
train_data = lightgbm.Dataset(train_x, label=train_y)
 test_data = lightgbm.Dataset(val_x, label=val_y )<train_model>
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
para={'boosting_type': 'gbdt',
'colsample_bytree': 0.85,
'learning_rate': 0.1,
'max_bin': 512,
'max_depth': -1,
'metric': 'multi_error',
'min_child_samples': 8,
'min_child_weight': 1,
'min_split_gain': 0.5,
'nthread': 3,
'num_class': 9,
'num_leaves': 31,
'objective': 'multiclass',
'reg_alpha': 0.8,
'reg_lambda': 1.2,
'scale_pos_weight': 1,
'subsample': 0.7,
'subsample_for_bin': 200,
'subsample_freq': 1}
model = lightgbm.train(para,
train_data,
valid_sets=test_data,
num_boost_round=5000,
early_stopping_rounds=50,
)<predict_on_test>
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
y_pred = model.predict(test )<create_dataframe>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
class_prediction=pd.DataFrame(y_pred ).idxmax(axis=1 )<count_values>
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts
submission1 = pd.DataFrame({
"series_id": test.index,
"surface": class_prediction,
})
 submission1.surface.value_counts()<categorify>
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts
submission1.surface=label_encoder.inverse_transform(submission1.surface )<count_values>
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;
submission1.surface.value_counts()<save_to_csv>
data = pd.read_pickle('data.pkl' )
submission1.to_csv('submission1.csv', index=False )<import_modules>
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train = train_data.copy()
test = test_data.copy()<drop_column>
del data
gc.collect() ;
train.drop(['PassengerId'],axis=1,inplace=True)
 test.drop(['PassengerId'],axis=1,inplace=True)
 pred = train_data['Survived']<count_missing_values>
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts
train.isnull().sum()<count_missing_values>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))