Kaggle competition: Predict Future Sales
clean_all_df.drop(rm_cols, axis=1, inplace=True )<drop_column>
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )
for col in rm_cols:
for _list in [ind_bool_col, ind_ordered_col, hh_bool_col, hh_cont_col, hh_ordered_col]:
if col in _list:
_list.remove(col )<categorify>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]
roof_list = ["techozinc", "techoentrepiso", "techocane", "techootro", "cielorazo"]
clean_all_df["cielorazo"] = clean_all_df[roof_list[:-1]].sum(axis=1 ).map({1:0, 0:1} )<groupby>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
no_head_household_ids = clean_all_df.groupby("idhogar" ).sum() [clean_all_df.groupby("idhogar" ).sum() ["parentesco1"] == 0].index<drop_column>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
clean_drop_no_head_df = clean_all_df.copy()
for x in no_head_household_ids:
clean_drop_no_head_df.drop(clean_drop_no_head_df.loc[clean_drop_no_head_df.idhogar == x, :].index, axis=0, inplace=True )<categorify>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )
clean_heads_df = clean_drop_no_head_df.loc[clean_drop_no_head_df.parentesco1 == 1]
heads_dict = clean_heads_df[["idhogar", "Target"]].set_index("idhogar" ).to_dict() ["Target"]<concatenate>
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )
def synchronise_diff_target(x):
x.Target = heads_dict[x.idhogar]
return x
clean_drop_no_head_df = clean_drop_no_head_df.apply(synchronise_diff_target, axis=1 )<define_variables>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
aggregation_list = ["max", "sum", "min", "mean", "std"]<groupby>
train['revenue'] = train['item_price'] * train['item_cnt_day']
hh_agg_features = clean_drop_no_head_df[["idhogar"] + ind_bool_col + ind_ordered_col].groupby("idhogar" ).agg(aggregation_list)
 hh_agg_features.columns = ["_".join(x)for x in hh_agg_features.columns.ravel() ]
hh_agg_features = hh_agg_features.fillna(0 )<concatenate>
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts
clean_heads_df = clean_drop_no_head_df[clean_drop_no_head_df.parentesco1 == 1].loc[:, id_col + hh_bool_col + hh_cont_col + hh_ordered_col].set_index("idhogar")
 clean_heads_df = pd.concat([clean_heads_df, hh_agg_features], axis=1, sort=False ).reset_index().rename(columns={"index":'idhogar'} )<prepare_x_and_y>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
X = clean_heads_df[clean_heads_df.Target.notna() ].reset_index(drop=True)
 y = X.Target
X_predict = clean_heads_df[clean_heads_df.Target.isna() ].reset_index(drop=True )<split>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
all_predict_df = clean_all_df[clean_all_df.Target.isnull() ]<import_modules>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
from sklearn.preprocessing import MinMaxScaler<normalization>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
scaler = MinMaxScaler()
X.iloc[:, 3:], X_predict.iloc[:, 3:] = scaler.fit_transform(X.iloc[:, 3:]), scaler.fit_transform(X_predict.iloc[:, 3:] )<import_modules>
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest, chi2<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
numeric_train_df = X.iloc[:, 3:]
label_series = X.Target.reset_index(drop=True )<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
pca = PCA(n_components=3, whiten=True, svd_solver='auto', random_state=166446054 )<categorify>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
pca_train_array = pca.fit_transform(numeric_train_df )<create_dataframe>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
pca_train_df = pd.DataFrame(pca_train_array)
 pca_train_df = pd.concat([pca_train_df, label_series], axis=1 )<import_modules>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
from sklearn.cluster import KMeans
from tempfile import mkdtemp
from shutil import rmtree
from sklearn.pipeline import Pipeline<find_best_params>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
result_dict = search_optimised_k([4, 8, 16, 32, 64, 128, 259], random_state=16446054, X=X.iloc[:, 3:] )<predict_on_test>
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
km_predict = KMeans(n_jobs=-1, n_clusters=4, random_state=16446054 ).fit_predict(pca_train_df)
 km_X = pd.concat([pd.Series(km_predict, name="k_means_predict" ).map({0:"centroid1",
1:"centroid2",
2: "centroid3",
3: "centroid4",
4: "centroid5",
5: "centroid6",
6: "centroid7",
7: "centroid8"
}), X.iloc[:, 2:]], axis=1)
 plot_hist(km_X, "k_means_predict", x_label="KMeans Prediction", title="K Means Clustering Prediction" )<import_modules>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
from sklearn.model_selection import train_test_split<import_modules>
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
from sklearn.model_selection import train_test_split<feature_engineering>
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
my_knn_agent = knn_agent()<split>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
X_train, X_test, y_train, y_test = train_test_split(X.iloc[:, 3:], y, test_size=0.2, random_state=16446054, stratify=y )<train_model>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
my_knn_agent.optimise_params(X_train=X_train, y_train=y_train, X_dev=X_test, y_dev=y_test, max_k_num=50 )<compute_test_metric>
matrix['month'] = matrix['date_block_num'] % 12
uniform_f1, distance_f1 = my_knn_agent.evaluate(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)
 print("Unifrom test set f1 score: ", uniform_f1, " Distance test set f1 score: ", distance_f1 )<predict_on_test>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
uniform_prediction, distance_prediction = my_knn_agent.predict(X_train=X_train, y_train=y_train, X_predict=X_predict.iloc[:, 3:] )<import_modules>
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
from sklearn.model_selection import GridSearchCV<import_modules>
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
from sklearn.model_selection import GridSearchCV<categorify>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
cachedir = mkdtemp()
knn_pipe = Pipeline([('reduce_dim', PCA(random_state=16446054)) ,('classify', my_knn_agent.uniform_knn)], memory=cachedir )<define_variables>
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts
num_features_list = np.arange(10, 200, 20 )<define_variables>
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts
knn_param_grid = [
{
'reduce_dim': [PCA(random_state=16446054)],
'reduce_dim__n_components': num_features_list,
},
{
'reduce_dim': [SelectKBest(chi2)],
'reduce_dim__k': num_features_list,
}
]
reducer_labels = ['PCA', 'KBest(chi2)']<choose_model_class>
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;
knn_grid = GridSearchCV(estimator=knn_pipe, cv=5, n_jobs=4, param_grid=knn_param_grid, scoring='f1_macro' )<train_model>
matrix = matrix[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]
_ = knn_grid.fit(X.iloc[:, 3:], y )<create_dataframe>
X_train = matrix[matrix.date_block_num < 34].drop(['item_cnt_month'], axis=1)
 y_train = matrix[matrix.date_block_num < 34]['item_cnt_month']
X_test = matrix[matrix.date_block_num == 34].drop(['item_cnt_month'], axis=1)
knn_params_results = pd.DataFrame(knn_grid.cv_results_ )<concatenate>
import lightgbm as lgb
from sklearn.linear_model import LinearRegression
knn_params_results = pd.concat([knn_params_results.param_reduce_dim__n_components.fillna(knn_params_results.param_reduce_dim__k ).rename("num_features"),
knn_params_results.drop(["param_reduce_dim__k", "param_reduce_dim__n_components"], axis=1)], axis=1 )<feature_engineering>
lr = LinearRegression()
lr.fit(X_train.values, y_train)
 pred_lr = lr.predict(X_test.values )
knn_params_results.loc[:, "param_reduce_dim"] = knn_params_results.param_reduce_dim.apply(lambda x: "PCA" if str(x ).startswith("PCA")else "SelectKBest" ).rename("algorithm" )<drop_column>
lgb_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread':1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 2**7,
'bagging_freq':1,
'verbose':0
}
model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)
 pred_lgb = model.predict(X_test )
rmtree(cachedir )<import_modules>
X_test_level2 = np.c_[pred_lr, pred_lgb]
from sklearn.svm import SVC
from sklearn.utils.class_weight import compute_class_weight<define_variables>
dates = matrix['date_block_num']
dates_train = dates[dates < 34]
dates_test = dates[dates == 34]
cachedir = mkdtemp()
svc_pipe = Pipeline([('reduce_dim', PCA(random_state=166446054)) ,('classify', SVC(random_state=16446054, class_weight="balanced")) ], memory=cachedir )<define_search_space>
dates_train_level2 = dates_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]
y_train_level2 = y_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]
svc_param_grid = [
{
'reduce_dim__n_components': [10, 70, 140, 200],
'classify__kernel': ['rbf', 'poly'],
'classify__C': [0.1, 1, 10, 50],
}
]<train_on_grid>
X_train_level2 = np.zeros([y_train_level2.shape[0], 2])
 for cur_block_num in [27, 28, 29, 30, 31, 32, 33]:
print(cur_block_num)

X_train = matrix.loc[dates < cur_block_num].drop(['item_cnt_month'], axis=1)
 X_test = matrix.loc[dates == cur_block_num].drop(['item_cnt_month'], axis=1)
 y_train = matrix.loc[dates < cur_block_num, 'item_cnt_month'].values
lr.fit(X_train.values, y_train)
 pred_lr_2 = lr.predict(X_test.values)
 model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)
 pred_lgb_2 = model.predict(X_test)
 X_train_level2[dates_train_level2==cur_block_num] = np.c_[pred_lr_2,pred_lgb_2]
svc_grid = GridSearchCV(svc_pipe, cv=5, n_jobs=4, param_grid=svc_param_grid, scoring='f1_macro', refit=True)
 _ = svc_grid.fit(X.iloc[:, 3:], y )<create_dataframe>
alphas_to_try = np.linspace(0, 1, 1001)
 best_alpha = 0
r2_train_simple_mix = 0
for alpha in alphas_to_try:
mix = alpha*X_train_level2[:,0] +(1-alpha)*X_train_level2[:,1]
r2 = r2_score(y_train_level2, mix)
 if r2 > r2_train_simple_mix:
best_alpha = alpha
r2_train_simple_mix = r2
svc_params_results = pd.DataFrame(svc_grid.cv_results_ )<set_options>
test_preds = best_alpha*X_test_level2[:,0] +(1-best_alpha)*X_test_level2[:,1]
test_preds = test_preds.clip(0,20 )
rmtree(cachedir )<import_modules>
submissions = pd.DataFrame({
"ID": test.index,
"item_cnt_month": test_preds
})
 submissions.item_cnt_month = submissions.item_cnt_month.fillna(0.0 )
<choose_model_class><EOS> 
submissions.to_csv('submission.csv',index=False )