Kaggle competition: Predict Future Sales
%%time
ridge = linear_model.Ridge(normalize=True)
 parameter_grid = {'alpha': [0.1, 1.0, 10.0]}
grid_search = GridSearchCV(ridge, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
 grid_search.fit(X, y)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
ridge = linear_model.Ridge(**grid_search.best_params_, normalize=True)
 result_dict_ridge = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=ridge)
<train_on_grid>
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
%%time
knn = neighbors.KNeighborsRegressor()
parameter_grid = {'n_neighbors': [50, 100]}
grid_search = GridSearchCV(knn, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
 grid_search.fit(X, y)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
knn = neighbors.KNeighborsRegressor(**grid_search.best_params_)
 result_dict_knn = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=knn )<train_on_grid>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]
train["item_cnt_day"]=train["item_cnt_day"].fillna(0)
 train = train[train.item_cnt_day>0]
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
%%time
lasso = linear_model.Lasso(normalize=True)
 parameter_grid = {'alpha': [0.1, 1.0, 10.0]}
grid_search = GridSearchCV(lasso, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
 grid_search.fit(X, y)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
lasso = linear_model.Lasso(**grid_search.best_params_, normalize=True)
 result_dict_lasso = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=lasso )<train_on_grid>
def check_nulls(df_train):
for d in df_train.columns:
if df_train[d].isnull().values.any() :
print("column "+d)
 if df_train[d].dtype.kind in 'bifc':
df_train[d].fillna(0,inplace = True)
 else:
print("column "+d)
 df_train[d].fillna("NULL_VALUE", inplace = True )
%%time
etr = ExtraTreesRegressor()
etr = ExtraTreesRegressor(n_estimators=1000, max_depth=10)
 result_dict_etr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=etr )<train_on_grid>
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
%%time
adr = AdaBoostRegressor()
parameter_grid = {'n_estimators': [10, 50],
}
grid_search = GridSearchCV(adr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
 grid_search.fit(X, y)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
adr = AdaBoostRegressor(**grid_search.best_params_)
 result_dict_adr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=adr )<init_hyperparams>
check_count = shops.groupby(["shop_id"])['shop_id'].size()
print(check_count[check_count > 1])
 check_count = cats.groupby(["item_category_id"])['item_category_id'].size()
print(check_count[check_count > 1])
params = {'num_leaves': 128,
'min_child_samples': 79,
'objective': 'gamma',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"subsample_freq": 5,
"subsample": 0.9,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501,
'colsample_bytree': 0.2
}
result_dict_lgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb',
eval_metric='mae', plot_feature_importance=True )<train_model>
check_count = train.groupby(["date","date_block_num","shop_id","item_id"])['shop_id'].size()
print(check_count[check_count > 1])
xgb_params = {'eta': 0.03,
'max_depth': 9,
'subsample': 0.85,
'colsample_bytree': 0.3,
'objective': 'reg:linear',
'eval_metric': 'mae',
'silent': True,
'nthread': -1}
result_dict_xgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=xgb_params, folds=folds, model_type='xgb' )<save_to_csv>
train.query("date_block_num==16 and shop_id==50 and item_id==3423" )
submission['time_to_failure'] =(result_dict_lgb['prediction'] + result_dict_etr['prediction'] + result_dict_xgb['prediction'])/ 3
print(submission.head())
 submission.to_csv('blending.csv' )<create_dataframe>
test["date_block_num"]=34
test = pd.merge(test, shops, on=['shop_id'], how='left',suffixes=('', '_x'))
test = pd.merge(test, items, on=['item_id'], how='left',suffixes=('', '_y'))
test = pd.merge(test, cats, on=['item_category_id'], how='left')
 test.columns
train_stack = np.vstack([result_dict_rfr['oof'], result_dict_ridge['oof'], result_dict_knn['oof'], result_dict_lasso['oof'], result_dict_etr['oof'],
result_dict_adr['oof'], result_dict_lgb['oof'], result_dict_xgb['oof'], result_dict_etr['oof']] ).transpose()
train_stack = pd.DataFrame(train_stack, columns=['rfr', 'ridge', 'knn', 'lasso', 'etr', 'adr', 'lgb', 'xgb', 'etr'])
 test_stack = np.vstack([result_dict_rfr['prediction'], result_dict_ridge['prediction'], result_dict_knn['prediction'], result_dict_lasso['prediction'], result_dict_etr['prediction'],
result_dict_adr['prediction'], result_dict_lgb['prediction'], result_dict_xgb['prediction'], result_dict_etr['prediction']] ).transpose()
test_stack = pd.DataFrame(test_stack, columns=['rfr', 'ridge', 'knn', 'lasso', 'etr', 'adr', 'lgb', 'xgb', 'etr'] )<train_model>
check_count = test.groupby(["shop_id","item_id"])['shop_id'].size()
print(check_count[check_count > 1])
params = {'num_leaves': 8,
'objective': 'regression',
'max_depth': 2,
'learning_rate': 0.01,
"boosting": "gbdt",
"bagging_seed": 11,
"metric": 'rmse',
"verbosity": -1}
result_dict_lgb_stack = artgor_utils.train_model_regression(X=train_stack, X_test=test_stack, y=y, params=params, folds=folds, model_type='lgb',
eval_metric='mae', plot_feature_importance=False,
columns=(list(train_stack.columns)) )<save_to_csv>
train_2 = pd.DataFrame(np.vstack(train_2))
submission['time_to_failure'] = result_dict_lgb_stack['prediction']
print(submission.head())
 submission.to_csv('stacking.csv' )<set_options>
train_2.rename(columns={0: "date_block_num", 1: "shop_id", 2: "item_id"} , inplace = True)
 train_2
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
<train_on_grid>
cols=['date_block_num','shop_id','item_id']
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=cols, how='left')
 train_2['item_cnt_month'] =(train_2['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float32))
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change )<load_from_csv>
train_2 = pd.merge(train_2, shops, on=['shop_id'], how='left',suffixes=('', '_x'))
train_2 = pd.merge(train_2, items, on=['item_id'], how='left',suffixes=('', '_y'))
train_2 = pd.merge(train_2, cats, on=['item_category_id'], how='left')
class FeatureGenerator(object):
def __init__(self, dtype, n_jobs=1, chunk_size=None):
self.chunk_size = chunk_size
self.dtype = dtype
self.filename = None
self.n_jobs = n_jobs
self.test_files = []
if self.dtype == 'train':
self.filename = '.. /input/train.csv'
self.total_data = int(629145481 / self.chunk_size)
 else:
submission = pd.read_csv('.. /input/sample_submission.csv')
 for seg_id in submission.seg_id.values:
self.test_files.append(( seg_id, '.. /input/test/' + seg_id + '.csv'))
self.total_data = int(len(submission))
def read_chunks(self):
if self.dtype == 'train':
iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,
dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})
 for counter, df in enumerate(iter_df):
x = df.acoustic_data.values
y = df.time_to_failure.values[-1]
seg_id = 'train_' + str(counter)
 del df
yield seg_id, x, y
else:
for seg_id, f in self.test_files:
df = pd.read_csv(f, dtype={'acoustic_data': np.float64})
 x = df.acoustic_data.values[-self.chunk_size:]
del df
yield seg_id, x, -999
def get_features(self, x, y, seg_id):

 x = pd.Series(x)
 zc = np.fft.fft(x)
 realFFT = pd.Series(np.real(zc))
imagFFT = pd.Series(np.imag(zc))
main_dict = self.features(x, y, seg_id)
 r_dict = self.features(realFFT, y, seg_id)
 i_dict = self.features(imagFFT, y, seg_id)
 for k, v in r_dict.items() :
if k not in ['target', 'seg_id']:
main_dict[f'fftr_{k}'] = v
for k, v in i_dict.items() :
if k not in ['target', 'seg_id']:
main_dict[f'ffti_{k}'] = v
return main_dict
def features(self, x, y, seg_id):
feature_dict = dict()
feature_dict['target'] = y
feature_dict['seg_id'] = seg_id
percentiles = [1, 5, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 95, 99]
hann_windows = [50, 150, 1500, 15000]
spans = [300, 3000, 30000, 50000]
windows = [10, 50, 100, 500, 1000, 10000]
borders = list(range(-4000, 4001, 1000))
peaks = [10, 20, 50, 100]
coefs = [1, 5, 10, 50, 100]
lags = [10, 100, 1000, 10000]
autocorr_lags = [5, 10, 50, 100, 500, 1000, 5000, 10000]
feature_dict['mean'] = x.mean()
feature_dict['std'] = x.std()
feature_dict['max'] = x.max()
feature_dict['min'] = x.min()
feature_dict['mean_change_abs'] = np.mean(np.diff(x))
feature_dict['abs_max'] = np.abs(x ).max()
feature_dict['abs_mean'] = np.abs(x ).mean()
feature_dict['abs_std'] = np.abs(x ).std()
feature_dict['hmean'] = stats.hmean(np.abs(x[np.nonzero(x)[0]]))
feature_dict['gmean'] = stats.gmean(np.abs(x[np.nonzero(x)[0]]))
for i in range(1, 5):
feature_dict[f'kstat_{i}'] = stats.kstat(x, i)
 feature_dict[f'moment_{i}'] = stats.moment(x, i)
 for i in [1, 2]:
feature_dict[f'kstatvar_{i}'] = stats.kstatvar(x, i)
 for agg_type, slice_length, direction in product(['std', 'min', 'max', 'mean'], [1000, 10000, 50000], ['first', 'last']):
if direction == 'first':
feature_dict[f'{agg_type}_{direction}_{slice_length}'] = x[:slice_length].agg(agg_type)
 elif direction == 'last':
feature_dict[f'{agg_type}_{direction}_{slice_length}'] = x[-slice_length:].agg(agg_type)
 feature_dict['max_to_min'] = x.max() / np.abs(x.min())
 feature_dict['max_to_min_diff'] = x.max() - np.abs(x.min())
 feature_dict['count_big'] = len(x[np.abs(x)> 500])
 feature_dict['sum'] = x.sum()
feature_dict['mean_change_rate'] = calc_change_rate(x)
 for slice_length, direction in product([1000, 10000, 50000], ['first', 'last']):
if direction == 'first':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[:slice_length])
 elif direction == 'last':
feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[-slice_length:])
 for p in percentiles:
feature_dict[f'percentile_{p}'] = np.percentile(x, p)
 feature_dict[f'abs_percentile_{p}'] = np.percentile(np.abs(x), p)
 feature_dict['trend'] = add_trend_feature(x)
 feature_dict['abs_trend'] = add_trend_feature(x, abs_values=True)
 feature_dict['mad'] = x.mad()
feature_dict['kurt'] = x.kurtosis()
feature_dict['skew'] = x.skew()
feature_dict['med'] = x.median()
feature_dict['Hilbert_mean'] = np.abs(hilbert(x)).mean()
for hw in hann_windows:
feature_dict[f'Hann_window_mean_{hw}'] =(convolve(x, hann(hw), mode='same')/ sum(hann(hw)) ).mean()
feature_dict['classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
feature_dict['classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
feature_dict['classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
feature_dict['classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
feature_dict['classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000 ).mean()
feature_dict['classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
feature_dict['classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666 ).mean()
feature_dict['classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000 ).mean()
ewma = pd.Series.ewm
for s in spans:
feature_dict[f'exp_Moving_average_{s}_mean'] =(ewma(x, span=s ).mean(skipna=True)).mean(skipna=True)
 feature_dict[f'exp_Moving_average_{s}_std'] =(ewma(x, span=s ).mean(skipna=True)).std(skipna=True)
 feature_dict[f'exp_Moving_std_{s}_mean'] =(ewma(x, span=s ).std(skipna=True)).mean(skipna=True)
 feature_dict[f'exp_Moving_std_{s}_std'] =(ewma(x, span=s ).std(skipna=True)).std(skipna=True)
 feature_dict['iqr'] = np.subtract(*np.percentile(x, [75, 25]))
feature_dict['iqr1'] = np.subtract(*np.percentile(x, [95, 5]))
feature_dict['ave10'] = stats.trim_mean(x, 0.1)
 for slice_length, threshold in product([50000, 100000, 150000],
[5, 10, 20, 50, 100]):
feature_dict[f'count_big_{slice_length}_threshold_{threshold}'] =(np.abs(x[-slice_length:])> threshold ).sum()
feature_dict[f'count_big_{slice_length}_less_threshold_{threshold}'] =(np.abs(x[-slice_length:])< threshold ).sum()
feature_dict['range_minf_m4000'] = feature_calculators.range_count(x, -np.inf, -4000)
 feature_dict['range_p4000_pinf'] = feature_calculators.range_count(x, 4000, np.inf)
 for i, j in zip(borders, borders[1:]):
feature_dict[f'range_{i}_{j}'] = feature_calculators.range_count(x, i, j)
 for autocorr_lag in autocorr_lags:
feature_dict[f'autocorrelation_{autocorr_lag}'] = feature_calculators.autocorrelation(x, autocorr_lag)
 feature_dict[f'c3_{autocorr_lag}'] = feature_calculators.c3(x, autocorr_lag)
 for p in percentiles:
feature_dict[f'binned_entropy_{p}'] = feature_calculators.binned_entropy(x, p)
 feature_dict['num_crossing_0'] = feature_calculators.number_crossing_m(x, 0)
 for peak in peaks:
feature_dict[f'num_peaks_{peak}'] = feature_calculators.number_peaks(x, peak)
 for c in coefs:
feature_dict[f'spkt_welch_density_{c}'] = list(feature_calculators.spkt_welch_density(x, [{'coeff': c}])) [0][1]
feature_dict[f'time_rev_asym_stat_{c}'] = feature_calculators.time_reversal_asymmetry_statistic(x, c)
 for w in windows:
x_roll_std = x.rolling(w ).std().dropna().values
x_roll_mean = x.rolling(w ).mean().dropna().values
feature_dict[f'ave_roll_std_{w}'] = x_roll_std.mean()
feature_dict[f'std_roll_std_{w}'] = x_roll_std.std()
feature_dict[f'max_roll_std_{w}'] = x_roll_std.max()
feature_dict[f'min_roll_std_{w}'] = x_roll_std.min()
for p in percentiles:
feature_dict[f'percentile_roll_std_{p}_window_{w}'] = np.percentile(x_roll_std, p)
 feature_dict[f'av_change_abs_roll_std_{w}'] = np.mean(np.diff(x_roll_std))
feature_dict[f'av_change_rate_roll_std_{w}'] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 feature_dict[f'abs_max_roll_std_{w}'] = np.abs(x_roll_std ).max()
feature_dict[f'ave_roll_mean_{w}'] = x_roll_mean.mean()
feature_dict[f'std_roll_mean_{w}'] = x_roll_mean.std()
feature_dict[f'max_roll_mean_{w}'] = x_roll_mean.max()
feature_dict[f'min_roll_mean_{w}'] = x_roll_mean.min()
for p in percentiles:
feature_dict[f'percentile_roll_mean_{p}_window_{w}'] = np.percentile(x_roll_mean, p)
 feature_dict[f'av_change_abs_roll_mean_{w}'] = np.mean(np.diff(x_roll_mean))
feature_dict[f'av_change_rate_roll_mean_{w}'] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 feature_dict[f'abs_max_roll_mean_{w}'] = np.abs(x_roll_mean ).max()
return feature_dict
def generate(self):
feature_list = []
res = Parallel(n_jobs=self.n_jobs,
backend='threading' )(delayed(self.get_features )(x, y, s)
 for s, x, y in tqdm_notebook(self.read_chunks() , total=self.total_data))
for r in res:
feature_list.append(r)
 return pd.DataFrame(feature_list )<prepare_x_and_y>
check_count = train_2.groupby(["date_block_num","shop_id","item_id"])['shop_id'].size()
print(check_count[check_count > 1])
training_fg = FeatureGenerator(dtype='train', n_jobs=20, chunk_size=150000)
 training_data = training_fg.generate()
test_fg = FeatureGenerator(dtype='test', n_jobs=20, chunk_size=150000)
 test_data = test_fg.generate()
X = training_data.drop(['target', 'seg_id'], axis=1)
 X_test = test_data.drop(['target', 'seg_id'], axis=1)
 test_segs = test_data.seg_id
y = training_data.target<sort_values>
cols = ['shop_id','item_id']
train_2 = pd.concat([train_2, test], ignore_index=True, sort=False, keys=cols )
np.abs(X.corrwith(y)).sort_values(ascending=False ).head(12 )<feature_engineering>
def sum_df(df,columns,target):
x= []
name=target+"_sum"
data = columns.split(",")
 for col in data:
x.append(col)
 print(x)
 return df.groupby(x)[target].sum().to_frame(name ).reset_index()
def mean_df(df,columns,target):
x= []
name=target+"_mean"
data = columns.split(",")
 for col in data:
x.append(col)
 print(x)
 return df.groupby(x)[target].mean().to_frame(name ).reset_index()
means_dict = {}
for col in X.columns:
if X[col].isnull().any() :
print(col)
 mean_value = X.loc[X[col] != -np.inf, col].mean()
X.loc[X[col] == -np.inf, col] = mean_value
X[col] = X[col].fillna(mean_value)
 means_dict[col] = mean_value<feature_engineering>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
for col in X_test.columns:
if X_test[col].isnull().any() :
X_test.loc[X_test[col] == -np.inf, col] = means_dict[col]
X_test[col] = X_test[col].fillna(means_dict[col] )<choose_model_class>
group = train_2.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num'], how='left')
 train_2['date_avg_item_cnt'] = train_2['date_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_avg_item_cnt')
 train_2.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 group = train_2.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num','item_id'], how='left')
 train_2['date_item_avg_item_cnt'] = train_2['date_item_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1,2,3,6,12], 'date_item_avg_item_cnt')
 train_2.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 group = train_2.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num','shop_id'], how='left')
 train_2['date_shop_avg_item_cnt'] = train_2['date_shop_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 train_2.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 group = train_2.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num','item_category_id'], how='left')
 train_2['date_cat_avg_item_cnt'] = train_2['date_cat_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_cat_avg_item_cnt')
 train_2.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11 )<split>
group = train_2.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'city_code'], how='left')
 train_2['date_city_avg_item_cnt'] = train_2['date_city_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_city_avg_item_cnt')
 train_2.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 group = train_2.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 train_2['date_item_city_avg_item_cnt'] = train_2['date_item_city_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_item_city_avg_item_cnt')
 train_2.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 group = train_2.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'type_code'], how='left')
 train_2['date_type_avg_item_cnt'] = train_2['date_type_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_type_avg_item_cnt')
 train_2.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 group = train_2.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'subtype_code'], how='left')
 train_2['date_subtype_avg_item_cnt'] = train_2['date_subtype_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_subtype_avg_item_cnt')
 train_2.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True )
def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if type(X)== np.ndarray:
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae',
verbose=10000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='MAE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction, scores
else:
return oof, prediction, scores<init_hyperparams>
group = train_2.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 train_2['date_shop_cat_avg_item_cnt'] = train_2['date_shop_cat_avg_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_shop_cat_avg_item_cnt')
 train_2.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True )
params = {'num_leaves': 128,
'min_data_in_leaf': 79,
'objective': 'gamma',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": 0.8126672064208567,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501,
'feature_fraction': 0.2
}
oof_lgb, prediction_lgb, feature_importance = train_model(X,
X_test,
y,
params=params,
model_type='lgb',
plot_feature_importance=True )<save_to_csv>
group = train_2.groupby(['date_block_num', 'shop_id', 'item_id'] ).agg({'item_cnt_month': ['median']})
 group.columns = ['date_shop_item_med_item_cnt']
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 train_2['date_shop_item_med_item_cnt'] = train_2['date_shop_item_med_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1,2,3,6,12], 'date_shop_item_med_item_cnt')
 train_2.drop(['date_shop_item_med_item_cnt'], axis=1, inplace=True)
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 submission['time_to_failure'] = prediction_lgb
print(submission.head())
 submission.to_csv('submission.csv' )<save_to_csv>
group = train_2.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['median']})
 group.columns = ['date_shop_med_item_cnt']
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'shop_id'], how='left')
 train_2['date_shop_med_item_cnt'] = train_2['date_shop_med_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_shop_med_item_cnt')
 train_2.drop(['date_shop_med_item_cnt'], axis=1, inplace=True )
X.to_csv('train_features.csv', index=False)
 X_test.to_csv('test_features.csv', index=False)
 pd.DataFrame(y ).to_csv('y.csv', index=False )<load_from_csv>
group = train_2.groupby(['date_block_num', 'shop_id','item_category_id'] ).agg({'item_cnt_month': ['median']})
 group.columns = ['date_shop_cat_med_item_cnt']
group.reset_index(inplace=True)
 train_2 = pd.merge(train_2, group, on=['date_block_num', 'shop_id','item_category_id'], how='left')
 train_2['date_shop_cat_med_item_cnt'] = train_2['date_shop_cat_med_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_shop_cat_med_item_cnt')
 train_2.drop(['date_shop_cat_med_item_cnt'], axis=1, inplace=True )
class FeatureGenerator(object):
def __init__(self, dtype, n_jobs=1, chunk_size=None):
self.chunk_size = chunk_size
self.dtype = dtype
self.filename = None
self.n_jobs = n_jobs
self.test_files = []
if self.dtype == 'train':
self.filename = '.. /input/train.csv'
self.total_data = int(629145481 / self.chunk_size)
 else:
submission = pd.read_csv('.. /input/sample_submission.csv')
 for seg_id in submission.seg_id.values:
self.test_files.append(( seg_id, '.. /input/test/' + seg_id + '.csv'))
self.total_data = int(len(submission))
def read_chunks(self):
if self.dtype == 'train':
iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,
dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})
 for counter, df in enumerate(iter_df):
x = df.acoustic_data.values
y = df.time_to_failure.values[-1]
seg_id = 'train_' + str(counter)
 del df
yield seg_id, x, y
else:
for seg_id, f in self.test_files:
df = pd.read_csv(f, dtype={'acoustic_data': np.float64})
 x = df.acoustic_data.values[-self.chunk_size:]
del df
yield seg_id, x, -999
def features(self, x, y, seg_id):
feature_dict = dict()
feature_dict['target'] = y
feature_dict['seg_id'] = seg_id
feature_dict['mean'] = np.mean(x)
 feature_dict['max'] = np.max(x)
 feature_dict['min'] = np.min(x)
 feature_dict['std'] = np.std(x)
 feature_dict['var'] = np.var(x)
 feature_dict['ptp'] = np.ptp(x)
 feature_dict['percentile_10'] = np.percentile(x, 10)
 feature_dict['percentile_20'] = np.percentile(x, 20)
 feature_dict['percentile_30'] = np.percentile(x, 30)
 feature_dict['percentile_40'] = np.percentile(x, 40)
 feature_dict['percentile_50'] = np.percentile(x, 50)
 feature_dict['percentile_60'] = np.percentile(x, 60)
 feature_dict['percentile_70'] = np.percentile(x, 70)
 feature_dict['percentile_80'] = np.percentile(x, 80)
 feature_dict['percentile_90'] = np.percentile(x, 90)
 feature_dict['skew'] = sp.stats.skew(x)
 feature_dict['kurtosis'] = sp.stats.kurtosis(x)
 feature_dict['kstat_1'] = sp.stats.kstat(x, 1)
 feature_dict['kstat_2'] = sp.stats.kstat(x, 2)
 feature_dict['kstat_3'] = sp.stats.kstat(x, 3)
 feature_dict['kstat_4'] = sp.stats.kstat(x, 4)
 feature_dict['moment_1'] = sp.stats.moment(x, 1)
 feature_dict['moment_2'] = sp.stats.moment(x, 2)
 feature_dict['moment_3'] = sp.stats.moment(x, 3)
 feature_dict['moment_4'] = sp.stats.moment(x, 4)
 feature_dict['abs_energy'] = feature_calculators.abs_energy(x)
 feature_dict['abs_sum_of_changes'] = feature_calculators.absolute_sum_of_changes(x)
 feature_dict['count_above_mean'] = feature_calculators.count_above_mean(x)
 feature_dict['count_below_mean'] = feature_calculators.count_below_mean(x)
 feature_dict['mean_abs_change'] = feature_calculators.mean_abs_change(x)
 feature_dict['mean_change'] = feature_calculators.mean_change(x)
 feature_dict['var_larger_than_std_dev'] = feature_calculators.variance_larger_than_standard_deviation(x)
 feature_dict['range_minf_m4000'] = feature_calculators.range_count(x, -np.inf, -4000)
 feature_dict['range_m4000_m3000'] = feature_calculators.range_count(x, -4000, -3000)
 feature_dict['range_m3000_m2000'] = feature_calculators.range_count(x, -3000, -2000)
 feature_dict['range_m2000_m1000'] = feature_calculators.range_count(x, -2000, -1000)
 feature_dict['range_m1000_0'] = feature_calculators.range_count(x, -1000, 0)
 feature_dict['range_0_p1000'] = feature_calculators.range_count(x, 0, 1000)
 feature_dict['range_p1000_p2000'] = feature_calculators.range_count(x, 1000, 2000)
 feature_dict['range_p2000_p3000'] = feature_calculators.range_count(x, 2000, 3000)
 feature_dict['range_p3000_p4000'] = feature_calculators.range_count(x, 3000, 4000)
 feature_dict['range_p4000_pinf'] = feature_calculators.range_count(x, 4000, np.inf)
 feature_dict['ratio_unique_values'] = feature_calculators.ratio_value_number_to_time_series_length(x)
 feature_dict['first_loc_min'] = feature_calculators.first_location_of_minimum(x)
 feature_dict['first_loc_max'] = feature_calculators.first_location_of_maximum(x)
 feature_dict['last_loc_min'] = feature_calculators.last_location_of_minimum(x)
 feature_dict['last_loc_max'] = feature_calculators.last_location_of_maximum(x)
 feature_dict['time_rev_asym_stat_10'] = feature_calculators.time_reversal_asymmetry_statistic(x, 10)
 feature_dict['time_rev_asym_stat_100'] = feature_calculators.time_reversal_asymmetry_statistic(x, 100)
 feature_dict['time_rev_asym_stat_1000'] = feature_calculators.time_reversal_asymmetry_statistic(x, 1000)
 feature_dict['autocorrelation_5'] = feature_calculators.autocorrelation(x, 5)
 feature_dict['autocorrelation_10'] = feature_calculators.autocorrelation(x, 10)
 feature_dict['autocorrelation_50'] = feature_calculators.autocorrelation(x, 50)
 feature_dict['autocorrelation_100'] = feature_calculators.autocorrelation(x, 100)
 feature_dict['autocorrelation_1000'] = feature_calculators.autocorrelation(x, 1000)
 feature_dict['c3_5'] = feature_calculators.c3(x, 5)
 feature_dict['c3_10'] = feature_calculators.c3(x, 10)
 feature_dict['c3_100'] = feature_calculators.c3(x, 100)
 feature_dict['fft_1_real'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 1, 'attr': 'real'}])) [0][1]
feature_dict['fft_1_imag'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 1, 'attr': 'imag'}])) [0][1]
feature_dict['fft_1_ang'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 1, 'attr': 'angle'}])) [0][1]
feature_dict['fft_2_real'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 2, 'attr': 'real'}])) [0][1]
feature_dict['fft_2_imag'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 2, 'attr': 'imag'}])) [0][1]
feature_dict['fft_2_ang'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 2, 'attr': 'angle'}])) [0][1]
feature_dict['fft_3_real'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 3, 'attr': 'real'}])) [0][1]
feature_dict['fft_3_imag'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 3, 'attr': 'imag'}])) [0][1]
feature_dict['fft_3_ang'] = list(feature_calculators.fft_coefficient(x, [{'coeff': 3, 'attr': 'angle'}])) [0][1]
feature_dict['long_strk_above_mean'] = feature_calculators.longest_strike_above_mean(x)
 feature_dict['long_strk_below_mean'] = feature_calculators.longest_strike_below_mean(x)
 feature_dict['cid_ce_0'] = feature_calculators.cid_ce(x, 0)
 feature_dict['cid_ce_1'] = feature_calculators.cid_ce(x, 1)
 feature_dict['binned_entropy_5'] = feature_calculators.binned_entropy(x, 5)
 feature_dict['binned_entropy_10'] = feature_calculators.binned_entropy(x, 10)
 feature_dict['binned_entropy_20'] = feature_calculators.binned_entropy(x, 20)
 feature_dict['binned_entropy_50'] = feature_calculators.binned_entropy(x, 50)
 feature_dict['binned_entropy_80'] = feature_calculators.binned_entropy(x, 80)
 feature_dict['binned_entropy_100'] = feature_calculators.binned_entropy(x, 100)
 feature_dict['num_crossing_0'] = feature_calculators.number_crossing_m(x, 0)
 feature_dict['num_peaks_10'] = feature_calculators.number_peaks(x, 10)
 feature_dict['num_peaks_50'] = feature_calculators.number_peaks(x, 50)
 feature_dict['num_peaks_100'] = feature_calculators.number_peaks(x, 100)
 feature_dict['num_peaks_500'] = feature_calculators.number_peaks(x, 500)
 feature_dict['spkt_welch_density_1'] = list(feature_calculators.spkt_welch_density(x, [{'coeff': 1}])) [0][1]
feature_dict['spkt_welch_density_10'] = list(feature_calculators.spkt_welch_density(x, [{'coeff': 10}])) [0][1]
feature_dict['spkt_welch_density_50'] = list(feature_calculators.spkt_welch_density(x, [{'coeff': 50}])) [0][1]
feature_dict['spkt_welch_density_100'] = list(feature_calculators.spkt_welch_density(x, [{'coeff': 100}])) [0][1]
feature_dict['time_rev_asym_stat_1'] = feature_calculators.time_reversal_asymmetry_statistic(x, 1)
 feature_dict['time_rev_asym_stat_10'] = feature_calculators.time_reversal_asymmetry_statistic(x, 10)
 feature_dict['time_rev_asym_stat_100'] = feature_calculators.time_reversal_asymmetry_statistic(x, 100)
 return feature_dict
def generate(self):
feature_list = []
res = Parallel(n_jobs=self.n_jobs,
backend='threading' )(delayed(self.features )(x, y, s)
 for s, x, y in tqdm(self.read_chunks() , total=self.total_data))
for r in res:
feature_list.append(r)
 return pd.DataFrame(feature_list)
 training_fg = FeatureGenerator(dtype='train', n_jobs=10, chunk_size=150000)
 training_data = training_fg.generate()
test_fg = FeatureGenerator(dtype='test', n_jobs=10, chunk_size=150000)
 test_data = test_fg.generate()<prepare_x_and_y>
train_2.columns
X = training_data.drop(['target', 'seg_id'], axis=1)
 X_test = test_data.drop(['target', 'seg_id'], axis=1)
 test_segs = test_data.seg_id
y = training_data.target<define_variables>
group = train_2.groupby(['date_block_num', 'shop_id','item_id'])['item_cnt_month'].quantile (.25)
 group.columns = ['date_shop_item_q25_item_cnt']
dfx=pd.DataFrame()
dfx["date_shop_item_q25_item_cnt"]=group
print(train_2.columns)
 train_2 = pd.merge(train_2, dfx, on=['date_block_num', 'shop_id','item_id'], how='left')
 train_2['date_shop_item_q25_item_cnt'] = train_2['date_shop_item_q25_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_shop_item_q25_item_cnt')
 train_2.drop(['date_shop_item_q25_item_cnt'], axis=1, inplace=True )
folds = KFold(n_splits=5, shuffle=True, random_state=42)
 oof_preds = np.zeros(( len(X), 1))
test_preds = np.zeros(( len(X_test), 1))<init_hyperparams>
group = train_2.groupby(['date_block_num', 'shop_id','item_id'])['item_cnt_month'].quantile (.75)
 group.columns = ['date_shop_item_q75_item_cnt']
dfx=pd.DataFrame()
dfx["date_shop_item_q75_item_cnt"]=group
print(train_2.columns)
 train_2 = pd.merge(train_2, dfx, on=['date_block_num', 'shop_id','item_id'], how='left')
 train_2['date_shop_item_q75_item_cnt'] = train_2['date_shop_item_q75_item_cnt'].astype(np.float16)
 train_2 = lag_feature(train_2, [1], 'date_shop_item_q75_item_cnt')
 train_2.drop(['date_shop_item_q75_item_cnt'], axis=1, inplace=True )
params = {
"learning_rate": 0.01,
"max_depth": 4,
"n_estimators": 10000,
"min_child_weight": 1,
"colsample_bytree": 0.9,
"subsample": 1.0,
"nthread": 12,
"random_state": 42,
}<find_best_model_class>
train_2['month'] = train_2['date_block_num'] % 12
for fold_,(trn_, val_)in enumerate(folds.split(X)) :
print("Current Fold: {}".format(fold_))
trn_x, trn_y = X.iloc[trn_], y.iloc[trn_]
val_x, val_y = X.iloc[val_], y.iloc[val_]
clf = xgb.XGBRegressor(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric='mae',
verbose=150,
early_stopping_rounds=100 
)
 val_pred = clf.predict(val_x, ntree_limit=clf.best_ntree_limit)
 test_fold_pred = clf.predict(X_test, ntree_limit=clf.best_ntree_limit)
 print("MAE = {}".format(metrics.mean_absolute_error(val_y, val_pred)))
 oof_preds[val_, :] = val_pred.reshape(( -1, 1))
test_preds += test_fold_pred.reshape(( -1, 1))
test_preds /= 5
oof_score = metrics.mean_absolute_error(y, oof_preds)
 print("Mean MAE = {}".format(oof_score))<save_to_csv>
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
train_2 = fill_na(train_2 )
submission = pd.DataFrame(columns=['seg_id', 'time_to_failure'])
 submission.seg_id = test_segs
submission.time_to_failure = test_preds
submission.to_csv('submission.csv', index=False)
<define_variables>
final=train_2[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1', 'item_cnt_month_lag_2', 'item_cnt_month_lag_3', 'item_cnt_month_lag_6', 'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2', 'date_item_avg_item_cnt_lag_3', 'date_item_avg_item_cnt_lag_6', 'date_cat_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1', 'date_subtype_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1', 'date_shop_item_med_item_cnt_lag_1',
'date_shop_item_med_item_cnt_lag_2', 'date_shop_item_med_item_cnt_lag_3',
'date_shop_item_med_item_cnt_lag_6', 'date_shop_cat_med_item_cnt_lag_1',
'date_shop_item_q25_item_cnt_lag_1', 'date_shop_item_q75_item_cnt_lag_1'
]]
random_seed = 4126
cast = {
'acoustic_data': 'int',
'time_to_failure': 'float'
}
def df_fragments(path, length, skip=1):
with open(path, 'r')as f:
m = {}
cols = []
count = 0
index = 0
for line in f:
if len(cols)== 0:
for col in line.strip("
\r " ).split(','):
cols.append(col)
 continue
if count == 0:
for col in cols:
m[col] = []
if index % skip == 0:
for j, cell in enumerate(line.strip("
\r " ).split(',')) :
col = cols[j]
m[col].append(getattr(builtins, cast[col] )(cell))
count += 1
if count == length:
if index % skip == 0:
yield pd.DataFrame(m)
 index += 1
count = 0
def count_rows(path):
with open(path, 'r')as f:
i = -1
for _ in f:
i += 1
return i<define_variables>
def One_way_ANOVA(df):
lister12 = []
for col in df.columns:
if df[col].dtype.kind not in 'bifc':
model = ols('item_cnt_month ~'+col,data=df ).fit()
table = sm.stats.anova_lm(model, typ=2)
 if table["PR(>F)"][0] < 0.05:
lister12.append(col)
 return lister12
print('counting total...')
 total = count_rows('.. /input/train.csv')
 print('total: {}'.format(total))<feature_engineering>
train_check=train_2[(train_2.date_block_num<=5)]
train_check['item_category_id_char']=train_check['item_category_id'].apply(str)
 train_check['type_code_char']=train_check['type_code'].apply(str)
 train_check['subtype_code_char']=train_check['subtype_code'].apply(str )
print('generating train data...')
 fragment_size = 150000
skip = 1
mfcc_ttf_map = collections.defaultdict(list)
 for df in tqdm(df_fragments('.. /input/train.csv', length=fragment_size, skip=skip), total=(total//fragment_size)//skip):
mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(each_mfcc_mean)
 key = 'time_to_failure'
if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(df.iloc[-1][key])
 mfcc_ttf_df = pd.DataFrame(mfcc_ttf_map)
 fname = 'mfcc_train.csv'
mfcc_ttf_df.to_csv(fname, index=False)
 print('saved {}.'.format(fname)) 
<compute_train_metric>
One_way_ANOVA(train_check[["city","shop_name","item_category_id_char","subtype_code_char","type_code_char","item_cnt_month"]] )
def report_cv(model):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=10)
 print('Cross Validation scores: {}'.format(abs(scores)))
 print('Average score: {}'.format(abs(scores.mean())))
 parameter_grid = {'n_estimators': [50, 60],
'max_depth': [5, 10]
}
report_cv(GridSearchCV(RandomForestRegressor() , param_grid=parameter_grid,n_jobs = -1))<set_options>
final.rename(columns = {'item_cnt_month':'item_monthly_sum_x'}, inplace = True)
 final["item_monthly_sum_x"] = final["item_monthly_sum_x"]
X_train = final[final.date_block_num < 33].drop(['item_monthly_sum_x'], axis=1)
 Y_train = final[final.date_block_num < 33]['item_monthly_sum_x']
X_valid = final[final.date_block_num == 33].drop(['item_monthly_sum_x'], axis=1)
 Y_valid = final[final.date_block_num == 33]['item_monthly_sum_x']
X_test = final[final.date_block_num == 34].drop(['item_monthly_sum_x'], axis=1 )
report_cv(CatBoostRegressor(verbose=False))<feature_engineering>
Y_test = model.predict(X_test)
 submission = pd.DataFrame({
"ID": test.ID,
"item_cnt_month": Y_test.clip(0, 20)
 })
 submission.to_csv('xgb_submission.csv', index=False)