Kaggle competition: Grupo Bimbo Inventory Demand
<SOS> metric: RMSLE Kaggle data source: grupo-bimbo-inventory-demand<load_from_csv>
types = {'Semana':np.uint8, 'Agencia_ID':np.uint16, 'Canal_ID':np.uint8,
'Ruta_SAK':np.uint16, 'Cliente_ID':np.uint32, 'Producto_ID':np.uint16,
'Demanda_uni_equil':np.uint32}
train=pd.read_csv('.. /input/train.csv',usecols=types.keys() ,dtype=types,nrows=1000)
 print(train.dtype)
 print(train.info(memery_usage=True))
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<train_on_grid>
types = {'Semana':np.uint8, 'Agencia_ID':np.uint16, 'Canal_ID':np.uint8,
'Ruta_SAK':np.uint16, 'Cliente_ID':np.uint32, 'Producto_ID':np.uint16,
'Demanda_uni_equil':np.uint32}
train=pd.read_csv('.. /input/train.csv',usecols=types.keys() ,dtypes=types,nrows=1000)
 print(train.dtype)
 print(train.info(memery_usage=True))
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
X_tr = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_tr = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 total_mean = train['acoustic_data'].mean()
total_std = train['acoustic_data'].std()
total_max = train['acoustic_data'].max()
total_min = train['acoustic_data'].min()
total_sum = train['acoustic_data'].sum()
total_abs_sum = np.abs(train['acoustic_data'] ).sum()
for segment in tqdm_notebook(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x = pd.Series(seg['acoustic_data'].values)
 y = seg['time_to_failure'].values[-1]
y_tr.loc[segment, 'time_to_failure'] = y
X_tr.loc[segment, 'mean'] = x.mean()
X_tr.loc[segment, 'std'] = x.std()
X_tr.loc[segment, 'max'] = x.max()
X_tr.loc[segment, 'min'] = x.min()
X_tr.loc[segment, 'mean_change_abs'] = np.mean(np.diff(x))
X_tr.loc[segment, 'mean_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_tr.loc[segment, 'abs_max'] = np.abs(x ).max()
X_tr.loc[segment, 'abs_min'] = np.abs(x ).min()
X_tr.loc[segment, 'std_first_50000'] = x[:50000].std()
X_tr.loc[segment, 'std_last_50000'] = x[-50000:].std()
X_tr.loc[segment, 'std_first_10000'] = x[:10000].std()
X_tr.loc[segment, 'std_last_10000'] = x[-10000:].std()
X_tr.loc[segment, 'avg_first_50000'] = x[:50000].mean()
X_tr.loc[segment, 'avg_last_50000'] = x[-50000:].mean()
X_tr.loc[segment, 'avg_first_10000'] = x[:10000].mean()
X_tr.loc[segment, 'avg_last_10000'] = x[-10000:].mean()
X_tr.loc[segment, 'min_first_50000'] = x[:50000].min()
X_tr.loc[segment, 'min_last_50000'] = x[-50000:].min()
X_tr.loc[segment, 'min_first_10000'] = x[:10000].min()
X_tr.loc[segment, 'min_last_10000'] = x[-10000:].min()
X_tr.loc[segment, 'max_first_50000'] = x[:50000].max()
X_tr.loc[segment, 'max_last_50000'] = x[-50000:].max()
X_tr.loc[segment, 'max_first_10000'] = x[:10000].max()
X_tr.loc[segment, 'max_last_10000'] = x[-10000:].max()
X_tr.loc[segment, 'max_to_min'] = x.max() / np.abs(x.min())
 X_tr.loc[segment, 'max_to_min_diff'] = x.max() - np.abs(x.min())
 X_tr.loc[segment, 'count_big'] = len(x[np.abs(x)> 500])
 X_tr.loc[segment, 'sum'] = x.sum()
X_tr.loc[segment, 'mean_change_rate_first_50000'] = np.mean(np.nonzero(( np.diff(x[:50000])/ x[:50000][:-1])) [0])
 X_tr.loc[segment, 'mean_change_rate_last_50000'] = np.mean(np.nonzero(( np.diff(x[-50000:])/ x[-50000:][:-1])) [0])
 X_tr.loc[segment, 'mean_change_rate_first_10000'] = np.mean(np.nonzero(( np.diff(x[:10000])/ x[:10000][:-1])) [0])
 X_tr.loc[segment, 'mean_change_rate_last_10000'] = np.mean(np.nonzero(( np.diff(x[-10000:])/ x[-10000:][:-1])) [0])
 X_tr.loc[segment, 'q95'] = np.quantile(x, 0.95)
 X_tr.loc[segment, 'q99'] = np.quantile(x, 0.99)
 X_tr.loc[segment, 'q05'] = np.quantile(x, 0.05)
 X_tr.loc[segment, 'q01'] = np.quantile(x, 0.01)
 X_tr.loc[segment, 'abs_q95'] = np.quantile(np.abs(x), 0.95)
 X_tr.loc[segment, 'abs_q99'] = np.quantile(np.abs(x), 0.99)
 X_tr.loc[segment, 'abs_q05'] = np.quantile(np.abs(x), 0.05)
 X_tr.loc[segment, 'abs_q01'] = np.quantile(np.abs(x), 0.01)
 X_tr.loc[segment, 'trend'] = add_trend_feature(x)
 X_tr.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_tr.loc[segment, 'abs_mean'] = np.abs(x ).mean()
X_tr.loc[segment, 'abs_std'] = np.abs(x ).std()
X_tr.loc[segment, 'mad'] = x.mad()
X_tr.loc[segment, 'kurt'] = x.kurtosis()
X_tr.loc[segment, 'skew'] = x.skew()
X_tr.loc[segment, 'med'] = x.median()
X_tr.loc[segment, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()
X_tr.loc[segment, 'Hann_window_mean'] =(convolve(x, hann(150), mode='same')/ sum(hann(150)) ).mean()
X_tr.loc[segment, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
X_tr.loc[segment, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
X_tr.loc[segment, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
X_tr.loc[segment, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
X_tr.loc[segment, 'Moving_average_700_mean'] = x.rolling(window=700 ).mean().mean(skipna=True)
 X_tr.loc[segment, 'Moving_average_1500_mean'] = x.rolling(window=1500 ).mean().mean(skipna=True)
 X_tr.loc[segment, 'Moving_average_3000_mean'] = x.rolling(window=3000 ).mean().mean(skipna=True)
 X_tr.loc[segment, 'Moving_average_6000_mean'] = x.rolling(window=6000 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X_tr.loc[segment, 'exp_Moving_average_300_mean'] =(ewma(x, span=300 ).mean() ).mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000 ).mean().mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_30000_mean'] = ewma(x, span=6000 ).mean().mean(skipna=True)
 no_of_std = 2
X_tr.loc[segment, 'MA_700MA_std_mean'] = x.rolling(window=700 ).std().mean()
X_tr.loc[segment,'MA_700MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_700MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_400MA_std_mean'] = x.rolling(window=400 ).std().mean()
X_tr.loc[segment,'MA_400MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_400MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_1000MA_std_mean'] = x.rolling(window=1000 ).std().mean()
X_tr.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_tr.loc[segment, 'q999'] = np.quantile(x,0.999)
 X_tr.loc[segment, 'q001'] = np.quantile(x,0.001)
 X_tr.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_tr.loc[segment, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_tr.loc[segment, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_tr.loc[segment, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_tr.loc[segment, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_tr.loc[segment, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_tr.loc[segment, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_tr.loc[segment, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_tr.loc[segment, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_tr.loc[segment, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_tr.loc[segment, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_tr.loc[segment, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_tr.loc[segment, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_tr.loc[segment, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_tr.loc[segment, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_tr.loc[segment, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_tr.loc[segment, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_tr.loc[segment, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_tr.loc[segment, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()<compute_test_metric>
print(check_output(['ls','.'] ).decode('utf8'))
submission=pd.read_csv('.. /input/sample_submission.csv')
 print(submission.shape)
 print(submission.columns)
 print(submission.head(20))
np.abs(X_tr.corrwith(y_tr['time_to_failure'])).sort_values(ascending=False ).head(12 )<load_from_csv>
def evalerror(preds, dtrain):
labels = dtrain.get_label()
assert len(preds)== len(labels)
 labels = labels.tolist()
preds = preds.tolist()
terms_to_sum = [(math.log(labels[i] + 1)- math.log(max(0,preds[i])+ 1)) ** 2.0 for i,pred in enumerate(labels)]
return 'error',(sum(terms_to_sum)*(1.0/len(preds)))** 0.5
nrows=10000
train=pd.read_csv('.. /input/train.csv',nrows=nrows)
 test=pd.read_csv('.. /input/test.csv',nrows=nrows)
 print(train.columns)
 print(test.columns)
 print(test.columns.values)
 ids=test['id']
test=test.drop(['id'],axis=1)
 y=train['Demanda_uni_equil']
X=train[test.columns.values]
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=1000)
 params = {}
params['objective'] = "reg:linear"
params['eta'] = 0.05
params['max_depth'] = 5
params['subsample'] = 0.8
params['colsample_bytree'] = 0.6
params['silent'] = True
print('')
 test_preds = np.zeros(test.shape[0])
 xg_train = xgb.DMatrix(X_train, label=y_train)
 xg_test = xgb.DMatrix(X_test)
 watchlist = [(xg_train, 'train')]
num_rounds = 100
xgclassifier = xgb.train(params, xg_train, num_rounds, watchlist, feval = evalerror, early_stopping_rounds= 20, verbose_eval = 10)
 preds = xgclassifier.predict(xg_test, ntree_limit=xgclassifier.best_iteration)
 print('RMSLE Score:', rmsle(y_test, preds))
fxg_test = xgb.DMatrix(test)
 fold_preds = np.around(xgclassifier.predict(fxg_test, ntree_limit=xgclassifier.best_iteration), decimals = 1)
 test_preds += fold_preds
submission = pd.DataFrame({'id':ids, 'Demanda_uni_equil': test_preds})
 submission.to_csv('submission.csv', index=False )
<feature_engineering><EOS> 
nrows=100000
train=pd.read_csv('.. /input/train.csv',nrows=nrows)
 print(train.shape)
 print(train.columns)
 data=train.copy()
data['target']=data['Demanda_uni_equil']
data.drop(['Demanda_uni_equil'],axis=1,inplace=True)
 nCliente_ID = pd.DataFrame(pd.groupby(data,['Cliente_ID','Semana'])['target'].count())
 print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID = nCliente_ID.reset_index()
print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID.rename(columns={'target': 'nCliente_ID'}, inplace=True)
 print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID = pd.DataFrame(pd.groupby(nCliente_ID,['Cliente_ID'])['nCliente_ID'].mean())
 print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID = nCliente_ID.reset_index()
print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
data = pd.merge(data, nCliente_ID,
how='left',
left_on=['Cliente_ID'],
right_on=['Cliente_ID'],
left_index=False, right_index=False, sort=True,
suffixes=('_x', '_y'), copy=False)
 print(data.columns)
 print(data.head(50))
del nCliente_ID
gc.collect()
print('merge completo nCliente_ID')
 print(data.shape[0] )