Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<feature_engineering>
warnings.filterwarnings('ignore' )
def transformData(data, is_train=True):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
for column in data.columns:
if column in ['row_id','series_id','measurement_number', 'surface']:
continue
df[column + '_mean'] = data.groupby(['series_id'])[column].mean()
df[column + '_median'] = data.groupby(['series_id'])[column].median()
df[column + '_std'] = data.groupby(['series_id'])[column].std()
df[column + '_min'] = data.groupby(['series_id'])[column].min()
df[column + '_max'] = data.groupby(['series_id'])[column].max()
df[column + '_range'] = df[column + '_max'] - df[column + '_min']
df[column + '_maxtoMin'] = df[column + '_max'] / df[column + '_min']
df[column + '_mean_abs_chg'] = data.groupby(['series_id'])[column].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[column + '_abs_max'] = data.groupby(['series_id'])[column].apply(lambda x: np.max(np.abs(x)))
 df[column + '_abs_min'] = data.groupby(['series_id'])[column].apply(lambda x: np.min(np.abs(x)))
 df[column + '_abs_avg'] =(df[column + '_abs_min'] + df[column + '_abs_max'])/2
if is_train:
surface_groupby = data.groupby(['series_id'] ).surface.unique().apply(lambda x: x[0])
 df.loc[:,'surface'] = surface_groupby.values
return df<feature_engineering>
data_dir = '/kaggle/input/competitive-data-science-predict-future-sales'
os.listdir(data_dir )
def fe(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 +
data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 +
data['linear_acceleration_Z'])**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 +
data['orientation_Z'])**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
for col in data.columns:
if col in ['row_id','series_id','measurement_number', 'surface']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df<feature_engineering>
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 train.head()
train = fe(X_train)
 test = fe(X_test )<categorify>
item_catg = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 item_catg.info()
train.fillna(0, inplace = True)
 train.replace(-np.inf,0,inplace=True)
 train.replace(np.inf,0,inplace=True)
 test.fillna(0, inplace = True)
 test.replace(-np.inf,0,inplace=True)
 test.replace(np.inf,0,inplace=True )<categorify>
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 items.info()
le = LabelEncoder()
y = le.fit_transform(y_train['surface'])
 X_train = train<split>
shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales//shops.csv')
 shops.head()
<find_best_model_class>
def k_folds(X, y, X_test, k):
folds = StratifiedKFold(n_splits = k, shuffle=True, random_state=2019)
 y_test = np.zeros(( X_test.shape[0], 9))
y_oof = np.zeros(( X.shape[0]))
score = 0
for i,(train_idx, val_idx)in enumerate(folds.split(X, y)) :
clf = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(X.iloc[train_idx], y[train_idx])
 y_oof[val_idx] = clf.predict(X.iloc[val_idx])
 y_test += clf.predict_proba(X_test)/ folds.n_splits
score += clf.score(X.iloc[val_idx], y[val_idx])
 print('Fold: {} score: {}'.format(i,clf.score(X.iloc[val_idx], y[val_idx])))
 print('Avg Accuracy', score / folds.n_splits)
 return y_oof, y_test<compute_test_metric>
median = train[(train.item_price>0)].item_price.median()
train.loc[train.item_price<0,'item_price'] = median
train.drop_duplicates(subset =['date','date_block_num','item_id','item_price','item_cnt_day'],keep='first',inplace=True )
y_oof, y_test = k_folds(X_train, y, test, k= 50 )<compute_test_metric>
train.loc[train.shop_id==0,'shop_id']=57
test.loc[test.shop_id ==0,'shop_id']=57
train.loc[train.shop_id==1,'shop_id']=58
test.loc[test.shop_id ==1,'shop_id']=58
train.loc[train.shop_id==10,'shop_id']=11
test.loc[test.shop_id ==10,'shop_id']=11
confusion_matrix(y_oof,y )<prepare_x_and_y>
train = train.merge(test[['shop_id']].drop_duplicates() ,how = 'inner' )
<save_to_csv>
shops['city']=shops.shop_name.str.split(' ' ).map(lambda x:x[0])
 shops.loc[shops.city=='!Якутск','city'] = 'Якутск'
shops['shop_city'] = LabelEncoder().fit_transform(shops.city)
 shops['names'] = shops['shop_name'].apply(lambda x:x.lower() ).str.replace('[^\w\s]','' ).str.replace('\d+','' ).str.strip()
shops['shop_new_name']=LabelEncoder().fit_transform(shops.names )
y_test = np.argmax(y_test, axis=1)
 submission = pd.DataFrame(
 {'surface': le.inverse_transform(y_test)},
index=test.index 
)
 submission.to_csv('submission.csv', index=True)
 submission.head(10 )<init_hyperparams>
shops['type']= shops.names.apply(lambda x: 'МТРЦ' if 'МТРЦ' in x
else 'ТРЦ' if 'ТРЦ' in x
else 'ТЦ' if 'ТЦ' in x
else 'ТРК' if 'ТРК' in x
else 'ТК' if 'ТК' in x
else 'No_data')
 shops['shop_type'] = shops.type.map({'No_data':0,'МТРЦ' :1,'ТРЦ':2,
'ТЦ':3, 'ТРК':4,'ТК' :5})
 shops = shops[['shop_id','shop_city','shop_new_name','shop_type']]
<load_from_csv>
item_catg['split'] = item_catg.item_category_name.str.split('-')
 item_catg['type'] = item_catg['split'].map(lambda x:x[0].strip())
 item_catg['item_type'] = LabelEncoder().fit_transform(item_catg['type'])
 item_catg['subtype']=item_catg.split.map(lambda x:x[1].strip() if len(x)>1 else x[0].strip())
 item_catg['item_subtype'] = LabelEncoder().fit_transform(item_catg.subtype)
 item_catg = item_catg[['item_category_id','item_type','item_subtype']]
testing = True
X_train = pd.read_csv('.. /input/X_train.csv')
 y_train = pd.read_csv('.. /input/y_train.csv')
 X_train = pd.merge(X_train,y_train,on='series_id')
 le = LabelEncoder()
X_train['surface'] = le.fit_transform(X_train['surface'])
 if(testing):
X_test = pd.read_csv('.. /input/X_test.csv')
 X_test['series_id'] = X_test['series_id']+3810
X_test['group_id'] = 0
X_test['surface'] = 0
frames = [X_train,X_test]
X_train = pd.concat(frames)
 X_train.reset_index(drop=True,inplace=True)
<drop_column>
items['name1'],items['name2']=items.item_name.str.split('[',1 ).str
items['name1'],items['name3']=items.item_name.str.split('(',1 ).str
items.name2=items.name2.str.replace('[A-Za-z0-9A-Яа-я+]',' ' ).str.lower()
items.name3=items.name3.str.replace('[A-Za-z0-9A-Яа-я+]',' ' ).str.lower()
items= items.fillna('0')
 items['name_1'] =LabelEncoder().fit_transform(items.name1)
 items['name_2'] =LabelEncoder().fit_transform(items.name2)
 items['name_3'] =LabelEncoder().fit_transform(items.name3)
 items = items[['item_id','item_category_id','name_1','name_2','name_3']]
cols = list(X_train.columns.values)
 cols.remove('orientation_W')
 cols.insert(3,'orientation_W')
 X_train = X_train[cols]<count_unique_values>
matrix = []
cols = [ 'date_block_num', 'shop_id', 'item_id']
for i in train.date_block_num.unique() :
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i],sales.shop_id.unique() ,sales.item_id.unique())) ,dtype = 'int16'))
matrix = pd.DataFrame(np.vstack(matrix),columns = cols)
 matrix.date_block_num = matrix.date_block_num.astype(np.int8)
 matrix.shop_id = matrix.shop_id.astype(np.int8)
 matrix.item_id = matrix.item_id.astype(np.int16)
 matrix.sort_values(cols,inplace = True )
num_meas = 128
num_series = X_train['series_id'].nunique()<prepare_x_and_y>
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day':['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True )
def q_to_angle(q_val):
roll = np.arctan2(2*(q_val[0]*q_val[1] + q_val[2]*q_val[3]),1 - 2*(q_val[1]*q_val[1] + q_val[2]*q_val[2]))
pitch = np.arcsin(2*(q_val[0]*q_val[2] - q_val[3]*q_val[1]))
yaw = np.arctan2(2*(q_val[0]*q_val[3] + q_val[1]*q_val[2]),1 - 2*(q_val[2]*q_val[2] + q_val[3]*q_val[3]))
return np.array([roll, pitch, yaw] )<feature_engineering>
matrix = pd.merge(matrix,group,on=cols,how='left')
 matrix['item_cnt_month'] = matrix.item_cnt_month.fillna(0 ).clip(0,20 ).astype(np.float16 )
quat_arr = np.array(X_train[['orientation_W','orientation_X','orientation_Y','orientation_Z']])
 euler_arr = np.zeros([quat_arr.shape[0],3])
 for n,arr in enumerate(quat_arr):
euler_arr[n] = q_to_angle(arr )<prepare_x_and_y>
test['date_block_num'] = 34
matrix = pd.concat([matrix,test],ignore_index=True,sort=False,keys=cols)
 matrix.fillna(0,inplace=True)
 gc.collect()
X_train['roll'] = euler_arr[:,0]
X_train['pitch'] = euler_arr[:,1]
X_train['yaw'] = euler_arr[:,2]<drop_column>
matrix = pd.merge(matrix,shops,on=['shop_id'],how = 'left')
 matrix = pd.merge(matrix,items,on=['item_id'],how='left')
 matrix = pd.merge(matrix,item_catg,on = ['item_category_id'],how='left')
 del shops,items,item_catg
gc.collect()
cols = list(X_train.columns.values)
 cols.remove('group_id')
 cols.append('group_id')
 cols.remove('surface')
 cols.append('surface')
 X_train = X_train[cols]<prepare_x_and_y>
def lag_feature(df,lags,col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id',col+'_lag_'+ str(i)]
shifted['date_block_num'] += i
df = pd.merge(df,shifted,on=['date_block_num','shop_id','item_id'],how='left')
 return df
def add(list_names,list_num):
global matrix
str1 = list_names[0]
for i in range(1,len(list_names)) :
str1 = str1+'_and_'+list_names[i]
str1 = str1+'_avg_item_cnt'
group = matrix.groupby(list_names ).agg({'item_cnt_month': ['mean']})
 group.columns = [str1]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=list_names, how='left')
 matrix[str1] = matrix[str1].astype(np.float16)
 matrix = lag_feature(matrix, list_num, str1)
 matrix.drop([str1], axis=1, inplace=True )
feat_cols = ['roll','pitch','yaw','angular_velocity_X','angular_velocity_Y','angular_velocity_Z','linear_acceleration_X','linear_acceleration_Y','linear_acceleration_Z']
feat_array = np.array(X_train[feat_cols])
 feat_array = np.reshape(feat_array,[num_series,128,len(feat_cols)])
 group_array = np.array(X_train['group_id'])
 group_array = np.reshape(group_array,[num_series,128])
 group_array = group_array[:,0]
target_array = np.array(X_train['surface'])
 target_array = np.reshape(target_array,[num_series,128])
 target_array = target_array[:,0]<feature_engineering>
matrix = lag_feature(matrix,[1,2,3],'item_cnt_month')
 add(['date_block_num'],[1])
 add(['date_block_num','item_id'],[1,2,3])
 add(['date_block_num','shop_id'],[1,2,3])
 add(['date_block_num','shop_city'],[1])
 add(['date_block_num','shop_new_name'],[1])
 add(['date_block_num','shop_type'],[1])
 add(['date_block_num','item_category_id'],[1])
 add(['date_block_num','item_type'],[1])
 add(['date_block_num','item_subtype'],[1])
 add(['date_block_num','shop_id','item_id'],[1])
 add(['date_block_num','shop_id','name_1'],[1])
 add(['date_block_num','shop_id','name_2'],[1])
 add(['date_block_num','shop_id','name_3'],[1])
 add(['date_block_num','shop_id','item_category_id'],[1] )
delta_cols = ['roll','pitch','yaw']
for dc in delta_cols:
iia = feat_cols.index(dc)
 np_arr = feat_array[:,:,iia]
roll_arr = np.copy(np_arr)
 roll_arr[:,1:] = roll_arr[:,:-1]
np_arr = np_arr - roll_arr
feat_array[:,:,iia] = np_arr<feature_engineering>
group = train.groupby(['item_id'] ).agg({'item_price':['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace = True)
 matrix = pd.merge(matrix,group,on=['item_id'],how = 'left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True )
norm_cols = ['linear_acceleration_X','linear_acceleration_Y','linear_acceleration_Z','angular_velocity_X','angular_velocity_Y','angular_velocity_Z']
for norm in norm_cols:
iia = feat_cols.index(norm)
 np_arr = feat_array[:,:,iia]
mean_arr = np.mean(np_arr,1)
 mean_arr = np.expand_dims(mean_arr,1)
 mean_arr = np.repeat(mean_arr,num_meas,1)
 np_arr = np_arr - mean_arr
feat_array[:,:,iia] = np_arr<concatenate>
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
def absfft(x):
return np.abs(np.fft.rfft(x))
feat_fft_array = np.copy(feat_array[:,:,3:])
 feat_fft_array = np.apply_along_axis(absfft,1,feat_fft_array)
<feature_engineering>
matrix['item_first_sale']=matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min' )
num_sensor = feat_array.shape[2]
for i in range(num_sensor):
mean_s = np.mean(feat_array[:,:,i])
 sd_s = np.std(feat_array[:,:,i])
 feat_array[:,:,i] =(feat_array[:,:,i]-mean_s)/sd_s
num_sensor_fft = feat_fft_array.shape[2]
for i in range(num_sensor_fft):
mean_s = np.mean(feat_fft_array[:,:,i])
 sd_s = np.std(feat_fft_array[:,:,i])
 feat_fft_array[:,:,i] =(feat_fft_array[:,:,i]-mean_s)/sd_s<train_on_grid>
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if 'item_cnt' in col:
df[col].fillna(0,inplace=True)
 return df
matrix = matrix[matrix.date_block_num > 3]
matrix = fill_na(matrix)
 matrix=downcast_dtypes(matrix )
kr = None
num_groups = np.unique(group_array ).shape[0]
num_surfaces = np.unique(target_array ).shape[0]
def get_net_with_fft_mag_only(dp):
inputs_t = Input(shape=(128,len(feat_cols)))
 x = SeparableConv1D(32,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(inputs_t)
 x = Dropout(dp )(x)
 x = SeparableConv1D(64,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = SeparableConv1D(256,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Reshape(( 256,))(x)
 x = Dropout(dp )(x)
 x = Dense(64, activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = Dense(64, activation='relu' )(x)
 inputs_f = Input(shape=(feat_fft_array.shape[1],feat_fft_array.shape[2]))
y = SeparableConv1D(32,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(inputs_f)
 y = Dropout(dp )(y)
 y = SeparableConv1D(64,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(256,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Reshape(( 256,))(y)
 y = Dropout(dp )(y)
 y = Dense(64, activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = Dense(64, activation='relu' )(y)
 inputs = [inputs_t,inputs_f]
z = Concatenate()([x,y])
 z = Dense(64, activation='relu' )(z)
 predictions = Dense(num_surfaces, activation='softmax' )(z)
 model = Model(inputs=inputs, outputs=predictions)
 model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return model<split>
matrix['year'] = matrix['date_block_num']/12+2013
matrix['month'] = matrix['date_block_num']%12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8)
 matrix['year'] = matrix.year.astype(np.int16 )
depthwise = False
fft_net = True
if(not(testing)) :
gkf = GroupKFold(3)
 train_gen = gkf.split(X=feat_array,groups=group_array)
 preds = np.zeros_like(target_array)
 for train_idx,test_idx in train_gen:
t_feats = feat_array[train_idx]
t_feats_fft = feat_fft_array[train_idx]
v_feats = feat_array[test_idx]
v_feats_fft = feat_fft_array[test_idx]
t_vals = target_array[train_idx]
v_vals = target_array[test_idx]
pred_classes = np.zeros([v_vals.shape[0],num_surfaces,5])
 for k in range(5):
nnet = get_net_with_fft_mag_only(0.5)
 nnet.fit(x=[t_feats,t_feats_fft],y=t_vals,batch_size=256,epochs=3000,validation_data=([v_feats,v_feats_fft],v_vals),verbose=2)
 pred_classes[:,:,k] = nnet.predict([v_feats,v_feats_fft])
 pred_classes = np.mean(pred_classes,axis=2)
 pred_classes = np.argmax(pred_classes,axis=1)
 preds[test_idx] = pred_classes
print('Val accuracy: ',accuracy_score(v_vals,pred_classes))
pred_classes = nnet.predict([t_feats,t_feats_fft])
 pred_classes = np.argmax(pred_classes,axis=1)
 print('Train accuracy: ',accuracy_score(t_vals,pred_classes))
print('5 Fold accuracy: ', accuracy_score(target_array,preds))
else:
t_feats = feat_array[:3810]
t_feats_fft = feat_fft_array[:3810]
t_vals = target_array[:3810]
v_feats = feat_array[3810:]
v_feats_fft = feat_fft_array[3810:]
pred_classes = np.zeros([v_feats.shape[0],num_surfaces,3])
 for k in range(3):
nnet = get_net_with_fft_mag_only(0.5)
 nnet.fit(x=[t_feats,t_feats_fft],y=t_vals,batch_size=256,epochs=3000,verbose=0)
 pred_classes[:,:,k] = nnet.predict([v_feats,v_feats_fft])
 pred_classes = np.mean(pred_classes,axis=2)
 pred_classes = list(np.argmax(pred_classes,axis=1))
pred_classes = [le.inverse_transform([i])[0] for i in pred_classes]
sub_df = pd.read_csv('.. /input/sample_submission.csv')
 sub_df['surface'] = pred_classes
sub_df.to_csv('submission.csv',index=False )<import_modules>
weekary = []
t = 2
count = 0
for w in range(3):
for i in [31,28,31,30,31,30,31,31,30,31,30,31]:
a = [0,0,0,0,0,0,0,count]
count += 1
for j in range(i):
a[t] += 1
if t ==6:
t = -1
t += 1
weekary.append(a)
 weekary = pd.DataFrame(np.vstack(weekary),columns = ['week0','week1','week2','week3','week4','week5','week6','date_block_num'])
 matrix = pd.merge(matrix,weekary,on=['date_block_num'],how='left' )
le = preprocessing.LabelEncoder()
RAW_DATA_PATH = ".. /input/career-con-2019"
warnings.filterwarnings('ignore')
 %matplotlib inline 
<load_from_csv>
matrix.to_pickle('data2.pkl')
 del matrix
gc.collect()
data_target = pd.read_csv("{}/y_train.csv".format(RAW_DATA_PATH))
train_X = pd.read_csv('{}/X_train.csv'.format(RAW_DATA_PATH)).iloc[:,3:]
test_X = pd.read_csv('{}/X_test.csv'.format(RAW_DATA_PATH)).iloc[:,3:]
train_X = train_X.append(test_X)
 train_X = train_X.values.reshape(-1,128,10 )<find_best_params>
data = pd.read_pickle('data2.pkl')
 data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'shop_type',
'item_category_id',
'name_1',
'name_2',
'name_3',
'item_subtype',
'year',
'month',
'days',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'date_block_num_avg_item_cnt_lag_1',
'date_block_num_and_item_id_avg_item_cnt_lag_1',
'date_block_num_and_item_id_avg_item_cnt_lag_2',
'date_block_num_and_item_id_avg_item_cnt_lag_3',
'date_block_num_and_shop_id_avg_item_cnt_lag_1',
'date_block_num_and_shop_id_avg_item_cnt_lag_2',
'date_block_num_and_shop_id_avg_item_cnt_lag_3',
'date_block_num_and_item_category_id_avg_item_cnt_lag_1',
'date_block_num_and_shop_id_and_item_category_id_avg_item_cnt_lag_1',
'date_block_num_and_shop_id_and_item_id_avg_item_cnt_lag_1',
'delta_price_lag',
'item_shop_last_sale',
'item_last_sale',
'item_first_sale'
]]
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_zong = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_zong = data[data.date_block_num < 33]['item_cnt_month']
X_train, X_test, y_train, y_test = train_test_split(X_valid,Y_valid,test_size = 0.25)
 X_tests = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 xx = pd.concat([X_zong,X_train])
 yy = pd.concat([Y_zong,y_train])
 del data,X_valid,Y_valid,X_train,y_train,X_zong,Y_zong
gc.collect()
train_left_edges, train_left_linked = find_run_edges(train_X, edge='left')
 train_right_edges, train_right_linked = find_run_edges(train_X, edge='right')
 print('Found', len(train_left_edges), 'left edges and', len(train_right_edges), 'right edges.' )<compute_test_metric>
xx.to_pickle('xx.pkl')
 yy.to_pickle('yy.pkl')
 X_test.to_pickle('X_test.pkl')
 y_test.to_pickle('y_test.pkl' )
train_runs = find_runs(train_X, train_left_edges, train_right_edges )<define_variables>
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=100,
min_child_weight=0.5,
colsample_bytree=0.9,
subsample=0.8,
num_round = 10000,
nthread = 16,
eta=0.1,
seed=1)
 model.fit(xx, yy, eval_metric="rmse", eval_set=[(xx,yy),(X_test,y_test)], verbose=True, early_stopping_rounds = 10)
 time.time() - ts
lost_samples = np.array([ i for i in range(len(train_X)) if i not in np.concatenate(train_runs)])
 print(lost_samples)
 print(len(lost_samples))<define_variables>
Y_tests = model.predict(X_tests ).clip(0, 20 )
first_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==821:
print(i)
 first_double_surface = i<drop_column>
test = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_tests
})
 submission.to_csv('xgb_submission1.csv', index=False )
new_train_runs = [821, 974, 328, 1548, 172,
355, 957, 1481, 1046, 1650, 857, 724, 164, 1092, 1017, 1300,
1212, 536, 531, 1032, 994, 1501, 588, 579, 1177, 812, 1333,
1253]
train_runs[first_double_surface] = train_runs[first_double_surface][0:-len(new_train_runs)]<concatenate>
%%time
train.date = pd.to_datetime(train.date,format='%d.%m.%Y')
 train.date
train_runs.append(np.array(new_train_runs))<define_variables>
%%time
data = train.groupby([train.date.apply(lambda x:x.strftime('%Y-%m')) ,'item_id','shop_id'] ).sum().reset_index()
data.head()
second_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==3055:
print(i)
 second_double_surface = i<categorify>
data = data[['date','item_id','shop_id','item_cnt_day']]
data.head()
new_train_runs2 = [3055, 3360, 3662,
3780, 3663, 3091, 3769, 3175, 1957, 2712, 2063, 2708, 3139, 2722]
train_runs[second_double_surface] = train_runs[second_double_surface][0:-len(new_train_runs2)]<concatenate>
data = data.pivot_table(index=['item_id','shop_id'],columns='date',values='item_cnt_day',fill_value=0 ).reset_index()
data.head()
train_runs.append(np.array(new_train_runs2))<define_variables>
test = pd.merge(test,data,on=['item_id','shop_id'],how='left' ).fillna(0)
 test.head()
third_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==2484:
print(i)
 third_double_surface = i<filter>
X_train = test.drop(['item_id','shop_id','2015-10'],axis = 1)
 y_train = test['2015-10']
X_test = test.drop(['item_id','shop_id','2013-01'],axis = 1)
 print('shape of X_train: ',X_train.shape)
 print('shape of y_train: ',y_train.shape)
 print('shape of X_test: ',X_test.shape)
<define_variables><EOS> 
X_train,X_val, y_train,y_val = train_test_split(X_train,y_train,test_size = 0.2,random_state=10 )