Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<concatenate>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 100)
 %matplotlib inline
sns.set(style="darkgrid")
 pd.set_option('display.float_format', lambda x: '%.2f' % x)
 warnings.filterwarnings("ignore")
 sys.version_info
train_runs.append(np.array(new_train_runs3))<define_variables>
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 items = items.drop('item_name', axis=1)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 test['date_block_num'] = 34
fourth_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==3501:
print(i)
 fourth_double_surface = i<filter>
def compress_columns(df,columns,keyword,search_type,datatype):
if search_type=='in':
valid_features = [x for x in columns if keyword in x]
elif search_type=='start':
valid_features = [x for x in columns if x.startswith(keyword)]
if len(valid_features):
for f in valid_features:
df[f] = df[f].round().astype(datatype)
 return df
def data_compression(df):
features = df.columns.values
if 'date_block_num' in features:
df['date_block_num'] = df['date_block_num'].astype(np.int8)
 if 'shop_id' in features:
df['shop_id'] = df['shop_id'].astype(np.int8)
 if 'item_category_id' in features:
df['item_category_id'] = df['item_category_id'].astype(np.int8)
 if 'item_id' in features:
df['item_id'] = df['item_id'].astype(np.int16)
 if 'item_price' in features:
df['item_price'] = df['item_price'].astype(np.float32)
 if 'item_id_avg_item_price' in features:
df['item_id_avg_item_price'] = df['item_id_avg_item_price'].astype(np.float32)
 df = compress_columns(df,features,'item_id_sum_item_cnt_day','in',np.int16)
 df = compress_columns(df,features,'item_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'shop_id_avg_item_price','in',np.float16)
 df = compress_columns(df,features,'shop_id_sum_item_cnt_day','in',np.int16)
 df = compress_columns(df,features,'shop_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'item_category_id_avg_item_price','in',np.float16)
 df = compress_columns(df,features,'item_category_id_sum_item_cnt_day','in',np.int32)
 df = compress_columns(df,features,'item_category_id_avg_item_cnt_day','in',np.float16)
 df = compress_columns(df,features,'item_cnt_day','start',np.int16)
 return df
train_runs[fourth_double_surface]<prepare_x_and_y>
train = data_compression(train)
 items = data_compression(items)
 test = data_compression(test )
new_train_runs4 = [3501, 2785]
train_runs[fourth_double_surface] = train_runs[fourth_double_surface][0:-len(new_train_runs4)]
train_runs.append(np.array(new_train_runs4))<feature_engineering>
train = pd.merge(train,items,on='item_id',how='left')
 test = pd.merge(test,items, on='item_id', how='left' )
df_train_y = pd.DataFrame()
df_train_y['run_id'] = 0
df_train_y['run_pos'] = 0
for run_id in range(len(train_runs)) :
for run_pos in range(len(train_runs[run_id])) :
series_id = train_runs[run_id][run_pos]
df_train_y.at[ series_id, 'run_id' ] = run_id
df_train_y.at[ series_id, 'run_pos' ] = run_pos
df_train_y.tail()<feature_engineering>
train = train[(train['item_price']<100000)&(train['item_price']>=0)]
train = train[(train['item_cnt_day']<1000)&(train['item_cnt_day']>=0)]
df_train_y['index'] = df_train_y.index
df_train_y = df_train_y.sort_values('index')
 df_train_y.rename(columns={'index':'series_id'}, inplace=True)
 df_train_y['run_id'] = df_train_y['run_id'].apply(lambda x: int(x))
df_train_y['run_pos'] = df_train_y['run_pos'].apply(lambda x: int(x))
df_train_y.tail()<split>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
run_id_train = df_train_y['run_id'][0:3810].values
run_id_test = df_train_y['run_id'][3810:7626].values<categorify>
train_m = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day':'sum','item_price':np.mean} ).reset_index()
train_m = pd.merge(grid,train_m,on=['date_block_num','shop_id','item_id'],how='left' ).fillna(0)
 train_m = pd.merge(train_m,items,on='item_id',how='left')
 train_m = data_compression(train_m )
data_target['run_id'] = run_id_train
mapping_leak = {}
for i in range(0,3810):
cur_data = data_target.iloc[i]
mapping_leak.update({cur_data['run_id']: cur_data['surface']})
 unknown_run = []
ans_test = []
known_series = []
unknown_series = []
for i in range(0,3816):
if run_id_test[i] in mapping_leak:
ans_test.append(mapping_leak[run_id_test[i]])
 known_series.append(i)
 else:
ans_test.append('unknown')
 unknown_series.append(i)
 unknown_run.append(run_id_test[i] )<load_from_csv>
for type_id in ['item_id', 'shop_id', 'item_category_id']:
for column_id, aggregator, aggtype in [('item_price',np.mean,'avg'),('item_cnt_day',np.sum,'sum'),('item_cnt_day',np.mean,'avg')]:
mean_df = train.groupby([type_id,'date_block_num'] ).aggregate(aggregator ).reset_index() [[column_id,type_id,'date_block_num']]
mean_df.columns = [type_id+'_'+aggtype+'_'+column_id,type_id,'date_block_num']
train_m = pd.merge(train_m, mean_df, on=['date_block_num',type_id], how='left')
 del mean_df
gc.collect()
sub = pd.read_csv("{}/sample_submission.csv".format(RAW_DATA_PATH))
sub['surface'] = ans_test
sub.head()<load_from_csv>
del train
gc.collect()
best = pd.read_csv('.. /input/best-73/submission_24.csv')
 map_best_ans = {}
for i in range(0, best.shape[0]):
map_best_ans.update({best.iloc[i]['series_id'] : best.iloc[i]['surface'] })
<define_variables>
for f in train_m.columns:
if 'item_cnt' in f:
train_m[f] = train_m[f].fillna(0)
 elif 'item_price' in f:
train_m[f] = train_m[f].fillna(train_m[f].median())
 train_m = data_compression(train_m)
 train_m.info(verbose=False )
result = []
for i in range(0, sub.shape[0]):
if(sub.surface[i] == 'unknown'):
result.append(map_best_ans[i])
 else:
result.append(sub.surface[i] )<save_to_csv>
lag_features = list(train_m.columns[6:])+['item_cnt_day']
lags = [1,2,3,6]
for lag in lags:
train_new_df = train_m.copy()
train_new_df['date_block_num'] += lag
train_new_df = train_new_df[['date_block_num','shop_id','item_id']+lag_features]
train_new_df.columns = ['date_block_num','shop_id','item_id'] + [x+'_lag_'+str(lag)for x in lag_features]
train_m = pd.merge(train_m,train_new_df,on=['date_block_num','shop_id','item_id'],how='left')
 del train_new_df
gc.collect()
print('lag %s processed' %lag )
sub.surface = result
sub.to_csv('final_submission.csv', index=False )<count_values>
for f in train_m.columns:
if 'item_cnt' in f:
train_m[f] = train_m[f].fillna(0)
 elif 'item_price' in f:
train_m[f] = train_m[f].fillna(train_m[f].median())
 train_m = data_compression(train_m)
 train_m.info(verbose=False )
sub.surface.value_counts()<set_options>
max_clip = 30
train_m['item_cnt_day'] = train_m['item_cnt_day'].clip(0,max_clip ).astype(np.float16 )
import pandas as pd
import numpy as np
import zipfile
import os
from matplotlib import pyplot as plt
from sklearn.linear_model import LogisticRegression,LinearRegression
from sklearn.model_selection import train_test_split,StratifiedKFold
from sklearn.metrics import accuracy_score
from scipy import fftpack
import xgboost
import warnings
from sklearn.model_selection import cross_val_predict, cross_validate
import seaborn as sns; sns.set()
from collections import Counter
from sklearn.neighbors import KNeighborsClassifier
from keras.layers import LSTM,Input,Dense,Flatten,SpatialDropout1D,Dropout,CuDNNLSTM,Reshape,Concatenate
from keras.layers import Lambda,concatenate,BatchNormalization
from keras.models import Model
from keras.utils import to_categorical
from keras.optimizers import Adam
from keras import backend as K
from sklearn.preprocessing import LabelEncoder<load_from_csv>
for lag in lags:
train_new_df = train_m.copy()
train_new_df['date_block_num'] += lag
train_new_df = train_new_df[['date_block_num','shop_id','item_id']+lag_features]
train_new_df.columns = ['date_block_num','shop_id','item_id'] + [x+'_lag_'+str(lag)for x in lag_features]
test = pd.merge(test,train_new_df,on=['date_block_num','shop_id','item_id'],how='left')
 del train_new_df
gc.collect()
print('lag %s processed' %lag )
directory = ".. /input/"
X_test_path = os.path.join(directory,"X_test.csv")
 X_train_path = os.path.join(directory,"X_train.csv")
 X_test_data = pd.read_csv(X_test_path)
 X_train_data = pd.read_csv(X_train_path)
 y_train_data = pd.read_csv(os.path.join(directory,"y_train.csv"))
sample_submission = pd.read_csv(os.path.join(directory,"sample_submission.csv"))<categorify>
for f in test.columns:
if 'item_cnt' in f:
test[f] = test[f].fillna(0)
 elif 'item_price' in f:
test[f] = test[f].fillna(test[f].median())
 test = data_compression(test )
le = LabelEncoder()
le.fit(list(y_train_data["surface"]))
y_train_dataset_for_nn = to_categorical(le.transform(list(y_train_data["surface"])) )<categorify>
cols_to_drop = lag_features[:-1] + ['item_price']
print('Columns to drop')
 print(cols_to_drop )
def dataset_for_nn(X_dataset):
num_samples = X_dataset.shape[0]//128
X_dataset_for_nn = np.zeros(( num_samples,128,10))
for i in range(num_samples):
subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 X_dataset_for_nn[i,:,:] = subset
return X_dataset_for_nn<categorify>
train_cols = train_m.columns.values
test_cols = test.columns.values
for c in cols_to_drop:
if c in train_cols:
train_m = train_m.drop(c,axis=1)
 if c in test_cols:
test = test.drop(c,axis=1 )
X_train_for_nn = dataset_for_nn(X_train_data)
 X_test_for_nn = dataset_for_nn(X_test_data )<feature_engineering>
train_m['month'] = train_m['date_block_num']%12
train_m['month'] = train_m['month'].astype(np.int8)
 days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 train_m['days'] = train_m['month'].map(days ).astype(np.int8)
 test['month'] = 11
test['month'] = test['month'].astype(np.int8)
 test['days'] = 30
test['days'] = test['days'].astype(np.int8 )
def freqs(dataset,width):
X = np.abs(fftpack.fft(dataset))
squeezed_dataset = []
for i in range(64//width):
squeezed_dataset.append(np.mean(X[i*width:(i+1)*width]))
return squeezed_dataset<feature_engineering>
print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 33]['item_cnt_day'])
 print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 32]['item_cnt_day'])
 print(train_m[train_m['shop_id'] == 5][train_m['item_id'] == 5037][train_m['date_block_num'] == 31]['item_cnt_day'] )
def X_features(X_dataset,width=3):
num_samples = len(list(set(X_dataset["series_id"])))
 num_cols = 64//width
features = np.zeros(( num_samples,40+10*num_cols))
for i in range(num_samples):
X_train_subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 features[i,:10] = np.mean(X_train_subset,axis=0)
 features[i,10:20] = np.std(X_train_subset,axis=0)
 features[i,20:30] = np.max(X_train_subset,axis=0)-np.min(X_train_subset,axis=0)
 features[i,30:40] = X_train_subset[-1,:]-X_train_subset[0,:]
for j in range(X_train_subset.shape[1]):
features[i,40+j*num_cols:40+(j+1)*num_cols] = freqs(X_train_subset[:,j],width)
 return features<feature_engineering>
train_m = train_m[train_m['date_block_num']>12]
X_train_features = X_features(X_train_data)
 X_test_features = X_features(X_test_data )<choose_model_class>
train_set = train_m[train_m['date_block_num']<33]
val_set = train_m[train_m['date_block_num']==33]
def LSTM_NN(drop):
inp = Input(shape=(128,10))
x = SpatialDropout1D(0.1 )(inp)
 inp_2 = Input(shape=(250,))
x_2 = Dense(250, input_shape=(250,), activation="sigmoid" )(inp_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(120, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(60, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = BatchNormalization()(x_2)
 x = CuDNNLSTM(units=200, return_sequences=True, return_state=False, go_backwards=False )(x)
 x = Dropout(drop )(x)
 x = CuDNNLSTM(units=100, return_sequences=False, return_state=False, go_backwards=False )(x)
 x = concatenate([x,x_2])
 x = Dropout(drop )(x)
 outp = Dense(9, activation="sigmoid" )(x)
 model = Model(inputs=[inp,inp_2], outputs=outp)
 model.compile(loss='categorical_crossentropy', optimizer=Adam())
 return model<split>
train_set.to_pickle('train.pkl')
 val_set.to_pickle('val.pkl')
 test.to_pickle('test.pkl' )
%%time
n_splits=5
kfold = StratifiedKFold(n_splits=n_splits, random_state=10, shuffle=True)
 y_test = np.zeros(( X_test_for_nn.shape[0],9*n_splits))
train_preds = np.zeros(( X_train_for_nn.shape[0],9))
X = X_train_for_nn
X_test = X_test_for_nn
Y = np.array(list(y_train_data["surface"]))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val = X[list(train_index),:,:],X[list(valid_index),:,:]
X_train_feat, X_val_feat = X_train_features[list(train_index),:],X_train_features[list(valid_index),:]
Y_train, Y_val = Y[list(train_index)], Y[list(valid_index)]
Y_train = to_categorical(le.transform(Y_train))
Y_val = to_categorical(le.transform(Y_val))
model = LSTM_NN(0.15)
 model.fit([X_train,X_train_feat], Y_train, epochs=120, validation_data=([X_val,X_val_feat], Y_val), verbose=2)
 y_pred = model.predict([X_val,X_val_feat], verbose=2)
 y_test[:,i*9:(i+1)*9] = model.predict([X_test,X_test_features])
 train_preds[list(valid_index),:] = np.squeeze(y_pred )<compute_test_metric>
del train_m
gc.collect()
res_train = np.argmax(train_preds,axis=1)
 ans_train = np.argmax(to_categorical(le.transform(Y)) ,axis=1)
 print("CV score",round(accuracy_score(res_train,ans_train),4))<feature_engineering>
train_x = train_set.drop(['item_cnt_day'],axis=1)
 train_y = train_set['item_cnt_day']
val_x = val_set.drop(['item_cnt_day'],axis=1)
 val_y = val_set['item_cnt_day']
def most_frequent(List):
occurence_count = Counter(List)
 return occurence_count.most_common(1)[0][0]
res_test_inter = np.zeros(( y_test.shape[0],n_splits))
res_test=[]
for i in range(n_splits):
inter_arr = y_test[:,i*9:(i+1)*9]
res_test_inter[:,i] = np.argmax(inter_arr,axis=1)
 for j in range(y_test.shape[0]):
res_test.append(int(most_frequent(res_test_inter[j,:])) )<save_to_csv>
del train_set
del val_set
gc.collect()
test_for_sub=le.inverse_transform(res_test)
 print(test_for_sub[:5])
 test_size = len(list(set(X_test_data["series_id"])))
 Y_test_pred_array = np.zeros(( test_size,2))
Y_test_for_submission = pd.DataFrame(Y_test_pred_array,columns = ["series_id","surface"])
 Y_test_for_submission.iloc[:,0] = list(range(test_size))
Y_test_for_submission.iloc[:,1] = test_for_sub
Y_test_for_submission.to_csv("submission.csv",index=None )<import_modules>
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error
FileLink('submission.csv' )<import_modules>
gc.collect()
ts = time.time()
lm = linear_model.LinearRegression()
lm.fit(train_x,train_y)
 print('Training time: %s' %(time.time() - ts))
import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
import itertools
from sklearn.metrics import *<load_from_csv>
train_pred1 = lm.predict(train_x ).clip(0, 20)
 val_pred1 = lm.predict(val_x ).clip(0, 20)
 test_pred1 = lm.predict(test ).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred1))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred1))
print("Val RMSE: %f" %(val_rmse))
sub_df1 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred1})
 sub_df1.to_csv('lm_submission.csv',index=False)
 pickle.dump(train_pred1,open('lm_train.pickle','wb'))
pickle.dump(val_pred1,open('lm_val.pickle','wb'))
pickle.dump(test_pred1,open('lm_test.pickle','wb'))
X_train = pd.read_csv(".. /input/X_train.csv")
 X_test = pd.read_csv(".. /input/X_test.csv")
 y_train = pd.read_csv(".. /input/y_train.csv")
 sub = pd.read_csv(".. /input/sample_submission.csv" )<feature_engineering>
import xgboost as xgb
from xgboost import XGBRegressor
from xgboost import plot_importance
def fe(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 +
data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 +
data['linear_acceleration_Z'])**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 +
data['orientation_Z'])**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df 
<categorify>
gc.collect()
ts = time.time()
xgbtrain = xgb.DMatrix(train_x.values, train_y.values)
 param = {'max_depth':10,
'subsample':1,
'min_child_weight':0.5,
'eta':0.3,
'num_round':1000,
'seed':1,
'verbosity':2,
'eval_metric':'rmse'}
bst = xgb.train(param, xgbtrain)
 print('Training time: %s' %(time.time() - ts))
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )<categorify>
train_pred2 = bst.predict(xgb.DMatrix(train_x.values)).clip(0, 20)
 val_pred2 = bst.predict(xgb.DMatrix(val_x.values)).clip(0, 20)
 test_pred2 = bst.predict(xgb.DMatrix(test.values)).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred2))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred2))
print("Val RMSE: %f" %(val_rmse))
sub_df2 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred2 })
 sub_df2.to_csv('xgb_submission.csv',index=False)
 pickle.dump(train_pred2,open('xgb_train.pickle','wb'))
pickle.dump(val_pred2,open('xgb_val.pickle','wb'))
pickle.dump(test_pred2,open('xgb_test.pickle','wb'))
X_train.fillna(0, inplace = True)
 X_test.fillna(0, inplace = True)
 X_train.replace(-np.inf, 0, inplace = True)
 X_train.replace(np.inf, 0, inplace = True)
 X_test.replace(-np.inf, 0, inplace = True)
 X_test.replace(np.inf, 0, inplace = True )<find_best_model_class>
from sklearn.ensemble import RandomForestRegressor
def k_folds(X, y, X_test, k):
folds = StratifiedKFold(n_splits = k, shuffle=True, random_state=2019)
 y_test = np.zeros(( X_test.shape[0], 9))
y_oof = np.zeros(( X.shape[0]))
score = 0
for i,(train_idx, val_idx)in enumerate(folds.split(X, y)) :
clf = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(X_train.iloc[train_idx], y[train_idx])
 y_oof[val_idx] = clf.predict(X.iloc[val_idx])
 y_test += clf.predict_proba(X_test)/ folds.n_splits
score += clf.score(X.iloc[val_idx], y[val_idx])
 print('Fold: {} score: {}'.format(i,clf.score(X.iloc[val_idx], y[val_idx])))
 print('Avg Accuracy', score / folds.n_splits)
 return y_oof, y_test<compute_test_metric>
from sklearn.ensemble import RandomForestRegressor
y_oof, y_test = k_folds(X_train, y_train['surface'], X_test, k= 50 )<compute_test_metric>
gc.collect()
ts = time.time()
rf = RandomForestRegressor(
 bootstrap=True,
max_depth=100,
max_features= 3,
min_samples_leaf=5,
min_samples_split=12,
n_estimators=100,
random_state=42,
verbose=1,
n_jobs=-1 
)
 rf.fit(train_x.values,train_y.values)
 print('Training time: %s' %(time.time() - ts))
confusion_matrix(y_oof,y_train['surface'] )<save_to_csv>
train_pred3 = rf.predict(train_x ).clip(0, 20)
 val_pred3 = rf.predict(val_x ).clip(0, 20)
 test_pred3 = rf.predict(test ).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred3))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred3))
print("Val RMSE: %f" %(val_rmse))
sub_df3 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred3})
 sub_df3.to_csv('rf_submission.csv',index=False)
 pickle.dump(train_pred3,open('rf_train.pickle','wb'))
pickle.dump(val_pred3,open('rf_val.pickle','wb'))
pickle.dump(test_pred3,open('rf_test.pickle','wb'))
y_test = np.argmax(y_test, axis=1)
 submission = pd.read_csv(os.path.join(".. /input/",'sample_submission.csv'))
submission['surface'] = le.inverse_transform(y_test)
 submission.to_csv('submission.csv', index=False)
 submission.head(10 )<load_from_csv>
train_stack = pd.DataFrame({'lm':train_pred1,'xgboost':train_pred2,'rf':train_pred3})
 val_stack = pd.DataFrame({'lm':val_pred1,'xgboost':val_pred2,'rf':val_pred3})
 test_stack = pd.DataFrame({'lm':test_pred1,'xgboost':test_pred2,'rf':test_pred3} )
warnings.filterwarnings('ignore')
 data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape)) 
<install_modules>
gc.collect()
ts = time.time()
stack_lm = linear_model.LinearRegression()
stack_lm.fit(train_stack,train_y)
 print('Training time: %s' %(time.time() - ts))
!pip install.. /input/iterativestratification 
<categorify>
train_pred4 = stack_lm.predict(train_stack ).clip(0, 20)
 val_pred4 = stack_lm.predict(val_stack ).clip(0, 20)
 test_pred4 = stack_lm.predict(test_stack ).clip(0, 20)
 train_rmse = np.sqrt(mean_squared_error(train_y, train_pred4))
print("Train RMSE: %f" %(train_rmse))
val_rmse = np.sqrt(mean_squared_error(val_y, val_pred4))
print("Val RMSE: %f" %(val_rmse))
sub_df4 = pd.DataFrame({'ID':test.index,'item_cnt_month': test_pred4})
 sub_df4.to_csv('stack_submission.csv',index=False)
 pickle.dump(train_pred4,open('stack_train.pickle','wb'))
pickle.dump(val_pred4,open('stack_val.pickle','wb'))
pickle.dump(test_pred4,open('stack_test.pickle','wb'))
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<define_variables>
<concatenate><EOS>