Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_from_csv>
%matplotlib inline
df_train = pd.read_csv('.. /input/dog-breed-identification/labels.csv')
 df_test = pd.read_csv('.. /input/dog-breed-identification/sample_submission.csv' )<set_options>
all_data = pd.read_pickle('.. /input/extended-train-data/all_data_final.pkl')
 all_data.head()
%matplotlib inline 
<define_variables>
print('mean for whole train set: {0}'.format(np.mean(all_data.loc[all_data['date_block_num']<34, 'target'].astype(np.float32))))
print('mean for validation train set: {0}'.format(np.mean(all_data.loc[all_data['date_block_num']<33, 'target'].astype(np.float32))))
print('mean for validation test set: {0}'.format(np.mean(all_data.loc[all_data['date_block_num']==33, 'target'].astype(np.float32))))
train_files = glob('.. /input/dog-breed-identification/train/*.jpg')
 test_files = glob('.. /input/dog-breed-identification/test/*.jpg' )<load_from_csv>
missing_shop_item_count = 15876
index_cols = ['shop_id', 'item_id', 'date_block_num']
grid = []
for block_num in all_data.loc[all_data['date_block_num']<34, 'date_block_num'].unique() :
print(block_num)
 zero_target_df = all_data[(all_data['date_block_num'] == block_num)&(all_data['target']==0)& 
(all_data['item_id'].isin(items_in_train_and_not_in_test)) ]
idx_to_delete = zero_target_df.sample(missing_shop_item_count, random_state=block_num ).index
all_data.drop(idx_to_delete, inplace=True)
 temp = np.array(list(product(*[test_shops, items_in_test_and_not_in_train, [block_num]])) ,dtype='int32')
 grid.append(temp)
 del zero_target_df
del idx_to_delete
del temp
gc.collect()
grid = pd.DataFrame(np.vstack(grid), columns = index_cols,dtype=np.int32 )
plt.imshow(plt.imread(train_files[100]))<categorify>
grid['shop_id'] = grid['shop_id'].astype(np.int16)
 grid['item_id'] = grid['item_id'].astype(np.int32)
 grid['date_block_num'] = grid['date_block_num'].astype(np.int8 )
targets_series = pd.Series(df_train['breed'])
 one_hot = pd.get_dummies(targets_series, sparse = True)
 one_hot_labels = np.asarray(one_hot )<define_variables>
all_data = pd.concat([all_data, grid], ignore_index=True, sort=False, keys=index_cols)
 all_data[['item_shop_last_sale', 'item_last_sale']].fillna(-1, inplace=True)
 all_data.fillna(0, inplace=True)
 del grid
del test_items
del test_shops
del train_items
del items_in_test_and_not_in_train
del items_in_train_and_not_in_test
gc.collect()
im_size = 224<prepare_x_and_y>
all_data['is_december'] = all_data['is_december'].astype(np.int8)
 all_data['item_category_id'] = all_data['item_category_id'].astype(np.int8)
 all_data['type_code'] = all_data['type_code'].astype(np.int8)
 all_data['subtype_code'] = all_data['subtype_code'].astype(np.int8)
 all_data['city_code'] = all_data['city_code'].astype(np.int16)
 all_data['month'] = all_data['month'].astype(np.int8)
 all_data['days'] = all_data['days'].astype(np.int8)
 all_data['item_shop_last_sale'] = all_data['item_shop_last_sale'].astype(np.int8)
 all_data['item_last_sale'] = all_data['item_last_sale'].astype(np.int8)
 all_data['item_shop_first_sale'] = all_data['item_shop_first_sale'].astype(np.int8)
 all_data['item_first_sale'] = all_data['item_first_sale'].astype(np.int8 )
y_train = []
y_val = []
x_train_raw = bcolz.zeros(( 0,im_size,im_size,3),np.float32)
 x_val_raw = bcolz.zeros(( 0,im_size,im_size,3),np.float32 )<categorify>
print('mean for whole train set: {0}'.format(np.mean(all_data.loc[all_data['date_block_num']<34, 'target'].astype(np.float32))))
print('mean for validation train set: {0}'.format(np.mean(all_data.loc[all_data['date_block_num']<33, 'target'].astype(np.float32))))
print('mean for validation test set: {0}'.format(np.mean(all_data.loc[all_data['date_block_num']==33, 'target'].astype(np.float32))))
i = 0
for f, breed in tqdm(df_train.values):
image = load_img('.. /input/dog-breed-identification/train/{}.jpg'.format(f), target_size=(im_size, im_size))
image = img_to_array(image)
 label = one_hot_labels[i]
if random.randint(1,101)< 80:
x_train_raw.append(image)
 y_train.append(label)
 else:
x_val_raw.append(image)
 y_val.append(label)
 i += 1<prepare_x_and_y>
all_test_data = all_data[all_data['date_block_num'] == 34]
all_data = all_data[all_data['date_block_num'] < 34]
all_data.sort_values(['date_block_num'], inplace=True)
 all_data = pd.concat([all_data, all_test_data], ignore_index=True, sort=False, keys=index_cols)
 del all_test_data
gc.collect()
y_train_raw = np.array(y_train, np.uint8)
 y_val_raw = np.array(y_val, np.uint8)
 del(y_train,y_val)
 gc.collect()<define_variables>
dates = all_data['date_block_num']
last_block = dates.max()
print('Test `date_block_num` is {0}'.format(last_block))
X_train = all_data.loc[dates < last_block]
X_test = all_data.loc[dates == last_block]
y_train = all_data.loc[dates < last_block, 'target'].values
y_test = all_data.loc[dates == last_block, 'target'].values
X_valid_train = all_data.loc[dates < last_block-1]
X_valid_test = all_data.loc[dates == last_block-1]
y_valid_train = all_data.loc[dates < last_block-1, 'target'].values
y_valid_test = all_data.loc[dates == last_block-1, 'target'].values
all_data.to_pickle('all_data.pkl')
 del dates
del all_data
gc.collect()
num_class = y_train_raw.shape[1]<choose_model_class>
columns_to_delete = ['date_block_num', 'target']
X_valid_train = X_valid_train.drop(columns_to_delete, axis=1)
 X_valid_test = X_valid_test.drop(columns_to_delete, axis=1)
 X_train = X_train.drop(columns_to_delete, axis=1)
 X_test = X_test.drop(columns_to_delete, axis=1 )
base_model = VGG19(weights = 'imagenet', include_top=False, input_shape=(im_size, im_size, 3))
base_model.summary()<load_pretrained>
def validate(estimator, X_train_, y_train_, X_val_, y_val_, grid_params):
keys = grid_params.keys()
vals = grid_params.values()
parameters = []
rmses = []
rmses_train = []
return_obj ={}
prods = product(*vals)
 for idx, instance in enumerate(prods):
print('-'*50)
 print('model {0}:'.format(idx))
model_params = dict(zip(keys, instance))
parameters.append(model_params)
 print(model_params)
 model = estimator(**model_params)
 model.fit(X_train_, y_train_)
 pred_test = model.predict(X_val_)
 mse = mean_squared_error(y_val_, pred_test)
 rmse = np.sqrt(mse)
 print('RMSE: {0}'.format(rmse))
rmses = rmses + [rmse]
best_rmse_so_far = np.min(rmses)
 print('Best rmse so far: {0}'.format(best_rmse_so_far))
best_model_params_so_far = parameters[np.argmin(rmses)]
print('Best model params so far: {0}'.format(best_model_params_so_far))
del best_rmse_so_far
del best_model_params_so_far
del pred_test
del model
gc.collect()
rmses = np.array(rmses)
 best_rmse = np.min(rmses)
 print('Best rmse: {0}'.format(best_rmse))
best_model_params = parameters[np.argmin(rmses)]
print('Best model params: {0}'.format(best_model_params))
return_obj['rmses'] = rmses
return_obj['best_rmse'] = best_rmse
return_obj['best_model_params'] = best_model_params
return return_obj
fine_tuning_layers = 0
layers_to_freeze = len(base_model.layers)- fine_tuning_layers
print(layers_to_freeze)
 for layer in base_model.layers[0:layers_to_freeze]:
layer.trainable = False<choose_model_class>
alphas = [10, 100, 2000, 5000]
grid_params = {'alpha':alphas}
val_res = validate(Ridge, X_valid_train, y_valid_train,
X_valid_test, y_valid_test, grid_params )
x = base_model.layers[layers_to_freeze-1+fine_tuning_layers].output
x = GlobalAveragePooling2D()(x)
 model = Model(inputs=base_model.input, outputs=x)
 model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.summary()<define_variables>
best_params = {'learning_rate': 0.16, 'n_estimators': 500,
'max_depth': 6, 'min_child_weight': 7,
'subsample': 0.9, 'colsample_bytree': 0.7, 'nthread': -1,
'scale_pos_weight': 1, 'random_state': 42,
'tree_method': 'gpu_hist', 'predictor': 'gpu_predictor', 'gpu_id': 0}
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, data, labels, im_size = 224, batch_size=32, shuffle=True, data_augment = False, test = False):
'Initialization'
self.batch_size = batch_size
self.list_IDs = np.arange(0,data.shape[0])
 self.shuffle = shuffle
if self.shuffle == True:
np.random.shuffle(self.list_IDs)
 self.data = data
self.data_augment = data_augment
self.test = test
if self.test == False:
self.labels = labels
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.list_IDs[index*self.batch_size:(index+1)*self.batch_size]
if self.test == False:
X, y = self.__data_generation(indexes)
 return preprocess_input(X), y
else:
X = self.__data_generation(indexes)
 return preprocess_input(X)
 def on_epoch_end(self):
'Updates indexes after each epoch'
if self.shuffle == True:
np.random.shuffle(self.list_IDs)
 def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
X = np.zeros(( list_IDs_temp.shape[0],im_size,im_size,3), dtype=np.float32)
 if self.test == False:
y = np.zeros(( list_IDs_temp.shape[0],self.labels.shape[1]), dtype=np.uint8)
 for i, ID in enumerate(list_IDs_temp):
if self.data_augment == True:
if random.randint(1,101)< 50:
flip_horizontal = True
else:
flip_horizontal = False
if random.randint(1,101)< 50:
flip_vertical = True
else:
flip_vertical = False
tx = im_size*random.randint(1,2)/100.0
ty = im_size*random.randint(1,2)/100.0
shear = random.randint(1,10)/100.0
zx = random.randint(80,120)/100.0
zy = random.randint(80,120)/100.0
brightness = random.randint(1,2)/100.0
channel_shift_intensity = random.randint(1,10)/100.0
X[i,] = self.data[ID,]
else:
X[i,] = self.data[ID,]
if self.test == False:
y[i,] = self.labels[ID,]
if self.test == False:
return X, y
else:
return X<train_model>
ts = time.time()
model = XGBRegressor(**best_params)
 model.fit(X_valid_train.values,
y_valid_train,
eval_metric="rmse",
eval_set=[(X_valid_test.values, y_valid_test)],
verbose=True,
early_stopping_rounds = 50)
 time.time() - ts
batch_size = 32
params_trn = {
'im_size': im_size,
'batch_size': batch_size,
'shuffle': True,
'data_augment' : True,
'test' : False
}
params_val = {
'im_size': im_size,
'batch_size': batch_size,
'shuffle': True,
'data_augment' : False,
'test' : False
}
training_generator = DataGenerator(x_train_raw, y_train_raw, **params_trn)
 validation_generator = DataGenerator(x_val_raw, y_val_raw, **params_val )<predict_on_test>
joblib.dump(model, 'xgboost_model.pkl')
 del model
del X_train
del X_test
del y_train
del y_test
del X_valid_train
del X_valid_test
del y_valid_train
del y_valid_test
gc.collect()
preds_trn = model.predict_generator(training_generator, steps = x_train_raw.shape[0], verbose=1)
 preds_val = model.predict_generator(validation_generator, steps = x_train_raw.shape[0], verbose=1 )<save_to_csv>
all_data = pd.read_pickle('all_data.pkl' )
sub = pd.DataFrame(preds_trn)
 sub.to_csv("preds_trn.csv",index =False)
 sub = pd.DataFrame(y_train_raw)
 sub.to_csv("labels_trn.csv",index =False)
 sub = pd.DataFrame(preds_val)
 sub.to_csv("preds_val.csv",index =False)
 sub = pd.DataFrame(y_val_raw)
 sub.to_csv("labels_val.csv",index =False)
<load_from_csv>
def compute_prediction_for_specifi_month(monthes_before, df, estimator, params, prefix):
predictions = pd.DataFrame(columns=['date_block_num', 'pred_'+prefix])
 for before in monthes_before:
last_valid_month = np.max(df['date_block_num'])
 print('train: 12 to {0}'.format(last_valid_month-before-1))
print('test: {0}'.format(last_valid_month-before))
cur_train = df[df['date_block_num'] < last_valid_month-before]
cur_test = df[df['date_block_num'] == last_valid_month-before]
cur_y_train = cur_train['target']
cur_train.drop('target', axis=1, inplace=True)
 cur_test.drop('target', axis=1, inplace=True)
 model = estimator(**params)
 model.fit(cur_train.values, cur_y_train.values)
 pred = model.predict(cur_test.values)
 cur_df = pd.DataFrame(columns=['date_block_num', 'pred_'+prefix])
 cur_df['pred_'+prefix] = pred
cur_df['date_block_num'] =(last_valid_month-before)
 predictions = pd.concat([predictions, cur_df])
 del model
del pred
del cur_test
del cur_df
del cur_y_train
gc.collect()
return predictions
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data<groupby>
best_param = {
'learning_rate' :0.16,
'n_estimators':500,
'max_depth':6,
'min_child_weight':7,
'subsample':0.9,
'colsample_bytree':0.7,
'nthread':-1,
'scale_pos_weight':1,
'random_state':42,
'nthread': -1,
'tree_method':'gpu_hist',
'predictor':'gpu_predictor',
'gpu_id': 0,
}
ts= time.time()
monthes_before = [6, 5, 4, 3, 2, 1, 0]
stacks_xgb = compute_prediction_for_specifi_month(monthes_before, all_data, XGBRegressor, best_params,'xgb')
 time.time() -ts
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women*100)
 men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived", rate_men*100)
 passengers = train_data.Survived.values
rate_passengers = sum(passengers)/len(passengers)
 print("
% of all passengers who survived", rate_passengers*100)
 print("% of survivors who were women: ", 100*sum(women)/sum(passengers))
print("% of survivors who were men: ", 100*sum(men)/sum(passengers))
print("
% of passengers who were women:", 100*len(train_data.loc[train_data.Sex == 'female'])/len(train_data))
print("% of passengers who were men:",100*len(train_data.loc[train_data.Sex == 'male'])/len(train_data)) 
<filter>
stacks_xgb['pred_xgb'] = stacks_xgb['pred_xgb'].clip(0, 20 )
withsbsp=train_data.loc[train_data.Survived == 0]["SibSp"]
print("Average number of siblings/spouses for those who died: {}".format(sum(withsbsp)/len(withsbsp)))
 survivedsbsp = train_data.loc[train_data.Survived == 1]["SibSp"]
print("Average number of siblings/spouses for those who survived: {}".format(sum(survivedsbsp)/len(survivedsbsp)))
 nmen = train_data.loc[train_data.Survived == 0][train_data.Sex == 'male']["Parch"]
ymen = train_data.loc[train_data.Survived == 1][train_data.Sex == 'male']["Parch"]
print("
Average number of children/parents per man who did not survive: {}".format(sum(nmen)/len(nmen)))
 print("Average number of children/parents per man who survived: {}".format(sum(ymen)/len(ymen)))
<define_variables>
ts = time.time()
monthes_before = [6, 5, 4, 3, 2, 1, 0]
best_param ={'alpha': 2000}
stacks_lin_reg = compute_prediction_for_specifi_month(monthes_before, all_data, Ridge, best_param,'lin')
 time.time() - ts
male=train_data.loc[train_data.Sex=='male']
nmen = train_data.loc[train_data.Survived == 0][train_data.Sex == 'male'][train_data.Parch >=1]
ymen = train_data.loc[train_data.Survived == 1][train_data.Sex == 'male'][train_data.Parch >=1]
mediann=np.median(nmen.Parch)
 mediany=np.median(ymen.Parch)
 print("
Average number of children/Fathers per man who did not survive: {}
The median is {}" 
.format(sum(nmen.Parch)/len(nmen),mediann))
print("Average number of children/Fathers per man who survived: {}
The median is {}" 
.format(sum(ymen.Parch)/len(ymen),mediany))
print("% of men with children/parents: ", 100*(len(nmen)+len(ymen)) /len(male))<categorify>
stacks_lin_reg['pred_lin'] = stacks_lin_reg['pred_lin'].clip(0, 20 )
train_data1 = train_data.drop(['Name', 'Ticket','Cabin','Embarked','PassengerId'], axis = 1)
 train_data1
X_test = test_data.drop(['Name', 'Ticket','Cabin','Embarked','PassengerId'], axis = 1)
 train_data1 = pd.get_dummies(train_data1)
 train_data1 = train_data1.drop(['Sex_male'],axis=1)
 X_test=pd.get_dummies(X_test)
 X_test= X_test.drop(['Sex_male'],axis=1)
<feature_engineering>
X_valid_train = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] < 33],
stacks_lin_reg[stacks_lin_reg['date_block_num'] < 33]], axis=1)
 X_valid_test = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] == 33],
stacks_lin_reg[stacks_lin_reg['date_block_num'] == 33]], axis=1)
 y_valid_train = all_data.loc[(all_data['date_block_num']<33)&(all_data['date_block_num']>27), 'target']
y_valid_test = all_data.loc[all_data['date_block_num']==33, 'target']
c1a = int(( train_data1.loc[train_data1.Pclass ==1])['Age'].mean())
 c2a = int(( train_data1.loc[train_data1.Pclass ==2])['Age'].mean())
 c3a = int(( train_data1.loc[train_data1.Pclass ==3])['Age'].mean())
 print(c1a,c2a,c3a)
 a =train_data1.loc[train_data.Pclass==1].fillna({'Age':c1a})
 train_data1.loc[train_data.Pclass==1]=a
a2=X_test.loc[X_test.Pclass==1].fillna({'Age':c1a})
 X_test.loc[X_test.Pclass==1] =a2
b=train_data1.loc[train_data.Pclass==2].fillna({'Age':c2a})
 train_data1.loc[train_data.Pclass==2]=b
b2=X_test.loc[X_test.Pclass==2].fillna({'Age':c2a})
 X_test.loc[X_test.Pclass==2] =b2
c=train_data1.loc[train_data.Pclass==3].fillna({'Age':c3a})
 train_data1.loc[train_data.Pclass==3]=c
c2=X_test.loc[X_test.Pclass==3].fillna({'Age':c3a})
 X_test.loc[X_test.Pclass==3] =c2
X_test['Fare'] = X_test['Fare'].fillna(20)
<prepare_x_and_y>
X_train = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] < 34],
stacks_lin_reg[stacks_lin_reg['date_block_num'] < 34]], axis=1)
 X_test = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] == 34],
stacks_lin_reg[stacks_lin_reg['date_block_num'] == 34]], axis=1)
 y_train = all_data.loc[(all_data['date_block_num']<34)&(all_data['date_block_num']>27), 'target']
X_train = train_data1.drop(columns=["Survived"])
 y_train = train_data1.Survived
params = {'n_estimators': [1,200],
'max_features': [1,6],
'max_depth': [1,None],
'min_samples_split': [2,10],
'min_samples_leaf': [1,10],
'criterion':['gini','entropy']
}
forest = RandomForestClassifier()
forest_cv = RandomizedSearchCV(forest, params, cv=5)
 forest_cv.fit(X_train,y_train)
 print("Tuned Decision Tree Parameters: {}".format(forest_cv.best_params_))
print("Best score is {}".format(forest_cv.best_score_)) 
<save_to_csv>
X_valid_train.drop('date_block_num', axis=1, inplace=True)
 X_valid_test.drop('date_block_num', axis=1, inplace=True)
 X_train.drop('date_block_num', axis=1, inplace=True)
 X_test.drop('date_block_num', axis=1, inplace=True )
y_pred = forest_cv.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': y_pred})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
model = LinearRegression()
model.fit(X_valid_train, y_valid_train)
 pred = model.predict(X_valid_test ).clip(0,20)
 rmse = np.sqrt(mean_squared_error(y_valid_test, pred))
print('RMSE on valid set: {0}'.format(rmse))
%matplotlib inline
warnings.filterwarnings('ignore' )<load_from_csv>
model = LinearRegression()
model.fit(X_train, y_train)
 pred = model.predict(X_test ).clip(0,20 )
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train.describe(include="all" )<define_variables>
submit = pd.DataFrame({'ID':range(len(pred)) , 'item_cnt_month': pred})
 submit.to_csv('submit.csv', index=False )