Kaggle competition: Titanic - Machine Learning from Disaster
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 print('Use time:', time.time() - ts)
<data_type_conversions>
data_path = '/kaggle/input/titanic/'
train_df = pd.read_csv(data_path+'train.csv')
 test_df = pd.read_csv(data_path+'test.csv' )
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['shop_city'] = matrix['shop_city'].astype(np.int8)
 matrix['shop_category'] = matrix['shop_category'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 print('Use time:', time.time() - ts)
<merge>
combined_df = pd.concat([train_df,test_df])
 train_df.name = 'Training Dataset'
test_df.name = 'Test Dataset'
combined_df.name = 'Combined Dataset'
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'-lag'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<groupby>
def missing_columns(df):
for col in df.columns.tolist() :
print('{} column missing values: {}'.format(col, df[col].isnull().sum()))
print('
')
 for df in [train_df,test_df]:
print('{}'.format(df.name))
missing_columns(df )
dict_simple = {'date_block_num': 'date', 'item_id': 'item', 'shop_id': 'shop',
'item_category_id': 'itemcate', 'item_price':'price',
'item_cnt_month': 'cnt', }
def sum_names(name_list):
names = ''
for x in name_list:
names += x+'+'
return names
def group_agg(matrix, groupby_feats, transform_feat, aggtype='mean'):
group = matrix.groupby(groupby_feats ).agg({transform_feat: [aggtype]})
 groupby_feats_simple = [dict_simple[x] if x in dict_simple.keys() else x
for x in groupby_feats]
transform_feat_simple = dict_simple[transform_feat] \
if transform_feat in dict_simple.keys() else transform_feat
group_name = f'{sum_names(groupby_feats_simple)[:-1]}-{aggtype.upper() }-{transform_feat_simple}'
group.columns = [ group_name ]
group.reset_index(inplace=True)
 return group, group_name
def add_groupmean_lag(matrix, groupby_feats, transform_feat, lags):
group, group_name = group_agg(matrix, groupby_feats, transform_feat)
 matrix = pd.merge(matrix, group, on=groupby_feats, how='left')
 matrix[group_name] = matrix[group_name].astype(np.float16)
 if lags != []:
matrix = lag_feature(matrix, lags, group_name)
 matrix.drop([group_name], axis=1, inplace=True)
 return matrix 
<groupby>
%matplotlib inline
ts = time.time()
transform_feat = 'item_cnt_month'
groupby_feats = ['date_block_num']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'item_id']
lags = [1,2,3]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id']
lags = [1,2,3]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'item_category_id']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id', 'item_category_id']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id', 'item_id']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id', 'subtype_code']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_city']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'item_id', 'shop_city']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 print('Use time:', time.time() - ts)
<categorify>
mask_m1 =(train_df.Sex == "male")&(train_df.Pclass == 1)
 mask_m2 =(train_df.Sex == "male")&(train_df.Pclass == 2)
 mask_m3 =(train_df.Sex == "male")&(train_df.Pclass == 3)
 mask_f1 =(train_df.Sex == "female")&(train_df.Pclass == 1)
 mask_f2 =(train_df.Sex == "female")&(train_df.Pclass == 2)
 mask_f3 =(train_df.Sex == "female")&(train_df.Pclass == 3)
 m_age_class1_male = combined_df.loc[(combined_df.Sex == "male")&(combined_df.Pclass == 1),'Age'].dropna().median()
m_age_class2_male = combined_df.loc[(combined_df.Sex == "male")&(combined_df.Pclass == 2),'Age'].dropna().median()
m_age_class3_male = combined_df.loc[(combined_df.Sex == "male")&(combined_df.Pclass == 3),'Age'].dropna().median()
m_age_class1_female = combined_df.loc[(combined_df.Sex == "female")&(combined_df.Pclass == 1),'Age'].dropna().median()
m_age_class2_female = combined_df.loc[(combined_df.Sex == "female")&(combined_df.Pclass == 2),'Age'].dropna().median()
m_age_class3_female = combined_df.loc[(combined_df.Sex == "female")&(combined_df.Pclass == 3),'Age'].dropna().median()
train_df.loc[mask_m1,'Age'] = train_df.loc[mask_m1,'Age'].fillna(m_age_class1_male)
 train_df.loc[mask_m2,'Age'] = train_df.loc[mask_m2,'Age'].fillna(m_age_class2_male)
 train_df.loc[mask_m3,'Age'] = train_df.loc[mask_m3,'Age'].fillna(m_age_class3_male)
 train_df.loc[mask_f1,'Age'] = train_df.loc[mask_f1,'Age'].fillna(m_age_class1_female)
 train_df.loc[mask_f2,'Age'] = train_df.loc[mask_f2,'Age'].fillna(m_age_class2_female)
 train_df.loc[mask_f3,'Age'] = train_df.loc[mask_f3,'Age'].fillna(m_age_class3_female)
ts = time.time()
fetures_to_drop = []
transform_feat = 'item_price'
groupby_feats = ['item_id']
group, mean_price_col = group_agg(train, groupby_feats,
transform_feat, aggtype='mean')
 matrix = pd.merge(matrix, group, on=groupby_feats, how='left')
 matrix[mean_price_col] = matrix[mean_price_col].astype(np.float16)
 transform_feat = 'item_price'
groupby_feats = ['date_block_num','item_id']
group, mean_monthlyprice_col = group_agg(train, groupby_feats,
transform_feat, aggtype='mean')
 matrix = pd.merge(matrix, group, on=groupby_feats, how='left')
 matrix[mean_monthlyprice_col] = matrix[mean_monthlyprice_col].astype(np.float16)
 lags = [1,2,3]
matrix = lag_feature(matrix, lags, mean_monthlyprice_col)
 for i in lags:
matrix['delta_price-lag'+str(i)] = \ 
(matrix[f'{mean_monthlyprice_col}-lag'+str(i)] - matrix[mean_price_col])\
/ matrix[mean_price_col]
matrix['delta_price-lag']=0
bool_loc = np.ones(len(matrix)) ==1
for i in lags:
matrix.loc[bool_loc, 'delta_price-lag'] = matrix.loc[bool_loc,'delta_price-lag'+str(i)]
bool_loc &= matrix['delta_price-lag'+str(i)]==0
matrix['delta_price-lag'] = matrix['delta_price-lag'].astype(np.float16)
 matrix['delta_price-lag'].fillna(0, inplace=True)
 fetures_to_drop.append(mean_price_col)
 fetures_to_drop.append(mean_monthlyprice_col)
 for i in lags:
fetures_to_drop += [f'{mean_monthlyprice_col}-lag'+str(i)]
fetures_to_drop += ['delta_price-lag'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 print('Use time:', time.time() - ts)
<merge>
mask_m1 =(test_df.Sex == "male")&(test_df.Pclass == 1)
 mask_m2 =(test_df.Sex == "male")&(test_df.Pclass == 2)
 mask_m3 =(test_df.Sex == "male")&(test_df.Pclass == 3)
 mask_f1 =(test_df.Sex == "female")&(test_df.Pclass == 1)
 mask_f2 =(test_df.Sex == "female")&(test_df.Pclass == 2)
 mask_f3 =(test_df.Sex == "female")&(test_df.Pclass == 3)
 test_df.loc[mask_m1,'Age'] = test_df.loc[mask_m1,'Age'].fillna(m_age_class1_male)
 test_df.loc[mask_m2,'Age'] = test_df.loc[mask_m2,'Age'].fillna(m_age_class2_male)
 test_df.loc[mask_m3,'Age'] = test_df.loc[mask_m3,'Age'].fillna(m_age_class3_male)
 test_df.loc[mask_f1,'Age'] = test_df.loc[mask_f1,'Age'].fillna(m_age_class1_female)
 test_df.loc[mask_f2,'Age'] = test_df.loc[mask_f2,'Age'].fillna(m_age_class2_female)
 test_df.loc[mask_f3,'Age'] = test_df.loc[mask_f3,'Age'].fillna(m_age_class3_female )
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 print('Use time:', time.time() - ts)
<feature_engineering>
train_df[train_df.Embarked.isna() ]
total_block_num = 35
date_block_num = np.arange(total_block_num)
 date_block = [pd.Timestamp(2013, 1, 1)+pd.DateOffset(months=x)for x in date_block_num]
df_date = pd.DataFrame(date_block_num, columns=['date_block_num'])
 df_date['date_block'] = date_block
df_date['year'] = df_date['date_block'].dt.year
df_date['month'] = df_date['date_block'].dt.month
for i in range(len(df_date)) :
day_to_count = 0
calendar_matrix = calendar.monthcalendar(df_date['year'].iloc[i],df_date['month'].iloc[i])
 for j in range(7):
num_days = sum(1 for x in calendar_matrix if x[j] != 0)
 df_date.loc[i, f'week{j}'] = num_days
df_date = df_date[['date_block_num', 'year','month','week0','week1',
'week2','week3','week4','week5','week6']]
df_date['days'] = df_date[['week0','week1','week2','week3','week4','week5','week6']].sum(axis=1)
 df_date['year'] = df_date['year']-2012
df_date = df_date.astype(np.int8)
 matrix = pd.merge(matrix, df_date, on=['date_block_num'], how='left')
<categorify>
train_df['Embarked'] = train_df['Embarked'].fillna('C' )
matrix['item_shop_first_sale'] = \
matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = \
matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
<filter>
median_class3_fare = combined_df.loc[(combined_df.Pclass == 3),'Fare'].dropna().median()
test_df.Fare.fillna(median_class3_fare,inplace=True)
matrix = matrix[matrix.date_block_num > 3] 
<data_type_conversions>
train_df['SibSp_cat'] =(train_df['SibSp'] > 0)* 1
train_df['Parch_cat'] =(train_df['Parch'] > 0)* 1
train_df['Gender'] =(train_df['Sex'] == 'female')* 1
train_df['Class 1'] =(train_df['Pclass'] == 1)* 1
train_df['Class 2'] =(train_df['Pclass'] == 2)* 1
train_df['Class 3'] =(train_df['Pclass'] == 3)* 1
train_df['IsAlone'] =(( train_df['SibSp'] == 0)&(train_df['Parch'] == 0)) * 1
train_df['High_Fare'] =(train_df['Fare'] >= 200.0)* 1
train_df['Family_Members'] = train_df.SibSp + train_df.Parch
def fill_na(df):
for col in df.columns:
if('-lag' in col)&(df[col].isnull().any()):
print(col)
 if('cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix )<count_missing_values>
test_df['SibSp_cat'] =(test_df['SibSp'] > 0)* 1
test_df['Parch_cat'] =(test_df['Parch'] > 0)* 1
test_df['Gender'] =(test_df['Sex'] == 'female')* 1
test_df['Class 1'] =(test_df['Pclass'] == 1)* 1
test_df['Class 2'] =(test_df['Pclass'] == 2)* 1
test_df['Class 3'] =(test_df['Pclass'] == 3)* 1
test_df['IsAlone'] =(( test_df['SibSp'] == 0)&(test_df['Parch'] == 0)) * 1
test_df['High_Fare'] =(test_df['Fare'] >= 200.0)* 1
test_df['Family_Members'] = test_df.SibSp + test_df.Parch
matrix.isna().sum()<load_pretrained>
y_train = train_df['Survived']
features = ['Gender','Age','Class 1','Class 2','Class 3','Family_Members','Embarked','Fare']
X_train = train_df[features]
X_test = test_df[features]
del group
del items
del shops
del cats
del train
gc.collect() ;
matrix.to_pickle('.. /working/data.pkl')
 del matrix
gc.collect() ;<load_from_csv>
X_train_1hot = pd.get_dummies(X_train)
 X_train_1hot
data = pd.read_pickle('.. /working/data.pkl')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 print(len(data.columns))
data.columns<prepare_x_and_y>
X_test_1hot = pd.get_dummies(X_test)
 X_test_1hot
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del data
gc.collect() ;<create_dataframe>
base_model = RandomForestClassifier(random_state=42,n_estimators=100,max_depth=5)
 fit = base_model.fit(X_train_1hot,y_train )
lgb_train = lgb.Dataset(X_train, Y_train)
 lgb_eval = lgb.Dataset(X_valid, Y_valid, reference=lgb_train)
 del X_train
gc.collect() ;<compute_test_metric>
fit.score(X_train_1hot,y_train )
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
params = {'num_leaves': 2000, 'max_depth': 19, 'max_bin': 107, 'n_estimators': 3747,
'bagging_freq': 1, 'bagging_fraction': 0.7135681370918421,
'feature_fraction': 0.49446461478601994, 'min_data_in_leaf': 88,
'learning_rate': 0.015980721586917768, 'num_threads': 3,
'min_sum_hessian_in_leaf': 6,
'random_state' : RANDOM_SEED,
'verbosity' : 1,
'bagging_seed' : RANDOM_SEED,
'boost_from_average' : 'true',
'boost' : 'gbdt',
'metric' : 'rmse',}
model = lgb.train(params,
lgb_train,
num_boost_round=20,
valid_sets=[lgb_train,lgb_eval],
early_stopping_rounds=20,
verbose_eval=1, 
)
 y_pred = model.predict(X_valid)
 rmsle(Y_valid, y_pred )<save_to_csv>
n_estimators = [int(x)for x in np.linspace(start = 100, stop = 1000, num = 10)]
max_features = ['auto', 'sqrt']
max_depth = [5,6,7,8,9,10]
max_depth.append(None)
 min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
random_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
pp = pprint.PrettyPrinter(indent=4)
 pp.pprint(random_grid )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('lgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('lgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('lgb_test.pickle', 'wb'))<import_modules>
rf = RandomForestClassifier()
rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 200, cv = 4, verbose=2, random_state=42, n_jobs = -1)
 rf_random.fit(X_train_1hot, y_train)
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,concatenate,Concatenate)
<set_options>
rf_random.best_params_
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 300
NUM_CLASSES = 5<load_from_csv>
rf_best = rf_random.best_estimator_
best_fit = rf_best.fit(X_train_1hot, y_train)
 best_fit.score(X_train_1hot,y_train )
<split><EOS> 
predictions = base_model.predict(X_test_1hot)
 output = pd.DataFrame({'PassengerId': test_df.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )