Kaggle competition: IESB Sul - IGM - Maio 2019
all_data = all_data.merge(period, on = ['date_block_num'], how = 'left')
 all_data = all_data.merge(shops.drop('shop_name', axis=1), on = ['shop_id'], how = 'left')
 all_data = all_data.merge(items.drop('item_name', axis=1), on = ['item_id'], how ='left' )<load_pretrained>
df = pd.read_csv('.. /input/train.csv')
 df.shape
all_data = downcast_dtypes(all_data)
 all_data.to_pickle('all_data.pkl')
 del grid, gb , cur_items
gc.collect() ;<create_dataframe>
df_test = pd.read_csv('.. /input/test.csv')
 df_test.shape
all_data = agg_and_lag(all_data,col_groupby=['date_block_num'],
target_col='item_price_month',
new_col_name='date_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_item_price_mean',
agg_function='mean',shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['shop_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_shop_price_mean',
agg_function='mean',shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['sub_cat','shop_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_shop_subcat_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_category_id','date_block_num'],
target_col='item_price_month',
new_col_name='date_itemcat_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['main_cat','date_block_num'],
target_col='item_price_month',
new_col_name='date_maincat_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['shop_type','date_block_num'],
target_col='item_price_month',
new_col_name='date_shop_type_price_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_id','shop_city','date_block_num'],
target_col='item_price_month',
new_col_name='date_item_shopcity_price_mean',
agg_function='mean',shift_range=[1] )<groupby>
df = df.append(df_test, ignore_index=True, sort=False)
 df.shape
all_data = agg_and_lag(all_data,col_groupby=['item_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_month_mean',
agg_function='mean', shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['shop_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_shop_month_mean',
agg_function='mean', shift_range=[1,2])
 all_data = agg_and_lag(all_data,col_groupby=['item_category_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_itemcat_month_mean',
agg_function='mean', shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['shop_id','item_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_shop_month_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['sub_cat','shop_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_subcat_month_mean',
agg_function='mean', shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['main_cat','shop_id','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_maincat_month_mean',
agg_function='mean', shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['item_id','shop_city','date_block_num'],
target_col='item_cnt_month',
new_col_name='date_item_shopcity_month_mean',
agg_function='mean',shift_range=[1])
 all_data = agg_and_lag(all_data,col_groupby=['date_block_num'],
target_col='item_cnt_month',
new_col_name='date_month_mean',
agg_function='mean', shift_range=[1] )<categorify>
df['populacao'] = df['populacao'].str.replace('.', '')
 df['populacao'] = df['populacao'].str.replace('\(\)', '')
 df['populacao'] = df['populacao'].str.replace('\(1\)', '')
 df['populacao'] = df['populacao'].str.replace('\(2\)', '')
 df['populacao'] = df['populacao'].str.replace(',', '' ).astype('float')
 df['densidade_dem'] = df['densidade_dem'].str.replace(',', '' ).astype('float')
 df['area'] = df['area'].str.replace(',', '' ).astype('float')
 df['comissionados_por_servidor'] = df['comissionados_por_servidor'].str.replace('
df['comissionados_por_servidor'] = df['comissionados_por_servidor'].str.replace('%', '' ).astype('float' )
index_cols = ['shop_id', 'item_id', 'date_block_num']
all_data = create_lags(all_data, index_cols=index_cols,
column_to_lag='item_cnt_month', shift_range=[1,2,3,12])
 all_data = create_lags(all_data, index_cols=index_cols,
column_to_lag='item_price_month', shift_range=[1,2] )<drop_column>
df['in_na_densidade_dem'] = df['densidade_dem'].isna().astype('int')
 df['densidade_dem'].fillna(df['densidade_dem'].mean() , inplace=True)
 df['in_na_participacao_transf_receita'] = df['participacao_transf_receita'].isna().astype('int')
 df['participacao_transf_receita'].fillna(-1, inplace=True)
 df['in_na_servidores'] = df['servidores'].isna().astype('int')
 df['servidores'].fillna(-1, inplace=True)
 df['in_na_perc_pop_econ_ativa'] = df['perc_pop_econ_ativa'].isna().astype('int')
 df['perc_pop_econ_ativa'].fillna(-1, inplace=True)
 df['in_na_gasto_pc_saude'] = df['gasto_pc_saude'].isna().astype('int')
 df['gasto_pc_saude'].fillna(-1, inplace=True)
 df['in_na_hab_p_medico'] = df['hab_p_medico'].isna().astype('int')
 df['hab_p_medico'].fillna(-1, inplace=True)
 df['in_na_exp_vida'] = df['exp_vida'].isna().astype('int')
 df['exp_vida'].fillna(df['exp_vida'].mean() , inplace=True)
 df['in_na_gasto_pc_educacao'] = df['gasto_pc_educacao'].isna().astype('int')
 df['gasto_pc_educacao'].fillna(-1, inplace=True)
 df['in_na_exp_anos_estudo'] = df['exp_anos_estudo'].isna().astype('int')
 df['exp_anos_estudo'].fillna(-1, inplace=True )
del sales, item_cats, items<filter>
ordem_porte = ['Pequeno porte 1', 'Pequeno porte 2', 'Médio porte', 'Grande porte']
df['porte'] = pd.Categorical(df['porte'], categories=ordem_porte ).codes
df['div_gasto'] = df['gasto_pc_saude'] / df['pib']
df['exp_pib'] = df['pib'] / df['exp_vida']
df['exp_pib_estudos'] = df['pib'] / df['exp_anos_estudo']
all_data = all_data[all_data['date_block_num'] >= 3]
gc.collect() ;<load_pretrained>
for col in df.columns:
if df[col].dtype == 'object':
df[col] = df[col].astype('category' ).cat.codes
all_data.to_pickle('all_data_lags.pkl' )<categorify>
test = df[df['nota_mat'].isnull() ]
df = df[~df['nota_mat'].isnull() ]
<prepare_x_and_y>
df = df[df['populacao'] > 5000].copy()
date = all_data['date_block_num']
target = all_data['item_cnt_month']<drop_column>
train, valid = train_test_split(df, test_size=0.2, random_state=42 )
all_data = all_data.drop('item_cnt_month', axis = 1)
 all_data = all_data.drop('item_price_month', axis = 1 )<categorify>
feats = [c for c in df.columns if c not in ['nota_mat', 'codigo_mun', 'municipio']]
def correlatedDropper(df_data, thresh = 0.95):

 df = df_data.copy()
corr_matrix = df.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
cols_to_drop = [column for column in upper.columns if any(upper[column] > thresh)]
df = df.drop(cols_to_drop, axis=1)
 return df, cols_to_drop<choose_model_class>
from sklearn.ensemble import RandomForestClassifier
test, col_drop = correlatedDropper(all_data, 0.95 )<prepare_x_and_y>
rf = RandomForestClassifier(n_estimators=200, min_samples_split=5, max_depth=4, random_state=42 )
last_block_num = 33
X_test = all_data[date == 34]
X_valid = all_data[date == last_block_num]
X_train = all_data[date < last_block_num]
y_train = target.loc[date < last_block_num].values
y_valid = target.loc[date == last_block_num].values
categorical_feat_ind = np.where(X_train.dtypes != np.float32)[0]
print(categorical_feat_ind)
<set_options>
rf.fit(train[feats], train['nota_mat'] )
del all_data, target, date
gc.collect()<compute_train_metric>
preds = rf.predict(valid[feats] )
def catboost_rmse(X_train,y_train,
X_valid, y_valid,
loss_function='RMSE',
iterations=400,
random_seed=0,
learning_rate=0.15,
depth=5,
l2_leaf_reg=1,
one_hot_max_size=200,
min_data_in_leaf = 1,
bagging_temperature = 1,
border_count = 32,
max_ctr_complexity = 2 
):

 catmodel = CatBoostRegressor(
 loss_function= loss_function,
iterations=iterations,
random_seed=random_seed,
learning_rate=learning_rate,
depth=depth,
l2_leaf_reg=l2_leaf_reg,
one_hot_max_size=one_hot_max_size,
min_data_in_leaf = min_data_in_leaf,
bagging_temperature= bagging_temperature,
border_count=border_count,
max_ctr_complexity=max_ctr_complexity,
thread_count=-1,
od_type='Iter',
od_wait= 20,
task_type='GPU',
devices='0',
verbose = 100 
)
 catmodel.fit(X_train,
y_train,
eval_set=(X_valid,y_valid),
plot=False 
)
 predict = catmodel.predict(X_valid ).clip(0,20)
 best_rmse_train = catmodel.best_score_['learn']['RMSE']
best_rmse_valid = catmodel.best_score_['validation']['RMSE']
print('train_rmse : {:.4f}, test_rmse : {:.4f}'.format(best_rmse_train, best_rmse_valid))
r2 = r2_score(y_valid,predict)
 print('the R2 score is {:.4f}'.format(r2))
del predict
return - best_rmse_valid<init_hyperparams>
accuracy_score(valid['nota_mat'], preds )
params_probe = {'bagging_temperature': 0.28300538794137076,
'border_count': 247.51179202909668,
'depth': 8.478936311173683,
'l2_leaf_reg': 7.73213368476749,
'learning_rate': 0.10290800942292017,
'max_ctr_complexity': 2.9452851946784433,
'min_data_in_leaf': 2.383458113416781,
'one_hot_max_size': 209.4502766887087}<train_on_grid>
valid['nota_mat'].value_counts()
print(Colours.green())
 optimizer_cat = optimize_catboost(X_train=X_train, y_train=y_train,
X_valid=X_valid, y_valid=y_valid,
param_probe=params_probe )<init_hyperparams>
pred_test = rf.predict(test[feats] )
params_catboost = {'bagging_temperature': 0.28300538794137076,
'border_count': 247,
'depth': 8,
'l2_leaf_reg': 7.73213368476749,
'learning_rate': 0.10290800942292017,
'max_ctr_complexity': 2,
'min_data_in_leaf': 2.383458113416781,
'one_hot_max_size': 209,
'random_seed' : 0,
'thread_count' : -1,
'task_type' : 'GPU',
'devices' : '0',
'od_type' : 'Iter',
'od_wait' : 20,
'verbose' : 50
}<train_model>
r1 = pd.DataFrame({'codigo_mun': test['codigo_mun'], 'nota_mat': pred_test})
 r1.to_csv('r1.csv', index=False )
catmodel = CatBoostRegressor(**params_catboost)
 catmodel.fit(X_train,y_train,
eval_set=(X_valid,y_valid),
plot=False 
)
 predict = catmodel.predict(X_valid ).clip(0,20)
 r2 = r2_score(y_valid,predict)
 print('r2_score est {}'.format(r2))<filter>
def cv(df, test, feats, y_name, k=5):
preds, score, fis = [], [], []
chunk = df.shape[0] // k
for i in range(k):
if i+1 < k:
valid = df.iloc[i*chunk:(i+1)*chunk]
train = df.iloc[:i*chunk].append(df.iloc[(i+1)*chunk:])
 else:
valid = df.iloc[i*chunk:]
train = df.iloc[:i*chunk]
rf = RandomForestClassifier(random_state=42, n_jobs=-1, n_estimators=200)
 rf.fit(train[feats], train[y_name])
 score.append(accuracy_score(valid[y_name], rf.predict(valid[feats])))
 preds.append(rf.predict(test[feats]))
fis.append(rf.feature_importances_)
 print(i, 'OK')
 return score, preds, fis
feature_useless_pred = feature_pred[feature_pred['Score'] < 0.01]
feature_useless_pred.Feature<drop_column>
score, preds, fis = cv(df, test, feats, 'nota_mat' )
X_train_pred = X_train.drop(feature_useless_pred.Feature[:-1], axis = 1)
 X_valid_pred = X_valid.drop(feature_useless_pred.Feature[:-1], axis = 1)
 X_test_pred = X_test.drop(feature_useless_pred.Feature[:-1], axis = 1)
<filter>
r2 = pd.DataFrame({'codigo_mun': test['codigo_mun'], 'nota_mat': preds[1]})
 r2.to_csv('r2.csv', index=False )
feature_useless_loss = feature_loss[feature_loss['Score'] < 0.0]
feature_useless_loss.Feature<train_model>
import lightgbm as lgb
catmodel_pred = CatBoostRegressor(**params_catboost)
 catmodel_pred.fit(X_train_pred,y_train,
eval_set=(X_valid_pred,y_valid),
plot=False 
)
 predict = catmodel_pred.predict(X_valid_pred ).clip(0,20)
 r2 = r2_score(y_valid,predict)
 print('r2_score est {}'.format(r2))<train_model>
def cv(df, test, feats, y_name, k=5):
preds, score, fis = [], [], []
chunk = df.shape[0] // k
for i in range(k):
if i+1 < k:
valid = df.iloc[i*chunk:(i+1)*chunk]
train = df.iloc[:i*chunk].append(df.iloc[(i+1)*chunk:])
 else:
valid = df.iloc[i*chunk:]
train = df.iloc[:i*chunk]
rf = lgb.LGBMClassifier(min_child_samples=500)
 rf.fit(train[feats], train[y_name], eval_set=[(train[feats], train['nota_mat']),(valid[feats], valid['nota_mat'])], early_stopping_rounds=50)
 score.append(accuracy_score(valid[y_name], rf.predict(valid[feats])))
 preds.append(rf.predict(test[feats]))
fis.append(rf.feature_importances_)
 print(i, 'OK')
 return score, preds, fis
print(Colours.green())
 optimizer_cat_loss = optimize_catboost(X_train=X_train_pred, y_train=y_train,
X_valid=X_valid_pred, y_valid=y_valid,
param_probe=params_probe )<drop_column>
score, preds, fis = cv(df, test, feats, 'nota_mat' )
<load_pretrained><EOS> 
r3 = pd.DataFrame({'codigo_mun': test['codigo_mun'], 'nota_mat': preds[1]})
 r3.to_csv('r3.csv', index=False )