Kaggle competition: COVID19 Global Forecasting (Week 4)
df = downcast(df)
 int8_cols = [
'item_cnt','month','group_id','shop_type',
'shop_city','shop_id','date_block_num','category_id',
'item_age',
]
int16_cols = [
'item_id','item_name_first4',
'item_name_first6','item_name_first11'
]
for col in int8_cols:
df[col] = df[col].astype('int8')
 for col in int16_cols:
df[col] = df[col].astype('int16' )<merge>
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
def nearby_item_data(df,col):
if col in ['item_cnt_unclipped_lag1','item_cnt_lag1to12']:
cols = ['date_block_num', 'shop_id', 'item_id']
temp = df[cols + [col]]
else:
cols = ['date_block_num', 'item_id']
temp = df.groupby(cols ).agg({col:'first'} ).reset_index() [cols + [col]]
temp.columns = cols + [f'below_{col}']
temp['item_id'] += 1
df = pd.merge(df, temp, on=cols, how='left')
 temp.columns = cols + [f'above_{col}']
temp['item_id'] -= 2
df = pd.merge(df, temp, on=cols, how='left')
 return df
item_cols = ['item_cnt_unclipped_lag1','item_cnt_lag1to12',
'item_cnt_all_shops_lag1','item_cnt_all_shops_lag1to12']
for col in item_cols:
df = nearby_item_data(df,col)
 del temp<feature_engineering>
train.rename(columns={'Country_Region':'Countries'}, inplace=True)
 test.rename(columns={'Country_Region':'Countries'}, inplace=True)
 train.rename(columns={'Province_State':'States'}, inplace=True)
 test.rename(columns={'Province_State':'States'}, inplace=True)
 train['Date'] = pd.to_datetime(train['Date'], infer_datetime_format=True)
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True )
results = Counter()
items['item_name'].str.split().apply(results.update)
 words = []
cnts = []
for key, value in results.items() :
words.append(key)
 cnts.append(value)
 counts = pd.DataFrame({'word':words,'count':cnts})
 common_words = counts.query('count>200' ).word.to_list()
for word in common_words:
items[f'{word}_in_name'] = items['item_name'].str.contains(word ).astype('int8')
 drop_cols = [
'item_id','category_id','item_name','item_name_first4',
'item_name_first6','item_name_first11',
'category_name','group_name','group_id'
]
items = items.drop(columns=drop_cols )<merge>
na = "na"
def filtrar_state(states, countries):
if states == na:
return countries
else:
return states
df = df.join(items, on='item_id' )<categorify>
train_1 = train.copy()
train_1['States'].fillna(na, inplace=True)
 train_1['States'] = train_1.loc[:, ['States', 'Countries']].apply(lambda x : filtrar_state(x['States'], x['Countries']), axis=1)
 train_1.loc[:, 'Date'] = train_1.Date.dt.strftime("%m%d")
 train_1["Date"] = train_1["Date"].astype(int)
 train_1.head()
def binary_encode(df, letters, cols):
encoder = ce.BinaryEncoder(cols=[f'item_name_first{letters}'], return_df=True)
 temp = encoder.fit_transform(df[f'item_name_first{letters}'])
 df = pd.concat([df,temp], axis=1)
 del df[f'item_name_first{letters}_0']
name_cols = [f'item_name_first{letters}_{x}' for x in range(1,cols)]
df[name_cols] = df[name_cols].astype('int8')
 return df
df = binary_encode(df, 11, 15)
 del df['item_name_first4'], df['item_name_first6']<categorify>
te = test.copy()
te['States'].fillna(na, inplace=True)
 te['States'] = te.loc[:, ['States', 'Countries']].apply(lambda x : filtrar_state(x['States'], x['Countries']), axis=1)
 te.loc[:, 'Date'] = te.Date.dt.strftime("%m%d")
 te["Date"] = te["Date"].astype(int)
 te.head()
df.to_pickle('df_complete.pkl' )<set_options>
le = preprocessing.LabelEncoder()
train_1.Countries = le.fit_transform(train_1.Countries)
 train_1['States'] = le.fit_transform(train_1['States'])
 train_1.head()
%reset -f<set_options>
te.Countries = le.fit_transform(te.Countries)
 te['States'] = le.fit_transform(te['States'])
 te.head()
pd.set_option('display.max_rows', 160)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 30)
 warnings.filterwarnings("ignore" )<prepare_x_and_y>
train.loc[train.Countries == 'Afghanistan', :]
df = pd.read_pickle('.. /input/feature-engineering-lightgbm-exploring-performance/df_complete.pkl')
 X_train = df[~df.date_block_num.isin([0,1,33,34])]
y_train = X_train['item_cnt']
del X_train['item_cnt']
X_val = df[df['date_block_num']==33]
y_val = X_val['item_cnt']
del X_val['item_cnt']
X_test = df[df['date_block_num']==34].drop(columns='item_cnt')
 X_test = X_test.reset_index()
del X_test['index']
del df<train_model>
output = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 le = preprocessing.LabelEncoder()
countr = train_1.Countries.unique()
for countries in countr:
states = train_1.loc[train_1.Countries == countries, :].States.unique()
for state in states:
X_tra = train_1.loc[(train_1.Countries == countries)&(train_1.States == state), ['State', 'Countries', 'Date',
'ConfirmedCases','Fatalities']]
y_cc_train = X_tra.loc[:, 'ConfirmedCases']
y_fat_train = X_tra.loc[:, 'Fatalities']
X_tra = X_tra.loc[:, ['States', 'Countries', 'Date']]
X_tra.Countries = le.fit_transform(X_tra.Countries)
 X_tra['States'] = le.fit_transform(X_tra['States'])
 X_te = te.loc[(te.Countries == countries)&(te.States == state), ['States', 'Countries', 'Date',
'ForecastId']]
X_te_Id = X_te.loc[:, 'ForecastId']
X_te = X_te.loc[:, ['States', 'Countries', 'Date']]
X_te.Countries = le.fit_transform(X_te.Countries)
 X_te['States'] = le.fit_transform(X_te['States'])
 xmodel1 = XGBRegressor(learning_rate =0.01,n_estimators=2000,objective='count:poisson',max_depth=6,min_child_weight=4,gamma=0,
subsample=0.8,colsample_bytree=0.8,nthread=4,scale_pos_weight=1)
 xmodel1.fit(X_tra, y_cc_train, eval_metric='auc')
 y_cc_pred = xmodel1.predict(X_te)
 xmodel2 = XGBRegressor(learning_rate =0.01,n_estimators=2000,objective = 'count:poisson',max_depth=6,min_child_weight=4,gamma=0,
subsample=0.8,colsample_bytree=0.8,nthread=4,scale_pos_weight=1)
 xmodel2.fit(X_tra, y_fat_train, eval_metric='auc')
 y_fat_pred = xmodel2.predict(X_te)
 dt = pd.DataFrame({'ForecastId': X_te_Id, 'ConfirmedCases': y_cc_pred, 'Fatalities': y_fat_pred})
 output = pd.concat([output, dt], axis=0 )
def build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features):
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_val = lgb.Dataset(X_val, y_val)
 model = lgb.train(params=params, train_set=lgb_train, valid_sets=(lgb_train, lgb_val), verbose_eval=50,
categorical_feature=cat_features)
 return model<train_model>
output.ForecastId = output.ForecastId.astype('int')
 output.tail()
params = {
'objective': 'rmse',
'metric': 'rmse',
'num_leaves': 1023,
'min_data_in_leaf':10,
'feature_fraction':0.7,
'learning_rate': 0.01,
'num_rounds': 1000,
'early_stopping_rounds': 30,
'seed': 1
}
cat_features = ['category_id','month','shop_id','shop_city']
lgb_model = build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features)
<save_to_csv>
output.to_csv('submission.csv', index=False )