Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<categorify>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
import datetime as dt
import folium
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()<feature_engineering>
!kaggle kaggle competitions download -c covid19-global-forecasting-week-1
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )<feature_engineering>
!pip install kaggle
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique() :
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
 df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
 df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
 df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
 df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
 df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2 )<feature_engineering>
!mkdir.kaggle
add_sg(weather_train_df )<define_variables>
token = {"username":'nitingrover425','key':'c22685e02df7d46edd199e441980c448'}
with open('/content/.kaggle/kaggle.json', 'w')as file:
json.dump(token, file )
category_cols = ['building_id', 'site_id', 'primary_use', 'IsHoliday']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'day',
] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr',
'sea_level_pressure',
'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72',
'wind_speed_mean_lag72',
'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3',
'sea_level_pressure_mean_lag3',
'year_cnt', 'bid_cnt',
'dew_smooth', 'air_smooth',
'dew_diff', 'air_diff',
'dew_diff2', 'air_diff2',
]<merge>
!kaggle config set -n path -v{/content}
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train<train_model>
!kaggle competitions download -c covid19-global-forecasting-week-1 -p /content
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<choose_model_class>
train_data = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv" )
seed = 666
shuffle = False
kf = GroupKFold(n_splits=folds )<groupby>
train_data.isnull().sum()
def get_groups(df, meter):
if folds == 12:
return df[df.meter==meter].month -1
elif folds == 6:
return(df[df.meter==meter].month -1)// 2
elif folds == 3:
return(df[df.meter==meter].month -1)// 4<split>
train_data = train_data.rename(columns = {"Province/State":"State" , "Country/Region":"Country" } )
oof_total = 0
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print(mindex)
 print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append([mindex, model])
 gc.collect()
if debug:
break
oof0 = mean_squared_error(y_train, y_valid_pred_total)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()<split>
train_data = train_data.fillna("Not Available")
 train_data.isnull().sum()
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append([mindex, model])
 gc.collect()
if debug:
break
oof1 = mean_squared_error(y_train, y_valid_pred_total)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()<split>
train_data[train_data['ConfirmedCases']<0]
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append([mindex, model])
 gc.collect()
if debug:
break
oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 del X_train, y_train
gc.collect()<split>
train_data[train_data['Fatalities']<0]
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append([mindex, model])
 gc.collect()
if debug:
break
oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 oof_total = oof_total / len(train_df)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 del X_train, y_train
gc.collect()<compute_test_metric>
train_data['Date'] = pd.to_datetime(train_data['Date'] )
print('oof0=', np.sqrt(oof0))
print('oof1=', np.sqrt(oof1))
print('oof2=', np.sqrt(oof2))
print('oof3=', np.sqrt(oof3))
print('total=', np.sqrt(oof_total))<drop_column>
train_data[train_data['ConfirmedCases'] == train_data['ConfirmedCases'].max() ]
del train_df, weather_train_df
gc.collect()<feature_engineering>
date_country_total = train_data.groupby(['Date'] ).sum()
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather_test_df = weather_test_df.drop_duplicates(['timestamp', 'site_id'])
 set_local(weather_test_df)
 add_holiyday(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_sg(weather_test_df)
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 test_df['bid_cnt'] = test_df.building_id.map(bid_map)
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
print(test_df.shape )<load_from_disk>
date_country_total['Total Cases'] = date_country_total['ConfirmedCases'].cumsum()
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission)
 print(sample_submission.shape )<merge>
country_cases = train_data.groupby(['Country' , 'Lat' , 'Long'] ).sum()
country_cases.reset_index(inplace=True)
 country_cases
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols + ['month']]
return X_test<predict_on_test>
columns = train_data.columns.tolist()
columns
train_data['Date'] = pd.to_numeric(train_data['Date'] )
def pred_all(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i,(mindex, model)in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def pred(X_test, models, batch_size=1000000):
if predmode == 'valid':
print('valid pred')
 return pred_valid(X_test, models, batch_size=1000000)
 elif predmode == 'train':
print('train pred')
 return pred_train(X_test, models, batch_size=1000000)
 else:
print('all pred')
 return pred_all(X_test, models, batch_size=1000000 )<feature_engineering>
columns = [c for c in columns if c not in['Id','ConfirmedCases','Fatalities','State','Country']]
X_train = train_data[columns]
Y1_train = train_data['ConfirmedCases']
Y2_train = train_data['Fatalities']
print(X_train.shape)
 print(Y1_train.shape)
 print(Y2_train.shape )
print(sample_submission.loc[test_df['meter'] == 0, 'meter_reading'].shape,np.expm1(y_test0 ).shape)
 print(sample_submission.loc[test_df['meter'] == 1, 'meter_reading'].shape,np.expm1(y_test1 ).shape)
 print(sample_submission.loc[test_df['meter'] == 2, 'meter_reading'].shape,np.expm1(y_test2 ).shape)
 print(sample_submission.loc[test_df['meter'] == 3, 'meter_reading'].shape,np.expm1(y_test3 ).shape)
 sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )<feature_engineering>
regressor = RandomForestRegressor(n_estimators = 100 ,random_state=0)
 regressor.fit(X_train,Y1_train)
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118<feature_engineering>
test_data = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv' )
if clip0:
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0<save_to_csv>
columnst = test_data.columns.tolist()
columnst
test_data['Date'] = pd.to_datetime(test_data['Date'] )
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<feature_engineering>
test_data['Date'] = pd.to_numeric(test_data['Date'] )
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 print(leak_df.duplicated().sum())
 print(leak_df.meter.value_counts())
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )<compute_test_metric>
columnst = [c for c in columnst if c not in ['ForecastId', 'Province/State', 'Country/Region']]
columnst
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))<feature_engineering>
y1_pred = regressor.predict(test_data[columnst] )
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<save_to_csv>
regressor.fit(X_train,Y2_train)
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )<compute_test_metric>
y2_pred = regressor.predict(test_data[columnst] )
print('oof score meter0 =', np.sqrt(oof0))
print('oof score meter1 =', np.sqrt(oof1))
print('oof score meter2 =', np.sqrt(oof2))
print('oof score meter3 =', np.sqrt(oof3))
print('oof score total =', np.sqrt(oof_total))<compute_test_metric>
pred1 = pd.DataFrame(y1_pred)
 pred2 = pd.DataFrame(y2_pred)
 sub_df = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv')
 sub_df.head()
<set_options><EOS> 
datasets = pd.concat([sub_df['ForecastId'],pred1,pred2],axis=1)
 datasets.columns = ['ForecastId','ConfirmedCases','Fatalities']
datasets.to_csv('submission.csv',index=False )