Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<define_variables>
import numpy as np
import pandas as pd
import xgboost as xgb
from xgboost import plot_importance, plot_tree
from sklearn.metrics import mean_squared_error, mean_absolute_error
from google.cloud import bigquery
def make_is_bad_zero(Xy_subset, min_interval=48, summer_start=3000, summer_end=7500):
meter = Xy_subset.meter_id.iloc[0]
is_zero = Xy_subset.meter_reading == 0
if meter == 0:
return is_zero
transitions =(is_zero != is_zero.shift(1))
all_sequence_ids = transitions.cumsum()
ids = all_sequence_ids[is_zero].rename("ids")
 if meter in [2, 3]:
keep = set(ids[(Xy_subset.timestamp < summer_start)| 
(Xy_subset.timestamp > summer_end)].unique())
 is_bad = ids.isin(keep)&(ids.map(ids.value_counts())>= min_interval)
 elif meter == 1:
time_ids = ids.to_frame().join(Xy_subset.timestamp ).set_index("timestamp" ).ids
is_bad = ids.map(ids.value_counts())>= min_interval
jan_id = time_ids.get(0, False)
 dec_id = time_ids.get(8283, False)
 if(jan_id and dec_id and jan_id == time_ids.get(500, False)and
dec_id == time_ids.get(8783, False)) :
is_bad = is_bad &(~(ids.isin(set([jan_id, dec_id]))))
else:
raise Exception(f"Unexpected meter type: {meter}")
 result = is_zero.copy()
result.update(is_bad)
 return result
def find_bad_zeros(X, y):

 Xy = X.assign(meter_reading=y, meter_id=X.meter)
 is_bad_zero = Xy.groupby(["building_id", "meter"] ).apply(make_is_bad_zero)
 return is_bad_zero[is_bad_zero].index.droplevel([0, 1])
 def find_bad_sitezero(X):

 return X[(X.timestamp < 3378)&(X.site_id == 0)&(X.meter == 0)].index
def find_bad_building1099(X, y):

 return X[(X.building_id == 1099)&(X.meter == 2)&(y > 3e4)].index
def find_bad_rows(X, y):
return find_bad_zeros(X, y ).union(find_bad_sitezero(X)).union(find_bad_building1099(X, y))<save_to_csv>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv" )
X, y = combined_data(train,weather_train)
 bad_rows = find_bad_rows(X, y)
 pd.Series(bad_rows.sort_values() ).to_csv("rows_to_drop.csv", header=False, index=False)
 X = X.drop(index=bad_rows)
 y = y.reindex_like(X)
 X = _add_time_features(X)
 X = compress_dataframe(X)
 X = X.drop(columns="timestamp")
 del bad_rows,train,weather_train
gc.collect() ;<prepare_output>
%%time
client = bigquery.Client()
dataset_ref = client.dataset("noaa_gsod", project="bigquery-public-data")
 dataset = client.get_dataset(dataset_ref)
 tables = list(client.list_tables(dataset))
table_ref = dataset_ref.table("stations")
 table = client.get_table(table_ref)
 stations_df = client.list_rows(table ).to_dataframe()
table_ref = dataset_ref.table("gsod2020")
 table = client.get_table(table_ref)
 twenty_twenty_df = client.list_rows(table ).to_dataframe()
stations_df['STN'] = stations_df['usaf'] + '-' + stations_df['wban']
twenty_twenty_df['STN'] = twenty_twenty_df['stn'] + '-' + twenty_twenty_df['wban']
cols_1 = ['STN', 'mo', 'da', 'temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']
cols_2 = ['STN', 'country', 'state', 'call', 'lat', 'lon', 'elev']
weather_df = twenty_twenty_df[cols_1].join(stations_df[cols_2].set_index('STN'), on='STN')
 weather_df.tail(10 )
y = np.log1p(y )<split>
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in train.index:
df = train.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.Id, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 train['closest_station'] = C
train = train.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 train.sort_values(by=['Id'], inplace=True)
 train.head()
n_splits = 3
for val in X['meter'].unique() :
X1 = X[X['meter'] == val].drop(columns=['meter'])
 kf = StratifiedKFold(n_splits=n_splits,random_state=42)
 t = 0
for train_index, test_index in kf.split(X1, X1['tm_hour_of_day']):
train_features = X1.iloc[train_index]
train_target = y[X1.iloc[train_index].index]
test_features = X1.iloc[test_index]
test_target = y[X1.iloc[test_index].index]

 gbm = LGBMRegressor(
 boosting_type='gbdt',
objective='regression',
metric='rmse',
learning_rate=0.03,
num_leaves=40,
n_estimators=1000,
subsample=0.8 
)
 gbm.fit(train_features,train_target)
 joblib.dump(gbm, 'loan_model{}_fold{}.pkl'.format(val,t))
t += 1
del X1
del train_features, test_features
gc.collect() ;<merge>
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in test.index:
df = test.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.ForecastId, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 test['closest_station'] = C
test = test.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 test.sort_values(by=['ForecastId'], inplace=True)
 test.head()
X = compress_dataframe(test.join(building, on="building_id" ).join(weather_test, on=["site_id", "timestamp"] ).fillna(-1))
X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 del test, weather_test
gc.collect() ;<choose_model_class>
train["wdsp"] = pd.to_numeric(train["wdsp"])
 test["wdsp"] = pd.to_numeric(test["wdsp"] )
result = np.zeros(len(X))
for val in X['meter'].unique() :
ix = np.nonzero(( X['meter'] == val ).to_numpy())
 for i in tqdm(range(n_splits)) :
gbm = joblib.load('loan_model{}_fold{}.pkl'.format(val,i))
result[ix] += gbm.predict(X.iloc[ix].drop(columns=['meter']), num_iteration=gbm.best_iteration_)/n_splits
gc.collect() ;
predictions = pd.DataFrame({
"row_id": X.index,
"meter_reading": np.clip(np.expm1(result), 0, None)
 })
 predictions.to_csv("submission.csv", index=False, float_format="%.4f" )<define_variables>
train["fog"] = pd.to_numeric(train["fog"])
 test["fog"] = pd.to_numeric(test["fog"] )
folds = 3
debug = False
num_rounds = 1000<set_options>
X_train = train.drop(["Fatalities", "ConfirmedCases"], axis=1 )
py.init_notebook_mode(connected=True)
<set_options>
countries = X_train["Country/Region"]
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_disk>
X_train = X_train.drop(["Id"], axis=1)
 X_test = test.drop(["ForecastId"], axis=1 )
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<feature_engineering>
X_train['Date']= pd.to_datetime(X_train['Date'])
 X_test['Date']= pd.to_datetime(X_test['Date'] )
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )<filter>
X_train = X_train.set_index(['Date'])
 X_test = X_test.set_index(['Date'] )
building_meta_df[building_meta_df.site_id == 0]<filter>
def create_time_features(df):

 df['date'] = df.index
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<feature_engineering>
create_time_features(X_train)
 create_time_features(X_test )
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_localtime(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )<feature_engineering>
X_train.drop("date", axis=1, inplace=True)
 X_test.drop("date", axis=1, inplace=True )
set_localtime(weather_train_df )<feature_engineering>
world_happiness_index = pd.read_csv(".. /input/world-bank-datasets/World_Happiness_Index.csv" )
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek<categorify>
world_happiness_grouped = world_happiness_index.groupby('Country name' ).nth(-1 )
preprocess(train_df )<categorify>
world_happiness_grouped.drop("Year", axis=1, inplace=True )
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group<count_missing_values>
X_train = pd.merge(left=X_train, right=world_happiness_grouped, how='left', left_on='Country/Region', right_on='Country name')
 X_test = pd.merge(left=X_test, right=world_happiness_grouped, how='left', left_on='Country/Region', right_on='Country name' )
weather_train_df.isna().sum()<groupby>
malaria_world_health = pd.read_csv(".. /input/world-bank-datasets/Malaria_World_Health_Organization.csv" )
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<groupby>
X_train = pd.merge(left=X_train, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country' )
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<groupby>
X_train.drop("Country", axis=1, inplace=True)
 X_test.drop("Country", axis=1, inplace=True )
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<categorify>
human_development_index = pd.read_csv(".. /input/world-bank-datasets/Human_Development_Index.csv" )
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()<feature_engineering>
X_train = pd.merge(left=X_train, right=human_development_index, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=human_development_index, how='left', left_on='Country/Region', right_on='Country' )
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )<define_variables>
X_train.drop(["Country", "Gross national income(GNI)per capita 2018"], axis=1, inplace=True)
 X_test.drop(["Country", "Gross national income(GNI)per capita 2018"], axis=1, inplace=True )
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed',]<prepare_x_and_y>
night_ranger_predictors = pd.read_csv(".. /input/covid19-demographic-predictors/covid19_by_country.csv" )
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train<train_model>
night_ranger_predictors = night_ranger_predictors[night_ranger_predictors.Country != "Georgia"]
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<choose_model_class>
X_train = pd.merge(left=X_train, right=night_ranger_predictors, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=night_ranger_predictors, how='left', left_on='Country/Region', right_on='Country' )
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0<split>
X_train.drop(["Country", "Restrictions","Quarantine", "Schools", "Total Infected", "Total Deaths", "Total Recovered"], axis=1, inplace=True)
 X_test.drop(["Country", "Restrictions","Quarantine", "Schools", "Total Infected", "Total Deaths", "Total Recovered"], axis=1, inplace=True )
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()<split>
X_train = pd.concat([X_train,pd.get_dummies(X_train['Province/State'], prefix='ps')],axis=1)
 X_train.drop(['Province/State'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Province/State'], prefix='ps')],axis=1)
 X_test.drop(['Province/State'],axis=1, inplace=True )
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()<split>
X_train = pd.concat([X_train,pd.get_dummies(X_train['Country/Region'], prefix='cr')],axis=1)
 X_train.drop(['Country/Region'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Country/Region'], prefix='cr')],axis=1)
 X_test.drop(['Country/Region'],axis=1, inplace=True )
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()<split>
y_train = train["Fatalities"]
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()<define_variables>
reg = xgb.XGBRegressor(n_estimators=1000 )
oof_total = oof_total / len(train_df )<drop_column>
reg.fit(X_train, y_train, verbose=True )
del train_df, weather_train_df, building_meta_df
gc.collect()<feature_engineering>
y_train = train.groupby(["Country/Region"] ).Fatalities.pct_change(periods=1 )
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 set_localtime(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<load_from_disk>
y_train = y_train.replace(np.nan, 0 )
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
sample_submission = reduce_mem_usage(sample_submission )<merge>
y_train = y_train.replace(np.inf, 0 )
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test<predict_on_test>
reg = xgb.XGBRegressor(n_estimators=1000 )
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total 
<feature_engineering>
reg.fit(X_train, y_train, verbose=True )
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )<save_to_csv>
y_train = train["ConfirmedCases"]
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<define_variables>
reg = xgb.XGBRegressor(n_estimators=1000 )
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 1000
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'<set_options>
reg.fit(X_train, y_train, verbose=True )
py.init_notebook_mode(connected=True)
<set_options>
y_train = train.groupby(["Country/Region"] ).ConfirmedCases.pct_change(periods=1 )
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<feature_engineering>
y_train = y_train.replace(np.nan, 0 )
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_local(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )<load_from_disk>
y_train = y_train.replace(np.inf, 0 )
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 root_black = Path('.. /input/ashrae-local-datatime-and-black-count')
 train_df = pd.read_feather(root/'train.feather')
 train_df_black = pd.read_feather(root_black/'train_black.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<filter>
reg = xgb.XGBRegressor(n_estimators=1000 )
train_df = train_df [ train_df['building_id'] != 1099 ]<feature_engineering>
reg.fit(X_train, y_train, verbose=True )
building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count<feature_engineering>
y_train = train["ConfirmedCases"]
confirmed_reg = xgb.XGBRegressor(n_estimators=1000)
 confirmed_reg.fit(X_train, y_train, verbose=True)
 preds = confirmed_reg.predict(X_test)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0 )
en_holidays = holidays.England()
ir_holidays = holidays.Ireland()
ca_holidays = holidays.Canada()
us_holidays = holidays.UnitedStates()
def add_holiyday(df_weather):
en_idx = df_weather.query('site_id == 1 or site_id == 5' ).index
ir_idx = df_weather.query('site_id == 12' ).index
ca_idx = df_weather.query('site_id == 7 or site_id == 11' ).index
us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15' ).index
df_weather['IsHoliday'] = 0
df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))
df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))
df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))
df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))
holiday_idx = df_weather['IsHoliday'] != 0
df_weather.loc[holiday_idx, 'IsHoliday'] = 1
df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8 )<feature_engineering>
preds = np.array(preds )
set_local(weather_train_df)
 add_holiyday(weather_train_df )<filter>
submissionOrig = pd.read_csv(".. /input/covid19-global-forecasting-week-1/submission.csv" )
train_df = train_df[train_df_black.black_count < 24*black_day]<set_options>
submissionOrig["ConfirmedCases"]=pd.Series(preds )
del train_df_black
gc.collect()<filter>
for index, row in submissionOrig.iterrows() :
if index >= 1:
if submissionOrig.iloc[index, 'ConfirmedCases'] < submissionOrig.iloc[index - 1, 'ConfirmedCases']:
submissionOrig.at[index, 'ConfirmedCases'] = submissionOrig.iloc[index - 1,'ConfirmedCases']
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<data_type_conversions>
y_train = train["Fatalities"]
confirmed_reg = xgb.XGBRegressor(n_estimators=1000)
 confirmed_reg.fit(X_train, y_train, verbose=True)
 preds = confirmed_reg.predict(X_test)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0)
 submissionOrig["Fatalities"]=pd.Series(preds )
( train_df[train_df.building_id == 954].meter_reading==0 ).astype(int ).plot()<filter>
for index, row in submissionOrig.iterrows() :
if index >= 1:
if submissionOrig.iloc[index, 'Fatalities'] < submissionOrig.iloc[index - 1, 'Fatalities']:
submissionOrig.at[index, 'Fatalities'] = submissionOrig.iloc[index - 1,'Fatalities']
<feature_engineering><EOS> 
submissionOrig.to_csv('submission.csv',index=False )