Kaggle competition: COVID19 Global Forecasting (Week 4)
all_data = pd.merge(all_data, match_data)
 del match_data
gc.collect()
all_data['enemy.players'] = all_data['m.players'] - all_data['players']
for c in sum_col:
all_data['p.max_msum.' + c] = all_data['max.' + c] / all_data['m.sum.' + c]
all_data['p.max_mmax.' + c] = all_data['max.' + c] / all_data['m.max.' + c]
all_data.drop(['m.sum.' + c, 'm.max.' + c], axis=1, inplace=True)
 fillInf(all_data, 0)
 print(all_data.shape )<feature_engineering>
df_train=pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_test=pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 df_sub=pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(df_train.shape)
 print(df_test.shape)
 print(df_sub.shape )
match = all_data.groupby('matchId')
 matchRank = match[numcols].rank(pct=True ).rename(columns=lambda s: 'rank.' + s)
 all_data = reduce_mem_usage(pd.concat([all_data, matchRank], axis=1))
rank_col = matchRank.columns
del matchRank
gc.collect()
match = all_data.groupby('matchId')
 matchRank = match[rank_col].max().rename(columns=lambda s: 'max.' + s ).reset_index()
all_data = pd.merge(all_data, matchRank)
 for c in numcols:
all_data['rank.' + c] = all_data['rank.' + c] / all_data['max.rank.' + c]
all_data.drop(['max.rank.' + c], axis=1, inplace=True)
 del matchRank
gc.collect()
print(all_data.shape )<merge>
print(len(df_train.Country_Region.unique()), len(df_train.Province_State.unique()))
print(len(df_test.Country_Region.unique()), len(df_test.Province_State.unique()))
killMinorRank = all_data[['matchId','min.kills','max.killPlace']].copy()
group = killMinorRank.groupby(['matchId','min.kills'])
 killMinorRank['rank.minor.maxKillPlace'] = group.rank(pct=True ).values
all_data = pd.merge(all_data, killMinorRank)
 killMinorRank = all_data[['matchId','max.kills','min.killPlace']].copy()
group = killMinorRank.groupby(['matchId','max.kills'])
 killMinorRank['rank.minor.minKillPlace'] = group.rank(pct=True ).values
all_data = pd.merge(all_data, killMinorRank)
 del killMinorRank
gc.collect()<count_unique_values>
df_train["Unique_Region"] = df_train.Country_Region
df_train.Unique_Region[df_train.Province_State.isna() == False] = df_train.Province_State+" , "+df_train.Country_Region
df_train.sample(5 )
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )<feature_engineering>
df_train.drop(labels=["Id","Province_State","Country_Region"], axis=1, inplace=True )
all_data['matchType'] = all_data['matchType'].apply(mapper)
 all_data = pd.concat([all_data, pd.get_dummies(all_data['matchType'])], axis=1)
 all_data.drop(['matchType'], axis=1, inplace=True)
 all_data['matchId'] = all_data['matchId'].apply(lambda x: int(x,16))
all_data['groupId'] = all_data['groupId'].apply(lambda x: int(x,16))<sort_values>
df_test["Unique_Region"] = df_test.Country_Region
df_test.Unique_Region[df_test.Province_State.isna() == False] = df_test.Province_State+" , "+df_test.Country_Region
df_test.sample(5 )
null_cnt = all_data.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )<feature_engineering>
df_test.drop(labels=["Province_State","Country_Region"], axis=1, inplace=True)
 df_test.sample(5 )
cols = [col for col in all_data.columns if col not in ['Id','matchId','groupId']]
for i, t in all_data.loc[:, cols].dtypes.iteritems() :
if t == object:
all_data[i] = pd.factorize(all_data[i])[0]
all_data = reduce_mem_usage(all_data)
 all_data.head()<prepare_x_and_y>
print(list(df_train.Date.unique()))
print(len(df_train.Date.unique()))
X_train = all_data[all_data['winPlacePerc'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['winPlacePerc'].isnull() ].drop(['winPlacePerc'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
Y_train = X_train.pop('winPlacePerc')
 X_test_grp = X_test[['matchId','groupId']].copy()
train_matchId = X_train['matchId']
X_train.drop(['matchId','groupId'], axis=1, inplace=True)
 X_test.drop(['matchId','groupId'], axis=1, inplace=True)
 print(X_train.shape, X_test.shape )<create_dataframe>
print(list(df_test.Date.unique()))
print(len(df_test.Date.unique()))
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )<train_model>
world_list_confirmed = []
world_list_fatality = []
for date in list(df_train.Date.unique()):
confirmed = df_train.ConfirmedCases[df_train.Date == date]
world_list_confirmed.append(sum(confirmed))
fatality = df_train.Fatalities[df_train.Date == date]
world_list_fatality.append(sum(fatality))
params={'learning_rate': 0.05,
'objective':'mae',
'metric':'mae',
'num_leaves': 128,
'verbose': 1,
'random_state':42,
'bagging_fraction': 0.7,
'feature_fraction': 0.7
}
reg = lgb.LGBMRegressor(**params, n_estimators=10000)
 reg.fit(X_train, Y_train)
 pred = reg.predict(X_test, num_iteration=reg.best_iteration_ )<concatenate>
Date_submission = date_list[71:84]
Date_prediction = date_test[13:]
print(Date_submission)
 print(Date_prediction)
 print(len(Date_prediction))
X_test_grp['_nofit.winPlacePerc'] = pred
group = X_test_grp.groupby(['matchId'])
 X_test_grp['winPlacePerc'] = pred
X_test_grp['_rank.winPlacePerc'] = group['winPlacePerc'].rank(method='min')
 X_test = pd.concat([X_test, X_test_grp], axis=1 )<feature_engineering>
def submit_from_pred(date_list, value_list, Unique_Region, test_df, submission_df, num):
if len(date_list)!= len(value_list):
print("Error, the length of these two lists are not equal")
 else:
for i in range(len(date_list)) :
pred = value_list[i]
selected_df = test_df[(test_df["Date"] == date_list[i])&(test_df["Unique_Region"] == Unique_Region)]
forecastID = selected_df["ForecastId"].iloc[0]
if num == 0:
submission_df.ConfirmedCases[submission_df["ForecastId"] == forecastID] = pred
else:
submission_df.Fatalities[submission_df["ForecastId"] == forecastID] = pred
fullgroup =(X_test['numGroups'] == X_test['maxPlace'])
 subset = X_test.loc[fullgroup]
X_test.loc[fullgroup, 'winPlacePerc'] =(subset['_rank.winPlacePerc'].values - 1)/(subset['maxPlace'].values - 1)
 subset = X_test.loc[~fullgroup]
gap = 1.0 /(subset['maxPlace'].values - 1)
 new_perc = np.around(subset['winPlacePerc'].values / gap)* gap
X_test.loc[~fullgroup, 'winPlacePerc'] = new_perc
X_test['winPlacePerc'] = X_test['winPlacePerc'].clip(lower=0,upper=1 )<groupby>
def from_diff_to_var(diff_series, start_value):
value = start_value
for i in range(len(diff_series)) :
value += diff_series[i]
diff_series[i] = value
return diff_series
_=<feature_engineering>
regions = list(df_train.Unique_Region.unique())
 for region in regions:
selected_df = df_train[df_train.Unique_Region == region]
data_df = selected_df.iloc[71:84]
confirmed = list(data_df["ConfirmedCases"])
 fatality = list(data_df["Fatalities"])
 submit_from_pred(Date_submission, confirmed, region, df_test, df_sub, 0)
 submit_from_pred(Date_submission, fatality, region, df_test, df_sub, 1)
<save_to_csv><EOS> 
df_sub.to_csv("submission.csv", index = False )