Kaggle competition: COVID19 Global Forecasting (Week 4)
print('Number of players who used more than 35 heals/boosts: ' + str(len(df_train[df_train['weaponsAcquired'] > 35])) )<drop_column>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.tsa.arima_model as arima
from sklearn.linear_model import LinearRegression
from sklearn import linear_model
def remove_outliers_heals_boosts(data):
print('Number of heals/boosts used outliers: ' + str(len(data.loc[data['weaponsAcquired'] > 35])))
 data.drop(data.loc[data['heals_boosts'] > 35].index, inplace=True)
 return str('Heals/boosts outliers removed.' )<drop_column>
warnings.filterwarnings('ignore')
 pd.set_option("display.max_rows", None, "display.max_columns", None )
remove_outliers_heals_boosts(df_train )<feature_engineering>
training_data_original = pd.read_csv('/kaggle/input/inputs3/train.csv')
 training_data_original['Date'] = pd.to_datetime(training_data_original['Date'])
 training_data = training_data_original[~training_data_original['Country_Region'].isin(['Diamond Princess', 'MS Zaandam'])]
countries = training_data['Country_Region'].unique()
countries_main = ['China', 'US', 'Australia', 'Canada']
states = training_data['Province_State'].unique()
winPoints_clean = df_train.drop(df_train[df_train['winPoints'] < 1].index)
 winPoints_clean = winPoints_clean['winPoints']
winPoints_mean = winPoints_clean.mean()
df_train['winPoints_imp'] = df_train['winPoints'].replace({0 : winPoints_mean} )<feature_engineering>
training_data[training_data['Country_Region'] == 'Belize']
rankPoints_clean = df_train.drop(df_train[df_train['rankPoints'] < 1].index)
 rankPoints_clean = rankPoints_clean['rankPoints']
rankPoints_mean = rankPoints_clean.mean()
df_train['rankPoints_imp'] = df_train['rankPoints'].replace({-1 : rankPoints_mean,
0 : rankPoints_mean} )<drop_column>
state_metadata = pd.read_excel('/kaggle/input/externaldata3/states.xlsx')
 country_metadata = pd.read_excel('/kaggle/input/externaldata3/countries.xlsx' )
df_train = df_train.drop(columns=matchType_drop, errors='ignore' )<feature_engineering>
def data_preparation(training_data, countries, states, country_metadata, state_metadata, n_days_case,
n_days_fatal, min_num_cases = 2):
training_data_trun = training_data[training_data['ConfirmedCases'] >= min_num_cases]
conf_cases_dict = {}
fatal_dict = {}
for country in countries:
if country not in countries_main:
training_data_trun_loc = training_data_trun[(training_data_trun['Country_Region'] == country)&(pd.isnull(training_data_trun.Province_State)) ]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[country] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / country_metadata[country_metadata['Countries'] == country]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[country] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
for state in states:
training_data_trun_loc = training_data_trun[training_data_trun['Province_State'] == state]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[state] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / state_metadata[state_metadata['States'] == state]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[state] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
return pd.DataFrame(conf_cases_dict), pd.DataFrame(fatal_dict )
def playersJoined_norm(list_of_cols, df):
for column in list_of_cols:
df[column] = df[column]*(( 100-df['match_size'])/ 100 + 1)
<define_variables>
def fts_training(input_df, rank = 3):
matrix = input_df.values
u, s, v = np.linalg.svd(matrix, full_matrices=False)
 scores = np.matmul(u[:, :rank], np.diag(s[:rank]))
pcs = v[:rank, :]
return scores, pcs
to_pJnorm = ['DBNOs', 'kills', 'maxPlace']<compute_test_metric>
def forecast_trajectories(training_data, countries, states, country_metadata,
state_metadata, loc = None, n_days_case = 30, n_days_fatal = 10,
forecast_days = 10, min_num_cases = 2, model_type = 'ARIMA',
components_modeling = True, rank = 3):
kf = KalmanFilter(initial_state_mean = np.zeros(rank ).tolist() , n_dim_obs = rank)
 conf_cases_df, fatal_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case,
n_days_fatal = n_days_case,
min_num_cases = min_num_cases)
 pred_countries = conf_cases_df.columns.tolist()
pred_countries_fatal = fatal_df.columns.tolist()
conf_cases_exog_df, fatal_exog_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case + forecast_days,
n_days_fatal = n_days_case + forecast_days,
min_num_cases = min_num_cases)
 scores_exog, pcs_exog = fts_training(conf_cases_exog_df, rank = rank)
 if len(scores_exog)> 0:
scores_exog = kf.em(scores_exog ).smooth(scores_exog)[0]
scores, pcs = fts_training(conf_cases_df, rank = rank)
 forecasted_scores = []
idx = 0
for score in scores.T:
if components_modeling:
exog = scores_exog[:n_days_case, idx] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, idx] if len(scores_exog)> 0 else None
y = score
else:
exog = scores_exog[:n_days_case, :] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, :] if len(scores_exog)> 0 else None
y = conf_cases_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
pred_traj = np.matmul(np.array(forecasted_scores ).T, pcs)
 pred_traj_df = pd.DataFrame(pred_traj, columns = pred_countries)
 for loc in pred_countries:
if loc in country_metadata['Countries'].values.tolist() :
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj_df[loc]
if loc in state_metadata['States'].values.tolist() :
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj_df[loc]
else:
pred_traj_df = pd.DataFrame()
if loc in countries:
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj
elif loc in states:
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj
fatal_scores_exog, fatal_pcs_exog = fts_training(fatal_exog_df, rank = rank)
 if len(fatal_pcs_exog)> 0:
fatal_scores_exog = kf.em(fatal_scores_exog ).smooth(fatal_scores_exog)[0]
fatal_scores, fatal_pcs = fts_training(fatal_df, rank = rank)
 forecasted_fatal_scores = []
idx = 0
for fatal_score in fatal_scores.T:
if components_modeling:
exog = fatal_scores_exog[:n_days_fatal, idx] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, idx] if len(fatal_scores_exog)> 0 else None
y = fatal_score
else:
exog = fatal_scores_exog[:n_days_fatal, :] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, :] if len(fatal_scores_exog)> 0 else None
y = fatal_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
fatal_pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_fatal_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
fatal_pred_traj = np.matmul(np.array(forecasted_fatal_scores ).T, fatal_pcs)
 fatal_pred_traj_df = pd.DataFrame(fatal_pred_traj, columns = pred_countries_fatal)
 else:
fatal_pred_traj_df = pd.DataFrame()
if loc in countries:
fatal_pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * fatal_pred_traj
elif loc in states:
fatal_pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * fatal_pred_traj
return pred_traj_df, fatal_pred_traj_df
playersJoined_norm(to_pJnorm, df_train )<feature_engineering>
def generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases):
for country in countries:
print(country)
 if country not in countries_main and country not in excl_list:
test_loc_df = test_data[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) ].reset_index()
train_loc_df = training_data[(training_data['Country_Region'] == country)&(pd.isnull(training_data.Province_State)) ].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 min_num_cases_temp = min_num_cases
if len(effective_df)> 0:
while min_num_cases_temp > 1:
try:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases_temp].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = country,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases_temp,
rank = 4)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'Fatalities'] = test_loc_df['Fatalities'].values
break
except:
min_num_cases_temp = math.floor(min_num_cases_temp / 2)
 continue
for state in states:
print(state)
 if str(state)not in excl_list:
test_loc_df = test_data[(test_data['Province_State'] == state)].reset_index()
train_loc_df = training_data[(training_data['Province_State'] == state)].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 min_num_cases_temp = min_num_cases
if len(effective_df)> 0:
while min_num_cases_temp > 1:
try:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases_temp].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = state,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases_temp,
rank = 4)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'Fatalities'] = test_loc_df['Fatalities'].values
break
except:
min_num_cases_temp = math.floor(min_num_cases_temp / 2)
 continue
return test_data
def matchDuration_wt(list_of_cols, df):
for column in list_of_cols:
df[column] =(df[column] * df[column])/ df['matchDuration'].mean()
df[column].clip(0)
<define_variables>
test_data = pd.read_csv('/kaggle/input/inputs3/test.csv')
 test_data['Date'] = pd.to_datetime(test_data['Date'] )
to_matchDwt = ['heals_boosts']<compute_test_metric>
test_data['ConfirmedCases'] = None
test_data['Fatalities'] = None
matchDuration_wt(to_matchDwt, df_train )<feature_engineering>
excl_list = ['Belize', 'Botswana', 'Diamond Princess', 'MS Zaandam', 'Angola',
'Turks and Caicos Islands', 'Burma', 'Burundi', 'Chad', 'Eritrea',
'Grenada', 'Guinea-Bissau', 'Holy See', 'Kosovo',
'Laos', 'Libya', 'Mali', 'Mozambique', 'Saint Kitts and Nevis', 'Somalia',
'Syria', 'nan', 'Saint Barthelemy', 'Virgin Islands', 'Montserrat',
'Saint Vincent and the Grenadines', 'Sierra Leone', 'Northwest Territories',
'Yukon', 'Anguilla', 'British Virgin Islands', 'Papua New Guinea', 'Bhutan',
'Congo(Brazzaville)', 'Gabon', 'Guinea', 'Guyana', 'Haiti', 'Namibia',
'Saint Lucia', 'Seychelles', 'Curacao', 'Cayman Islands',
'Central African Republic', 'Liberia', 'Mauritania', 'Nepal',
'Nicaragua', 'Sudan']
def engineer_features(data):
data['max_possible_kills'] = data.match_size - data.team_size
data['total_items_acquired'] = data.boosts + data.heals + data.weaponsAcquired
data['items_per_distance'] = data.total_items_acquired/data.total_distance
data['kills_per_distance'] = data.kills/data.total_distance
data['knocked_per_distance'] = data.DBNOs/data.total_distance
data['damage_per_distance'] = data.damageDealt/data.total_distance
data['headshot_kill_rate'] = data.headshotKills/data.kills
data['headshot_kill_rate'] = data['headshot_kill_rate'].fillna(data['headshot_kill_rate'].mean())
 data['max_kills_by_team'] = data.groupby(['matchId','groupId'] ).kills.transform('max')
 data['total_team_damage'] = data.groupby(['matchId','groupId'] ).damageDealt.transform('sum')
 data['total_team_kills'] = data.groupby(['matchId','groupId'] ).kills.transform('sum')
 data['total_team_items'] = data.groupby(['matchId','groupId'] ).total_items_acquired.transform('sum')
 data['pct_killed'] = data.kills/data.max_possible_kills
data['pct_knocked'] = data.DBNOs/data.max_possible_kills
data['pct_team_killed'] = data.total_team_kills/data.max_possible_kills
data['team_kill_points'] = data.groupby(['matchId','groupId'] ).killPoints.transform('sum')
 data['team_kill_rank'] = data.groupby(['matchId','groupId'] ).killPlace.transform('mean')
 data['max_kills_match'] = data.groupby('matchId' ).kills.transform('max')
 data['total_kills_match'] = data.groupby('matchId' ).kills.transform('sum')
 data = data.join(pd.get_dummies(data['game_mode']))
data.fillna(0, axis=1, inplace=True)
 return data 
<feature_engineering>
predictions = generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases = 100)
engineer_features(df_train )<feature_engineering>
def fill_excl_pred(predictions, training_data_original, test_data):
for country in missing_countries:
print(country)
 pred_loc_df = predictions[predictions['Country_Region'] == country]
train_loc_df = training_data_original[training_data_original['Country_Region'] == country]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Country_Region'] == country]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['Fatalities']
if len(series_comf_cases)> 0:
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
else:
comf_cases_pred = []
fatal_pred = []
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Country_Region'] == country, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Country_Region'] == country, 'Fatalities'] = fatal_loc.astype(int)
 for state in missing_states:
print(state)
 pred_loc_df = predictions[predictions['Province_State'] == state]
train_loc_df = training_data_original[training_data_original['Province_State'] == state]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Province_State'] == state]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['Fatalities']
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(13, 43 ).reshape(-1, 1))
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Province_State'] == state, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Province_State'] == state, 'Fatalities'] = fatal_loc.astype(int)
 return predictions
df_train = reduce_mem_usage(df_train )<filter>
prediction_final = fill_excl_pred(predictions, training_data_original, test_data )
def engineer_features2(is_train=True):
test_idx = None
if is_train:
print("processing train")
 df = df_train
df = df[df['maxPlace'] > 1]
else:
print("processing test")
 df = df_test
test_idx = df['Id']
print("selected features to generate aggregates for")
 target = 'winPlacePerc'
features = [
'assists',
'damageDealt',
'DBNOs',
'killPlace',
'killPoints',
'kills',
'killStreaks',
'longestKill',
'maxPlace',
'numGroups',
'revives',
'teamKills',
'vehicleDestroys',
'weaponsAcquired',
'team_size',
'match_size',
'team_indicator_1',
'team_indicator_2',
'team_indicator_3',
'team_indicator_4',
'total_distance',
'heals_boosts',
'winPoints_imp',
'rankPoints_imp',
'max_possible_kills',
'total_items_acquired',
'items_per_distance',
'kills_per_distance',
'knocked_per_distance',
'damage_per_distance',
'headshot_kill_rate',
'total_team_damage',
'total_team_kills',
'total_team_items',
'pct_killed',
'pct_knocked',
'pct_team_killed',
'team_kill_points',
'team_kill_rank',
'total_kills_match',
]
y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
prediction_final[prediction_final['Country_Region'] == 'Germany']
df_train, y_train, train_columns, _ = engineer_features2(is_train=True )<feature_engineering>
predictions_csv = pd.DataFrame()
predictions_csv['ForecastId'] = prediction_final['ForecastId'].astype(int)
 predictions_csv['ConfirmedCases'] = prediction_final['ConfirmedCases'].astype(float)
 predictions_csv['Fatalities'] = prediction_final['Fatalities'].astype(float )
<define_variables><EOS> 
predictions_csv.to_csv('submission.csv', index = False )