Kaggle competition: COVID19 Global Forecasting (Week 4)
df_train = reduce_mem_usage(df_train )<load_from_csv>
train_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv" )
df_test = pd.read_csv('.. /input/test_V2.csv', index_col = 'Id')
 df_test.shape<feature_engineering>
india = train_df[train_df["Country_Region"]=="India"]
df_test = reduce_mem_usage(df_test )<create_dataframe>
def province(state,country):
if state == "nan":
return country
return state
df_test_id = pd.DataFrame(index=df_test.index )<define_variables>
train_df = train_df.fillna("nan" )
part_train = 0.05
part_valid = 0.05<count_unique_values>
train_df["Province_State"] = train_df.apply(lambda x: province(x["Province_State"],x["Country_Region"]),axis=1 )
match_ids = df_train['matchId'].unique()
match_ids_train = np.random.choice(match_ids, int(part_train * len(match_ids)))
 len(match_ids_train )<filter>
italy = train_df[train_df["Country_Region"]=="Italy"]
df_train_train = df_train[df_train['matchId'].isin(match_ids_train)]
df_train_train.shape[0]<compute_test_metric>
from datetime import datetime
match_ids_valid = np.random.choice(np.setdiff1d(match_ids, match_ids_train), int(part_valid * len(match_ids)))
 len(match_ids_valid )<drop_column>
train_df["Date"] = pd.to_datetime(train_df["Date"] )
del df_train
del match_ids
del match_ids_train<feature_engineering>
train_df["month"] = train_df["Date"].dt.month
def feature_engineering(df, is_train=True):
df['rankPoints'] = np.where(df['rankPoints'] <= 0, 0, df['rankPoints'])
 features = list(df.columns)
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 if 'winPlacePerc' in features:
features.remove('winPlacePerc')
 y = None
if is_train:
y = df.groupby(['matchId','groupId'])['winPlacePerc'].agg('mean')
 elif 'winPlacePerc' in df.columns:
y = df['winPlacePerc']
agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train:
df_out = agg.reset_index() [['matchId','groupId']]
else:
df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 del agg, agg_rank
return df_out, y<prepare_x_and_y>
train_df['day'] = train_df['Date'].dt.day
%%time
fe_x_train, fe_y_train = feature_engineering(df_train_train, is_train=True )<drop_column>
train_df.drop('Date',axis=1,inplace=True )
del df_train_train<feature_engineering>
from sklearn import preprocessing
%%time
fe_x_test, t = feature_engineering(df_test, is_train=False )<drop_column>
def labelencoder(data):
le = preprocessing.LabelEncoder()
new_data = le.fit_transform(data)
 return new_data
del df_test<train_model>
train_df["Country_Region"] = labelencoder(train_df["Country_Region"].values)
 train_df["Province_State"] = labelencoder(train_df["Province_State"].values )
%%time
rf = RandomForestRegressor(n_estimators=30, criterion='mae', n_jobs=-1)
 rf.fit(fe_x_train, fe_y_train)
 rf_y_pred = rf.predict(fe_x_test )<categorify>
test_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv" )
def fix_pred(x, pred):
updated_pred = []
for i in range(len(x)) :
winPlacePerc = pred[i]
maxPlace = int(x.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
updated_pred.append(winPlacePerc)
 return updated_pred<predict_on_test>
test_df["Date"] = pd.to_datetime(test_df["Date"] )
%%time
fixed_rf_y_pred = fix_pred(fe_x_test, rf_y_pred )<save_to_csv>
test_df['day'] = test_df['Date'].dt.day
def make_submission(x, y, filename):
submission = pd.DataFrame(index=x.index)
 submission['winPlacePerc'] = np.clip(y, a_min=0, a_max=1)
 submission.to_csv(filename, index_label='Id' )<compute_test_metric>
test_df["month"] = test_df["Date"].dt.month
make_submission(df_test_id, fixed_rf_y_pred, 'random_forest_baseline.csv' )<load_from_csv>
test_df.drop('Date',axis=1,inplace=True)
 test_df.head()
train_data_df = pd.read_csv('.. /input/train.csv')
 test_data_df = pd.read_csv('.. /input/test.csv' )<normalization>
test_df.fillna("nan",inplace=True)
 test_df["Province_State"] = test_df.apply(lambda x: province(x["Province_State"],x["Country_Region"]),axis=1 )
train_data_df = reduce_mem_usage(train_data_df)
 test_data_df = reduce_mem_usage(test_data_df )<groupby>
test_df["Country_Region"] = labelencoder(test_df["Country_Region"].values)
 test_df["Province_State"] = labelencoder(test_df["Province_State"].values )
mean_match_features = train_data_df.groupby(['matchId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
size_match_features = pd.DataFrame(train_data_df.groupby(['matchId'])[train_data_df.columns[3]].agg('size' ).reset_index() [train_data_df.columns[3]])
 size_match_features.columns = ['match_size'] 
<groupby>
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import MinMaxScaler
mean_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
mean_group_rank_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('mean' ).groupby(['matchId'])[train_data_df.columns[3:-1]].rank(pct=True ).reset_index()
mean_group_features = mean_group_features.join(mean_group_rank_features, rsuffix='_rank')
 max_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('max' ).reset_index().loc[:, 'assists':'winPoints']
max_group_rank_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('max' ).groupby(['matchId'])[train_data_df.columns[3:-1]].rank(pct=True ).reset_index()
max_group_features = max_group_features.join(max_group_rank_features, rsuffix='_rank')
 min_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('min' ).reset_index().loc[:, 'assists':'winPoints']
min_group_rank_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('min' ).groupby(['matchId'])[train_data_df.columns[3:-1]].rank(pct=True ).reset_index()
min_group_features = min_group_features.join(min_group_rank_features, rsuffix='_rank')
 size_group_features = pd.DataFrame(train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3]].agg('size' ).reset_index() [train_data_df.columns[3]])
 size_group_features.columns = ['group_size']<merge>
poly_reg_cc = PolynomialFeatures(degree = 4)
 poly_reg_ft = PolynomialFeatures(degree = 4)
 reg_cc = LinearRegression()
reg_ft = LinearRegression()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = train_df[train_df["Country_Region"]==country]["Province_State"].unique()
for state in states:
train_df_filt = train_df[(train_df["Country_Region"]==country)&(train_df["Province_State"]==state)]
y_train_cc = train_df_filt["ConfirmedCases"].values
y_train_ft = train_df_filt["Fatalities"].values
X_train = train_df_filt[["month","day"]]
test_df_filt = test_df[(test_df["Country_Region"]==country)&(test_df["Province_State"]==state)]
X_test = test_df_filt.drop('ForecastId',axis=1)
 X_test = X_test[["month","day"]]
test_Id = test_df_filt["ForecastId"].values
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 scaler_cc = MinMaxScaler()
scaler_ft = MinMaxScaler()
y_train_cc=y_train_cc.reshape(-1,1)
 y_train_ft=y_train_ft.reshape(-1,1)
 y_train_cc=scaler_cc.fit_transform(y_train_cc)
 y_train_ft=scaler_ft.fit_transform(y_train_ft)
 y_train_cc = y_train_cc.flatten()
y_train_ft = y_train_ft.flatten()
X_train_poly = poly_reg_cc.fit_transform(X_train)
 reg_cc.fit(X_train_poly,y_train_cc)
 X_test_poly = poly_reg_cc.fit_transform(X_test)
 test_cc = reg_cc.predict(X_test_poly)
 test_cc = test_cc.reshape(-1,1)
 test_cc = scaler_cc.inverse_transform(test_cc)
 test_cc = test_cc.flatten()
X_train_poly = poly_reg_ft.fit_transform(X_train)
 reg_ft.fit(X_train_poly,y_train_ft)
 X_test_poly = poly_reg_ft.fit_transform(X_test)
 test_ft = reg_ft.predict(X_test_poly)
 test_ft = test_ft.reshape(-1,1)
 test_ft = scaler_ft.inverse_transform(test_ft)
 test_ft = test_ft.flatten()
df = pd.DataFrame({'ForecastId': test_Id, 'ConfirmedCases': test_cc, 'Fatalities': test_ft})
 df_out = pd.concat([df_out, df], axis=0 )
features_three = mean_group_features.join(max_group_features, lsuffix='_group_mean', rsuffix='_group_max')
 features_four = min_group_features.join(size_group_features, lsuffix='_group_min', rsuffix='_group_size')
 features_2 = features_three.join(features_four)
 features_2['matchId'] = train_data_df.groupby(['matchId', 'groupId'])['matchId'].agg('mean' ).values
features_2 = features_2.fillna(0.0)
 features_2 = features_2.replace([np.inf, -np.inf], 0.0 )<merge>
df_out["Fatalities"] = df_out["Fatalities"].apply(int)
 df_out["ConfirmedCases"] = df_out["ConfirmedCases"].apply(int)
 df_out[:10]
features = pd.merge(features_2, features_1, on='matchId', how='right' )<drop_column>
df_out["ForecastId"] = df_out["ForecastId"].astype('int32')
 df_out["Fatalities"] = df_out["Fatalities"].astype('int32')
 df_out["ConfirmedCases"] = df_out["ConfirmedCases"].astype('int32')
 df_out.info()
df_out.to_csv("submission.csv",index=False)
 sub = pd.read_csv("submission.csv")
 sub[:20]
features = features.drop(['matchId'], axis=1)
 features<groupby>
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv")
 train["Province_State"] = train["Province_State"].fillna('')
 test["Province_State"] = test["Province_State"].fillna('')
 train["Month"], train["Day"] = 0, 0
for i in range(len(train)) :
train["Month"][i] =(train["Date"][i] ).split("-")[1]
train["Day"][i] =(train["Date"][i] ).split("-")[2]
test["Month"], test["Day"] = 0, 0
for i in range(len(test)) :
test["Month"][i] =(test["Date"][i] ).split("-")[1]
test["Day"][i] =(test["Date"][i] ).split("-")[2]
targets = train_data_df.groupby(['matchId', 'groupId'])['winPlacePerc'].agg('mean' ).reset_index() ['winPlacePerc']
targets<define_variables>
for i in range(len(train)) :
if train["Province_State"][i] != '':
train["Country_Region"][i] = str(train["Province_State"][i])+ "(" + str(train["Country_Region"][i])+ ")"
for i in range(len(test)) :
if test["Province_State"][i] != '':
test["Country_Region"][i] = str(test["Province_State"][i])+ "(" + str(test["Country_Region"][i])+ ")"
train.drop(columns = "Province_State", inplace=True)
 test.drop(columns = "Province_State", inplace=True)
 train.rename(columns = {"Country_Region" : "Country/State"}, inplace=True)
 test.rename(columns = {"Country_Region" : "Country/State"}, inplace=True)
<import_modules>
i = 0
for value in train["Country/State"].unique() :
if i < len(train):
j = 1
while(train["Country/State"][i] == value):
train["Day"][i] = j
j += 1; i += 1
if i == len(train):
break
i = 0
for value in test["Country/State"].unique() :
if i < len(test):
j = 72
while(test["Country/State"][i] == value):
test["Day"][i] = j
j += 1; i += 1
if i == len(test):
break
import sklearn<normalization>
train = train.drop(columns = ["Date"])
 test = test.drop(columns = ["Date"] )
scaler = MinMaxScaler(feature_range=(-1, 1)).fit(features.values)
<import_modules>
poly_reg_cc = PolynomialFeatures(degree = 4)
 poly_reg_ft = PolynomialFeatures(degree = 4)
 reg_cc = LinearRegression()
reg_ft = LinearRegression()
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for value in countries:
train_temp = train.loc[train["Country/State"] == value]
test_temp = test.loc[test["Country/State"] == value]
train_temp_cc = train_temp["ConfirmedCases"].loc[train["Country/State"] == value].to_frame()
train_temp_ft = train_temp["Fatalities"].loc[train["Country/State"] == value].to_frame()
train_temp_X = train_temp.iloc[:, 4:6]
test_temp_X = test_temp.iloc[:, 2:4]
sc1 = StandardScaler()
train_temp_X = sc1.fit_transform(train_temp_X)
 test_temp_X = sc1.transform(test_temp_X)
 sc_cc = StandardScaler()
sc_ft = StandardScaler()
train_temp_cc = sc_cc.fit_transform(train_temp_cc)
 train_temp_ft = sc_ft.fit_transform(train_temp_ft)
 X_poly = poly_reg_cc.fit_transform(train_temp_X)
 reg_cc.fit(X_poly, train_temp_cc)
 test_cc = sc_cc.inverse_transform(reg_cc.predict(poly_reg_cc.fit_transform(test_temp_X)))
 X_poly = poly_reg_ft.fit_transform(train_temp_X)
 reg_ft.fit(X_poly, train_temp_ft)
 test_ft = sc_ft.inverse_transform(reg_ft.predict(poly_reg_ft.fit_transform(test_temp_X)))
 a = int(train["Day"].loc[train["Country/State"] == "India"].max())
 b = int(a - test_temp["Day"].min())
 test_cc[0:b+1] = sc_cc.inverse_transform(train_temp_cc)[(a-b-1):(a)]
test_ft[0:b+1] = sc_ft.inverse_transform(train_temp_ft)[(a-b-1):(a)]
test_cc = test_cc.flatten()
test_ft = test_ft.flatten()
sub_temp = pd.DataFrame({'ForecastId': test_temp["ForecastId"].loc[test["Country/State"] == value],
'ConfirmedCases': test_cc, 'Fatalities': test_ft})
 sub = pd.concat([sub, sub_temp], axis = 0 )
import ultimate
from ultimate.mlp import MLP<train_model>
sub["ForecastId"] = sub["ForecastId"].astype('int32' )
epoch_train = 15
model = MLP(layer_size=[train_data_df.shape[1], 32, 32, 32, 1], regularization=1, output_shrink=0.1, output_range=[0,1], loss_type="hardmse")
 model.train(scaler.transform(features.values), targets.values, iteration_log=20000, rate_init=0.08, rate_decay=0.8, epoch_train=epoch_train, epoch_decay=1 )<groupby>
sub.to_csv("submission.csv", index = False )