Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<compute_test_metric>
warnings.filterwarnings("ignore" )
print("coefficients:", coefficients)
 print("validation score:", valid_score )<save_to_csv>
train=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
test_df.diagnosis = test_predictions
test_df.to_csv("submission.csv", index=None)
 test_df.head()<set_options>
%matplotlib inline
gc.enable()<load_from_csv>
X_train=train[['Id']]
test['Id']=test['ForecastId']
X_test=test[['Id']]
y_train_cc=train[['ConfirmedCases']]
y_train_ft=train[['Fatalities']]
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
print("remove some columns")
 target = 'winPlacePerc'
print("Adding Features")
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df["headshotKills"] + df["roadKills"]
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
X_tr=np.array_split(X_train,313)
 y_cc=np.array_split(y_train_cc,313)
 y_ft=np.array_split(y_train_ft,313)
 X_te=np.array_split(X_test,313 )
x_train, y_train, train_columns, _ = feature_engineering(True,False )<feature_engineering>
a=np.max(X_tr[0] ).values
b=a-71
b=b[0]
x_test, _, _ , test_idx = feature_engineering(False,True )<set_options>
X_te[0]=X_te[0]+a
for i in range(312):
X_te[i+1]=X_te[0]
warnings.filterwarnings("ignore")
 color = sns.color_palette() 
<train_model>
for i in range(312):
X_tr[i+1]=X_tr[0]
train_index = round(int(x_train.shape[0]*0.9))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
gc.collect() ;
def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':20000, 'early_stopping_rounds':100,
"num_leaves" : 31, "learning_rate" : 0.05, "bagging_fraction" : 0.9,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.7
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y, model
pred_test, model = run_lgb(dev_X, dev_y, val_X, val_y, x_test )<load_from_csv>
poly=PolynomialFeatures(2)
 y_pred_cc=[]
for i in range(313):
X_tr[i]=poly.fit_transform(X_tr[i])
 X_te[i]=poly.fit_transform(X_te[i])
 model=Lasso()
model.fit(X_tr[i],y_cc[i]);
y_pr_cc=model.predict(X_te[i])
 y_cc[i]= y_cc[i][71:]
y_pr_cc=y_pr_cc[b:]
y_pr_cc=np.append(y_cc[i], y_pr_cc)
 y_pred_cc.append(y_pr_cc);
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission_adjusted.csv", index=False )<load_from_csv>
y_pred_ft=[]
for i in range(313):
model=Lasso()
model.fit(X_tr[i],y_ft[i]);
y_pr_ft=model.predict(X_te[i])
 y_ft[i]= y_ft[i][71:]
y_pr_ft=y_pr_ft[b:]
y_pr_ft=np.append(y_ft[i], y_pr_ft)
 y_pred_ft.append(y_pr_ft);
random.seed(42)
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 def reload_train() :
gc.collect()
df = pd.read_csv('.. /input/train_V2.csv')
 invalid_match_ids = df[df['winPlacePerc'].isna() ]['matchId'].values
df = df[-df['matchId'].isin(invalid_match_ids)]
return df
def reload_test() :
gc.collect()
df = pd.read_csv('.. /input/test_V2.csv')
 return df<train_model>
y_pred_ft_1 = [item for sublist in y_pred_ft for item in sublist]
y_pred_cc_1 = [item for sublist in y_pred_cc for item in sublist]
def run_train(preprocess):
df = reload_train()
df.drop(columns=['matchType'], inplace=True)
 df = preprocess(df)
 target = 'winPlacePerc'
cols_to_drop = ['Id', 'groupId', 'matchId', target]
cols_to_fit = [col for col in df.columns if col not in cols_to_drop]
df = SelectKBest(chi2, k=30 ).fit_transform(df[cols_to_fit], df[target])
 col = df.columns
print(df.columns)
 model = XGBRegressor()
model.fit(df[cols_to_fit], df[target],verbose=False)
 return model,col
def run_test(preprocess):
df = reload_test()
df.drop(columns=['matchType'], inplace=True)
 df = preprocess(df)
 print(df.columns)
 return df<merge>
result=pd.DataFrame({'ForecastId':submission.ForecastId, 'ConfirmedCases':np.round(y_pred_cc_1), 'Fatalities':np.round(y_pred_ft_1)})
 result.to_csv('/kaggle/working/submission.csv', index=False)
 data=pd.read_csv('/kaggle/working/submission.csv')
 data.head(50 )
def rank_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'winPlacePerc']
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId', 'groupId'])[features].mean()
agg = agg.groupby('matchId')[features].rank(pct=True)
 return df.merge(agg, suffixes=['', '_mean_rank'], how='left', on=['matchId', 'groupId'] )<train_model>
warnings.filterwarnings("ignore" )
model,col = run_train(rank_by_team )<train_model>
train=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
test = run_test(rank_by_team )<predict_on_test>
%matplotlib inline
pred = model.predict(test)
 pred.shape<save_to_csv>
X_train=train[['Id']]
test['Id']=test['ForecastId']
X_test=test[['Id']]
y_train_cc=train[['ConfirmedCases']]
y_train_ft=train[['Fatalities']]
pred_df = pd.DataFrame({'Id' : test_id, 'winPlacePerc' : pred})
 pred_df.to_csv("submission.csv", index=False )<import_modules>
X_tr=np.array_split(X_train,313)
 y_cc=np.array_split(y_train_cc,313)
 y_ft=np.array_split(y_train_ft,313)
 X_te=np.array_split(X_test,313 )
<categorify>
a=np.max(X_tr[0] ).values
b=a-71
b=b[0]
train = pd.get_dummies(train,columns=['matchType'] )<correct_missing_values>
X_te[0]=X_te[0]+a
for i in range(312):
X_te[i+1]=X_te[0]
train =train.dropna()<categorify>
for i in range(312):
X_tr[i+1]=X_tr[0]
test = pd.get_dummies(test,columns=['matchType'] )<prepare_x_and_y>
poly=PolynomialFeatures(2)
 y_pred_cc=[]
for i in range(313):
X_tr[i]=poly.fit_transform(X_tr[i])
 X_te[i]=poly.fit_transform(X_te[i])
 model=Lasso()
model.fit(X_tr[i],y_cc[i]);
y_pr_cc=model.predict(X_te[i])
 y_cc[i]= y_cc[i][71:]
y_pr_cc=y_pr_cc[b:]
y_pr_cc=np.append(y_cc[i], y_pr_cc)
 y_pred_cc.append(y_pr_cc);
y_train =train['winPlacePerc']
x_train =train.drop(['Id','groupId','matchId','winPlacePerc'],axis=1 )<prepare_x_and_y>
y_pred_ft=[]
for i in range(313):
model=Lasso()
model.fit(X_tr[i],y_ft[i]);
y_pr_ft=model.predict(X_te[i])
 y_ft[i]= y_ft[i][71:]
y_pr_ft=y_pr_ft[b:]
y_pr_ft=np.append(y_ft[i], y_pr_ft)
 y_pred_ft.append(y_pr_ft);
X_train = x_train.values
Y_train = y_train.values<split>
y_pred_ft_1 = [item for sublist in y_pred_ft for item in sublist]
y_pred_cc_1 = [item for sublist in y_pred_cc for item in sublist]
validation_size = 0.20
seed = 1000
X_train, X_valid, Y_train, Y_valid = train_test_split(X_train, Y_train, test_size=validation_size, random_state=seed )<train_model>
result=pd.DataFrame({'ForecastId':submission.ForecastId, 'ConfirmedCases':np.round(y_pred_cc_1), 'Fatalities':np.round(y_pred_ft_1)})
 result.to_csv('/kaggle/working/submission.csv', index=False)
 data=pd.read_csv('/kaggle/working/submission.csv')
 data.head(50 )