Kaggle competition: COVID19 Global Forecasting (Week 1)
target_key = 'totals.transactionRevenue'<load_from_csv>
train_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv")
 submission_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/submission.csv")
 test_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv' )
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape<feature_engineering>
train_df["Date"] = train_df["Date"].apply(lambda x: datetime.strptime(x,'%Y-%m-%d'))
train_df["Date"] = train_df["Date"].apply(lambda x: x.timestamp())
 train_df["Date"] = train_df["Date"].astype(int )
train['totals.transactionRevenue'] = train['totals.transactionRevenue'].fillna(0 )<load_from_csv>
train_df.isnull().sum()
train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv',
low_memory=False, skiprows=6, dtype={"Client Id":'str'} )<data_type_conversions>
train_df = train_df.drop(['Province/State'],axis=1)
 train_df = train_df.dropna()
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(str)
 train_exdata = pd.concat([train_store_1, train_store_2], sort=False)
 test_exdata = pd.concat([test_store_1, test_store_2], sort=False )<data_type_conversions>
test_df.isnull().sum()
for df in [train, test, train_exdata, test_exdata]:
df["visitId"] = df["visitId"].astype(float ).astype(int)
 train_new = train.merge(train_exdata, how="left", on="visitId")
 test_new = test.merge(test_exdata, how="left", on="visitId")
 for df in [train_new, test_new]:
df.drop("Client Id", 1, inplace=True)
 for df in [train_new, test_new]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True)
 df["Revenue"] = np.log1p(df["Revenue"])
 df['is_high_hits'] = np.logical_or(df["totals.hits"]>4,df["totals.pageviews"]>4 ).astype(np.int32)
 df['views/hits']=df["totals.pageviews"]/df["totals.hits"].dropna(0 )<drop_column>
test_df["Date"] = test_df["Date"].apply(lambda x: datetime.strptime(x,'%Y-%m-%d'))
test_df["Date"] = test_df["Date"].apply(lambda x: x.timestamp())
 test_df["Date"] = test_df["Date"].astype(int)
 test_df = test_df.drop(['Province/State'],axis=1)
 test_df = test_df.dropna()
test_df.head()
%%time
train = process(train_new)
 test = process(test_new)
 del train_new, test_new<define_variables>
X = train_df[['Lat', 'Long', 'Date']]
Y1 = train_df[['ConfirmedCases']]
X_test = test_df[['Lat', 'Long', 'Date']]
Y2 = train_df[['Fatalities']]
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<categorify>
rf = RandomForestRegressor(n_estimators=100)
 rf.fit(X,Y1)
 pred1 = rf.predict(X_test)
 pred1 = pd.DataFrame(pred1)
 pred1.columns = ["ConfirmedCases_prediction"]
def freq_encoding(column):
counter = column.value_counts()
counter /= counter.sum()
return counter<categorify>
rf_fatalities_model = RandomForestRegressor(n_estimators=100)
 rf_fatalities_model.fit(X,Y2)
 pred2 = rf_fatalities_model.predict(X_test)
 pred2 = pd.DataFrame(pred2)
 pred2.columns = ["Death_prediction"]
def mean_encoding(column, target, alpha = 0.01):
global_mean = target.mean()
gb = target.groupby(column)
 mean = gb.mean()
count = gb.count()
mean_adj =(mean * count + global_mean * alpha)/(count + alpha)
 return mean_adj<categorify>
submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/submission.csv")
 submission.columns
sub = submission[["ForecastId"]]
def get_subfolds(train, fold_idx, n_subfolds):
subfolds = get_folds(df=train.loc[fold_trn_idx], n_splits=n_subfolds)
 for i in range(len(subfolds)) :
for j in range(len(subfolds[i])) :
subfolds[i][j] = fold_trn_idx[subfolds[i][j]]
return subfolds<categorify>
combined_preds = pd.concat([pred1,pred2,sub],axis=1)
 combined_preds.head()
combined_preds.columns = ['ConfirmedCases', 'Fatalities', 'ForecastId']
combined_preds = combined_preds[['ForecastId','ConfirmedCases', 'Fatalities']]
n_folds = 2
n_subfolds = 3
subfolds = []
folds = get_folds(df=train, n_splits=n_folds)
 for fold_trn_idx, fold_val_idx in tqdm_notebook(folds):
subfolds_cur = get_subfolds(train, fold_val_idx, n_subfolds)
 subfolds.append(subfolds_cur)
 
<categorify>
combined_preds["ConfirmedCases"] = combined_preds["ConfirmedCases"].astype(int)
 combined_preds["Fatalities"] = combined_preds["Fatalities"].astype(int )
<categorify>
combined_preds.to_csv("submission.csv",index=False )