Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<load_from_csv>
def create_time_features(df):

 df['date'] = pd.to_datetime(df['Date'] ).values
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
return df
def mean_absolute_percentage_error(y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
 return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 10
def wg_func(params):
a = params[0]
r = params[1]
c = params[2]
hcnb = params[3]
incr = [cdata[i] if i == 0 else cdata[i] - cdata[i-1] for i,item in enumerate(cdata)]
ptlt = [(1-(a/(a+item**c)) **r)*(1-hcnb)for item in ts]
ptwt = [ptlt[i] if i == 0 else ptlt[i] - ptlt[i-1] for i,item in enumerate(ptlt)]
return(np.sum([j * np.log(i)for i,j in zip(ptwt,incr)])
 +(pop*(1-hcnb)- np.max(cdata)) * np.log(1-np.max(ptlt))
+ np.log(hcnb)*pop*hcnb)*-1
def wg_func2(params):
a = params[0]
r = params[1]
c = params[2]
hcnb =.98
incr = [cdata[i] if i == 0 else cdata[i] - cdata[i-1] for i,item in enumerate(cdata)]
ptlt = [(1-(a/(a+item**c)) **r)*(1-hcnb)for item in ts]
ptwt = [ptlt[i] if i == 0 else ptlt[i] - ptlt[i-1] for i,item in enumerate(ptlt)]
return(np.sum([j * np.log(i)for i,j in zip(ptwt,incr)])
 +(pop*(1-hcnb)- np.max(cdata)) * np.log(1-np.max(ptlt))
+ np.log(hcnb)*pop*hcnb)*-1
def constraint1(inputs):
return inputs[0]
cons =({'type': 'ineq', "fun": constraint1})
 def calc_distance(lat1, lng1, lat2, lng2):
R = 6373.0
lat1 = radians(lat1)
 lng1 = radians(lng1)
 lat2 = radians(lat2)
 lng2 = radians(lng2)
 dlon = lng2 - lng1
dlat = lat2 - lat1
a = sin(dlat / 2)**2 + cos(lat1)* cos(lat2)* sin(dlon / 2)**2
c = 2 * atan2(sqrt(a), sqrt(1 - a))
return R * c
def fill_missing_coords(df):
print(' Filling in missing lat,lng')
 df.loc[df.Country_Region=='Zimbabwe', 'Lat'] = 19.0154
df.loc[df.Country_Region=='Zimbabwe', 'Long']= 29.1549
df.loc[(df.Country_Region=='Angola')&(df.Province_State==''), 'Lat'] = -11.2027
df.loc[(df.Country_Region=='Angola')&(df.Province_State==''), 'Long']= 17.8739
df.loc[(df.Country_Region=='Bahamas')&(df.Province_State==''), 'Lat'] = 25.0343
df.loc[(df.Country_Region=='Bahamas')&(df.Province_State==''), 'Long']= -77.3963
df.loc[(df.Country_Region=='Belize')&(df.Province_State==''), 'Lat'] = 17.1899
df.loc[(df.Country_Region=='Belize')&(df.Province_State==''), 'Long']= -88.4976
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State==''), 'Lat'] = 55.3781
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State==''), 'Long']= -3.4360
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Isle of Man'), 'Lat'] = 54.2361
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Isle of Man'), 'Long']= -4.5481
df.loc[(df.Country_Region=='Cabo Verde')&(df.Province_State==''), 'Lat'] = 16.5388
df.loc[(df.Country_Region=='Cabo Verde')&(df.Province_State==''), 'Long']= -23.0418
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Bermuda'), 'Lat'] = 32.3078
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Bermuda'), 'Long']= -64.7505
df.loc[(df.Country_Region=='Chad')&(df.Province_State==''), 'Lat'] = 15.4542
df.loc[(df.Country_Region=='Chad')&(df.Province_State==''), 'Long']= 18.7322
df.loc[(df.Country_Region=='Uganda')&(df.Province_State==''), 'Lat'] = 1.3733
df.loc[(df.Country_Region=='Uganda')&(df.Province_State==''), 'Long']= 32.2903
df.loc[(df.Country_Region=='Denmark')&(df.Province_State=='Greenland'), 'Lat'] = 71.7069
df.loc[(df.Country_Region=='Denmark')&(df.Province_State=='Greenland'), 'Long']= -42.6043
df.loc[(df.Country_Region=='Denmark')&(df.Province_State==''), 'Lat'] = 56.2639
df.loc[(df.Country_Region=='Denmark')&(df.Province_State==''), 'Long']= 9.5018
df.loc[(df.Country_Region=='Timor-Leste')&(df.Province_State==''), 'Lat'] = -8.8742
df.loc[(df.Country_Region=='Timor-Leste')&(df.Province_State==''), 'Long']= 125.7275
df.loc[(df.Country_Region=='Syria')&(df.Province_State==''), 'Lat'] = 34.8021
df.loc[(df.Country_Region=='Syria')&(df.Province_State==''), 'Long']= 38.9968
df.loc[(df.Country_Region=='Saint Kitts and Nevis')&(df.Province_State==''), 'Lat'] = 17.3578
df.loc[(df.Country_Region=='Saint Kitts and Nevis')&(df.Province_State==''), 'Long']= -62.7830
df.loc[(df.Country_Region=='Papua New Guinea')&(df.Province_State==''), 'Lat'] = -6.3150
df.loc[(df.Country_Region=='Papua New Guinea')&(df.Province_State==''), 'Long']= 143.9555
df.loc[(df.Country_Region=='Niger')&(df.Province_State==''), 'Lat'] = 17.6078
df.loc[(df.Country_Region=='Niger')&(df.Province_State==''), 'Long']= 8.0817
df.loc[(df.Country_Region=='El Salvador')&(df.Province_State==''), 'Lat'] = 13.7942
df.loc[(df.Country_Region=='El Salvador')&(df.Province_State==''), 'Long']= -88.8965
df.loc[(df.Country_Region=='Gambia')&(df.Province_State==''), 'Lat'] = 13.4432
df.loc[(df.Country_Region=='Gambia')&(df.Province_State==''), 'Long']= -15.3101
df.loc[(df.Country_Region=='Libya')&(df.Province_State==''), 'Lat'] = 26.3351
df.loc[(df.Country_Region=='Libya')&(df.Province_State==''), 'Long']= 17.2283
df.loc[(df.Country_Region=='Mali')&(df.Province_State==''), 'Lat'] = 17.5707
df.loc[(df.Country_Region=='Mali')&(df.Province_State==''), 'Long']= -3.9962
df.loc[(df.Country_Region=='Grenada')&(df.Province_State==''), 'Lat'] = 12.1165
df.loc[(df.Country_Region=='Grenada')&(df.Province_State==''), 'Long']= -61.6790
df.loc[(df.Country_Region=='Laos')&(df.Province_State==''), 'Lat'] = 19.8563
df.loc[(df.Country_Region=='Laos')&(df.Province_State==''), 'Long']= 102.4955
df.loc[(df.Country_Region=='Madagascar')&(df.Province_State==''), 'Lat'] = -18.7669
df.loc[(df.Country_Region=='Madagascar')&(df.Province_State==''), 'Long']= 46.8691
df.loc[(df.Country_Region=='Guinea-Bissau')&(df.Province_State==''), 'Lat'] = 11.8037
df.loc[(df.Country_Region=='Guinea-Bissau')&(df.Province_State==''), 'Long']= -15.1804
df.loc[(df.Country_Region=='Fiji')&(df.Province_State==''), 'Lat'] = -17.7134
df.loc[(df.Country_Region=='Fiji')&(df.Province_State==''), 'Long']= 178.0650
df.loc[(df.Country_Region=='Nicaragua')&(df.Province_State==''), 'Lat'] = 12.8654
df.loc[(df.Country_Region=='Nicaragua')&(df.Province_State==''), 'Long']= -85.2072
df.loc[(df.Country_Region=='Eritrea')&(df.Province_State==''), 'Lat'] = 15.1794
df.loc[(df.Country_Region=='Eritrea')&(df.Province_State==''), 'Long']= 39.7823
df.loc[(df.Country_Region=='Haiti')&(df.Province_State==''), 'Lat'] = 18.9712
df.loc[(df.Country_Region=='Haiti')&(df.Province_State==''), 'Long']= -72.2852
df.loc[(df.Country_Region=='Dominica')&(df.Province_State==''), 'Lat'] = 15.4150
df.loc[(df.Country_Region=='Dominica')&(df.Province_State==''), 'Long']= -61.3710
df.loc[(df.Country_Region=='Mozambique')&(df.Province_State==''), 'Lat'] = -18.6657
df.loc[(df.Country_Region=='Mozambique')&(df.Province_State==''), 'Long']= 35.5296
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State==''), 'Lat'] = 52.1326
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State==''), 'Long']= 5.2913
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State=='Sint Maarten'), 'Lat'] = 18.0425
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State=='Sint Maarten'), 'Long']= -63.0548
df.loc[(df.Country_Region=='France')&(df.Province_State==''), 'Lat'] = 46.2276
df.loc[(df.Country_Region=='France')&(df.Province_State==''), 'Long']= 2.2137
df.loc[(df.Country_Region=='France')&(df.Province_State=='New Caledonia'), 'Lat'] = -20.9043
df.loc[(df.Country_Region=='France')&(df.Province_State=='New Caledonia'), 'Long']= 165.6180
df.loc[(df.Country_Region=='France')&(df.Province_State=='Martinique'), 'Lat'] = 14.6415
df.loc[(df.Country_Region=='France')&(df.Province_State=='Martinique'), 'Long']= -61.0242
print('done', datetime.now())
pd.read_csv('submission.csv' )<load_from_csv>
PATH = '/kaggle/input/covid19-global-forecasting-week-2/'
train = pd.read_csv(PATH + 'train.csv')
 test = pd.read_csv(PATH + 'test.csv')
 df_geo = pd.read_csv('.. /input/df-geo/df_geo2.csv')
 df_geo.Province_State.fillna('', inplace=True)
 dfp = pd.read_csv('/kaggle/input/population3/' + 'population.csv')
 dfp.columns = ['Province_State', 'Country_Region', 'pop']
print(train.shape, dfp.shape)
 print('Cleaning Data')
 train.loc[train['Province_State'].isnull() , 'Province_State'] = ''
test.loc[test['Province_State'].isnull() , 'Province_State'] = ''
dfp.loc[dfp['Province_State'].isnull() , 'Province_State'] = ''
print('Joining Data')
 print(train.shape, test.shape)
 n = train.shape[0]
train = pd.merge(train, dfp, on=['Country_Region','Province_State'], how='left')
 assert train.shape[0] == n
n = test.shape[0]
test = pd.merge(test, dfp, on=['Country_Region','Province_State'], how='left')
 assert test.shape[0] == n
test.reset_index(drop=True, inplace=True)
 train.reset_index(drop=True, inplace=True)
 n = train.shape[0]
train = pd.merge(train, df_geo, on=['Country_Region','Province_State'], how='left')
 assert train.shape[0] == n
n = test.shape[0]
test = pd.merge(test, df_geo, on=['Country_Region','Province_State'], how='left')
 assert test.shape[0] == n
test.reset_index(drop=True, inplace=True)
 train.reset_index(drop=True, inplace=True)
 train.loc[train['pop'].isnull() ,'pop'] = 0
test.loc[test['pop'].isnull() ,'pop'] = 0
print('
Number of countries with missing Lat/Lng: ',
train[train.Lat.isnull() ]['Country_Region'].value_counts().shape[0])
 fill_missing_coords(train)
 fill_missing_coords(test)
 print('
Number of countries with missing Lat/Lng after fixing: ',
train[train.Lat.isnull() ]['Country_Region'].value_counts().shape[0])
 print(train.shape, test.shape)
 print('
Enriching Data')
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 create_time_features(train)
 create_time_features(test)
 print('done', datetime.now() )
X_train = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 X_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 y_train = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv' )<prepare_x_and_y>
country1 = 'Luxembourg'
lat1 = df_geo[df_geo.Country_Region==country1]['Lat'].item()
lng1 = df_geo[df_geo.Country_Region==country1]['Long'].item()
country2 = 'Singapore'
lat2 = df_geo[df_geo.Country_Region==country2]['Lat'].item()
lng2 = df_geo[df_geo.Country_Region==country2]['Long'].item()
print('
Distance between ' + country1, country2, calc_distance(lat1, lng1, lat2, lng2))
print('This should be 10,436km')
 print(' Label Encoding the geographic features...')
 df1 = train[['Country_Region','Province_State','Lat','Long']].copy()
df2 = test[['Country_Region','Province_State','Lat','Long']].copy()
geo = pd.concat([df1,df2], axis=0)
 geo = geo.groupby(['Country_Region','Province_State'])[['Lat','Long']].max().reset_index()
le_country = LabelEncoder().fit(geo.Country_Region)
 le_state = LabelEncoder().fit(geo.Province_State)
 train['country'] = le_country.transform(train.Country_Region)
 train['state'] = le_state.transform(train.Province_State)
 test['country'] = le_country.transform(test.Country_Region)
 test['state'] = le_state.transform(test.Province_State)
 print('done', train.shape)
 train['wg'] = 0
train['wga'] = 0
train['wgr'] = 0
train['wgc'] = 0
train['wghcnb'] = 0
test['wg'] = 0
test['wga'] = 0
test['wgr'] = 0
test['wgc'] = 0
test['wghcnb'] = 0
test['SARIMAX'] = 0
test['ARIMA'] = 0
countries = ['Afghanistan']
for country in train.Country_Region.unique() :
bool1 = train.Country_Region == country
print(country)
 for state in train[bool1].Province_State.unique() :
bool2 = bool1 &(train.Province_State == state)&(train.ConfirmedCases>0)
 pop = np.max(train[bool2]['pop'])
 data = train[ bool2 ].copy().reset_index()
data['ts'] = data.index+1
dfj = data.iloc[0]['day_from_jan_first']
cdata = data['ConfirmedCases'].values
ts = data['ts'].values
boolt =(test.Country_Region==country)&(test.Province_State== state)
 datat = test[boolt].copy().reset_index()
datat['ts'] = datat.day_from_jan_first - dfj
datat.loc[datat.ts<=0,'ts'] = 1
x0 = [1343, 5.440110881178935, 2.188935325131958, 0.9897823619555628]
sol = minimize(wg_func, x0, constraints = cons)
 a,r,c,hcnb = sol.x[0], sol.x[1], sol.x[2], sol.x[3]
if np.isnan(sol.x[0]):
train.loc[bool2, 'wg'] = np.NaN
train.loc[bool2,'wga'] = np.NaN
train.loc[bool2,'wgr'] = np.NaN
train.loc[bool2,'wgc'] = np.NaN
train.loc[bool2,'wghcnb'] = np.NaN
test.loc[boolt, 'wg'] = np.NaN
test.loc[boolt,'wga'] = np.NaN
test.loc[boolt,'wgr'] = np.NaN
test.loc[boolt,'wgc'] = np.NaN
test.loc[boolt,'wghcnb'] = np.NaN
else:
datat['wg'] = datat.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
data['wg'] = data.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
train.loc[bool2, 'wg'] = data['wg'].values
train.loc[bool2,'wga'] = a
train.loc[bool2,'wgr'] = r
train.loc[bool2,'wgc'] = c
train.loc[bool2,'wghcnb'] = hcnb
test.loc[boolt, 'wg'] = datat.wg.values
test.loc[boolt,'wga'] = a
test.loc[boolt,'wgr'] = r
test.loc[boolt,'wgc'] = c
test.loc[boolt,'wghcnb'] = hcnb
incr = [cdata[i] if i == 0 else cdata[i] - cdata[i-1] for i,item in enumerate(cdata)]
cc = np.max(data[data.day_from_jan_first <=np.min(datat.day_from_jan_first)]['ConfirmedCases'])
 try:
model_arima = ARIMA(incr, order=(1,1,0)).fit()
preds = [item if item >=0 else 0 for item in model_arima.predict(datat.ts[0].item() , datat.ts[-1:].item())]
cum_sum = cc
preds_cc = []
for item in preds:
cum_sum = cum_sum + item
preds_cc.append(cum_sum)
 test.loc[boolt, 'ARIMA'] = pd.Series(preds_cc ).values
except:
test.loc[boolt, 'ARIMA']= np.NaN
try:
model_SARIMAX = SARIMAX(incr, order=(1,1,0), seasonal_order=(1,1,0,12),enforce_stationarity=False ).fit()
preds = [item if item >=0 else 0 for item in model_SARIMAX.predict(datat.ts[0].item() , datat.ts[-1:].item())]
cum_sum = cc
preds_cc = []
for item in preds:
cum_sum = cum_sum + item
preds_cc.append(cum_sum)
 test.loc[boolt, 'SARIMAX']= pd.Series(preds_cc ).values
except:
test.loc[boolt, 'SARIMAX']= np.NaN
print('done', datetime.now() )
X_train.set_index('sig_id', inplace=True)
 X_test.set_index('sig_id', inplace=True)
 y_train.set_index('sig_id', inplace=True )<categorify>
train.loc[train.wg<0,'wg'] = 0
test.loc[test.wg<0,'wg'] = 0
train.loc[train.wg.isnull() ,'wg'] = 0
test.loc[test.wg.isnull() ,'wg'] = 0
train.loc[np.isinf(train.wg),'wg'] = 0
test.loc[np.isinf(test.wg),'wg'] = 0
print('done', datetime.now() )
X_train.cp_time = X_train.cp_time // 24
X_train.cp_dose = X_train.cp_dose.map({'D1': 0, 'D2': 1})
 X_test.cp_time = X_test.cp_time // 24
X_test.cp_dose = X_test.cp_dose.map({'D1': 0, 'D2': 1} )<drop_column>
train['wg2'] = 0
test['wg2'] = 0
train['SARIMAX2'] = 0
train['SARIMAX2'] = 0
data = train.groupby(['day_from_jan_first'])['ConfirmedCases','pop'].sum().reset_index()
pop = np.max(data['pop'])
 data['ts'] = data.index+1
dfj = data.iloc[0]['day_from_jan_first']
cdata = data['ConfirmedCases'].values
ts = data['ts'].values
x0 = [1343, 5.440110881178935, 2.188935325131958]
sol = minimize(wg_func2, x0, constraints = cons)
 a,r,c = sol.x[0], sol.x[1], sol.x[2]
print(a,r,c)
 for country in train.Country_Region.unique() :
bool1 = train.Country_Region == country
for state in train[bool1].Province_State.unique() :
bool2 = bool1 &(train.Province_State == state)
 pop = np.max(train[bool2]['pop'])
 data = train[ bool2 ].copy().reset_index()
data['ts'] = data.index+1
dfj = data.iloc[0]['day_from_jan_first']
cdata = data['ConfirmedCases'].values
ts = data['ts'].values
boolt =(test.Country_Region==country)&(test.Province_State== state)
 datat = test[boolt].copy().reset_index()
datat['ts'] = datat.day_from_jan_first - dfj
datat.loc[datat.ts<=0,'ts'] = 1
x0 = [1343, 5.440110881178935, 2.188935325131958, 0.9897823619555628]
sol = minimize(wg_func, x0, constraints = cons)
 a,r,c,hcnb = sol.x[0], sol.x[1], sol.x[2], sol.x[3]
if not np.isnan(sol.x[0]):
datat['wg2'] = datat.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
data[ 'wg2'] = data.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
train.loc[bool2, 'wg2'] = data['wg2'].values
test.loc[boolt, 'wg2'] = datat.wg2.values
try:
model_SARIMAX = SARIMAX(data.ConfirmedCases, order=(1,1,0), seasonal_order=(1,1,0,12),enforce_stationarity=False ).fit()
preds = [item if item >=0 else 0 for item in model_SARIMAX.predict(datat.ts[0].item() , datat.ts[-1:].item())]
preds2 = [item if item >=0 else 0 for item in model_SARIMAX.predict(data.ts[0].item() , data.ts[-1:].item())]
train.loc[bool2, 'SARIMAX2']= pd.Series(preds2 ).values
test.loc[boolt, 'SARIMAX2']= pd.Series(preds ).values
except:
test.loc[boolt, 'SARIMAX2']= np.NaN
print('done', datetime.now() )
X_train_moa = X_train[X_train.cp_type != 'ctl_vehicle'].drop(columns=['cp_type'])
 X_test_moa = X_test[X_test.cp_type != 'ctl_vehicle'].drop(columns=['cp_type'])
 y_train_moa = y_train.loc[X_train_moa.index]<compute_train_metric>
data = train.groupby(['day_from_jan_first'])['ConfirmedCases','pop'].sum().reset_index()
train['wg3'] = 0
test['wg3'] = 0
pop = np.max(data['pop'])
 data['ts'] = data.index+1
dfj = data.iloc[0]['day_from_jan_first']
cdata = data['ConfirmedCases'].values
ts = data['ts'].values
x0 = [1343, 5.440110881178935, 2.188935325131958]
sol = minimize(wg_func2, x0, constraints = cons)
 a,r,c = sol.x[0], sol.x[1], sol.x[2]
for country in train.Country_Region.unique() :
bool1 = train.Country_Region == country
for state in train[bool1].Province_State.unique() :
bool2 = bool1 &(train.Province_State == state)
 pop = np.max(train[bool2]['pop'])
 data = train[ bool2 ].copy().reset_index()
data['ts'] = data.index+1
dfj = data.iloc[0]['day_from_jan_first']
cdata = data['ConfirmedCases'].values
ts = data['ts'].values
boolt =(test.Country_Region==country)&(test.Province_State== state)
 datat = test[boolt].copy().reset_index()
datat['ts'] = datat.day_from_jan_first - dfj
datat.loc[datat.ts<=0,'ts'] = 1
datat['wg3'] = datat.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
data[ 'wg3'] = data.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
train.loc[bool2, 'wg3'] = data['wg3'].values
test.loc[boolt, 'wg3'] = datat.wg3.values
print('done', datetime.now())
 print(a,r,c )
def mean_predictions(models_dense, X):
y_pred_dense = [model.predict(X)for model in models_dense]
return np.mean(y_pred_dense, axis=0)
 def macro_log_loss(y_true, y_pred):
if len(y_true.shape)== 1:
return log_loss(y_true, y_pred, labels=[0, 1]), [log_loss(y_true, y_pred, labels=[0, 1])]
y_pred = np.maximum(np.minimum(y_pred, [[1 - 1e-15] * y_pred.shape[1]] * y_pred.shape[0]), [[1e-15] * y_pred.shape[1]] * y_pred.shape[0])
 losses = [log_loss(y_true[:, i], y_pred[:, i], labels=[0, 1])for i in range(y_true.shape[1])]
return np.mean(losses), losses<choose_model_class>
train.loc[train.wg2<0,'wg2'] = 0
test.loc[test.wg2<0,'wg2'] = 0
train.loc[train.wg2.isnull() ,'wg2'] = 0
test.loc[test.wg2.isnull() ,'wg2'] = 0
train.loc[np.isinf(train.wg2),'wg2'] = 0
test.loc[ np.isinf(test.wg2),'wg2'] = 0
train.loc[train.wg3<0,'wg3'] = 0
test.loc[test.wg3<0,'wg3'] = 0
train.loc[train.wg3.isnull() ,'wg3'] = 0
test.loc[test.wg3.isnull() ,'wg3'] = 0
train.loc[np.isinf(train.wg3),'wg3'] = 0
test.loc[ np.isinf(test.wg3),'wg3'] = 0
print('done', datetime.now() )
def create_dense_model(input_shape, output_shape):
inputs = keras.Input(shape=(input_shape,), name='drug')
 x = layers.BatchNormalization()(inputs)
 x = layers.Dropout(0.2 )(x)
 x = tfa.layers.WeightNormalization(layers.Dense(units=256, activation='relu', name='dense_1'))(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dropout(0.2 )(x)
 x = tfa.layers.WeightNormalization(layers.Dense(units=256, activation='relu', name='dense_2'))(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dropout(0.2 )(x)
 output = tfa.layers.WeightNormalization(layers.Dense(output_shape, activation='sigmoid', kernel_regularizer=keras.regularizers.l2(l2=1e-5),
name='predictions'))(x)
 model = keras.Model(inputs=inputs, outputs=output)
 opt = tfa.optimizers.AdamW(weight_decay=1e-5, learning_rate=1e-2)
 model.compile(optimizer=opt, loss='binary_crossentropy')
 return model<choose_model_class>
print(mean_squared_log_error(train.ConfirmedCases, train.wg))
print(mean_squared_log_error(train.ConfirmedCases, train.wg2))
print(mean_squared_log_error(train.ConfirmedCases, train.wg3))
callbacks = [
keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5,
patience=3,
verbose=0),
keras.callbacks.ReduceLROnPlateau(
 monitor="val_loss",
factor=0.1,
patience=2)
 ] 
<train_model>
for country in train.Country_Region.unique() :
bool1 = train.Country_Region == country
for state in train[bool1].Province_State.unique() :
bool2 = bool1 &(train.Province_State == state)
 data = train[ bool2 ].copy().reset_index()
wg1 = np.round(mean_squared_log_error(data.ConfirmedCases, data.wg),2)
 wg2 = np.round(mean_squared_log_error(data.ConfirmedCases, data.wg2),2)
 wg3 = np.round(mean_squared_log_error(data.ConfirmedCases, data.wg3),2)
 boolt =(test.Country_Region==country)&(test.Province_State== state)
 datat = test[boolt].copy().reset_index()
if wg1<wg2 and wg1<wg3:
data['best'] = data.wg
datat['best'] = datat.wg
elif wg2<wg1 and wg2 < wg3:
data['best'] = data.wg2
datat['best'] = datat.wg2
else:
data['best'] = data.wg3
datat['best'] = datat.wg3
train.loc[bool2, 'wg'] = data['best'].values
test.loc[boolt, 'wg'] = datat['best'].values
print('done', datetime.now() )
x_train_dense = X_train_moa.to_numpy()
y_train_dense = y_train_moa.to_numpy()
kf = MultilabelStratifiedKFold(n_splits=5, shuffle=True)
 mlls = []
models = []
i = 1
for train, test in kf.split(x_train_dense, y_train_dense):
print(f'FOLD {i}.', end=' ')
 i = i + 1
seed(train[0])
 tf.random.set_seed(train[0])
 dense = create_dense_model(x_train_dense.shape[1], y_train_dense.shape[1])
 history_dense = dense.fit(x_train_dense[train], y_train_dense[train],
batch_size=64,
epochs=100,
callbacks=callbacks,
validation_data=(x_train_dense[test], y_train_dense[test]),
shuffle=True, verbose=0)
 mll, _ = macro_log_loss(y_train_dense[test], dense.predict(x_train_dense[test]))
models.append(dense)
 mlls.append(mll)
 print('Dense macro log loss:', mll )<compute_test_metric>
print(mean_squared_log_error(train.ConfirmedCases, train.wg))
train.drop('wg2', inplace=True, axis=1)
 train.drop('wg3', inplace=True, axis=1)
 test.drop('wg2', inplace=True, axis=1)
 test.drop('wg3', inplace=True, axis=1)
 print('done', datetime.now() )
print(np.mean(mlls))<predict_on_test>
train_data = train.copy()
train_df = train_data
train_df['area'] = [str(i)+str(' - ')+str(j)for i,j in zip(train_data['Country_Region'], train_data['Province_State'])]
train_df['Date'] = pd.to_datetime(train_df['Date'])
 full_data = train_df
today = full_data['Date'].max() +timedelta(days=1)
 def get_country_data(train_df, area, metric):
country_data = train_df[train_df['area']==area]
country_data = country_data.drop(['Id','Province_State', 'Country_Region', 'Lat','Long'], axis=1)
 country_data = pd.pivot_table(country_data, values=['ConfirmedCases','Fatalities'], index=['Date'], aggfunc=np.sum)
 country_data = country_data[country_data[metric]!=0]
return country_data
area_info = pd.DataFrame(columns=['area', 'cases_start_date', 'deaths_start_date', 'init_ConfirmedCases', 'init_Fatalities'])
 for i in range(len(train_df['area'].unique())) :
area = train_df['area'].unique() [i]
area_cases_data = get_country_data(train_df, area, 'ConfirmedCases')
 area_deaths_data = get_country_data(train_df, area, 'Fatalities')
 cases_start_date = area_cases_data.index.min()
deaths_start_date = area_deaths_data.index.min()
if len(area_cases_data)> 0:
confirmed_cases = max(area_cases_data['ConfirmedCases'])
 else:
confirmed_cases = 0
if len(area_deaths_data)> 0:
fatalities = max(area_deaths_data['Fatalities'])
 else:
fatalities = 0
area_info.loc[i] = [area, cases_start_date, deaths_start_date, confirmed_cases, fatalities]
area_info = area_info.fillna(pd.to_datetime(today))
area_info['init_cases_day_no'] = pd.to_datetime(today)-area_info['cases_start_date']
area_info['init_cases_day_no'] = area_info['init_cases_day_no'].dt.days.fillna(0 ).astype(int)
 area_info['init_deaths_day_no'] = pd.to_datetime(today)-area_info['deaths_start_date']
area_info['init_deaths_day_no'] = area_info['init_deaths_day_no'].dt.days.fillna(0 ).astype(int)
 area_info.head()
def log_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)))
 def log_fit(train_df, area, metric):
area_data = get_country_data(train_df, area, metric)
 x_data = range(len(area_data.index))
y_data = area_data[metric]
if len(y_data)< 5:
estimated_k = -1
estimated_x_0 = -1
ymax = -1
elif max(y_data)== 0:
estimated_k = -1
estimated_x_0 = -1
ymax = -1
else:
try:
popt, pcov = curve_fit(log_curve, x_data, y_data, bounds=([0,0,0],np.inf), p0=[0.3,100,10000], maxfev=1000000)
 estimated_k, estimated_x_0, ymax = popt
except RuntimeError:
print(area)
 print("Error - curve_fit failed")
 estimated_k = -1
estimated_x_0 = -1
ymax = -1
estimated_parameters = pd.DataFrame(np.array([[area, estimated_k, estimated_x_0, ymax]]), columns=['area', 'k', 'x_0', 'ymax'])
 return estimated_parameters
def get_parameters(metric):
parameters = pd.DataFrame(columns=['area', 'k', 'x_0', 'ymax'], dtype=np.float)
 for area in train_df['area'].unique() :
estimated_parameters = log_fit(train_df, area, metric)
 parameters = parameters.append(estimated_parameters)
 parameters['k'] = pd.to_numeric(parameters['k'], downcast="float")
 parameters['x_0'] = pd.to_numeric(parameters['x_0'], downcast="float")
 parameters['ymax'] = pd.to_numeric(parameters['ymax'], downcast="float")
 parameters = parameters.replace({'k': {-1: parameters[parameters['ymax']>0].median() [0]},
'x_0': {-1: parameters[parameters['ymax']>0].median() [1]},
'ymax': {-1: parameters[parameters['ymax']>0].median() [2]}})
 return parameters
cases_parameters = get_parameters('ConfirmedCases')
 cases_parameters.head(20)
 deaths_parameters = get_parameters('Fatalities')
 deaths_parameters.head(20)
 fit_df = area_info.merge(cases_parameters, on='area', how='left')
 fit_df = fit_df.rename(columns={"k": "cases_k", "x_0": "cases_x_0", "ymax": "cases_ymax"})
 fit_df = fit_df.merge(deaths_parameters, on='area', how='left')
 fit_df = fit_df.rename(columns={"k": "deaths_k", "x_0": "deaths_x_0", "ymax": "deaths_ymax"})
 fit_df['init_ConfirmedCases_fit'] = log_curve(fit_df['init_cases_day_no'], fit_df['cases_k'], fit_df['cases_x_0'], fit_df['cases_ymax'])
 fit_df['init_Fatalities_fit'] = log_curve(fit_df['init_deaths_day_no'], fit_df['deaths_k'], fit_df['deaths_x_0'], fit_df['deaths_ymax'])
 fit_df['ConfirmedCases_error'] = fit_df['init_ConfirmedCases']-fit_df['init_ConfirmedCases_fit']
fit_df['Fatalities_error'] = fit_df['init_Fatalities']-fit_df['init_Fatalities_fit']
fit_df.head()
test_data = test.copy()
test_df = test_data
test_df['area'] = [str(i)+str(' - ')+str(j)for i,j in zip(test_data['Country_Region'], test_data['Province_State'])]
test_df = test_df.merge(fit_df, on='area', how='left')
 test_df['Date'] = pd.to_datetime(test_df['Date'])
 test_df['cases_start_date'] = pd.to_datetime(test_df['cases_start_date'])
 test_df['deaths_start_date'] = pd.to_datetime(test_df['deaths_start_date'])
 test_df['cases_day_no'] = test_df['Date']-test_df['cases_start_date']
test_df['cases_day_no'] = test_df['cases_day_no'].dt.days.fillna(0 ).astype(int)
 test_df['deaths_day_no'] = test_df['Date']-test_df['deaths_start_date']
test_df['deaths_day_no'] = test_df['deaths_day_no'].dt.days.fillna(0 ).astype(int)
 test_df['ConfirmedCases_fit'] = log_curve(test_df['cases_day_no'], test_df['cases_k'], test_df['cases_x_0'], test_df['cases_ymax'])
 test_df['Fatalities_fit'] = log_curve(test_df['deaths_day_no'], test_df['deaths_k'], test_df['deaths_x_0'], test_df['deaths_ymax'])
 test_df['ConfirmedCases_pred'] = round(test_df['ConfirmedCases_fit']+test_df['ConfirmedCases_error'])
 test_df['Fatalities_pred'] = round(test_df['Fatalities_fit']+test_df['Fatalities_error'])
 test_df.head()
train_df = train.copy()
train_df['area'] = [str(i)+str(' - ')+str(j)for i,j in zip(train_df['Country_Region'], train_df['Province_State'])]
train_df = train_df.merge(fit_df, on='area', how='left')
 train_df['Date'] = pd.to_datetime(train_df['Date'])
 train_df['cases_start_date'] = pd.to_datetime(train_df['cases_start_date'])
 train_df['deaths_start_date'] = pd.to_datetime(train_df['deaths_start_date'])
 train_df['cases_day_no'] = train_df['Date']-train_df['cases_start_date']
train_df['cases_day_no'] = train_df['cases_day_no'].dt.days.fillna(0 ).astype(int)
 train_df['deaths_day_no'] = train_df['Date']-train_df['deaths_start_date']
train_df['deaths_day_no'] = train_df['deaths_day_no'].dt.days.fillna(0 ).astype(int)
 train_df['ConfirmedCases_fit'] = log_curve(train_df['cases_day_no'], train_df['cases_k'], train_df['cases_x_0'], train_df['cases_ymax'])
 train_df['Fatalities_fit'] = log_curve(train_df['deaths_day_no'], train_df['deaths_k'], train_df['deaths_x_0'], train_df['deaths_ymax'])
 train_df['ConfirmedCases_pred'] = round(train_df['ConfirmedCases_fit']+train_df['ConfirmedCases_error'])
 train_df['Fatalities_pred'] = round(train_df['Fatalities_fit']+train_df['Fatalities_error'])
 train_df.head()
print('done', datetime.now() )
preds = mean_predictions(models, X_test_moa )<define_variables>
train['y_hat_log' ] = test_df.ConfirmedCases_fit
test['y_hat_log' ] = test_df.ConfirmedCases_pred
indexes = dict(zip(X_test_moa.index, range(X_test_moa.shape[0])) )<feature_engineering>
dropcols = ['Date', 'date', 'ConfirmedCases', 'Id', 'ForecastId', 'Fatalities']
dropcols = dropcols + ['Country_Region','Province_State']
dropcols = dropcols + ['eg', 'egr', 'mae_eg', 'wga', 'wgr', 'wgc','wghcnb', 'mae_wg', 'SARIMAX', 'ARIMA',
'cc_es',]
print('
Modeling...')
 features = [f for f in train.columns if f not in dropcols +
['shift4w', 'shift6w', 'dist'
, 'dayofyear','year','quarter','hour','month','dayofmonth','dayofweek','weekofyear', 'Lat', 'Long']]
print(features)
 X_train = train[features].copy()
X_test = test[features].copy()
X_train.reset_index(drop=True, inplace=True)
 X_test.reset_index(drop=True, inplace=True)
 y_train = train["Fatalities"]
y_train_cc = train["ConfirmedCases"]
print('done', datetime.now() )
for idx, row in submission.iterrows() :
sig_id = row.sig_id
if sig_id in indexes.keys() :
submission.loc[idx] = [sig_id] + [np.maximum(np.minimum(pred, 1 - 1e-3), 1e-3)for pred in preds[indexes[sig_id]]]
else:
submission.loc[idx] = [sig_id] + [0] * preds.shape[1]<save_to_csv>
isTraining = False
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.3
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
if isTraining:
X_TRAIN = X_train[features].values
kf = KFold(n_splits = 5, shuffle = True, random_state=2020)
 acc = []
for tr_idx, val_idx in kf.split(X_TRAIN, y_train_cc):
X_tr, X_vl = X_TRAIN[tr_idx], X_TRAIN[val_idx, :]
y_tr, y_vl = y_train_cc[tr_idx], y_train_cc[val_idx]
print(X_tr.shape)
 model_xgb_cc = xgb.XGBRegressor(**params_xgb)
 model_xgb_cc.fit(X_tr, y_tr, verbose=True)
 y_hat = model_xgb_cc.predict(X_vl)
 print('xgb mae :', mean_absolute_error(y_vl, y_hat))
acc.append(mean_absolute_error(y_vl, y_hat))
print('done', np.mean(acc))
print('done', datetime.now() )
submission.to_csv('submission.csv', index=False )<save_to_csv>
dropcols = ['Date', 'date', 'ConfirmedCases', 'Id', 'ForecastId', 'Fatalities']
dropcols = dropcols + ['Country_Region','Province_State']
dropcols = dropcols + ['eg', 'egr', 'mae_eg', 'wg', 'wga', 'wgr', 'wg_xgb', 'wgc','wghcnb', 'mae_wg', 'SARIMAX', 'ARIMA',
'cc_es',]
print('
Modeling...')
 features = [f for f in train.columns if f not in dropcols +
['shift4w', 'shift6w', 'dist'
, 'dayofyear','year','quarter','hour','month','dayofmonth','dayofweek','weekofyear']]
print(features)
 X_train = train[features].copy()
X_test = test[features].copy()
X_train.reset_index(drop=True, inplace=True)
 X_test.reset_index(drop=True, inplace=True)
 X_train.head()
print('done', datetime.now() )
submission.to_csv('submission.csv', index=False )<set_options>
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.3
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
isTraining = True
X_train.reset_index(drop=True, inplace=True)
 if isTraining:
print('Evaluating model...')
 booll = X_train.day_from_jan_first < 86
X_tr, X_vl = X_train[booll][features ], X_train[~booll][features]
y_tr, y_vl = train[booll]['ConfirmedCases'], train[~booll]['ConfirmedCases']
model_xgb_cc = xgb.XGBRegressor(**params_xgb)
 model_xgb_cc.fit(X_tr, y_tr, verbose=True)
 y_hat = model_xgb_cc.predict(X_vl)
 y_hat[y_hat<0] = 0
print('xgb mae :', mean_absolute_error(y_vl, y_hat), mean_squared_log_error(y_vl, y_hat), X_tr.shape, X_vl.shape)
 print('done', datetime.now() )
warnings.filterwarnings('ignore' )<load_from_csv>
y_train = train["Fatalities"]
y_train_cc = train["ConfirmedCases"]
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.3
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
model_xgb_cc = xgb.XGBRegressor(**params_xgb ).fit(X_train[features], y_train_cc, verbose=True)
 y_hat_xgb_c = model_xgb_cc.predict(X_test[features])
 print('done', datetime.now() )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_drug = pd.read_csv(".. /input/lish-moa/train_drug.csv")
<init_hyperparams>
params_xgb = {}
params_xgb['n_estimators'] = 1100
params_xgb['max_depth'] = 10
params_xgb['seed'] = 2020
params_xgb['colsample_bylevel'] = 1
params_xgb['colsample_bytree'] = 1
params_xgb['learning_rate'] = 0.300000012
params_xgb['reg_alpha'] = 0
params_xgb['reg_lambda'] = 1
params_xgb['subsample'] = 1
model_xgb_f = xgb.XGBRegressor(**params_xgb ).fit(X_train[features], y_train, verbose=True)
 y_hat_xgb_f = model_xgb_f.predict(X_test[features])
 print(np.mean(y_hat_xgb_f))
print('done', datetime.now() )
params = {"n_genes_pca": 50,
"n_cells_pca": 20,
"batch_size": 256,
"lr": 1e-3,
"weight_decay": 1e-5,
"n_folds": 5,
"early_stopping_steps": 5,
"hidden_size": 512,
"boost_rate": 1.0,
"num_nets": 20,
"epochs_per_stage": 1,
"correct_epoch": 1,
"model_order": "second"
}<define_variables>
test['y_hat_xgb_c'] = y_hat_xgb_c
test['y_hat_xgb_f'] = y_hat_xgb_f
test['y_hat_xgb_c2'] = y_hat_xgb_c2
print('Fixing Negative Predictions:'
, np.sum(test.y_hat_xgb_c < 0)
 , np.sum(test.y_hat_xgb_c2 < 0)
 , np.sum(test.y_hat_xgb_f< 0)
 , np.sum(test.wg< 0)
 , np.sum(test.ARIMA< 0)
 , np.sum(test.SARIMAX< 0)
 , np.sum(test.y_hat_log< 0))
test.loc[test.y_hat_xgb_c < 0, 'y_hat_xgb_c'] = 0
test.loc[test.y_hat_xgb_c2 < 0, 'y_hat_xgb_c2'] = 0
test.loc[test.y_hat_xgb_f < 0, 'y_hat_xgb_f'] = 0
test.loc[test.wg < 0, 'wg'] = 0
test.loc[test.ARIMA < 0, 'ARIMA'] = 0
test.loc[test.SARIMAX < 0, 'SARIMAX'] = 0
test.loc[test.y_hat_log < 0, 'y_hat_log'] = 0
print('done', datetime.now() )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<set_options>
print('Fixing Inf Predictions:'
, np.sum(test.y_hat_xgb_c.isnull())
 , np.sum(test.y_hat_xgb_c2.isnull())
 , np.sum(test.y_hat_xgb_f.isnull())
 , np.sum(test.wg.isnull())
 , np.sum(test.ARIMA.isnull())
 , np.sum(test.SARIMAX.isnull())
 , np.sum(test.y_hat_log.isnull()))
booll =(test['SARIMAX'].isnull())
 test.loc[booll, 'SARIMAX'] = test[booll]['y_hat_log']
booll =(test['ARIMA'].isnull())
 test.loc[booll, 'ARIMA'] = test[booll]['y_hat_log']
print('Fixing Inf Predictions:'
, np.sum(test.y_hat_xgb_c.isnull())
 , np.sum(test.y_hat_xgb_c2.isnull())
 , np.sum(test.y_hat_xgb_f.isnull())
 , np.sum(test.wg.isnull())
 , np.sum(test.ARIMA.isnull())
 , np.sum(test.SARIMAX.isnull())
 , np.sum(test.y_hat_log.isnull()))
print('done', datetime.now() )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<create_dataframe>
print('Fixing Unrealistic Predictions:',
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_c),
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_c2),
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_f),
np.sum(test['pop'] *(700/3100)< test.wg),
np.sum(test['pop'] *(700/3100)< test.ARIMA),
np.sum(test['pop'] *(700/3100)< test.SARIMAX),
np.sum(test['pop'] *(700/3100)< test.y_hat_log), 
)
 booll =(test['pop'] *(700/3100)< test.y_hat_xgb_c)
 test.loc[booll, 'y_hat_xgb_c'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.y_hat_xgb_c2)
 test.loc[booll, 'y_hat_xgb_c2'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.y_hat_xgb_f)
 test.loc[booll, 'y_hat_xgb_f'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.wg)
 test.loc[booll, 'wg'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.ARIMA)
 test.loc[booll, 'ARIMA'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.SARIMAX)
 test.loc[booll, 'SARIMAX'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.y_hat_log)
 test.loc[booll, 'y_hat_log'] = test[booll]['pop'] *(700/3100)
 print('Fixed Unrealistic Predictions:',
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_c),
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_f),
np.sum(test['pop'] *(700/3100)< test.wg),
np.sum(test['pop'] *(700/3100)< test.ARIMA),
np.sum(test['pop'] *(700/3100)< test.SARIMAX),
np.sum(test['pop'] *(700/3100)< test.y_hat_log), 
)
 print('done', datetime.now() )
n_comp = params["n_genes_pca"]
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
test['y_hat_ens'] = test.y_hat_xgb_c *.15 + test.wg *.05 + test.y_hat_log *.30 + test['SARIMAX'] *.03 + test.y_hat_xgb_c2 *.47
test[(test.Province_State=='')&(test.Country_Region=='France')][['Date','y_hat_xgb_c','y_hat_xgb_c2', 'wg','SARIMAX', 'SARIMAX2','y_hat_log', 'y_hat_ens']]
n_comp = params["n_cells_pca"]
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<categorify>
print('Empty Predictions?', np.sum(test.y_hat_ens.isnull()))
data = pd.concat([pd.DataFrame(train_features), pd.DataFrame(test_features)])
 data2 = pd.get_dummies(data, columns=["cp_time", "cp_dose"])
 train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_features = train2
test_features = test2
train_features<create_dataframe>
narf = train[train.day_from_jan_first== 79].copy()
narf = narf.merge(test[test.day_from_jan_first== 79], on=['Country_Region','Province_State','day_from_jan_first'])
 narf['err'] = np.abs(( narf.wg_y - narf.ConfirmedCases)/(1+narf.ConfirmedCases))
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
print('wg', mean_squared_log_error(narf.ConfirmedCases,narf.wg_y))
print('SARIMAX', mean_squared_log_error(narf.ConfirmedCases,narf.SARIMAX))
print('y_hat_log', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_log_y))
print('y_hat_xgb_c', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_xgb_c))
narf = narf[narf.err>100].copy()
if narf.shape[0]> 0:
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
else:
print('no fixes for wg_y')
 for index, row in narf.iterrows() :
country = row['Country_Region']
state = row['Province_State']
booll =(test.Country_Region==country)&(test.Province_State==state)
 test.loc[booll, 'wg'] =(test[booll].y_hat_log)*.3 +(test[booll].y_hat_xgb_c)*.7
print('done', datetime.now() )
feature_cols = train_features.columns[4:].tolist()
params["feat_d"] = len(feature_cols)
 train_features<merge>
test['y_hat_ens'] = test.y_hat_xgb_c *.15 + test.wg *.05 + test.y_hat_log *.30 + test['SARIMAX'] *.03 + test.y_hat_xgb_c2 *.47
test[(test.Province_State=='')&(test.Country_Region=='France')][['Date','y_hat_xgb_c','y_hat_xgb_c2', 'wg', 'ARIMA','SARIMAX','y_hat_log', 'y_hat_ens']]
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
narf = train[train.day_from_jan_first== 79].copy()
narf = narf.merge(test[test.day_from_jan_first== 79], on=['Country_Region','Province_State','day_from_jan_first'])
 narf['err'] = np.abs(( narf.y_hat_log_y - narf.ConfirmedCases)/(1+narf.ConfirmedCases))
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
print('wg', mean_squared_log_error(narf.ConfirmedCases,narf.wg_y))
print('SARIMAX', mean_squared_log_error(narf.ConfirmedCases,narf.SARIMAX))
print('y_hat_log', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_log_y))
print('y_hat_xgb_c', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_xgb_c))
narf = narf[narf.err>10].copy()
if narf.shape[0]> 0:
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
else:
print('no fixes for y_hat_log')
 for index, row in narf.iterrows() :
country = row['Country_Region']
state = row['Province_State']
booll =(test.Country_Region==country)&(test.Province_State==state)
 print('done', datetime.now() )
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
test[test.y_hat_ens==1521225]
test[(test.Province_State=='')&(test.Country_Region=='Turkey')][['Date','y_hat_xgb_c', 'wg', 'ARIMA','SARIMAX','y_hat_log', 'y_hat_ens']]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<merge>
test['y_hat_ens'] = test.y_hat_ens.astype(int)
 print(np.max(test['y_hat_ens']))
<categorify><EOS> 
submissionOrig = pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv")
 submissionOrig["ConfirmedCases"]= pd.Series(test.y_hat_ens)
 submissionOrig["Fatalities"] = pd.Series(test.y_hat_xgb_f)
 submissionOrig.to_csv('submission.csv',index=False)
 submissionOrig.head(25)
 print('done', datetime.now() )