Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<train_model>
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
from sklearn.multioutput import MultiOutputRegressor
from sklearn.impute import SimpleImputer
BATCH_SIZE = 32
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.15,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True, 
)
 data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)
 mixup_generator = MixupGenerator(x_train, y_train, batch_size=BATCH_SIZE, alpha=0.2, datagen=create_datagen() )()<compute_train_metric>
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 country_data = pd.read_csv('/kaggle/input/country-data-population/country_data_population.csv')
 submission_csv = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return<choose_model_class>
train_data = pd.merge(train_data,country_data, left_on=['Country_Region'],right_on=['Country'], how='left')
 train_data.drop(['Country','Lending Category','UrbanRuralDesignation','GeoRegion','Continent'], axis=1, inplace=True)
 test_data = pd.merge(test_data,country_data, left_on=['Country_Region'],right_on=['Country'], how='left')
 test_data.drop(['Country','Lending Category','UrbanRuralDesignation','GeoRegion','Continent'], axis=1, inplace=True)
 train_data['GeoSubregion'].fillna('', inplace=True)
 test_data['GeoSubregion'].fillna('', inplace=True)
 train_data['Income Group'].fillna('', inplace=True)
 test_data['Income Group'].fillna('', inplace=True)
 train_data['Province_State'].fillna('', inplace=True)
 test_data['Province_State'].fillna('', inplace=True)
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(224,224,3)
 )<choose_model_class>
train_data['Date'] = pd.to_datetime(train_data['Date'], infer_datetime_format=True)
 test_data['Date'] = pd.to_datetime(test_data['Date'], infer_datetime_format=True )
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model<train_model>
train_data.loc[:, 'Date'] = train_data.Date.dt.strftime('%y%m%d')
 train_data.loc[:, 'Date'] = train_data['Date'].astype(int)
 test_data.loc[:, 'Date'] = test_data.Date.dt.strftime('%y%m%d')
 test_data.loc[:, 'Date'] = test_data['Date'].astype(int )
kappa_metrics = Metrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=15,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics]
)<compute_train_metric>
all_data = pd.merge(train_data, test_data ,how='outer', on=['Country_Region','Date','Province_State'])
 pd.set_option('display.max_rows', None)
 all_data.sort_values(['Country_Region','Province_State','Date'], ascending = True, inplace = True)
 all_data = all_data.reset_index(drop=True)
 all_data.head(500 )
model.load_weights('model.h5')
 y_val_pred = model.predict(x_val)
 def compute_score_inv(threshold):
y1 = y_val_pred > threshold
y1 = y1.astype(int ).sum(axis=1)- 1
y2 = y_val.sum(axis=1)- 1
score = cohen_kappa_score(y1, y2, weights='quadratic')
 return 1 - score
simplex = scipy.optimize.minimize(
 compute_score_inv, 0.5, method='nelder-mead' 
)
 best_threshold = simplex['x'][0]<save_to_csv>
all_data['days_since_confirmedcases'] = 0.0
all_data['ConfirmedCases'].fillna(0.0, inplace=True)
 count = 0
save_province_state = ""
save_country_region = ""
save_confirmedcases = 0.0
first_confirmedcase = 0
for index, row in all_data.iterrows() :
if save_province_state != row['Province_State'] or save_country_region != row['Country_Region']:
save_province_state = row['Province_State']
save_country_region = row['Country_Region']
first_confirmedcase = 0
save_confirmedcases = 0
if(save_confirmedcases == 0.0)and(row['ConfirmedCases'] > 0.0):
save_confirmedcases = row['ConfirmedCases']
first_confirmedcase = count
if first_confirmedcase > 0:
all_data['days_since_confirmedcases'][count] = count - first_confirmedcase
count += 1
all_data.head(200 )
y_test = model.predict(x_test)> 0.5
y_test = y_test.astype(int ).sum(axis=1)- 1
test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False )<set_options>
train_data = pd.merge(train_data,all_data[['Province_State','Country_Region','Date','days_since_confirmedcases']], left_on=['Province_State','Country_Region','Date'],right_on=['Province_State','Country_Region','Date'], how='left')
 test_data = pd.merge(test_data,all_data[['Province_State','Country_Region','Date','days_since_confirmedcases']], left_on=['Province_State','Country_Region','Date'],right_on=['Province_State','Country_Region','Date'], how='left')
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline 
<load_pretrained>
convert_dict = {'Province_State': str}
train_data = train_data.astype(convert_dict)
 test_data = test_data.astype(convert_dict )
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )<load_from_csv>
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index )
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()<feature_engineering>
from sklearn.preprocessing import LabelEncoder
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True )<compute_test_metric>
label_encoder1 = LabelEncoder()
label_encoder2 = LabelEncoder()
label_encoder3 = LabelEncoder()
label_encoder4 = LabelEncoder()
train_data['Province_State'] = label_encoder1.fit_transform(train_data['Province_State'])
 test_data['Province_State'] = label_encoder1.transform(test_data['Province_State'])
 train_data['Country_Region'] = label_encoder2.fit_transform(train_data['Country_Region'])
 test_data['Country_Region'] = label_encoder2.transform(test_data['Country_Region'])
 train_data['GeoSubregion'] = label_encoder3.fit_transform(train_data['GeoSubregion'])
 test_data['GeoSubregion'] = label_encoder3.transform(test_data['GeoSubregion'])
 train_data['Income Group'] = label_encoder4.fit_transform(train_data['Income Group'])
 test_data['Income Group'] = label_encoder4.transform(test_data['Income Group'])
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )<load_pretrained>
Test_id = test_data.ForecastId
learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png')) 
<load_pretrained>
train_data.drop(['Id'], axis=1, inplace=True)
 test_data.drop('ForecastId', axis=1, inplace=True )
learn.load('abcdef');<compute_train_metric>
missing_val_count_by_column =(train_data.isnull().sum())
 print(missing_val_count_by_column[missing_val_count_by_column>0] )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<save_to_csv>
from xgboost import XGBRegressor
def run_subm(learn=learn, coefficients=[0.6, 1.6, 2.6, 3.6]):
opt = OptimizedRounder()
preds,y = learn.get_preds(DatasetType.Test)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )<set_options>
X_train = train_data[['Province_State','Country_Region','GeoSubregion','Income Group','days_since_confirmedcases','Date']]
y_train = train_data[['ConfirmedCases', 'Fatalities']]
X_test = test_data[['Province_State','Country_Region','GeoSubregion','Income Group','days_since_confirmedcases','Date']]
%matplotlib inline
gc.enable()<set_options>
y_train_confirm = y_train.ConfirmedCases
y_train_fatality = y_train.Fatalities
warnings.simplefilter("ignore")
<load_from_csv>
model1 = XGBRegressor(n_estimators=40000)
 model1.fit(X_train, y_train_confirm)
 y_pred_confirm = model1.predict(X_test )
df_train = pd.read_csv('.. /input/train_V2.csv', sep=',')
<feature_engineering>
model2 = XGBRegressor(n_estimators=20000)
 model2.fit(X_train,y_train_fatality)
 y_pred_fat = model2.predict(X_test )
df_train = reduce_mem_usage(df_train )<filter>
df_sub = pd.DataFrame()
df_sub['ForecastId'] = Test_id
df_sub['ConfirmedCases'] = y_pred_confirm
df_sub['Fatalities'] = y_pred_fat
df_sub.to_csv('submission.csv', index=False )