Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<set_options>
pd.plotting.register_matplotlib_converters()
%matplotlib inline
print('Setup complete' )
%matplotlib inline<define_variables>
covid_train=pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv', index_col='Id', parse_dates=['Date'])
 covid_test=pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv', index_col='ForecastId', parse_dates=['Date'])
 last_register= pd.to_datetime(covid_train["Date"].iloc[covid_train.shape[0]-1])
 print("Len train %d, Len test %d"%(covid_train.shape[0], covid_test.shape[0]))
print("Last train "Date": ", last_register )
np.random.seed(2019)
 tf.set_random_seed(2019 )<load_from_csv>
def adjustState(row):
if pd.isna(row['Province_State']):
row['Province_State'] = row['Country_Region']
return row
covid_train = covid_train.apply(adjustState, axis=1)
 covid_test = covid_test.apply(adjustState, axis=1)
 covid_train.fillna('NA', inplace=True)
 covid_test.fillna('NA', inplace=True)
 n_cases_cc = 50
n_cases_ft = 50
data_mark_date = pd.DataFrame(columns=['Country_Region', 'Province_State', 'Date_cc', 'Date_ft'])
 data_mark_date.set_index(['Country_Region', 'Province_State'])
 for country in covid_train['Country_Region'].unique() :
for state in covid_train[covid_train['Country_Region']==country]['Province_State'].unique() :
data_df = covid_train[(covid_train['Country_Region']==country)&(covid_train['Province_State']==state)]
if data_df[data_df['ConfirmedCases']>=n_cases_cc].shape[0]>0:
date_cc=data_df[data_df['ConfirmedCases']>=n_cases_cc].iloc[0]['Date']
else:
date_cc=last_register
if data_df[data_df['Fatalities']>=n_cases_ft].shape[0]>0:
date_ft=data_df[data_df['Fatalities']>=n_cases_ft].iloc[0]['Date']
else:
date_ft=last_register
data_state = pd.DataFrame({ 'Country_Region': [country], 'Province_State': [state],
'Date_cc': [date_cc], 'Date_ft': [date_ft]})
 data_state.set_index(['Country_Region', 'Province_State'])
 data_mark_date=data_mark_date.append(data_state.iloc[0])
 def mark_date(row):
data_df=data_mark_date[(data_mark_date['Country_Region']==row['Country_Region'])&(data_mark_date['Province_State']==row['Province_State'])].iloc[0]
if not pd.isna(data_df['Date_cc']):
row['Date_cc']=(row['Date']-data_df['Date_cc'] ).days
if not pd.isna(data_df['Date_ft']):
row['Date_ft']=(row['Date']-data_df['Date_ft'] ).days
return row
covid_train = covid_train[(covid_train['ConfirmedCases']>0)|(covid_train['Fatalities']>0)]
covid_train['Date_cc'] = [0 for i in range(covid_train.shape[0])]
covid_train['Date_ft'] = [0 for i in range(covid_train.shape[0])]
covid_train = covid_train.apply(mark_date, axis=1)
 covid_test['Date_cc'] = [0 for i in range(covid_test.shape[0])]
covid_test['Date_ft'] = [0 for i in range(covid_test.shape[0])]
covid_test = covid_test.apply(mark_date, axis=1)
 covid_train['Date_st'] = covid_train['Date'].map(lambda x: x.timestamp())
 covid_test['Date_st'] = covid_test['Date'].map(lambda x: x.timestamp())
 enc = LabelEncoder()
covid_train['Province_State_enc'] = enc.fit_transform(covid_train['Province_State'])
 covid_test['Province_State_enc'] = enc.transform(covid_test['Province_State'])
 enc = LabelEncoder()
covid_train['Country_Region_enc'] = enc.fit_transform(covid_train['Country_Region'])
 covid_test['Country_Region_enc'] = enc.transform(covid_test['Country_Region'])
 X_features = ['Province_State', 'Country_Region', 'Date_st', 'Date_cc', 'Date_ft']
X = covid_train[X_features]
y_cc = covid_train['ConfirmedCases']
y_ft = covid_train['Fatalities']
print("Adjust data complete" )
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(train_df.shape)
 print(test_df.shape)
 train_df.head()<train_model>
X_features_cc = ['Province_State_enc', 'Country_Region_enc', 'Date_st', 'Date_cc']
X_features_ft = ['Province_State_enc', 'Country_Region_enc', 'Date_st', 'Date_ft']
def get_pad_width(im, new_shape, is_rgb=True):
pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]
t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)
 l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)
 if is_rgb:
pad_width =(( t,b),(l,r),(0, 0))
else:
pad_width =(( t,b),(l,r))
return pad_width
def preprocess_image(image_path, desired_size=224):
im = Image.open(image_path)
 im = im.resize(( desired_size,)*2, resample=Image.LANCZOS)
 return im<prepare_x_and_y>
X_train, X_valid, y_train_cc, y_valid_cc = train_test_split(X, y_cc, random_state=42)
 y_train_ft = y_ft[y_train_cc.index]
y_valid_ft = y_ft[y_valid_cc.index]
N = train_df.shape[0]
x_train = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(train_df['id_code'])) :
x_train[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png'
)<prepare_x_and_y>
def preds_country(X_fit, X_pred, y_fit, n_estimators=5, learning_rate=0.1, n_jobs=4, predict_type='XGBR'):

 X_pred_copy=X_pred.copy()
X_pred_copy['preds']=[np.nan for i in range(X_pred_copy.shape[0])]
for country in X_fit['Country_Region'].unique() :
if country in X_pred['Country_Region'].unique() :
for state in X_fit[X_fit['Country_Region']==country]['Province_State'].unique() :
if state in X_pred[X_pred['Country_Region']==country]['Province_State'].unique() :
X_fit_country = X_fit[(X_fit['Country_Region']==country)&(X_fit['Province_State']==state)].copy().drop(['Country_Region', 'Province_State'], axis=1)
 X_pred_country = X_pred[(X_pred['Country_Region']==country)&(X_pred['Province_State']==state)].copy().drop(['Country_Region', 'Province_State'], axis=1)
 y_fit_country = y_fit[X_fit_country.index]
if predict_type=='XGBR':
model = XGBRegressor(n_estimators=n_estimators, learning_rate=learning_rate, n_jobs=n_jobs, random_state=42)
 elif predict_type=='RFR':
model = RandomForestRegressor(n_estimators=n_estimators, random_state=42)
 else:
raise NameError('Model not valid')
 model.fit(X_fit_country, y_fit_country)
 X_pred_country['preds'] = model.predict(X_pred_country)
 for col in X_pred_country.index:
if col in X_pred_copy.index.values:
X_pred_copy.loc[col]=X_pred_country.loc[col]['preds']
return X_pred_copy
N = test_df.shape[0]
x_test = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(test_df['id_code'])) :
x_test[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png'
)<categorify>
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )<categorify>
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))<split>
test_preds_cc = preds_country(covid_train[['Province_State', 'Country_Region', 'Date_st', 'Date_cc']], covid_test[['Province_State', 'Country_Region', 'Date_st', 'Date_cc']], covid_train['ConfirmedCases'], n_estimators=100, predict_type='RFR')
 test_preds_ft = preds_country(covid_train[['Province_State', 'Country_Region', 'Date_st', 'Date_ft']], covid_test[['Province_State', 'Country_Region', 'Date_st', 'Date_ft']], covid_train['Fatalities'], n_estimators=50, predict_type='RFR' )
<train_model><EOS> 
submission = pd.DataFrame({'ForecastId': test_preds_cc.index,'ConfirmedCases':test_preds_cc['preds'].round() ,'Fatalities':test_preds_ft['preds'].round() })
 filename = 'submission.csv'
submission.to_csv(filename,index=False )