Kaggle competition: COVID19 Global Forecasting (Week 4)
steps_per_epoch = int(np.ceil(X_train.shape[0]/BATCH_SIZE))
val_steps_per_epoch = int(np.ceil(X_val.shape[0]/BATCH_SIZE))<choose_model_class>
warnings.filterwarnings('ignore')
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'] )<train_model>
test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_pop = pd.read_csv('.. /input/population-by-country-2020/population_by_country_2020.csv')
 covid_pop = pd.read_csv('.. /input/covid19-global-forecasting-locations-population/locations_population.csv' )
history = model.fit_generator(train_gen, validation_data=(X_val, y_val), steps_per_epoch=steps_per_epoch,
callbacks=[kappa_cb], epochs=25, class_weight=class_weights )<load_pretrained>
print("Max Date: ", max(train['Date']), ", Min Date: ", min(train['Date']))
print("Max Id: ", max(train['Id']), ", Unique Ids: ", train['Id'].nunique())
 print("Max Date: ", max(test['Date']), ", Min Date: ", min(test['Date']))
model.load_weights('model.h5' )<predict_on_test>
train.isnull().sum()
val_preds = np.squeeze(model.predict_generator(val_gen, steps=val_steps_per_epoch))
list(zip(val_preds, y_val)) [:10]<compute_test_metric>
train.isnull().sum()
pred = val_preds > 0.5
pred = np.sum(pred.astype(np.int8), axis=1)-1
kappa, acc = metrics.accuracy_score(pred, y_val.sum(axis=1)-1), metrics.cohen_kappa_score(y_val.sum(axis=1)-1, pred, weights='quadratic')
 print('kappa score: {:.4f}, accuracy: {:.4f}'.format(kappa, acc))<compute_test_metric>
cond1 = train.Fatalities >=1
cond2 = train.ConfirmedCases >= 1
train_Fatal = train[['Country_Region','Date']][(cond1)]
train_Confirm = train[['Country_Region','Date']][(cond2)]
Confirm_Min_Max = train_Confirm.groupby("Country_Region", as_index=False)["Date"].agg(["min","max"])
 Fatal_Min_Max = train_Fatal.groupby("Country_Region", as_index=False)["Date"].agg(["min","max"])
 Country_Dates = pd.merge(Confirm_Min_Max, Fatal_Min_Max, how='left', on=['Country_Region'])
 Country_Dates = Country_Dates.rename(columns={'min_x': 'fst_Confirmed', 'max_x': 'last_Confirmed', 'min_y': 'fst_Fatal','max_y': 'last_Fatal'})
 Country_Dates['Fatality_Days'] = pd.to_datetime(Country_Dates['last_Fatal'])- pd.to_datetime(Country_Dates['fst_Fatal'])
 Country_Dates = Country_Dates.reset_index()
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<train_on_grid>
Country_Dates['Fatality_Days'] = Country_Dates['Fatality_Days'] / np.timedelta64(1, 'D')
 Country_Dates.head(5 )
optR = OptimizedRounder()
optR.fit(np.sum(val_preds,axis=1), np.sum(y_val, axis=1))
coefficients = optR.coefficients()<load_from_csv>
cond1 = train.Date == '2020-04-12'
Country_Counts = train[['Country_Region','Date','Fatalities','ConfirmedCases']][(cond1)]
df_Country2 = pd.merge(Country_Dates, Country_Counts, how='left', on=['Country_Region'])
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = df_Country2['Fatality_Rate'].fillna(0)
 df_Country2 = df_Country2.replace([np.inf, -np.inf], np.nan)
 df_Country3 = df_Country2[df_Country2['Fatality_Rate'] > 0].groupby(['Country_Region'])[["Fatality_Rate"]].sum().nlargest(8,'Fatality_Rate')
 df_Country3 = df_Country3.reset_index()
sample = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv" )<define_variables>
confirmed_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_China = confirmed_China.join(fatalities_China)
 confirmed_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Italy = confirmed_Italy.join(fatalities_Italy)
 confirmed_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Spain = confirmed_Spain.join(fatalities_Spain)
 confirmed_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Iran = confirmed_Iran.join(fatalities_Iran)
 confirmed_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_France = confirmed_France.join(fatalities_France)
 confirmed_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Netherlands = confirmed_Netherlands.join(fatalities_Netherlands)
 confirmed_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_UK = confirmed_UK.join(fatalities_UK)
 confirmed_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_USA = confirmed_USA.join(fatalities_USA)
test_gen = TestSequence(sample['id_code'], BATCH_SIZE, True )<predict_on_test>
df_pop_sel = df_pop[['Country(or dependency)','Population(2020)','Density(P/Km²)','Land Area(Km²)','Med.Age','Urban Pop %']]
df_pop_sel.columns = ['Country_Region', 'Population(2020)', 'Density(P/Km²)','Land Area(Km²)', 'Med.Age', 'Urban Pop %']
df_pop_sel['Urban Pop %'] = df_pop_sel['Urban Pop %'].str.rstrip('%')
 df_pop_sel.loc[df_pop_sel['Country_Region']=='United States', 'Country_Region'] = 'US'
df_country_pop = pd.merge(df_pop_sel, df_Country, how='inner', on=['Country_Region'])
 df_country_pop
prediction = np.empty(( 5,len(sample)))
 for i in tqdm(range(5)) :
pred = model.predict_generator(test_gen)> 0.5
prediction[i,:] = np.sum(pred.astype(np.int8), axis=1)-1
prediction = prediction.astype(np.int8 )<sort_values>
train_fil['Date'] = pd.to_datetime(train_fil['Date'])
 train_fil['Day_num'] = preprocessing.LabelEncoder().fit_transform(train_fil.Date)
 train_fil['Day'] = train_fil['Date'].dt.day
train_fil['Month'] = train_fil['Date'].dt.month
train_fil['Year'] = train_fil['Date'].dt.year
train_fil_copy = train_fil
train_fil['Province_State'].fillna("None", inplace=True)
 train_fil[train_fil.iloc[:,0:].isnull().any(axis = 1)].iloc[:,0:].head()
def vote(x):
return np.argmax(np.bincount(x))<concatenate>
df_train_pop = train_fil.merge(covid_pop_sel, left_on = ['Country_Region','Province_State'], right_on = ['Country.Region','Province.State'], how = 'left')
preds = np.apply_along_axis(vote, 0, prediction )<save_to_csv>
display(covid_pop_sel[covid_pop_sel['Country.Region']=='Italy'])
 display(train_fil[train_fil['Country_Region']=='Italy'] )
sample.diagnosis = preds
sample.to_csv("submission.csv", index=False )<set_options>
resize_size = 224
%matplotlib inline<define_variables>
np.random.seed(2019)
 tf.set_random_seed(2019 )<load_from_csv>
df_train_pop['Province.State'].fillna("None", inplace=True)
 df_train_pop['Country.Region'].fillna("None", inplace=True)
 df_train_pop['Population'] = df_train_pop['Population'].fillna(0 )
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(train_df.shape)
 print(test_df.shape)
 train_df.head()<train_model>
missings_count = {col:df_train_pop[col].isnull().sum() for col in df_train_pop.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))
def get_pad_width(im, new_shape, is_rgb=True):
pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]
t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)
 l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)
 if is_rgb:
pad_width =(( t,b),(l,r),(0, 0))
else:
pad_width =(( t,b),(l,r))
return pad_width
def preprocess_image(image_path, desired_size=224):
im = Image.open(image_path)
 im = im.resize(( desired_size,)*2, resample=Image.LANCZOS)
 return im<prepare_x_and_y>
df_fin_selc = df_train_pop[['Province_State','Country_Region','Date','ConfirmedCases','Fatalities','Day_num','Day','Month','Year','Population']]
N = train_df.shape[0]
x_train_rsz = np.empty(( N, resize_size, resize_size, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm_notebook(train_df['id_code'])) :
x_train_rsz[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png',desired_size = resize_size
)<prepare_x_and_y>
df_fin_selc['I'] = df_fin_selc['ConfirmedCases']-df_fin_selc['Fatalities']
df_fin_selc['R'] = df_fin_selc['Fatalities']
df_fin_selc['S'] = df_fin_selc['Population']-df_fin_selc['ConfirmedCases']
df_SIR = df_fin_selc[['Province_State','Country_Region','Population','Day_num','S','I','R']].reindex()
df_SIR[df_SIR['Country_Region'] == 'Italy'].tail(5 )
N = test_df.shape[0]
x_test_rsz = np.empty(( N, resize_size, resize_size, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm_notebook(test_df['id_code'])) :
x_test_rsz[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png',desired_size = resize_size
)<categorify>
import numpy as np
from scipy.integrate import odeint
from scipy import integrate, optimize
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )<concatenate>
def SIR_testmodel(y,t,bta,gmma):
S, I, R = y
dS_dt = -1*bta*I*S/N
dI_dt =(bta*I*S/N)- gmma*I
dR_dt = gmma*I
return([dS_dt, dI_dt, dR_dt] )
l = len(x_train)
 for j in range(2):
for i in tqdm_notebook(range(l)) :
if(y_train[i] == np.array([0,0,0,0,1])).all() :
x_train = np.vstack(( x_train, x_train[i].reshape(1,resize_size,resize_size,3)))
 y_train = np.vstack(( y_train, y_train[i].reshape(1,5)))
 print(len(x_train))
print(len(y_train))<categorify>
N = 1
S00 = 0.9
I00 = 0.1
R00 = 0.0
bta = 0.35
gmma = 0.1
t = np.linspace(0,100,1000)
 sol = odeint(SIR_testmodel,[S00,I00,R00],t,args =(bta,gmma))
sol = np.array(sol )
print(y_train.shape)
 y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))<split>
display(covid_pop_sel[covid_pop_sel['Country.Region']=='Italy'] )
print(x_train.shape)
 print(y_train_multi.shape)
 x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.15,
random_state=2019
)<train_model>
Country_Dates[Country_Dates['Country_Region'] == 'Italy']
class MixupGenerator() :
def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None):
self.X_train = X_train
self.y_train = y_train
self.batch_size = batch_size
self.alpha = alpha
self.shuffle = shuffle
self.sample_num = len(X_train)
 self.datagen = datagen
def __call__(self):
while True:
indexes = self.__get_exploration_order()
itr_num = int(len(indexes)//(self.batch_size * 2))
for i in range(itr_num):
batch_ids = indexes[i * self.batch_size * 2:(i + 1)* self.batch_size * 2]
X, y = self.__data_generation(batch_ids)
 yield X, y
def __get_exploration_order(self):
indexes = np.arange(self.sample_num)
 if self.shuffle:
np.random.shuffle(indexes)
 return indexes
def __data_generation(self, batch_ids):
_, h, w, c = self.X_train.shape
l = np.random.beta(self.alpha, self.alpha, self.batch_size)
 X_l = l.reshape(self.batch_size, 1, 1, 1)
 y_l = l.reshape(self.batch_size, 1)
 X1 = self.X_train[batch_ids[:self.batch_size]]
X2 = self.X_train[batch_ids[self.batch_size:]]
X = X1 * X_l + X2 *(1 - X_l)
 if self.datagen:
for i in range(self.batch_size):
X[i] = self.datagen.random_transform(X[i])
 X[i] = self.datagen.standardize(X[i])
 if isinstance(self.y_train, list):
y = []
for y_train_ in self.y_train:
y1 = y_train_[batch_ids[:self.batch_size]]
y2 = y_train_[batch_ids[self.batch_size:]]
y.append(y1 * y_l + y2 *(1 - y_l))
else:
y1 = self.y_train[batch_ids[:self.batch_size]]
y2 = self.y_train[batch_ids[self.batch_size:]]
y = y1 * y_l + y2 *(1 - y_l)
 return X, y<define_variables>
cond1 = Italy_SIR_df.I >= 1
test = Italy_SIR_df['Day_num'][(cond1)]
test
Italy_SIR_df.iloc[:10]
BATCH_SIZE = 32
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.15,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True,
brightness_range=[0.3,1.0] 
)
 data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)
 mixup_generator = MixupGenerator(x_train, y_train, batch_size=BATCH_SIZE, alpha=0.2, datagen=create_datagen() )()
print(y_train[0:5] )<compute_train_metric>
Italy_S = Italy_SIR_df['S']
Italy_I = Italy_SIR_df['I']
Italy_R = Italy_SIR_df['R']
Italy_s = np.array(Italy_S, dtype=float)
 Italy_i = np.array(Italy_I, dtype=float)
 Italy_r = np.array(Italy_R, dtype=float)
 display(Italy_s[9],Italy_i[9], Italy_r[9] )
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return<choose_model_class>
def SIR_model(y,t,beta,gamma):
S, I, R = y
N = 60480000
dS_dt = -1*beta*I*S/N
dI_dt =(beta*I*S/N)- gamma*I
dR_dt = gamma*I
return([dS_dt, dI_dt, dR_dt])
 def fit_odeint(x, beta, gamma):
return integrate.odeint(SIR_model,(S0, I0, R0), x, args=(beta, gamma)) [:,1]
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(resize_size,resize_size,3)
 )<choose_model_class>
xdata = Italy_SIR_df.Day_num
ydata = Italy_i
xdata = np.array(xdata, dtype=float)
 S0 = 60479998.0
I0 = 2.0
R0 = 0
y = S0, I0, R0
popt, pcov = optimize.curve_fit(fit_odeint, xdata, ydata)
 fitted = fit_odeint(xdata, *popt )
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model<choose_model_class>
comp_df = pd.read_csv('.. /input/corona-virus-report/covid_19_clean_complete.csv')
 comp_df.head(5 )
model = build_model() 
<train_model>
comp_df['Province/State'].fillna("None", inplace=True)
 comp_df[comp_df.iloc[:,0:].isnull().any(axis = 1)].iloc[:,0:].head()
kappa_metrics = Metrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=20,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics]
)<compute_train_metric>
comp_df['Date'] = pd.to_datetime(comp_df['Date'] )
y_val_pro=model.predict(x_val)
 print(y_val_pro[32])
 y_val_pre = y_val_pro > 0.5
print(y_val_pre[32])
 a=y_val_pre.astype(int)
 y_val_pre = y_val_pre.astype(int ).sum(axis=1)- 1
y_val_act=y_val.astype(int ).sum(axis=1)-1
diff=[]
diff=y_val_act-y_val_pre
diff_col=np.where(diff!=0)
 diff_col_ov2=np.where(np.abs(diff)>1)
 same_col=np.where(diff==0)
 QWK=cohen_kappa_score(y_val_act,y_val_pre,weights='quadratic')
 print('Accuracy=%.2f％(%.d/%.d)'%(accuracy_score(y_val_act, y_val_pre)*100,len(same_col[0]),len(y_val_act)))
 print('diff=2 is %.d'%len(diff_col_ov2[0]))
print('Cohen kappa=%.4f'%QWK )<predict_on_test>
covid_pop_sel.loc[covid_pop_sel['Country.Region']=='Korea, South', 'Country.Region'] = 'South Korea'
comp_df_pop = comp_df.merge(covid_pop_sel, left_on = ['Country/Region','Province/State'], right_on = ['Country.Region','Province.State'], how = 'left')
 comp_df_pop.tail(3 )
y_train_pro = model.predict(x_train)
 print("the output of CNN is the probability of diagnosis")
 print(y_train_pro[32] )<train_model>
comp_pop_sel = comp_df_pop[['Province/State','Country/Region','Date','Confirmed','Deaths','Recovered','Population']]
clf=RFC(n_estimators=20,max_depth=10)
 y_train_act=y_train.sum(axis=1)-1
print(y_train_act)
 clf.fit(y_train_pro, y_train_act )<compute_train_metric>
missings_count = {col:comp_pop_sel[col].isnull().sum() for col in comp_pop_sel.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))
y_val_pre2=clf.predict(y_val_pro)
 diff=[]
diff=y_val_act-y_val_pre2
same_col=np.where(diff==0)
 print('Accuracy=%.2f％(%.d/%.d)'%(accuracy_score(y_val_act, y_val_pre2)*100,len(same_col[0]),len(y_val_act)))
 print('diff=2 is %.d'%len(diff_col_ov2[0]))
QWK=cohen_kappa_score(y_val_act,y_val_pre2,weights='quadratic')
 print('Cohen kappa=%.4f'%QWK )<compute_test_metric>
comp_pop_sel['Population'] = comp_pop_sel['Population'].fillna(0 )
print(y_val_pre2)
 print(y_val_act)
 print(cohen_kappa_score(y_val_act,y_val_pre2,weights='quadratic'))
y_val_pre2[3] = 3
print(y_val_pre2)
 print(y_val_act)
 print(cohen_kappa_score(y_val_act,y_val_pre2,weights='quadratic'))<train_model>
comp_df_Italy = comp_pop_sel[comp_pop_sel['Country/Region']=='Italy']
comp_df_Italy = comp_df_Italy.reset_index(drop = True)
 comp_df_Italy.tail(5 )
train_data = lgb.Dataset(y_train_pro, label=y_train_act)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'multiclass',
'num_class': 5,
'verbose': 2,
}
gbm = lgb.train(
 params,
train_data,
num_boost_round=100,
verbose_eval=5,
)<predict_on_test>
comp_df_Italy['Day_num'] = preprocessing.LabelEncoder().fit_transform(comp_df_Italy.Date)
 comp_df_Italy.tail(3 )
preds = gbm.predict(y_val_pro)
 y_gbm_pre = []
for i in range(len(preds)) :
y_gbm_pre.append(np.argmax(preds[i]))<compute_train_metric>
comp_df_Italy['R'] = comp_df_Italy['Deaths']+comp_df_Italy['Recovered']
comp_df_Italy['I'] = comp_df_Italy['Confirmed']- comp_df_Italy['R']
comp_df_Italy['S'] = comp_df_Italy['Population']-comp_df_Italy['I']-comp_df_Italy['R']
comp_df_Italy.tail(3 )
model.load_weights('model.h5')
 y_val_pred = model.predict(x_val)
 def compute_score_inv(threshold):
y1 = y_val_pred > threshold
y1 = y1.astype(int ).sum(axis=1)- 1
y2 = y_val.sum(axis=1)- 1
score = cohen_kappa_score(y1, y2, weights='quadratic')
 return 1 - score
simplex = scipy.optimize.minimize(
 compute_score_inv, 0.5, method='nelder-mead' 
)
 best_threshold = simplex['x'][0]<save_to_csv>
comp_df_Italy_S = comp_df_Italy['S']
comp_df_Italy_I = comp_df_Italy['I']
comp_df_Italy_R = comp_df_Italy['R']
comp_df_Italy_s = np.array(comp_df_Italy_S, dtype=float)
 comp_df_Italy_i = np.array(comp_df_Italy_I, dtype=float)
 comp_df_Italy_r = np.array(comp_df_Italy_R, dtype=float)
 display(comp_df_Italy_s[9],comp_df_Italy_i[9], comp_df_Italy_r[9] )
y_test = model.predict(x_test)
 y_test = gbm.predict(y_test)
 y_gbm_pre = []
for i in range(len(y_test)) :
y_gbm_pre.append(np.argmax(y_test[i]))
test_df['diagnosis'] = y_gbm_pre
test_df.to_csv('submission.csv',index=False )<import_modules>
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 df_subm = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
import os
import cv2
import random
import numpy as np
import pandas as pd
import scipy as sp
import torch
from fastai.vision import *
import glob 
<set_options>
df_train_copy = df_train.copy()
df_test_copy = df_test.copy()
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 2019
seed_everything(SEED)
 print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )<define_variables>
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )
PATH = ".. /input/aptos2019-blindness-detection"
!mkdir -p /tmp/.cache/torch/checkpoints/
!cp.. /input/fastai-pretrained/densenet161-8d451a50.pth /tmp/.cache/torch/checkpoints/densenet161-8d451a50.pth
!cp.. /input/fastai-pretrained/densenet169-b2777c0a.pth /tmp/.cache/torch/checkpoints/densenet169-b2777c0a.pth<load_from_csv>
y1_df_Train = df_train.iloc[:, -2]
y2_df_Train = df_train.iloc[:, -1]
NOVAL = "NOVAL"
def handlenullstate(State, Country):
if State == NOVAL: return Country
return State
SIZE=300
train_df=pd.read_csv(PATH+'/train.csv')
 test_df=pd.read_csv(PATH+'/sample_submission.csv' )<define_variables>
df_train['State'].fillna(NOVAL, inplace=True)
 df_train['State'] = df_train.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_train.loc[:, 'Date'] = df_train.Date.dt.strftime("%m%d")
 df_train["Date"] = df_train["Date"].astype(int)
 df_test['State'].fillna(NOVAL, inplace=True)
 df_test['State'] = df_test.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_test.loc[:, 'Date'] = df_test.Date.dt.strftime("%m%d")
 df_test["Date"] = df_test["Date"].astype(int)
 df_test.head()
train = ImageList.from_df(train_df, path=PATH, cols='id_code', folder="train_images", suffix='.png')
 test = ImageList.from_df(test_df, path=PATH, cols='id_code', folder="test_images", suffix='.png' )<compute_test_metric>
covid_pop_taken = covid_pop[['Province.State','Country.Region','Population']]
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )<define_variables>
covid_pop_taken.rename(columns={'Province.State':'State'}, inplace=True)
 covid_pop_taken.rename(columns={'Country.Region':'Country'}, inplace=True )
data.show_batch(rows=4,figsize=(15,15))<choose_model_class>
covid_pop_taken['State'].fillna("None", inplace=True)
 NOVAL = 'None'
covid_pop_taken['State'] = covid_pop_taken.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 covid_pop_taken.head(3 )
learn_1 = cnn_learner(data, models.densenet169, metrics=[quadratic_kappa], pretrained=True ).mixup() 
<train_model>
df_train_copy2 = df_train.copy()
df_test_copy2 = df_test.copy()
df_test.head(3 )
learn_1.fit_one_cycle(10,1e-02)
 learn_1.recorder.plot_losses()
learn_1.recorder.plot_metrics() 
<save_model>
df_train_co_pop = df_train.merge(covid_pop_taken, left_on = ['Country','State'], right_on = ['Country','State'], how = 'left')
 df_test_co_pop = df_test.merge(covid_pop_taken, left_on = ['Country','State'], right_on = ['Country','State'], how = 'left')
 df_test_co_pop.head(5 )
learn_1.save("densenet161phase1")
 learn_1.export() 
<find_best_params>
df_train_co_pop[['Population']] = preprocessing.scale(df_train_co_pop[['Population']])
 df_test_co_pop[['Population']] = preprocessing.scale(df_test_co_pop[['Population']] )
learn_1.unfreeze() 
<find_best_params>
df_train = df_train_co_pop
df_test = df_test_co_pop
learn_1.lr_find()<train_model>
le = preprocessing.LabelEncoder()
df_train.Country = le.fit_transform(df_train.Country)
 df_train['State'] = le.fit_transform(df_train['State'])
 df_train.Country = le.fit_transform(df_train.Country)
 df_test.Country = le.fit_transform(df_test.Country)
 df_test['State'] = le.fit_transform(df_test['State'])
 df_test.head()
learn_1.fit_one_cycle(5, max_lr=slice(7e-05,1e-04))
learn_1.recorder.plot_losses()
learn_1.recorder.plot_metrics() 
<predict_on_test>
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = df_train.Country.unique()
valid_preds, valid_y = learn_1.get_preds(ds_type=DatasetType.Valid)
 test_preds, _ = learn_1.get_preds(ds_type=DatasetType.Test )<compute_test_metric>
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = df_train.loc[df_train.Country == country, :].State.unique()
for state in states:
x_train_CS = df_train.loc[(df_train.Country == country)&(df_train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities','Population']]
y1_train_CS = x_train_CS.loc[:, 'ConfirmedCases']
y2_train_CS = x_train_CS.loc[:, 'Fatalities']
x_train_CS = x_train_CS.loc[:, ['State', 'Country', 'Date','Population']]
x_train_CS.Country = le.fit_transform(x_train_CS.Country)
 x_train_CS['State'] = le.fit_transform(x_train_CS['State'])
 x_test_CS = df_test.loc[(df_test.Country == country)&(df_test.State == state), ['State', 'Country', 'Date', 'ForecastId','Population']]
x_test_CS_Id = x_test_CS.loc[:, 'ForecastId']
x_test_CS = x_test_CS.loc[:, ['State', 'Country', 'Date','Population']]
x_test_CS.Country = le.fit_transform(x_test_CS.Country)
 x_test_CS['State'] = le.fit_transform(x_test_CS['State'])
 xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(x_train_CS, y1_train_CS)
 y1_xpred = xmodel1.predict(x_test_CS)
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(x_train_CS, y2_train_CS)
 y2_xpred = xmodel2.predict(x_test_CS)
 xdata = pd.DataFrame({'ForecastId': x_test_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0)
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_train_metric>
xout.ForecastId = xout.ForecastId.astype('int')
 xout['ConfirmedCases'] = round(xout['ConfirmedCases'],1)
 xout['Fatalities'] = round(xout['Fatalities'],1)
 display(xout.head() )
optR = OptimizedRounder()
optR.fit(valid_preds, valid_y)
 coefficients = optR.coefficients()
valid_predictions = optR.predict(valid_preds, coefficients)[:,0].astype(int)
 test_predictions = optR.predict(test_preds, coefficients)[:,0].astype(int)
 valid_score = cohen_kappa_score(valid_y.numpy().astype(int), valid_predictions, weights="quadratic" )<compute_test_metric>
xout['ConfirmedCases'] = xout['ConfirmedCases'].apply(int)
 xout['Fatalities'] = xout['Fatalities'].apply(int )
valid_score = cohen_kappa_score(valid_y.numpy().astype(int), valid_predictions, weights="quadratic" )<compute_test_metric>
xout = xout.drop_duplicates()
xout.reindex()
<save_to_csv><EOS> 
xout.to_csv('submission.csv', index=False )