Kaggle competition: COVID19 Global Forecasting (Week 4)
TRAIN_DATA_PATH ='.. /input/train/train'
sample = TRAIN_DATA_PATH +'/'+ os.listdir(TRAIN_DATA_PATH)[0]
TEST_DATA_PATH = '.. /input/test/test'
np.array(Image.open(sample)).shape<filter>
PATH ='/kaggle/input/covid19-global-forecasting-week-4'
trainingdata = pd.read_csv(f'{PATH}/train.csv')
 testdata = pd.read_csv(f'{PATH}/test.csv' )
print('target 0:1->',len(df_train[df_train.has_cactus==0])/len(df_train[df_train.has_cactus==1]))<prepare_x_and_y>
train = trainingdata
test = testdata
%%time
tmp = []
for i in range(len(df_train)) :
tmp.append(np.array(Image.open(TRAIN_DATA_PATH +'/'+ df_train.iloc[i]['id'])))
 X_train, y_train = np.array(tmp)/255, df_train['has_cactus']
tmp = []
for i in range(len(df_test)) :
tmp.append(np.array(Image.open(TEST_DATA_PATH +'/'+ df_test.iloc[i]['id'])))
 X_test = np.array(tmp)/255
del tmp
print(X_train.shape, y_train.shape)
 print(X_test.shape )<choose_model_class>
trainingdate = pd.to_datetime(trainingdata["Date"])
 testdate = pd.to_datetime(testdata["Date"])
 print(trainingdate )
def build_model(input_shape):
model =Sequential()
model.add(Conv2D(64,(3,3),padding='same',input_shape=(input_shape)))
 model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(64,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(64,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(MaxPooling2D())
 model.add(Dropout(0.5))
model.add(Conv2D(128,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(128,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(128,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(MaxPooling2D())
 model.add(Dropout(0.5))
model.add(Conv2D(256,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(256,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(256,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(GlobalAveragePooling2D())
 model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile('adam',loss='binary_crossentropy',metrics=['accuracy'])
 return model<filter>
m = []
d = []
for i in range(0,ltrainingdate):
dx =(trainingdate[i].strftime("%d"))
mx =(trainingdate[i].strftime("%m"))
m.append(int(mx))
d.append(int(dx))
mt = []
dt = []
for i in range(0,ltestdate):
dtx =(testdate[i].strftime("%d"))
mtx =(testdate[i].strftime("%m"))
mt.append(int(mtx))
dt.append(int(dtx))
1.0 / len(df_train[df_train.has_cactus==0])* len(df_train[df_train.has_cactus==1] )<init_hyperparams>
m = []
d = []
for i in range(0,ltrainingdate):
dx =(trainingdate[i].strftime("%d"))
mx =(trainingdate[i].strftime("%m"))
m.append(int(mx))
d.append(int(dx))
mt = []
dt = []
for i in range(0,ltestdate):
dtx =(testdate[i].strftime("%d"))
mtx =(testdate[i].strftime("%m"))
mt.append(int(mtx))
dt.append(int(dtx))
%%time
histories = []
oof_pred = np.zeros(len(df_train))
sub_pred = np.zeros(len(df_test))
class_weights = {}
weights = [3.010082493125573,
1.0]
len(df_train[df_train.has_cactus==0])
 for i in range(2):
class_weights[i] = weights[i]
print('class_weights:',class_weights)
 checkpoint_name = '/checkpoint.file'
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
 BATCH_SIZE = 64
EPOCHS = 128
for fold_id,(train_index, val_index)in enumerate(skf.split(X_train, y_train)) :
print(f'fold id: {fold_id}')
 X_tr, y_tr = X_train[train_index], y_train[train_index]
X_val, y_val = X_train[val_index], y_train[val_index]
callbacks=[
keras.callbacks.ModelCheckpoint(
 checkpoint_name, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False),
keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.7, patience=5, verbose=1,min_delta=0.00005,),
keras.callbacks.EarlyStopping(monitor='val_loss', patience=15)
 ]
datagen = ImageDataGenerator(
 height_shift_range=0.1,
horizontal_flip=True 
)
 K.clear_session()
model = build_model(X_train.shape[1:])
 model.summary()
histories.append(
 model.fit_generator(
 datagen.flow(X_train, y_train, batch_size=BATCH_SIZE),
steps_per_epoch=int(np.ceil(len(X_train)/ BATCH_SIZE)) , validation_data=(X_val, y_val),
epochs=EPOCHS, class_weight=class_weights,
callbacks=callbacks, verbose=2)
)
 model = load_model(checkpoint_name)
 oof_pred[val_index] = model.predict(X_val ).flatten()
sub_pred += model.predict(X_test ).flatten() / skf.n_splits
print(roc_auc_score(y_val, oof_pred[val_index]))
plot_roc_auc(y_val, oof_pred[val_index])
 del callbacks
sub_pred = np.clip(sub_pred,0.0,1.0 )<compute_test_metric>
train.insert(6,"Month",m,False)
 train.insert(7,"Day",d,False)
 test.insert(4,"Month",mt,False)
 test.insert(5,"Day",dt,False )
print('Ambiguous image index:',np.where(( sub_pred<0.80)&(sub_pred>0.20)) [:32][0] )<save_to_csv>
print("Training Data")
 traindays = trainingdata['Date'].nunique()
print("Number of Country_Region: ", trainingdata['Country_Region'].nunique())
 print("Number of Province_State: ", trainingdata['Province_State'].nunique())
 print("Number of Days: ", traindays)
 notrain = trainingdata['Id'].nunique()
print("Number of datapoints in train:", notrain)
 lotrain = int(notrain/traindays)
 print("L Trains:", lotrain)
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['has_cactus'] = sub_pred
submission.to_csv('submission.csv', index=False)
 submission.head(50 )<load_from_csv>
print("Test Data")
 testdays = testdata['Date'].nunique()
print("Number of Days: ", testdays)
 notest = testdata['ForecastId'].nunique()
print("Number of datapoints in test:", notest)
 lotest = int(notest/testdays)
 print("L Test:", lotest)
!pip install pretrainedmodels
DatasetType, ImageList, Path, accuracy, cnn_learner, error_rate,
get_transforms, imagenet_stats, models, Learner)
 print(os.listdir(".. /input/"))
print(dir(tmodels))
print(dir(models))
train_dir = ".. /input/train/train"
test_dir = ".. /input/test/train"
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv(".. /input/sample_submission.csv")
 data_folder = Path(".. /input")
 ')
 test_img = ImageList.from_df(test, path=data_folder/'test', folder='test')
 def predict(learn: Learner, name: str):
preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 test['has_cactus'] = preds.numpy() [:, 0]
test.to_csv('submission_{}.csv'.format(name), index=False)
 print('Finish creating submission_{}.csv'.format(name))
id_ = range(len(learn.recorder.losses))
loss_df = pd.DataFrame(
 {'id': id_, 'loss': np.array(learn.recorder.losses)})
 loss_df.to_csv('loss_{}.csv'.format(name), index=False)
 print('Finish creating loss_{}.csv'.format(name))
preds, targets = learn.get_preds(ds_type=DatasetType.Train)
 preds_label = np.argmax(preds.numpy() , axis=1)
 id_ = range(len(preds))
train_pred_df = pd.DataFrame({'id': id_, 'preds': preds.numpy(
)[:, 0], 'preds_label': preds_label, 'targets': targets.numpy() })
 train_pred_df.to_csv('./train_pred_{}.csv'.format(name))
print('Finish creating train_pred_{}.csv'.format(name))
correct_count = np.equal(preds_label, targets.numpy() ).sum()
len_preds = len(preds)
 incorrect_count = len_preds - correct_count
fpr, tpr = metrics.roc_curve(preds[:, 0], targets)
 fpr, tpr = fpr.numpy() , tpr.numpy()
FP = np.floor(fpr * len_preds)
 FN = incorrect_count - FP
TP = np.floor(tpr * len_preds)
 TN = correct_count - TP
id_ = range(len(fpr))
train_index_df = pd.DataFrame(
 {'id': id_, 'fpr': fpr, 'tpr': tpr, 'TP': TP, 'TN': TN, 'FP': FP, 'FN': FN})
 train_index_df.to_csv('./train_index_{}.csv'.format(name))
print('Finish creating train_index_{}.csv'.format(name))
learn.export('./model_{}.pth'.format(name), destroy=True)
 def get_model(model_name: str, pretrained: bool, seq: bool = False, pname: str = 'imagenet', **kwargs):
pretrained = pname if pretrained else None
model = getattr(ptms, model_name )(
 pretrained=pretrained, **kwargs)
 return nn.Sequential(*model.children())if seq else model
def dpn68b(pretrained: bool = False): return get_model(
 'dpn68b', pretrained, pname='imagenet+5k', seq=True)
 def dpn92(pretrained: bool = False): return get_model(
 'dpn92', pretrained, pname='imagenet+5k', seq=True)
 def dpn107(pretrained: bool = False): return get_model(
 'dpn107', pretrained, pname='imagenet+5k', seq=True)
 def se_resnet50(pretrained: bool = False): return get_model(
 'se_resnet50', pretrained)
 def se_resnet101(pretrained: bool = False): return get_model(
 'se_resnet101', pretrained)
 def se_resnet152(pretrained: bool = False): return get_model(
 'se_resnet152', pretrained)
 model_normal_dict = {
'densenet': tmodels.densenet161,
'vgg': tmodels.vgg16_bn,
'resnet': tmodels.resnet101,
'resnext': tmodels.resnext101_32x8d,
'se_resnet': se_resnet101,
}
model_layer_dict = {
'densenet': {
'121': tmodels.densenet121,
'161': None,
'169': tmodels.densenet169
},
'vgg': {
'13_bn': tmodels.vgg13_bn,
'16_bn': None,
'19_bn': tmodels.vgg19_bn,
},
'resnet': {
'50': tmodels.resnet50,
'101': None,
'152': tmodels.resnet152,
},
'resnext': {
'101': tmodels.resnext101_32x8d,
'50': tmodels.resnext50_32x4d,
'169': None,
},
'se_resnet': {
'50': se_resnet50,
'101': None,
'152': se_resnet152,
}
}
lr_dict = {
'1e-1': 1e-1,
'1e-3': 1e-3,
}
optim_dict = {
'adam': torch.optim.Adam,
'rprop': torch.optim.Rprop,
}
one_cycle_dict = {
'not-use-one-cycle': False,
}
transform_dict = {
'dofilp': [True, True],
'nothing': [True, True],
} 
<load_from_csv>
zt = testdate[0]
daycount = []
for i in range(0,lotrain):
for j in range(1,traindays+1):
daycount.append(j)
def load_data(dataframe=None, batch_size=1024, mode='categorical'):
if dataframe is None:
dataframe = pd.read_csv('.. /input/aerial-cactus-identification/train.csv')
 dataframe['has_cactus'] = dataframe['has_cactus'].apply(str)
 gen = ImageDataGenerator(rescale=1./255.,
validation_split=0.1, horizontal_flip=True, vertical_flip=True,rotation_range = 3,
width_shift_range = 0.1,height_shift_range = 0.1,shear_range = 0.1)
 trainGen = gen.flow_from_dataframe(dataframe, directory='.. /input/aerial-cactus-identification/train/train', x_col='id', y_col='has_cactus', has_ext=True, target_size=(32, 32),
class_mode=mode, batch_size=batch_size, shuffle=True, subset='training')
 testGen = gen.flow_from_dataframe(dataframe, directory='.. /input/aerial-cactus-identification/train/train', x_col='id', y_col='has_cactus', has_ext=True, target_size=(32, 32),
class_mode=mode, batch_size=batch_size, shuffle=True, subset='validation')
 return trainGen, testGen<choose_model_class>
for i in range(traindays):
if(zt == trainingdate[i]):
zx = i
print(zx)
 daytest = []
for i in range(0,lotest):
for j in range(1,testdays+1):
jr = zx + j
daytest.append(jr)
def baseline_model() :
model = Sequential()
model.add(Conv2D(32,(3, 3), input_shape=(32, 32, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(GlobalAveragePooling2D())
 model.add(Dense(2, activation='softmax'))
return model<train_model>
train.insert(8,"DayCount",daycount,False)
 test.insert(6,"DayCount",daytest,False )
def train_baseline() :
batch_size = 1024
trainGen, valGen = load_data(batch_size=batch_size)
 model = baseline_model()
model.load_weights('.. /input/h5models/baseline.h5')
 opt = Adam(1e-4)
 model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
 cbs = [ReduceLROnPlateau(monitor='loss', factor=0.1, patience=1, min_lr=1e-8, verbose=1)]
model.fit_generator(trainGen, steps_per_epoch=1000, epochs=10, validation_data=valGen,
validation_steps=100, shuffle=True, callbacks=cbs)
 return model
def predict_baseline(model):
testdf = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv')
 pred = np.empty(( testdf.shape[0],))
for n in tqdm(range(testdf.shape[0])) :
data = np.array(Image.open('.. /input/aerial-cactus-identification/test/test/'+testdf.id[n]))
data = data.astype(np.float32)/ 255.
 pred[n] = model.predict(data.reshape(( 1, 32, 32, 3)))[0][1]
testdf['has_cactus'] = pred
testdf.to_csv('sample_submission.csv', index=False )<predict_on_test>
traincount = int(len(train["Date"]))
testcount = int(len(test["Date"]))
model = train_baseline()
model.save('baseline.h5')
 predict_baseline(model )<import_modules>
train.Province_State = train.Province_State.fillna(0)
 empty = 0
for i in range(0,traincount):
if(train.Province_State[i] == empty):
train.Province_State[i] = train.Country_Region[i]
from fastai import *
from pathlib import Path
from fastai.vision import *<define_variables>
test.Province_State = test.Province_State.fillna(0)
 empty = 0
for i in range(0,testcount):
if(test.Province_State[i] == empty):
test.Province_State[i] = test.Country_Region[i]
data_folder = Path(".. /input/" )<load_from_csv>
label = preprocessing.LabelEncoder()
train.Country_Region = label.fit_transform(train.Country_Region)
 train.Province_State = label.fit_transform(train.Province_State )
train_df = pd.read_csv(f"{data_folder}/train.csv")
 test_df = pd.read_csv(f"{data_folder}/sample_submission.csv" )<choose_model_class>
test.Country_Region = label.fit_transform(test.Country_Region)
 test.Province_State = label.fit_transform(test.Province_State)
learn = cnn_learner(data, models.resnet18, metrics=[error_rate, accuracy],model_dir="/tmp/model/" )<train_model>
X = np.c_[train["Province_State"], train["Country_Region"], train["DayCount"], train["Month"], train["Day"]]
Xt = np.c_[test["Province_State"], test["Country_Region"], test["DayCount"], test["Month"], test["Day"]]
learn.fit_one_cycle(2 )<train_model>
Y1 = train["ConfirmedCases"]
Y2 = train["Fatalities"]
learn.fit_one_cycle(2 )<drop_column>
regr = XGBRegressor(n_estimators = 1500, gamma = 0, learning_rate = 0.8, random_state = 42, max_depth = 50, subsample = 1, reg_lambda = 0, reg_alpha = 0.5)
 regr1 = XGBRegressor(n_estimators = 1500, gamma = 0, learning_rate = 0.8, random_state = 42, max_depth = 50, subsample = 1, reg_lambda = 0, reg_alpha = 0.5 )
data.add_test(ImageList.from_df(test_df,data_folder/'test',folder='test'))<predict_on_test>
def rmsle(y_true, y_pred):
return mean_squared_log_error(y_true, y_pred)**(1/2);
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
regr.fit(X,Y1.ravel())
 yscore = regr.predict(X)
 a = abs(yscore)
 b=abs(Y1)
 ascore =(rmsle(a,b))
print("RMSLE of Confirmed Cases is ",ascore )
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
ypred = regr.predict(Xt)
 ypred = pd.DataFrame({'ConfirmedCases' : ypred})
 ypred = round(ypred)
 ypred.head(20 )
test_df.to_csv('submission.csv', index=False )<set_options>
regr1.fit(X,Y2.ravel())
 ypred2 = regr1.predict(Xt)
 yptest = regr1.predict(X)
 yptest = np.round(yptest)
 c = abs(yptest)
 d = abs(Y2)
 ascore = rmsle(c,d)
 print("RMSLE of Fatalities is", ascore)
%matplotlib inline<import_modules>
ypred2 = pd.DataFrame({'Fatalities' : ypred2})
 ypred2 = round(ypred2)
 ypred2.head(20 )
import tensorflow as tf
from tensorflow import keras<define_variables>
ypc = pd.DataFrame()
forecast = test["ForecastId"]
random.seed(0 )<import_modules>
ypc.insert(0,"ForecastId",forecast,False)
 ypc.insert(1,"ConfirmedCases",ypred,False)
 ypc.insert(2, "Fatalities",ypred2,False )
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import InputLayer, Input
from tensorflow.keras.layers import Conv2D, Dense, Flatten, Dropout, Activation
from tensorflow.keras.layers import BatchNormalization, Reshape, MaxPooling2D, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau<prepare_x_and_y>
ypc.to_csv('submission.csv', index=False )