Kaggle competition: COVID19 Global Forecasting (Week 4)
batch_size = 64
target_size =(32,32)
 class_mode = 'binary'
epochs = 100
input_shape =(32,32,3)
 num_classes = 2
data_dir = ".. /input/train/train/"
validation_split = 0.8
color_mode = 'rgb'
x_col = 'id'
y_col = 'has_cactus'
dropout_dense_layer = 0.5<load_from_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from xgboost import XGBRegressor
from sklearn.tree import DecisionTreeClassifier
df = pd.read_csv(".. /input/train.csv" )<data_type_conversions>
from fastai import *
from fastai.tabular import *
from fastai.callbacks import *
df.has_cactus = df.has_cactus.astype(str )<split>
path1 = "/kaggle/input/covid19-global-forecasting-week-4"
path2 = "/kaggle/input/covid19-demographic-predictors"
path3 = "/kaggle/input/covid19-country-data-wk3-release"
path4 = "/kaggle/input/countryinfo"
part = np.random.rand(len(df)) <validation_split
train = df[part]
validation = df[~part]
print(validation_split)
 len(df )<create_dataframe>
train_df = pd.read_csv(f"{path1}/train.csv", parse_dates=['Date'])
 test_df = pd.read_csv(f"{path1}/test.csv", parse_dates=['Date'] )
train_datagen = ImageDataGenerator(rescale=1/255, horizontal_flip=True, vertical_flip=True)
 train_generator = train_datagen.flow_from_dataframe(train, directory=data_dir, x_col=x_col, y_col=y_col, target_size=target_size, color_mode=color_mode, class_mode=class_mode, batch_size=batch_size, shuffle=True)
 validation_generator = train_datagen.flow_from_dataframe(validation, directory=data_dir, x_col=x_col, y_col=y_col, target_size=target_size, color_mode=color_mode, class_mode=class_mode, batch_size=batch_size, shuffle=True )<load_pretrained>
add_datepart(train_df, 'Date', drop=False )
class_weights = class_weight.compute_class_weight('balanced', np.unique(train_generator.classes), train_generator.classes)
 print(class_weights )<choose_model_class>
add_datepart(test_df, 'Date', drop=False )
model = Sequential()
model.add(Conv2D(32,(3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32,(3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(Conv2D(64,(3, 3),padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(Conv2D(128,(3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(Conv2D(256,(3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D())
 model.add(GlobalAveragePooling2D())
 model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(dropout_dense_layer))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss=keras.losses.binary_crossentropy, optimizer='adam', metrics=['accuracy'])
 model.summary()<choose_model_class>
missed = "NA"
def State(state, country):
if state == missed: return country
return state
callbacks = [EarlyStopping(monitor='val_loss', patience=20),
ReduceLROnPlateau(patience=10, verbose=1),
ModelCheckpoint(filepath='best_model.h5', monitor='val_loss',
verbose=0, save_best_only=True)] 
<train_model>
metadata_df = pd.read_csv(f"{path3}/Data Join - RELEASE.csv", thousands=',' )
history = model.fit_generator(train_generator,
validation_data=validation_generator,
epochs=100,
verbose=1,
shuffle=True,
callbacks=callbacks,
class_weight=class_weights )<load_pretrained>
country_df = pd.read_csv(f"{path4}/covid19countryinfo.csv",thousands=",", parse_dates=['quarantine', 'schools', 'publicplace', 'gathering', 'nonessential'] )
model.load_weights("best_model.h5")
 history.history['val_acc'][np.argmin(history.history['val_loss'])]<define_variables>
testinfo = pd.read_csv(f'{path4}/covid19tests.csv', thousands="," )
test_folder = ".. /input/test/"
test_datagen = ImageDataGenerator(
 rescale=1./ 255)
 test_generator = test_datagen.flow_from_directory(
 directory=test_folder,
target_size=target_size,
batch_size=1,
class_mode=None,
shuffle=False )<save_to_csv>
country_df.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
 testinfo.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
 testinfo = testinfo.drop(['alpha3code', 'alpha2code', 'date'], axis=1 )
submission_file = pd.read_csv('.. /input/sample_submission.csv')
 filenames = [path.split('/')[-1] for path in test_generator.filenames]
probabilities = list(model.predict_generator(test_generator)[:,0])
 submission_file.id = filenames
submission_file.has_cactus = probabilities
submission_file.to_csv('submission_file.csv', index=False )<install_modules>
group_cols = ['Country_Region', 'Province_State']
!pip install autokeras
!pip install natsort<set_options>
group = train_df[train_df["Fatalities"] > 1].groupby(group_cols)
 res_df =(group.Fatalities.last() / group.ConfirmedCases.last() ).reset_index()
fatalities = res_df.rename(columns={0 : 'FatalityRate'} )
import autokeras as ak
import numpy as np
import pandas as pd
from glob import glob
from skimage.io import imread
import skimage.io as sio
import os
from natsort import natsorted
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score, roc_auc_score
from skimage.transform import resize, rotate
import warnings; warnings.filterwarnings("ignore" )<load_from_csv>
group = train_df[train_df['ConfirmedCases'] >= 1].groupby(group_cols)
 first_confirmed = group.Dayofyear.first().reset_index().rename(columns={'Dayofyear': "First_Confirmed"} )
root_dir = ".. /input"
train_dir = root_dir + "/train/train/"
test_dir = root_dir + "/test/test/"
csv_path = root_dir + "/train.csv"
sub_path = root_dir + "sample_submission.csv"
df = pd.read_csv(csv_path)
 x = np.array([ imread(train_dir+p)/255 for p in df.id.values])
 y = df.has_cactus.values<split>
group = train_df[train_df['ConfirmedCases'] >= 50].groupby(group_cols)
 first_50= group.Dayofyear.first().reset_index().rename(columns={'Dayofyear': "First_50"} )
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.20,stratify=y )<train_model>
group = train_df[train_df['ConfirmedCases'] >= 100].groupby(group_cols)
 first_hundred = group.Dayofyear.first().reset_index().rename(columns={'Dayofyear': "First_Hundred"} )
runFor = 5
model = ak.ImageClassifier(verbose=True, augment=True)
 model.fit(x_train, y_train, time_limit=4*60*60 )<compute_test_metric>
train_df = pd.merge(train_df, metadata_df, how='left')
 train_df = pd.merge(train_df, country_df, how='left')
 train_df = pd.merge(train_df, testinfo, how='left', left_on=group_cols, right_on=group_cols )
y_pred = model.predict(x_train)
 y_prob = getProb(model, x_train)
 print("training accuracy = ", accuracy_score(y_train, y_pred))
print("training recall = ", recall_score(y_train, y_pred))
print("training precision = ", precision_score(y_train, y_pred))
print("training auc score = ", roc_auc_score(y_train, y_prob))
print("training f1 score = ", f1_score(y_train, y_pred))
y_pred = model.predict(x_val)
 y_prob = getProb(model, x_val)
 print("validation accuracy = ", accuracy_score(y_val, y_pred))
print("validation recall = ", recall_score(y_val, y_pred))
print("validation precision = ", precision_score(y_val, y_pred))
print("validation auc score = ",roc_auc_score(y_val, y_prob))
print("validation f1 score = ", f1_score(y_val, y_pred))<save_to_csv>
train_df = pd.merge(train_df, fatalities, how='left')
 train_df = pd.merge(train_df, first_confirmed, how='left')
 train_df = pd.merge(train_df, first_50, how='left')
 train_df = pd.merge(train_df, first_hundred, how='left' )
df_test = pd.read_csv('.. /input/sample_submission.csv')
 x_test = np.array([ imread(test_dir+p)/255 for p in df_test.id.values])
 x_test = np.array(x_test)
 y_prob_test = getProb(model, x_test)
 df_test['has_cactus'] = y_prob_test
df_test.to_csv('cactus_net_submission.csv', index=False )<import_modules>
test_df = pd.merge(test_df, metadata_df, how='left')
 test_df = pd.merge(test_df, country_df, how='left')
 test_df = pd.merge(test_df, testinfo, how='left', left_on=group_cols, right_on=group_cols)
 test_df = pd.merge(test_df, fatalities, how='left')
 test_df = pd.merge(test_df, first_confirmed, how='left')
 test_df = pd.merge(test_df, first_50, how='left')
 test_df = pd.merge(test_df, first_hundred, how='left' )
from sklearn.model_selection import train_test_split
from keras import layers, models
from keras.utils import to_categorical
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.optimizers import Adam, Adamax
from keras.regularizers import l2
import glob
import cv2<define_variables>
for df in [train_df, test_df]:
df['Province_State'].fillna(missed, inplace=True)
 df['Province_State'] = df.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : State(x['Province_State'], x['Country_Region']), axis=1)
 df.loc[:, 'Date'] = df.Date.dt.strftime("%m%d")
 df["Date"] = df["Date"].astype(int )
path=".. /input/"
train_path=path+"train/train/"
test_path=path+"test/test/"<load_pretrained>
le = preprocessing.LabelEncoder()
for df in [train_df, test_df]:
df['Country_Region'] = le.fit_transform(df['Country_Region'])
 df['Province_State'] = le.fit_transform(df['Province_State'] )
files = sorted(glob.glob(train_path + '*.jpg'))
files[:3]
train_raw = np.array([cv2.imread(image)
 for image in files], dtype='int32' )<load_pretrained>
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 features = ['Country_Region', 'Province_State', 'Date']
for country in range(len(countries)) :
country_train = train_df.loc[train_df['Country_Region'] == countries[country]]
country_test = test_df.loc[test_df['Country_Region'] == countries[country]]
states = train_df.loc[train_df['Country_Region'] == countries[country],:].Province_State.unique()
for state in states:
train = country_train.loc[country_train['Province_State'] == state]
X_train = train[features].to_numpy()
y1_train = train[['ConfirmedCases']].to_numpy()
y2_train = train[['Fatalities']].to_numpy()
test = country_test.loc[country_test['Province_State'] == state]
X_test = test[features].to_numpy()
model1 = DecisionTreeClassifier()
model1.fit(X_train, y1_train)
 pred1 = np.round(model1.predict(X_test))
model2 = DecisionTreeClassifier()
model2.fit(X_train, y2_train)
 pred2 = np.round(model2.predict(X_test))
country_test_Id = test.loc[:, 'ForecastId']
pred = pd.DataFrame({'ForecastId': country_test_Id, 'ConfirmedCases': pred1, 'Fatalities': pred2})
 sub = pd.concat([sub, pred], axis=0 )
<define_variables><EOS> 
sub.ForecastId = sub.ForecastId.astype('int')
 sub.to_csv('submission.csv', index=False )