Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<concatenate>
print("Read in libraries")
data_all = pd.concat([train_features, test_features], ignore_index=True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ['sig_id', 'drug_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(data_all[cols_numeric].var() >= VAR_THRESHOLD ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[['sig_id', 'drug_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )<categorify>
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 sub = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 print("read in files")
 test.head()
def preprocess(df):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 df = pd.get_dummies(df, columns=['cp_time','cp_dose'])
 return df
data_all = preprocess(data_all )<count_values>
train = train.append(test[test['Date']>'2020-03-31'] )
train_features['drug_id'].value_counts()<merge>
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d')
 train['day_dist'] = train['Date']-train['Date'].min()
train['day_dist'] = train['day_dist'].dt.days
print(train['Date'].max())
 print(test['Date'].min())
 print(test['Date'].max())
 cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True)
 train['place'] = train['Province_State']+'_'+train['Country_Region']
def assign_folds(train, train_targets, targets, seed=SEED):
scored = train_targets.copy()
scored = pd.merge(scored, train[['sig_id', 'drug_id']], how='left', on='sig_id')
 vc = scored['drug_id'].value_counts()
vc1 = vc.loc[vc <= 18].index.sort_values()
vc2 = vc.loc[vc > 18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=seed)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=seed)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored['drug_id'].map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored['fold'].values
folds = assign_folds(train_features, train_targets, train_targets.columns.values[1:].tolist() , SEED)
 pd.DataFrame(folds ).value_counts()<define_variables>
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col])
 train.keys()
drop_cols = ['Id', 'ConfirmedCases','Date', 'ForecastId','Fatalities','day_dist', 'Province_State', 'Country_Region']
n = 0
tr = np.where(folds != n)[0]
te = np.where(folds == n)[0]
print(tr )<categorify>
val = train[(train['Date']>='2020-03-19')&(train['Id'].isnull() ==False)]
val
def target_category_maker(targets):
targets_category = {'agonist': [], 'antagonist': [], 'agent': [], 'others': []}
for t in targets:
if('_agonist' in t)|('_activator' in t)|('_stimulant' in t)|('_secretagogue' in t)|('_sensitizer' in t):
targets_category['agonist'].append(t)
 elif('_antagonist' in t)|('_inhibitor' in t)|('_blocker' in t):
targets_category['antagonist'].append(t)
 elif('_agent' in t)|('_medium' in t):
targets_category['agent'].append(t)
 else:
targets_category['others'].append(t)
 return targets_category
targets_category = target_category_maker(targets)
 non_targets_category = target_category_maker(non_targets )<define_variables>
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"]
g_feats = [f for f in data_all.columns.values.tolist() if f.startswith('g-')]
c_feats = [f for f in data_all.columns.values.tolist() if f.startswith('c-')]
cp_feats = [f for f in data_all.columns.values.tolist() if f.startswith('cp_')]
print(len(g_feats), len(c_feats), len(cp_feats))<feature_engineering>
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)))
 def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1))
%%time
def add_stats_feats(df, feat_list, n):
df[f'{n}stats-mean'] = df[feat_list].mean(axis=1)
 df[f'{n}stats-std'] = df[feat_list].std(axis=1)
 df[f'{n}stats-skew'] = df[feat_list].skew(axis=1)
 df[f'{n}stats-kurt'] = df[feat_list].kurt(axis=1)
 df[f'{n}stats-mad'] = df[feat_list].mad(axis=1)
 return df
data_all = add_stats_feats(data_all, g_feats, 'g')
 data_all = add_stats_feats(data_all, c_feats, 'c')
 data_all = add_stats_feats(data_all, g_feats+c_feats, 'gc' )<normalization>
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
}
%%time
scaler = StandardScaler()
data_all[g_feats+c_feats] = scaler.fit_transform(data_all[g_feats+c_feats])
 def dim_reducer(data_all, feats, n_components=100):
trans = PCA(n_components=n_components, random_state=SEED)
 train_dist = trans.fit_transform(data_all[feats].values)
 return train_dist
train_g = dim_reducer(data_all, g_feats, n_components=N_COMPONENTS[0])
 train_c = dim_reducer(data_all, c_feats, n_components=N_COMPONENTS[1])
 for i in range(train_g.shape[1]):
data_all[f'g-pca{i+1}'] = train_g[:, i]
for i in range(train_c.shape[1]):
data_all[f'c-pca{i+1}'] = train_c[:, i]<define_variables>
dates = dates[dates>'2020-03-31']
dates
feats = data_all.columns.values.tolist()
drops = ['sig_id', 'cp_type', 'drug_id']
feats = [f for f in feats if f not in drops]
print('{:,} features'.format(len(feats)))
 print(feats )<normalization>
train.isnull().sum()
%%time
t_feats = [f for f in feats if 'cp_' not in f]
pt = GaussRankScaler()
data_all[t_feats] = pt.fit_transform(data_all[t_feats])
 scaler = StandardScaler()
data_all[t_feats] = scaler.fit_transform(data_all[t_feats] )<split>
train.isnull().sum()
train = data_all.iloc[:len(train_features)]
test = data_all.iloc[len(train_features):]
del data_all
gc.collect()<define_variables>
test[test['Country_Region']=='Italy']
def seed_everything(seed : int)-> NoReturn :
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )<compute_test_metric>
test[test['Country_Region']=='Italy']
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -K.mean(y_true*K.log(y_pred)+(1-y_true)*K.log(1-y_pred))
def create_model(num_feats):

 inp = layers.Input(shape=(num_feats,), name="inp")
 x = layers.Dense(NUM_NEURON, activation=AF )(inp)
 if NN_NORM == 'layer':
x = layers.LayerNormalization()(x)
 elif NN_NORM == 'batch':
x = layers.BatchNormalization()(x)
 x = layers.Dropout(DROPOUT )(x)
 for i in range(NUM_HIDDEN_LAYER):
x = layers.Dense(NUM_NEURON // DECAY_FACTOR, activation=AF )(inp)
 if NN_NORM == 'layer':
x = layers.LayerNormalization()(x)
 elif NN_NORM == 'batch':
x = layers.BatchNormalization()(x)
 x = layers.Dropout(DROPOUT )(x)
 preds = layers.Dense(206, activation='sigmoid' )(x)
 model = models.Model(inp, preds)
 opt = optimizers.Adam(lr=LR)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001)
 model.compile(loss=loss, optimizer=opt, metrics=logloss)
 return model<compute_test_metric>
train_sub = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv" )
def metric(y_true, y_pred, targets):
metrics = []
for _target in targets:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels=[0,1]))
return np.mean(metrics )<train_model>
def fit_nfold(train, test, ss, train_targets, feats, targets, seed=SEED, n_splits=N_SPLITS):
res = train_targets.copy()
ss.loc[:, targets] = 0
res.loc[:, targets] = 0
folds = assign_folds(train, train_targets, targets, seed=seed)
 historys = dict()
for n in range(n_splits):
tr = np.where(folds != n)[0]
te = np.where(folds == n)[0]
print(f"======{train_targets.values[tr].shape}========{train_targets.values[te].shape}=====")
 print(f'Seed: {seed} => Fold: {n}')
 if DEBUG:
if n > 0:
print(f'Skip fold{n}')
 continue
model = create_model(len(feats))
checkpoint_path = f'repeat{seed}_fold{n}.hdf5'
reduce_lr_loss = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-5, patience=4, verbose=VERBOSE, mode='min')
 cb_checkpt = callbacks.ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 2, save_best_only = True,
save_weights_only = True, mode = 'min')
 early = callbacks.EarlyStopping(monitor="val_loss", mode="min", restore_best_weights=True, patience=8, verbose = VERBOSE)
 nn_callbacks = [reduce_lr_loss, cb_checkpt, early]
x_train = train[feats].values[tr]
x_val = train[feats].values[te]
x_test = test[feats].values
y_train = train_targets[targets].values[tr]
y_val = train_targets[targets].values[te]
history = model.fit(x_train, y_train,
validation_data=(x_val, y_val),
epochs=EPOCHS, batch_size=BATCH_SIZE,
callbacks=nn_callbacks, verbose=VERBOSE 
)
 historys[f'history_{n}'] = history
model.load_weights(checkpoint_path)
 test_predict = model.predict(x_test)
 val_predict = model.predict(x_val)
 res.loc[te, targets] = val_predict
ss.loc[test_g, targets] += test_predict / n_splits
print(f'OOF Metric For SEED {seed} => FOLD {n} : {metric(train_targets.loc[te, targets], pd.DataFrame(val_predict, columns=targets), targets)}')
 print('+-' * 10)
 print(f'OOF Metric: {metric(train_targets[targets], res[targets], targets)}')
 return ss, res, historys<categorify>
test = pd.merge(test,train_sub[['Province_State','Country_Region','Date','ConfirmedCases','Fatalities']], on=['Province_State','Country_Region','Date'], how='left' )
res = train_targets.copy()
res.loc[:, targets] = 0
ss.loc[:, targets] = 0
for s in range(N_STARTS):
ss_tmp = ss.copy()
ss_, res_, historys = fit_nfold(train, test, ss_tmp, train_targets, feats, targets, seed=SEED+s**2, n_splits=N_SPLITS)
 ss.loc[:, targets] += ss_[targets].values / N_STARTS
res.loc[:, targets] += res_[targets].values / N_STARTS<save_to_csv>
test.loc[test['ConfirmedCases_x'].isnull() ==True]
print(f'OOF Metric: {metric(train_targets, res, targets)}')
 if POSTPROCESS:
print('post-process...')
 ss.iloc[:,1:] = np.clip(ss.values[:, 1:], p_min, p_max)
 ss.iloc[control_g, 1:] = 0
ss.to_csv('submission.csv', index=False)
 print(ss.shape)
 ss.head()<load_from_csv>
test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_x'] = test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_y']
X_train = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 X_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 y_train = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv' )<prepare_x_and_y>
test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_x'] = test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_y']
X_train.set_index('sig_id', inplace=True)
 X_test.set_index('sig_id', inplace=True)
 y_train.set_index('sig_id', inplace=True )<categorify>
X_train.cp_time = X_train.cp_time // 24
X_train.cp_dose = X_train.cp_dose.map({'D1': 0, 'D2': 1})
 X_test.cp_time = X_test.cp_time // 24
X_test.cp_dose = X_test.cp_dose.map({'D1': 0, 'D2': 1} )<drop_column>
sub = test[['ForecastId', 'ConfirmedCases_x','Fatalities_x']]
X_train_moa = X_train[X_train.cp_type != 'ctl_vehicle'].drop(columns=['cp_type'])
 X_test_moa = X_test[X_test.cp_type != 'ctl_vehicle'].drop(columns=['cp_type'])
 y_train_moa = y_train.loc[X_train_moa.index]<compute_train_metric>
sub.columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']
def mean_predictions(models_dense, X):
y_pred_dense = [model.predict(X)for model in models_dense]
return np.mean(y_pred_dense, axis=0)
 def macro_log_loss(y_true, y_pred):
if len(y_true.shape)== 1:
return log_loss(y_true, y_pred, labels=[0, 1]), [log_loss(y_true, y_pred, labels=[0, 1])]
y_pred = np.maximum(np.minimum(y_pred, [[1 - 1e-15] * y_pred.shape[1]] * y_pred.shape[0]), [[1e-15] * y_pred.shape[1]] * y_pred.shape[0])
 losses = [log_loss(y_true[:, i], y_pred[:, i], labels=[0, 1])for i in range(y_true.shape[1])]
return np.mean(losses), losses<choose_model_class>
sub.loc[sub['ConfirmedCases']<0, 'ConfirmedCases'] = 0
def create_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(N_TARGETS, activation="sigmoid"))
])
 model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = LR), loss='binary_crossentropy', metrics=["accuracy"])
 return model<choose_model_class>
sub.loc[sub['Fatalities']<0, 'Fatalities'] = 0
def create_dense_model(input_shape, output_shape):
inputs = keras.Input(shape=(input_shape,), name='drug')
 x = layers.BatchNormalization()(inputs)
 x = layers.Dropout(0.3 )(x)
 x = tfa.layers.WeightNormalization(layers.Dense(units=1024, activation='relu', name='dense_1'))(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dropout(0.4 )(x)
 x = tfa.layers.WeightNormalization(layers.Dense(units=1024, activation='relu', name='dense_2'))(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dropout(0.3 )(x)
 output = tfa.layers.WeightNormalization(layers.Dense(output_shape, activation='sigmoid', kernel_regularizer=keras.regularizers.l2(l2=1e-5),
name='predictions'))(x)
 model = keras.Model(inputs=inputs, outputs=output)
 opt = tfa.optimizers.AdamW(weight_decay=1e-5, learning_rate=1e-2)
 model.compile(optimizer=opt, loss='binary_crossentropy')
 return model<choose_model_class>
callbacks = [
keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5,
patience=3,
verbose=0),
keras.callbacks.ReduceLROnPlateau(
 monitor="val_loss",
factor=0.1,
patience=2)
 ]<train_model>
print("Read in libraries")
 print("read in train file")
 df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv",
usecols=['Province_State','Country_Region','Date','ConfirmedCases','Fatalities'])
 print("fill blanks and add region for counting")
 df.fillna(' ',inplace=True)
 df['Lat']=df['Province_State']+df['Country_Region']
df.drop('Province_State',axis=1,inplace=True)
 df.drop('Country_Region',axis=1,inplace=True)
 countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2)
 df_submit=pd.concat([pd.Series(np.arange(1,1+len(submit_confirmed))),pd.Series(submit_confirmed),pd.Series(submit_fatal)],axis=1)
 df_submit=df_submit.fillna(method='pad' ).astype(int)
 df_submit.head()
df_submit.rename(columns={0: 'ForecastId', 1: 'ConfirmedCases',2: 'Fatalities',}, inplace=True )
<compute_test_metric><EOS> 
sub.rename(columns={'ForecastId': 'ForecastId1', "ConfirmedCases": 'Confirmed_lgbt',"Fatalities": 'Fatalities_lgbt',}, inplace=True)
 df_submit.rename(columns={'ForecastId': 'ForecastId2', "ConfirmedCases": 'Confirmed_arima',"Fatalities": 'Fatalities_arima',}, inplace=True)
 df_submit.shape
df_combine = pd.concat([sub, df_submit], axis=1, join='inner')
 cols = ['Confirmed_lgbt','Confirmed_arima']
df_combine['ConfirmedCases'] = df_combine[cols].astype(float ).mean(axis=1)
 cols = ['Fatalities_lgbt','Fatalities_arima']
df_combine['Fatalities'] = df_combine[cols].astype(float ).mean(axis=1)
 del df_combine['ForecastId2']
del df_combine['Confirmed_lgbt']
del df_combine['Fatalities_lgbt']
del df_combine['Confirmed_arima']
del df_combine['Fatalities_arima']
df_combine.head()
df_combine.rename(columns={'ForecastId1': 'ForecastId', "ConfirmedCases": 'ConfirmedCases',"Fatalities": 'Fatalities',}, inplace=True)
 df_combine.to_csv('submission.csv',index=False)
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 complete_test= pd.merge(test, df_combine, how="left", on="ForecastId")
 complete_test.to_csv('complete_test.csv',index=False)