Kaggle competition: COVID19 Global Forecasting (Week 2)
bureau_balance_by_client = agg_numeric(bureau_by_loan.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'client')
 bureau_balance_by_client.head()<merge>
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-2'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv' )
train = train.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left' )<train_model>
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True )
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )<set_options>
df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )
gc.enable()
del app_train, app_test, app_train_domain, app_test_domain, bureau, bureau_balance, bureau_agg, bureau_balance_agg, bureau_balance_counts, bureau_by_loan, bureau_balance_by_client
gc.collect()<drop_column>
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
def train_with_cv(train_data, test_data, n_folds, seed_varying):
train_ids = train_data['SK_ID_CURR']
test_ids = test_data['SK_ID_CURR']
train_labels = train_data['TARGET']
train_features = train_data.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_data.drop(columns = ['SK_ID_CURR'])
 feature_names = list(train_features.columns)
 feature_importance_values = np.zeros(len(feature_names))
train_features = np.array(train_features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 50+seed_varying)
 test_pred = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(train_features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(train_features):
x_train, y_train = train_features[train_indices], train_labels[train_indices]
x_valid, y_valid = train_features[valid_indices], train_labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1,
subsample = 0.8, n_jobs = -1, random_state = 50+seed_varying)
 model.fit(x_train, y_train, eval_metric = 'auc',
eval_set = [(x_valid, y_valid),(x_train, y_train)],
eval_names = ['valid', 'train'], categorical_feature = 'auto',
early_stopping_rounds = 100, verbose = -1)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_pred += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(x_valid, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, x_train, y_train, x_valid, y_valid
gc.collect()
pred_score = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_pred})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(train_labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return pred_score, feature_importances, metrics<train_on_grid>
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()
train_times = 3
n_folds = 5
i = 0
metrics_all = np.zeros(( train_times, 2))
for seed_varying in range(train_times):
print('
=======================================================')
 print('The ', seed_varying, ' time of train')
 print('
=======================================================')
 sub, fi, metrics = train_with_cv(train, test, n_folds, seed_varying)
 if i==0:
submission = sub
feat_
else:
submission['TARGET'] += sub['TARGET']
feat_import['importance'] += fi['importance']
metrics_all[i, :] = metrics.iloc[-1, 1:3]
i += 1
metrics_all_average = metrics_all.mean(axis = 0)
 metrics_all = np.row_stack([metrics_all, metrics_all_average])
 train_time_names = list(range(train_times))
train_time_names.append('Average')
 metrics_final = pd.DataFrame({'train_time': train_time_names,
'train': metrics_all[:,0],
'valid': metrics_all[:,1]})
 submission['TARGET'] = submission['TARGET'] / train_times
feat_import['importance'] = feat_import['importance'] / train_times
submission.to_csv('lightgbm_version_10.csv', index = False )<set_options>
X_Test = df_test.copy()
X_Test['State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head()
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
<load_pretrained>
le = preprocessing.LabelEncoder()
CACHE_PATH = '.. /input/mlp012003weights'
def save_pickle(dic, save_path):
with open(save_path, 'wb')as f:
pickle.dump(dic, f)
 def load_pickle(load_path):
with open(load_path, 'rb')as f:
message_dict = pickle.load(f)
 return message_dict
f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy' )<define_variables>
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()
feat_cols = [f'feature_{i}' for i in range(130)]
all_feat_cols = [col for col in feat_cols]
all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])
 target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']<import_modules>
X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 X_Test.head()
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torch.nn import CrossEntropyLoss, MSELoss
from torch.nn.modules.loss import _WeightedLoss
import torch.nn.functional as F<choose_model_class>
df_train.loc[df_train.Country == 'Afghanistan', :]
class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))
self.dropout0 = nn.Dropout(0.9)
 dropout_rate = 0.9
hidden_size = 256
self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)
 self.batch_norm1 = nn.BatchNorm1d(hidden_size)
 self.dropout1 = nn.Dropout(dropout_rate)
 self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropout_rate)
 self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropout_rate)
 self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(dropout_rate)
 self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm0(x)
 x = self.dropout0(x)
 x1 = self.dense1(x)
 x1 = self.batch_norm1(x1)
 x1 = self.LeakyReLU(x1)
 x1 = self.dropout1(x1)
 x = torch.cat([x, x1], 1)
 x2 = self.dense2(x)
 x2 = self.batch_norm2(x2)
 x2 = self.LeakyReLU(x2)
 x2 = self.dropout2(x2)
 x = torch.cat([x1, x2], 1)
 x3 = self.dense3(x)
 x3 = self.batch_norm3(x3)
 x3 = self.LeakyReLU(x3)
 x3 = self.dropout3(x3)
 x = torch.cat([x2, x3], 1)
 x4 = self.dense4(x)
 x4 = self.batch_norm4(x4)
 x4 = self.LeakyReLU(x4)
 x4 = self.dropout4(x4)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 return x<set_options>
filterwarnings('ignore' )
if torch.cuda.is_available() :
print('using device: cuda')
 torch.device("cuda:0")
 else:
print('using device: cpu')
 device = torch.device('cpu' )<load_pretrained>
le = preprocessing.LabelEncoder()
NFOLDS = 5
model_list = []
tmp = np.zeros(len(feat_cols))
for _fold in range(NFOLDS):
torch.cuda.empty_cache()
model = Model()
model.to(device)
 model_weights = f"{CACHE_PATH}/online_model{_fold}.pth"
model.load_state_dict(torch.load(model_weights, map_location=device))
model.eval()
model_list.append(model )<import_modules>
from xgboost import XGBRegressor
import lightgbm as lgb
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import tensorflow_addons as tfa<choose_model_class>
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out2.ForecastId = df_out2.ForecastId.astype('int' )
SEED = 1111
np.random.seed(SEED)
 def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
epochs = 200
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.8, 0.6, 0.4, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(len(feat_cols), 5, hidden_units, dropout_rates, label_smoothing, learning_rate )<load_pretrained>
df_out['ConfirmedCases'] =(1/2)*(df_out['ConfirmedCases'] + df_out2['ConfirmedCases'])
 df_out['Fatalities'] =(1/2)*(df_out['Fatalities'] + df_out2['Fatalities'] )
clf.load_weights('.. /input/jane-street-with-keras-nn-overfit/model.h5')
<categorify>
df_out['ConfirmedCases'] = df_out['ConfirmedCases'].round().astype(int)
 df_out['Fatalities'] = df_out['Fatalities'].round().astype(int )
<import_modules><EOS> 
df_out.to_csv('submission.csv', index=False )