Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<concatenate>
import numpy as np
import pandas as pd
import lightgbm as lgb
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
train_seq = np.hstack([train_seq, train_num_features])
 test_seq = np.hstack([test_seq, test_num_features] )<set_options>
train = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv')
 world_population = pd.read_csv('.. /input/population-by-country-2020/population_by_country_2020.csv' )
gc.collect()
gc.disable()<set_options>
world_population = world_population[['Country(or dependency)', 'Population(2020)']]
world_population.columns = ['Country(or dependency)', 'Population']
world_population.loc[world_population['Country(or dependency)']=='United States', 'Country(or dependency)'] = 'US'
train = train.merge(world_population, left_on='Country_Region', right_on='Country(or dependency)', how='left')
 test = test.merge(world_population, left_on='Country_Region', right_on='Country(or dependency)', how='left' )
if 1:
def get_model() :
session_conf = tf.ConfigProto(intra_op_parallelism_threads=4, inter_op_parallelism_threads=4)
 K.set_session(tf.Session(graph=tf.get_default_graph() , config=session_conf))
filter_nr = 32
filter_size = 3
max_pool_size = 3
max_pool_strides = 2
dense_nr = 64
spatial_dropout = 0.2
dense_dropout = 0.05
train_embed = False
project = Input(shape=(project_maxlen,), name='project')
 emb_project = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=train_embed )(project)
 emb_project = SpatialDropout1D(spatial_dropout )(emb_project)
 pj_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(emb_project)
 pj_block1 = BatchNormalization()(pj_block1)
 pj_block1 = PReLU()(pj_block1)
 pj_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(pj_block1)
 pj_block1 = BatchNormalization()(pj_block1)
 pj_block1 = PReLU()(pj_block1)
 pj_resize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear' )(emb_project)
 pj_resize_emb = PReLU()(pj_resize_emb)
 pj_block1_output = add([pj_block1, pj_resize_emb])
 for _ in range(pj_repeat):
pj_block1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides )(pj_block1_output)
 pj_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(pj_block1_output)
 pj_block2 = BatchNormalization()(pj_block2)
 pj_block2 = PReLU()(pj_block2)
 pj_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(pj_block2)
 pj_block2 = BatchNormalization()(pj_block2)
 pj_block2 = PReLU()(pj_block2)
 pj_block1_output = add([pj_block2, pj_block1_output])
 resouse = Input(shape=(resouse_max_len,), name='resouse')
 emb_resouse = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=train_embed )(resouse)
 emb_resouse = SpatialDropout1D(spatial_dropout )(emb_resouse)
 rs_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(emb_resouse)
 rs_block1 = BatchNormalization()(rs_block1)
 rs_block1 = PReLU()(rs_block1)
 rs_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(rs_block1)
 rs_block1 = BatchNormalization()(rs_block1)
 rs_block1 = PReLU()(rs_block1)
 rs_resize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear' )(emb_resouse)
 rs_resize_emb = PReLU()(rs_resize_emb)
 rs_block1_output = add([rs_block1, rs_resize_emb])
 for _ in range(rs_repeat):
rs_block1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides )(rs_block1_output)
 rs_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(rs_block1_output)
 rs_block2 = BatchNormalization()(rs_block2)
 rs_block2 = PReLU()(rs_block2)
 rs_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(rs_block2)
 rs_block2 = BatchNormalization()(rs_block2)
 rs_block2 = PReLU()(rs_block2)
 rs_block1_output = add([rs_block2, rs_block1_output])
 pj_output = GlobalMaxPooling1D()(pj_block1_output)
 pj_output = BatchNormalization()(pj_output)
 rs_output = GlobalMaxPooling1D()(rs_block1_output)
 rs_output = BatchNormalization()(rs_output)
 inp_num = Input(shape=(train_seq.shape[1]-maxlen,), name='num_input')
 bn_inp_num = BatchNormalization()(inp_num)
 conc = concatenate([pj_output, rs_output, bn_inp_num])
 output = Dense(dense_nr, activation='linear' )(conc)
 output = BatchNormalization()(output)
 output = PReLU()(output)
 output = Dropout(dense_dropout )(output)
 output = Dense(1, activation='sigmoid' )(output)
 model = Model(inputs=[project, resouse, inp_num], outputs=output)
 model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['accuracy'])
 return model<split>
train['State_Country'] = [s + '_' + c if s == s else c for s,c in train[['Province_State', 'Country_Region']].values ]
test['State_Country'] = [s + '_' + c if s == s else c for s,c in test[['Province_State', 'Country_Region']].values ]
class qiaofeng_dpcnn(qiaofeng_kfold_stack):
def model_fit_predict(self, train, train_target, valid):
self.model = get_model()
early_stopping = EarlyStopping(monitor='roc_auc_val', patience=1, mode='max',min_delta=0.0005)
 X_tra, X_val, y_tra, y_val = train_test_split(train, train_target, train_size=0.98, random_state=233)
 X_tra = { 'project' : X_tra[:,:project_maxlen], 'resouse' : X_tra[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : X_tra[:,maxlen:] }
X_val = { 'project' : X_val[:,:project_maxlen], 'resouse' : X_val[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : X_val[:,maxlen:] }
x_test = { 'project' : self.test[:,:project_maxlen], 'resouse' : self.test[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : self.test[:,maxlen:] }
valid = { 'project' : valid[:,:project_maxlen], 'resouse' : valid[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : valid[:,maxlen:] }
RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)
 hist = self.model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),
callbacks=[RocAuc, early_stopping], verbose=2)
 predict_train = None
predict_valid = self.model.predict(valid, batch_size=1024)[:, 0]
predict_test = self.model.predict(x_test, batch_size=1024)[:, 0]
return predict_train, predict_valid, predict_test
dpcnn_kfold_model = qiaofeng_dpcnn(train=train_seq, train_target=train_target, test=test_seq, kfolds=dpcnn_folds, split_target=train_target,
score_func=roc_auc_score, logger=print, model=None)
 dpcnn_kfold_model.model_train_with_kfold()
pred_valid_dpcnn = dpcnn_kfold_model.predict_valid_kfolds
pred_test_avg_dpcnn = dpcnn_kfold_model.predict_test_mean()
del dpcnn_kfold_model
gc.enable()
gc.collect()
gc.disable()<normalization>
train.loc[(train['Date']=='2020-03-24')&(train['State_Country']=='France'),'ConfirmedCases'] = 22654
train.loc[(train['Date']=='2020-03-24')&(train['State_Country']=='France'),'Fatalities'] = 1000
scaler = MinMaxScaler()
predict_valid_list = [pred_valid_rf, pred_valid_lgb, pred_valid_dpcnn, pred_valid_ftrl]
predict_test_list = [pred_test_avg_rf, pred_test_avg_lgb, pred_test_avg_dpcnn, pred_test_avg_ftrl]
valid_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_valid_list])
 test_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_test_list])
 train_features = np.hstack([valid_results ])
 test_features = np.hstack([test_results ])
 lgb_model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=400,
num_leaves=10,
boosting_type="gbdt",
learning_rate=0.01,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.4,
bagging_freq=5,
reg_lambda=0.2)
 X_tra, X_val, y_tra, y_val = train_test_split(train_features, train_target, train_size=0.8, random_state=233)
 lgb_model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
verbose=False)
 print('Valid Score is %.4f' % roc_auc_score(y_val, lgb_model.predict_proba(X_val)[:,1]))
final_predict = lgb_model.predict_proba(test_features)[:,1]
if quick_run == False:
sample_df = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 sample_df['project_is_approved'] = final_predict
sample_df.to_csv('submission.csv', index=False )<normalization>
for metric in ['ConfirmedCases', 'Fatalities']:
dict_values = train.groupby('State_Country')[metric].apply(np.array ).to_dict()
for country in dict_values:
if sum(np.diff(dict_values[country])< 0):
print(country, metric)
 new_val = [dict_values[country][-1]]
for val_1, val_2 in zip(dict_values[country][1:][::-1], dict_values[country][:-1][::-1]):
if val_2 <= new_val[-1]:
new_val += [val_2]
else:
new_val += [new_val[-1]]
new_val = np.array(new_val[::-1])
 train.loc[train.State_Country == country, metric] = new_val
scaler = MinMaxScaler()
predict_valid_list = [pred_valid_rf, pred_valid_lgb, pred_valid_dpcnn, pred_valid_ftrl, pred_valid_lgb_bool]
predict_test_list = [pred_test_avg_rf, pred_test_avg_lgb, pred_test_avg_dpcnn, pred_test_avg_ftrl, pred_test_avg_lgb_bool]
valid_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_valid_list])
 test_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_test_list])
 train_features = np.hstack([valid_results ])
 test_features = np.hstack([test_results ])
 lgb_model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=400,
num_leaves=10,
boosting_type="gbdt",
learning_rate=0.01,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.4,
bagging_freq=5,
reg_lambda=0.2)
 X_tra, X_val, y_tra, y_val = train_test_split(train_features, train_target, train_size=0.8, random_state=233)
 lgb_model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
verbose=False)
 print('Valid Score is %.4f' % roc_auc_score(y_val, lgb_model.predict_proba(X_val)[:,1]))
final_predict = lgb_model.predict_proba(test_features)[:,1]
if quick_run == False:
sample_df = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 sample_df['project_is_approved'] = final_predict
sample_df.to_csv('submission_with_lgb_bool.csv', index=False )<set_options>
train['max_case'] = train['State_Country'].map(train.groupby('State_Country' ).ConfirmedCases.max())
 train['pct_c'] = train.ConfirmedCases.pct_change()
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master/')
 %matplotlib inline
sns.set_style('ticks')
 sns.set_context("poster")
 sns.set_palette('colorblind')
 warnings.filterwarnings('ignore')
<set_options>
def predict_cc(data, country, len_predict, metrics, len_intersection):
country_data = data[data['State_Country']==country]
if country_data[metrics].values.max() > 1000:
start_people = 2
else:
start_people = 0
country_data = country_data.iloc[dict_case_date[country][start_people]:, :]
x_data = range(len(country_data.index))
y_data = country_data[metrics].values
if len(x_data)<= 1:
y_min = 0
y_max = 100
y_val = np.arange(len(x_data), len(x_data)+ len_predict)
 return [-1, -1, -1], log_curve(y_val, 0.3, 30, 100)
 else:
add_max =(1 + country_data.pct_c.values[-3:].mean())**(1 / 2)
 if y_data.max() > 10000:
add_min =(1 + country_data.pct_c.values[-3:].mean())**(1 / 5)
 elif y_data.max() > 1000:
add_min =(1 + country_data.pct_c.values[-3:].mean())**(1 / 3)
 else:
add_min =(1 + country_data.pct_c.values[-3:].mean())**(1 / 2)
 add_max =(1 + country_data.pct_c.values[-3:].mean())**(1)
 day_left = max(10, 50 - len(x_data))
y_min = y_data[-1] *(add_min ** day_left)
 if y_min > country_data['Population'].values[0] * 0.05:
y_min = country_data['Population'].values[0] * 0.05
y_max = y_data[-1] *(add_max ** day_left + 30)
 if y_max > country_data['Population'].values[0] * 0.15:
y_max = country_data['Population'].values[0] * 0.15
if add_min == add_max or add_max != add_max:
y_min = y_data[-1]
y_max = max(100, y_data[-1] *(1.3 ** 30))
if y_max > 100000 and y_data[-1] < 10000:
y_max = y_data[-1] * 10
y_min = y_data[-1]
popt, pcov = curve_fit(log_curve, x_data, y_data, bounds=([0.1, 10, y_min ],[0.35, 50, y_max]),
p0=[0.2,30,(y_min + y_max)/ 2], maxfev=10000)
 y_val = np.arange(len(x_data)- len_intersection, len(x_data)+ len_predict - len_intersection)
 return popt, log_curve(y_val, popt[0], popt[1], popt[2])
 def log_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)) )
plt.rcParams['figure.figsize'] =(20.0, 10.0 )<set_options>
metrics = 'ConfirmedCases'
dict_values = train.groupby('State_Country')[metrics].apply(np.array ).to_dict()
dict_case_date = {}
for country in dict_values:
dict_case_date[country] = []
for case in [1, 10, 100, 250, 500, 1000, 2500, 5000]:
try:
dict_case_date[country] += [np.where(dict_values[country] >= case)[0][0]]
except:
dict_case_date[country] += [-1]
dict_case_date[country] = np.array(dict_case_date[country])
 dict_predict = {}
data_train = train.copy()
data_val = test.copy()
len_predict = data_val[data_val.State_Country == country].shape[0]
len_intersection = len(set(data_train.Date.unique())& set(data_val.Date.unique()))
for country in train.State_Country.unique() :
popt, pred_values = predict_cc(data_train, country, len_predict, metrics, len_intersection)
 dict_predict[country] = pred_values
device =('cuda' if torch.cuda.is_available() else 'cpu' )<init_hyperparams>
test['ConfirmedCases'] = 0
for country in test['State_Country'].unique() :
test.loc[test.State_Country == country, 'ConfirmedCases'] = dict_predict[country]
params = {'device': device,
'n_comp_g': 450,
'n_comp_c': 45,
'var_thresh': 0.67,
'epochs': 25,
'batch_size': 128,
'lr': 1e-3,
'weight_decay': 1e-5,
'n_folds': 7,
'early_stopping_steps': 10,
'early_stop': False,
'in_size': None,
'out_size': None,
'hidden_size': 1500}<load_from_csv>
def predict_f(data, country, len_predict, metrics, len_intersection, dict_max_cc):
country_data = data[data['State_Country']==country]
if country_data[metrics].values.max() > 1000:
start_people = 2
else:
start_people = 0
country_data = country_data.iloc[dict_case_date[country][start_people]:, :]
x_data = range(len(country_data.index))
y_data = country_data[metrics].values
if len(x_data)<= 1:
y_val = np.arange(len(x_data), len(x_data)+ len_predict)
 return [-1, -1, -1], log_curve(y_val, 0.3, 30, 2)
 else:
y_min = max(1, dict_max_cc[country] * 0.001)
 y_max = max(y_min + 2, dict_max_cc[country] * 0.1)
 popt, pcov = curve_fit(log_curve, x_data, y_data, bounds=([0.1, 10, y_min ],[0.35, 50, y_max]),
p0=[0.2,30,(y_min + y_max)/ 2], maxfev=10000)
 y_val = np.arange(len(x_data)- len_intersection, len(x_data)+ len_predict - len_intersection)
 return popt, log_curve(y_val, popt[0], popt[1], popt[2] )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
dict_max_cc = test.groupby('State_Country' ).ConfirmedCases.max()
metrics = 'Fatalities'
dict_values = train.groupby('State_Country')[metrics].apply(np.array ).to_dict()
dict_case_date = {}
for country in dict_values:
dict_case_date[country] = []
for case in [1, 10, 100, 250, 500, 1000, 2500, 5000]:
try:
dict_case_date[country] += [np.where(dict_values[country] >= case)[0][0]]
except:
dict_case_date[country] += [-1]
dict_case_date[country] = np.array(dict_case_date[country])
 dict_predict = {}
data_train = train.copy()
data_val = test.copy()
len_predict = data_val[data_val.State_Country == country].shape[0]
len_intersection = len(set(data_train.Date.unique())& set(data_val.Date.unique()))
for country in train.State_Country.unique() :
popt, pred_values = predict_f(data_train, country, len_predict, metrics, len_intersection, dict_max_cc)
 dict_predict[country] = pred_values
g_features = [col for col in train_features.columns if col.startswith('g-')]
c_features = [col for col in train_features.columns if col.startswith('c-')]
g_c_features = g_features + c_features<feature_engineering>
test['Fatalities'] = 0
for country in test['State_Country'].unique() :
test.loc[test.State_Country == country, 'Fatalities'] = dict_predict[country]
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal" )<create_dataframe>
submit = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv')
 submit['Fatalities'] = test['Fatalities'].astype('float')
 submit['ConfirmedCases'] = test['ConfirmedCases'].astype('float')
 submit.to_csv('submission.csv',index=False )
trans_train_features = transformer.fit_transform(train_features[g_c_features])
 trans_test_features = transformer.transform(test_features[g_c_features])
 trans_train_df = pd.DataFrame(trans_train_features, columns = g_c_features)
 trans_test_df = pd.DataFrame(trans_test_features, columns = g_c_features)
 train_features = pd.concat([train_features.drop(columns=g_c_features), trans_train_df], axis=1)
 test_features = pd.concat([test_features.drop(columns=g_c_features), trans_test_df], axis=1 )<categorify>
def rmsle(true, pred):
true = np.array(true)
 pred = np.array(pred)
 return np.mean(( np.log1p(true)- np.log1p(pred)) ** 2)**(1/2 )