Kaggle competition: COVID19 Global Forecasting (Week 2)
floattypes = []
inttypes = []
stringtypes = []
for c in test.columns:
if(train[c].dtype=='object'):
train[c] = train[c].astype('str')
 stringtypes.append(c)
 elif(train[c].dtype=='int64'):
train[c] = train[c].astype('int32')
 inttypes.append(c)
 else:
train[c] = train[c].astype('float32')
 floattypes.append(c)
 train = train.reset_index(drop=True)
 test = test.reset_index(drop=True )<categorify>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
kf = KFold(n_splits=5, shuffle=True, random_state=42)
 for col in stringtypes:
train['te_'+col] = 0.
 test['te_'+col] = 0.
 SMOOTHING = test[~test[col].isin(train[col])].shape[0]/test.shape[0]
for f,(vis_index, blind_index)in enumerate(kf.split(train)) :
_, train.loc[blind_index, 'te_'+col] = target_encode(train.loc[vis_index, col],
train.loc[blind_index, col],
target=train.loc[vis_index,'TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0)
 _, x = target_encode(train.loc[vis_index, col],
test[col],
target=train.loc[vis_index,'TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0)
 test['te_'+col] += (.2*x )<categorify>
data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv',index_col='Date',parse_dates=True )
kf = KFold(n_splits=5, shuffle=True, random_state=42)
 for col in inttypes:
train['te_'+col] = 0.
 test['te_'+col] = 0.
 SMOOTHING = test[~test[col].isin(train[col])].shape[0]/test.shape[0]
for f,(vis_index, blind_index)in enumerate(kf.split(train)) :
_, train.loc[blind_index, 'te_'+col] = target_encode(train.loc[vis_index, col],
train.loc[blind_index, col],
target=train.loc[vis_index,'TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0)
 _, x = target_encode(train.loc[vis_index, col],
test[col],
target=train.loc[vis_index,'TARGET'],
min_samples_leaf=100,
smoothing=SMOOTHING,
noise_level=0.0)
 test['te_'+col] += (.2*x )<concatenate>
data = data.fillna(value='empty' )
ntrainrows = train.shape[0]
test.insert(1,'TARGET',-1)
 alldata = pd.concat([train,test])
 del train ,test
gc.collect()<feature_engineering>
data['state_with_country'] = data['Province_State'] +'_'+ data['Country_Region']
alldata['nans'] = alldata.isnull().sum(axis=1 )<merge>
data= data.drop(labels=['Province_State','Country_Region','Id'],axis=1 )
for col in inttypes[1:]:
x = alldata[col].value_counts().reset_index(drop=False)
 x.columns = [col,'cnt_'+col]
x['cnt_'+col]/=alldata.shape[0]
alldata = alldata.merge(x,on=col,how='left' )<data_type_conversions>
state_i_data =[]
for i in state_with_country_name:
state_i_data.append(i)
alldata[features] = alldata[features].astype('float32' )<normalization>
for i,j in enumerate(state_with_country_name):
state_i_data[i] = data[data['state_with_country'] == j ]
state_i_data[i] = state_i_data[i].drop(['state_with_country'],axis=1)
for c in features:
ss = StandardScaler()
alldata.loc[~alldata[c].isnull() ,c] = ss.fit_transform(alldata.loc[~alldata[c].isnull() ,c].values.reshape(-1,1))
alldata[c].fillna(alldata[c].mean() ,inplace=True )<prepare_x_and_y>
prev_sub = pd.read_csv('/kaggle/input/covid19-forecasting-using-rnn/submission.csv')
 test_whole = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv',index_col='Date',parse_dates=True)
 test_whole = test_whole.fillna(value='empty')
train = alldata[:ntrainrows]
test = alldata[ntrainrows:]<feature_engineering>
test_whole['state_with_country'] = test_whole['Province_State'] +'_'+ test_whole['Country_Region']
test_whole.head()
traintargets = train.TARGET.values
train = UseGPFeatures(train)
 test = UseGPFeatures(test)
 train['TARGET'] = traintargets<set_options>
prev_sub.index = test_whole.index
gc.enable()<define_variables>
prev_sub['state_with_country'] = test_whole['state_with_country']
folds = KFold(n_splits=5, shuffle=True, random_state=42)
 oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feats = [f for f in train.columns if f not in ['SK_ID_CURR','TARGET']]<prepare_x_and_y>
forecast = []
for i in state_with_country_name:
forecast.append(i )
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(train)) :
trn_x, trn_y = train[feats].iloc[trn_idx], train.iloc[trn_idx]['TARGET']
val_x, val_y = train[feats].iloc[val_idx], train.iloc[val_idx]['TARGET']
clf = LGBMClassifier(
 n_estimators=4000,
learning_rate=0.03,
num_leaves=30,
colsample_bytree=.8,
subsample=.9,
max_depth=7,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.01,
min_child_weight=100,
silent=-1,
verbose=-1, 
)
 clf.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc', verbose=100, early_stopping_rounds=100 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()<save_to_csv>
for i,j in enumerate(state_with_country_name):
forecast[i] = prev_sub[prev_sub['state_with_country'] == j ]
forecast[i] = forecast[i].iloc[13:21]
forecast[i] = forecast[i].drop(['ForecastId','state_with_country'],axis=1)
 state_i_data[i] = state_i_data[i].append(forecast[i] )
Submission = pd.DataFrame({ 'SK_ID_CURR': ID,'TARGET': sub_preds })
 Submission.to_csv("hybridII.csv", index=False )<load_from_csv>
test_whole = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv',index_col='Date',parse_dates=True)
 test_whole = test_whole.fillna(value='empty' )
PATH = ".. /input"
list_of_files = os.listdir(PATH)
 application_train = pd.read_csv(PATH+"/application_train.csv")
 application_test = pd.read_csv(PATH+"/application_test.csv")
 bureau = pd.read_csv(PATH+"/bureau.csv")
 bureau_balance = pd.read_csv(PATH+"/bureau_balance.csv")
 credit_card_balance = pd.read_csv(PATH+"/credit_card_balance.csv")
 installments_payments = pd.read_csv(PATH+"/installments_payments.csv")
 previous_application = pd.read_csv(PATH+"/previous_application.csv")
 POS_CASH_balance = pd.read_csv(PATH+"/POS_CASH_balance.csv" )<concatenate>
test_whole['state_with_country'] = test_whole['Province_State'] +'_'+ test_whole['Country_Region']
test_whole.head()
total_IDS = np.concatenate(( application_test["SK_ID_CURR"].values, application_train["SK_ID_CURR"].values))
print(len(np.unique(np.array(total_IDS)))== len(total_IDS))<define_variables>
state_i_data_for_test=[]
for i in state_with_country_name_for_test:
state_i_data_for_test.append(i)
POS_CASH_balance_IDS = POS_CASH_balance["SK_ID_CURR"].values
bureau_IDS = bureau["SK_ID_CURR"].values
credit_card_balance_IDS = credit_card_balance["SK_ID_CURR"].values
installments_payments_IDS = installments_payments["SK_ID_CURR"].values
previous_application_IDS = previous_application["SK_ID_CURR"].values
tot = len(total_IDS)
 print(tot)
 print(len(np.intersect1d(POS_CASH_balance_IDS, total_IDS)) /tot*100,
len(np.intersect1d(bureau_IDS, total_IDS)) /tot*100,
len(np.intersect1d(credit_card_balance_IDS, total_IDS)) /tot*100,
len(np.intersect1d(installments_payments_IDS, total_IDS)) /tot*100,
len(np.intersect1d(previous_application_IDS, total_IDS)) /tot*100 )<feature_engineering>
for i,j in enumerate(state_with_country_name_for_test):
state_i_data_for_test[i] = test_whole[test_whole['state_with_country'] == j ]
state_i_data_for_test[i] = state_i_data_for_test[i].iloc[21:]
state_i_data_for_test[i] = state_i_data_for_test[i].drop(['ForecastId','state_with_country','Province_State','Country_Region'],axis=1)
 state_i_data_for_test.append(state_i_data_for_test[i] )
prev = previous_application["SK_ID_PREV"].values
POS_CASH_balance_IDS_prev = POS_CASH_balance["SK_ID_PREV"].values
credit_card_balance_IDS_prev = credit_card_balance["SK_ID_PREV"].values
installments_payments_IDS_prev = installments_payments["SK_ID_PREV"].values
prev_num = len(prev)
 print(prev_num)
 print(len(np.intersect1d(POS_CASH_balance_IDS_prev, prev)) /prev_num*100,
len(np.intersect1d(credit_card_balance_IDS_prev, prev)) /prev_num*100,
len(np.intersect1d(installments_payments_IDS_prev, prev)) /prev_num*100)
<count_unique_values>
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,LSTM
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator
bureau_br = np.unique(bureau["SK_ID_BUREAU"].values)
 print(len(np.intersect1d(np.unique(bureau_balance["SK_ID_BUREAU"].values), bureau_br)) /len(bureau_br)*100 )<count_unique_values>
result = []
full_scaler = MinMaxScaler()
breau_total = np.unique(np.intersect1d(bureau_IDS, total_IDS))
bureau_filtered = bureau.loc[bureau["SK_ID_CURR"].isin(breau_total)]
b = np.intersect1d(np.unique(bureau_filtered["SK_ID_BUREAU"].values), np.unique(bureau_balance["SK_ID_BUREAU"].values))
bureau_filtered = bureau_filtered.loc[bureau_filtered["SK_ID_BUREAU"].isin(b)]
len(bureau_filtered["SK_ID_CURR"].values)
 bureau_filtered<count_unique_values>
for i in range(len(state_with_country_name_for_test)) :
scaled_full_data = full_scaler.fit_transform(state_i_data[i])
 length = 1
batch_size = 1
generator = TimeseriesGenerator(scaled_full_data, scaled_full_data, length=length, batch_size=1)
 model = Sequential()
model.add(LSTM(96,input_shape=(length,scaled_full_data.shape[1])))
 model.add(Dense(scaled_full_data.shape[1]))
model.compile(optimizer='adam', loss='mse')
 model.fit_generator(generator,epochs=6)
 n_features = scaled_full_data.shape[1]
test_predictions = []
first_eval_batch = scaled_full_data[-length:]
current_batch = first_eval_batch.reshape(( 1, length, n_features))
for j in range(len(state_i_data_for_test[i])) :
current_pred = model.predict(current_batch)[0]
test_predictions.append(current_pred)
 current_batch = np.append(current_batch[:,1:,:],[[current_pred]],axis=1)
 true_predictions = full_scaler.inverse_transform(test_predictions)
 true_predictions = true_predictions.round()
true_predictions = pd.DataFrame(data=true_predictions,columns=state_i_data[1].columns)
 result.append(true_predictions)
 print('count:-',i)
print(len(np.unique(bureau_filtered["SK_ID_CURR"].values)) /tot*100 )<prepare_x_and_y>
prediction = pd.DataFrame(data= state_i_data[0].iloc[-21:] ,columns=['ConfirmedCases','Fatalities'])
 prediction = prediction.append(result[0] )
train = application_train.drop(["TARGET"], axis = 1)
 train_target = application_train["TARGET"]
test= application_test.copy()
tr = len(application_train)
 print(all(i ==True for i in train.columns==test.columns)) 
<concatenate>
for i in range(1,len(result)) :
prediction = prediction.append(state_i_data[i].iloc[-21:])
 prediction = prediction.append(result[i] )
df = pd.concat([train, test])
 del train, test, application_train, application_test
gc.collect()
def categorical_features(data):
features = [i for i in list(data.columns)if data[i].dtype == 'object']
return features
categorical = categorical_features(df)
 numerical = [i for i in df.columns if i not in categorical]
numerical.remove("SK_ID_CURR")
 IDs = df["SK_ID_CURR"] 
<categorify>
prediction.index = range(0,len(prediction))
for feature in categorical:
df[feature].fillna("unidentified")
 print(f'Transforming {feature}...')
 encoder = LabelEncoder()
encoder.fit(df[feature].astype(str))
df[feature] = encoder.transform(df[feature].astype(str))
df.head()<feature_engineering>
sub_format = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )
for feats in df.columns:
df[feats] = df[feats].fillna(-1)
 df.head()<count_unique_values>
sub_format = sub_format['ForecastId']
POS_CASH_balance_G1 = POS_CASH_balance.loc[POS_CASH_balance["SK_ID_CURR"].isin(total_IDS)]
print(len(np.unique(POS_CASH_balance_G1["SK_ID_CURR"].values)))
 POS_CASH_balance_G1.head()<merge>
final = pd.concat([sub_format,prediction],axis=1 )
<count_unique_values><EOS> 
final.to_csv('submission.csv',index=False )