Kaggle competition: Santander Customer Transaction Prediction
clf = SVC()
clf.fit(X_train, Y_train )<predict_on_test>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv(".. /input/test.csv")
 base_features = [x for x in train_df.columns.values.tolist() if x.startswith('var_')]
y_train_pred = clf.predict(X_train)
 y_val_pred = clf.predict(X_val)
 print('train accuracy: {}%'.format(accuracy_score(y_train_pred, Y_train)))
 print('validation accuracy: {}%'.format(accuracy_score(y_val_pred, Y_val)) )<save_to_csv>
train_df['real'] = 1
for col in base_features:
test_df[col] = test_df[col].map(test_df[col].value_counts())
 a = test_df[base_features].min(axis=1)
 test_df = pd.read_csv('.. /input/test.csv')
 test_df['real'] =(a == 1 ).astype('int')
 train = train_df.append(test_df ).reset_index(drop=True)
 del test_df, train_df; gc.collect()
y_test = clf.predict(test)
 result = pd.read_csv('.. /input/titanic/gender_submission.csv')
 result['Survived'] = y_test
result.to_csv('submission.csv', index = False )<load_from_csv>
for col in tqdm(base_features):
train[col + 'size'] = train[col].map(train.loc[train.real==1, col].value_counts())
 cnt_features = [col + 'size' for col in base_features]
train_X = pd.read_csv(".. /input/titanic/train.csv")
 test_y = pd.read_csv(".. /input/titanic/test.csv")
 print("Test and Train FILES are loaded")
 train_X.columns<categorify>
for col in tqdm(base_features):
train.loc[train[col+'size']>1,col+'no_noise'] = train.loc[train[col+'size']>1,col]
noise1_features = [col + 'no_noise' for col in base_features]
features = ["Pclass", "Sex", "SibSp", "Parch"]
train_y = train_X['Survived']
train_f= train_X[features]
le = LabelEncoder()
train_f['Sex']=le.fit_transform(train_f['Sex'])
 test_y['Sex'] = le.fit_transform(test_y['Sex'])
<train_model>
train[noise1_features] = train[noise1_features].fillna(train[noise1_features].mean() )
model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(train_f,train_y)
 predictions = model.predict(test_y[features])
 predictions<save_to_csv>
for col in tqdm(base_features):
train.loc[train[col+'size']>2,col+'no_noise2'] = train.loc[train[col+'size']>2,col]
noise2_features = [col + 'no_noise2' for col in base_features]
output = pd.DataFrame({"PassengerId" :test_y['PassengerId'] , 'Survived':predictions})
 output.to_csv('submission1.csv',index=False)
 print("Your submission was successfully saved!" )<save_to_csv>
train[noise2_features] = train[noise2_features].fillna(train[noise2_features].mean() )
sub = pd.read_csv(".. /input/subsub/submission1.csv")
 sub.to_csv("submission.csv",index=False )<load_from_csv>
train_df = train[train['target'].notnull() ]
test_df = train[train['target'].isnull() ]
all_features = base_features + noise1_features + noise2_features
input1 = pd.read_csv('/kaggle/input/final1/submit.csv' )<save_to_csv>
scaler = preprocessing.StandardScaler().fit(train_df[all_features].values)
 df_trn = pd.DataFrame(scaler.transform(train_df[all_features].values), columns=all_features)
 df_tst = pd.DataFrame(scaler.transform(test_df[all_features].values), columns=all_features)
 y = train_df['target'].values
input1.to_csv('submission.csv', index=False )<set_options>
def get_keras_data(dataset, cols_info):
X = {}
base_feats, noise_feats, noise2_feats = cols_info
X['base'] = np.reshape(np.array(dataset[base_feats].values),(-1, len(base_feats), 1))
X['noise1'] = np.reshape(np.array(dataset[noise_feats].values),(-1, len(noise_feats), 1))
X['noise2'] = np.reshape(np.array(dataset[noise2_feats].values),(-1, len(noise2_feats), 1))
return X
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
 sns.set_palette(sns.color_palette('tab20', 20)) 
<load_from_csv>
cols_info = [base_features, noise1_features, noise2_features]
X_test = get_keras_data(df_tst[all_features], cols_info )
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1 
<sort_values>
def Convnet(cols_info, classes=1):
base_feats, noise1_feats, noise2_feats = cols_info
X_base_input = Input(shape=(len(base_feats), 1), name='base')
 X_base = Dense(16 )(X_base_input)
 X_base = Activation('relu' )(X_base)
 X_base = Flatten(name='base_last' )(X_base)
 X_noise1_input = Input(shape=(len(noise1_feats), 1), name='noise1')
 X_noise1 = Dense(16 )(X_noise1_input)
 X_noise1 = Activation('relu' )(X_noise1)
 X_noise1 = Flatten(name='nose1_last' )(X_noise1)
 X_noise2_input = Input(shape=(len(noise2_feats), 1), name='noise2')
 X_noise2 = Dense(16 )(X_noise2_input)
 X_noise2 = Activation('relu' )(X_noise2)
 X_noise2 = Flatten(name='nose2_last' )(X_noise2)
 X = concatenate([X_base, X_noise1, X_noise2])
 X = Dense(classes, activation='sigmoid' )(X)
 model = Model(inputs=[X_base_input, X_noise1_input, X_noise2_input],outputs=X)
 return model
model = Convnet(cols_info)
 model.summary()
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby(['Country_Region'] ).sum() [['ConfirmedCases',
'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state['DeathConfirmedRatio'] =(countries_latest_state.Fatalities + 1)/(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['DeathConfirmedRatio'] = countries_latest_state['DeathConfirmedRatio'].clip(0, 0.1)
 countries_latest_state.shape
countries_latest_state.head()<merge>
try:
del df_tst
except:
pass
gc.collect()
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]<count_unique_values>
SEED = 2019
n_folds = 5
debug_flag = True
folds = 5
skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=SEED )
train_clean['Location'].nunique()<groupby>
i = 0
result = pd.DataFrame({"ID_code": test_df.ID_code.values})
 val_aucs = []
valid_X = train_df[['target']]
valid_X['predict'] = 0
for train_idx, val_idx in skf.split(df_trn, y):
if i == folds:
break
i += 1
X_train, y_train = df_trn.iloc[train_idx], y[train_idx]
X_valid, y_valid = df_trn.iloc[val_idx], y[val_idx]
X_train = get_keras_data(X_train, cols_info)
 X_valid = get_keras_data(X_valid, cols_info)
 model_name = 'NN_fold{}.h5'.format(str(i))
model = Convnet(cols_info)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', 'binary_crossentropy', auc_2])
 checkpoint = ModelCheckpoint(model_name, monitor='val_auc_2', verbose=1,
save_best_only=True, mode='max', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='min', epsilon=0.0001)
 earlystop = EarlyStopping(monitor='val_auc_2', mode='max', patience=10, verbose=1)
 history = model.fit(X_train, y_train,
epochs=300,
batch_size=1024 * 2,
validation_data=(X_valid, y_valid),
callbacks=[checkpoint, reduceLROnPlat, earlystop])
 train_history = pd.DataFrame(history.history)
 train_history.to_csv('train_profile_fold{}.csv'.format(str(i)) , index=None)
 model.load_weights(model_name)
 y_pred_keras = model.predict(X_valid ).ravel()
valid_X['predict'].iloc[val_idx] = y_pred_keras
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_valid, y_pred_keras)
 auc_valid = roc_auc_score(y_valid, y_pred_keras)
 val_aucs.append(auc_valid)
 prediction = model.predict(X_test)
 result["fold{}".format(str(i)) ] = prediction
country_progress = train_clean.groupby(['Date', 'DateTime', 'Country_Region'] ).sum() [[
'ConfirmedCases', 'Fatalities', 'ConfirmedDelta', 'FatalitiesDelta']].reset_index()<merge>
for i in range(len(val_aucs)) :
print('Fold_%d AUC: %.6f' %(i+1, val_aucs[i]))
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )<merge>
auc_mean = np.mean(val_aucs)
 auc_std = np.std(val_aucs)
 auc_all = roc_auc_score(valid_X.target, valid_X.predict)
 print('%d-fold auc mean: %.9f, std: %.9f.All auc: %6f.' %(n_folds, auc_mean, auc_std, auc_all))
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()<save_to_csv>
y_all = result.values[:, 1:]
result['target'] = np.mean(y_all, axis = 1)
 to_submit = result[['ID_code', 'target']]
to_submit.to_csv('NN_submission.csv', index=None)
 result.to_csv('NN_all_prediction.csv', index=None)
 valid_X['ID_code'] = train_df['ID_code']
valid_X = valid_X[['ID_code', 'target', 'predict']].to_csv('NN_oof.csv', index=None )