Kaggle competition: IEEE-CIS Fraud Detection
train_period = 6
feature_Confirmed = ['ConfirmedCases'] + ['ConfirmedCases_B' + str(i)for i in range(1,train_period)]
feature_Fatalities = ['Fatalities'] + ['Fatalities_B' + str(i)for i in range(1,train_period)]
model = tf.keras.Sequential([
tf.keras.layers.Dense(256, activation='relu', input_shape=[len(feature_Confirmed)]),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(1)
 ])
 optimizer = "rmsprop"
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 EPOCHS = 10
BATCH_SIZE = 1
df_train = train.copy()
df_test = test.copy()
pred_data = []
for country in [country_plot,'Vietnam']:
for province in df_train[(df_train['Country/Region'] == country)]['Province/State'].unique() :
max_train_date = df_train[(df_train['Country/Region'] == country)&(df_train['Province/State'] == province)]['Date'].max()
for forcast in df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)]['ForecastId'].unique() :
testdate = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Date'].max()
data = df_train[(df_train['Country/Region'] == country)&(df_train['Province/State'] == province)&(df_train['ConfirmedCases'] > 0)&(df_train['Date']<testdate)][['Date','ConfirmedCases','Fatalities']]
for i in range(1,train_period):
data['ConfirmedCases_B' + str(i)] = data['ConfirmedCases'].shift(i)
 data['Fatalities_B' + str(i)] = data['Fatalities'].shift(i)
 data = data[(data.ConfirmedCases_B1 != data.ConfirmedCases)&(data.ConfirmedCases_B2 != data.ConfirmedCases)]
X_train_confirmed = data[feature_Confirmed][train_period-1:-1]
y_train_confirmed = data['ConfirmedCases'][train_period-1:-1]
if(len(X_train_confirmed)< 1):
ConfirmedCases_hat = 0
elif(len(X_train_confirmed)< train_period):
ConfirmedCases_hat = X_train_confirmed['ConfirmedCases'].values[-1:][0]
else:
model.fit(X_train_confirmed, y_train_confirmed ,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 X_pred_confirmed = data[feature_Confirmed][-1:]
ConfirmedCases_hat = model.predict(X_pred_confirmed)[-1][0]
data = data[(data.Fatalities_B1 != data.Fatalities)&(data.Fatalities_B2 != data.Fatalities)]
X_train_fatalities = data[feature_Fatalities][train_period-1:-1]
y_train_fatalities = data['Fatalities'][train_period-1:-1]
if(len(X_train_fatalities)< 1):
Fatalities_hat = 0
elif(len(X_train_fatalities)< train_period):
Fatalities_hat = X_train_fatalities['Fatalities'].values[-1:][0]
else:
model.fit(X_train_fatalities, y_train_fatalities ,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 X_pred_fatalities = data[feature_Fatalities][-1:]
Fatalities_hat = model.predict(X_pred_fatalities)[-1][0]
pred_data.append([testdate,country,province,forcast,ConfirmedCases_hat,Fatalities_hat])
 if(testdate > max_train_date):
lat = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Lat'].max()
long = df_test[(df_test['Country/Region'] == country)&(df_test['Province/State'] == province)&(df_test['ForecastId'] == forcast)]['Long'].max()
df_train.loc[len(df_train)] = [int(forcast),province,country,lat,long,testdate,float(ConfirmedCases_hat),float(Fatalities_hat)]<define_variables>
train_transaction = pd.read_csv(".. /input/ieee-fraud-detection/train_transaction.csv")
 train_identity = pd.read_csv(".. /input/ieee-fraud-detection/train_identity.csv" )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data['Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = pd.Timestamp('2020-12-31')
 for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
model = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='elu', input_shape=[len(feature_day)]),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(32, activation='elu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(1)
 ])
 optimizer = "rmsprop"
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 EPOCHS = 10
BATCH_SIZE = 1
pred_data = []
for country in ['Vietnam']:
for province in train[(train['Country/Region'] == country)]['Province/State'].unique() :
df_train = train[(train['Country/Region'] == country)&(train['Province/State'] == province)]
df_test = test[(test['Country/Region'] == country)&(test['Province/State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases']
y_train_fatalities = df_train['Fatalities']
X_pred = CreateInput(df_test)
 if(y_train_confirmed[-5:].mean() == y_train_confirmed.iloc[-1]):
y_hat_confirmed = np.repeat(y_train_confirmed.iloc[-1], len(X_pred))
else:
model.fit(X_train,y_train_confirmed,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 y_hat = model.predict(X_pred)
 y_hat_confirmed = [item[0] for item in y_hat]
if(y_train_fatalities[-5:].mean() == y_train_fatalities.iloc[-1]):
y_hat_fatalities = np.repeat(y_train_fatalities.iloc[-1], len(X_pred))
else:
model.fit(X_train,y_train_fatalities,batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = 0)
 y_hat = model.predict(X_pred)
 y_hat_fatalities = [item[0] for item in y_hat]
for i in range(0,len(y_hat_confirmed)) :
pred_data.append([df_test['Date'].iloc[i],country,province,df_test['ForecastId'].iloc[i],y_hat_confirmed[i],y_hat_fatalities[i]] )<feature_engineering>
pd.set_option('display.max_columns', 500)
 train_transaction.head(5 )
def CreateInput(data):
feature_day = [5,20,50,100,200,500,1000]
feature_name = ['Lat','Long']
for day in feature_day:
data['Number day from ' + str(day)+ ' case'] = 0
for country in data['Country/Region'].unique() :
for province in data[data['Country/Region'] == country]['Province/State'].unique() :
fromday = train[(train['Country/Region'] == country)&(train['Province/State'] == province)&(train['ConfirmedCases'] > day)]['Date'].min()
if(pd.isnull(fromday)== False):
data.loc[(data['Country/Region'] == country)&(data['Province/State'] == province),'Number day from ' + str(day)+ ' case'] =(data[(data['Country/Region'] == country)&(data['Province/State'] == province)]['Date'] - fromday ).dt.days
data.loc[(data['Number day from ' + str(day)+ ' case'] < 0),'Number day from ' + str(day)+ ' case'] = 0
feature_name = feature_name + ['Number day from ' + str(day)+ ' case']
return data[feature_name]
X_train = CreateInput(train.copy())
 y_train = train[['ConfirmedCases','Fatalities']]
model = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu', input_shape=[X_train.shape[1]]),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(2)
 ])
 optimizer = "rmsprop"
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 EPOCHS = 100
BATCH_SIZE = 24
model.fit(X_train,y_train, batch_size=BATCH_SIZE, epochs=EPOCHS,verbose = 0)
 X_pred = CreateInput(test.copy())
 y_hat = model.predict(X_pred)
<save_to_csv>
train_transaction['Transaction_dow'] = np.floor(( train_transaction['TransactionDT'] /(3600 * 24)- 1)% 7)
 train_transaction['Transaction_hour'] = np.floor(train_transaction['TransactionDT'] / 3600)% 24
df_forcast = pd.concat([test[['Date','ForecastId']],pd.DataFrame(y_hat)],axis = 1)
 df_forcast.columns = ['Date','ForecastId','ConfirmedCases','Fatalities']
df_forcast.loc[df_forcast['Fatalities'] < 0,'Fatalities'] = 0
df_forcast.loc[df_forcast['ConfirmedCases'] < 0,'ConfirmedCases'] = 0
df_past = pd.merge(test,train,on = ['Province/State','Date','Country/Region','Lat','Long'], how = 'inner' ).drop(['Id','Lat','Long'],axis=1)
 df_future = df_forcast[(df_forcast['Date'] > df_past['Date'].max())]
submission = df_past.append(df_future, sort = True)
 submission[['ForecastId','ConfirmedCases','Fatalities']]<merge>
na_columns = train_identity.isna().sum()
na_columns[na_columns==0]
df_val = pd.merge(test,df_forcast[['ForecastId','ConfirmedCases','Fatalities']], how='inner', on = 'ForecastId')
 df_val = pd.merge(df_val,train,on=['Date','Country/Region','Province/State','Lat','Long'], how='left')
 df_val = df_val.sort_values(by = ['Date','Country/Region'])
 df = df_val[df_val['Country/Region'] == 'Vietnam']
val_size = df_val[df_val['ConfirmedCases_y'].notnull() == True].shape[0]
fig, axes = plt.subplots(nrows=1,ncols=2,figsize=(20, 7))
axes[0].plot(df['Date'], df['ConfirmedCases_x'], label = 'ConfirmedCase forcast ' + df['Date'].max().strftime('%Y-%m-%d'))
axes[0].plot(df['Date'][0:val_size], df['ConfirmedCases_y'][0:val_size], label = 'ConfirmedCase ' + df[df['ConfirmedCases_y'].isnull() == False]['Date'].max().strftime('%Y-%m-%d'))
axes[0].legend()
axes[1].plot(df['Date'], df['Fatalities_x'], label = 'Fatalities forcast ' + df['Date'].max().strftime('%Y-%m-%d'))
axes[1].plot(df['Date'][0:val_size], df['Fatalities_y'][0:val_size], label = 'Fatalities ' + df[df['Fatalities_y'].isnull() == False]['Date'].max().strftime('%Y-%m-%d'))
axes[1].legend()
fig.autofmt_xdate()<compute_test_metric>
na_columns = train_transaction.isna().sum()
na_columns[na_columns==0]
RMSLE(df_val['ConfirmedCases_x'][0:val_size].values,df_val['ConfirmedCases_y'][0:val_size].values )<set_options>
categoricalCols = list(set(transaction_columns)- set(numericCols))
sns.set() 
<load_from_csv>
train_transaction[categoricalCols] = train_transaction[categoricalCols].replace({np.nan:'missing'})
 train_transaction[numericCols] = train_transaction[numericCols].replace({np.nan:-1})
train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
<groupby>
false_Fraud_Amt = np.mean(train_transaction.loc[train_transaction["isFraud"]==0]['TransactionAmt'])
 true_Fraud_Amt = np.mean(train_transaction.loc[train_transaction["isFraud"]==1]['TransactionAmt'])
 print(false_Fraud_Amt)
 print(true_Fraud_Amt)
train_confm_date=train.groupby('Date')['ConfirmedCases','Fatalities'].sum()<sort_values>
false_Fraud_Amt_max = np.max(train_transaction.loc[train_transaction["isFraud"]==0]['TransactionAmt'])
 true_Fraud_Amt_max = np.max(train_transaction.loc[train_transaction["isFraud"]==1]['TransactionAmt'])
 print(false_Fraud_Amt_max)
 print(true_Fraud_Amt_max )
train_confirm_country=train.groupby('Country/Region')['ConfirmedCases','Fatalities'].sum().reset_index().sort_values('ConfirmedCases',ascending=False )<feature_engineering>
false_Fraud_Amt_min = np.min(train_transaction.loc[train_transaction["isFraud"]==0]['TransactionAmt'])
 true_Fraud_Amt_min = np.min(train_transaction.loc[train_transaction["isFraud"]==1]['TransactionAmt'])
 print(false_Fraud_Amt_min)
 print(true_Fraud_Amt_min )
train_confirm_country['Fatality rate in %']=train_confirm_country['Fatalities']/train_confirm_country['ConfirmedCases']<sort_values>
maxCardData = {}
minCardData = {}
meanCardData = {}
for i,i_card in enumerate(cardTypes):
cardData = eval('train_transaction.loc[train_transaction["card4"]=="'+i_card+'" ]')
 maxCardData[i_card] = np.max(cardData['TransactionAmt'])
 minCardData[i_card] = np.min(cardData['TransactionAmt'])
 meanCardData[i_card] = np.mean(cardData['TransactionAmt'] )
train_confirm_country.sort_values('Fatality rate in %', ascending=False ).head(10 )<load_from_csv>
del fraudDataTransaction,normalDataTransaction,productData,cardData
test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 test.head()<drop_column>
identity_data_columns = train_identity.columns
numericCols = train_identity._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
print('The categorical columns in identity data are: ',categoricalCols)
 train_identity[categoricalCols] = train_identity[categoricalCols].replace({ np.nan:'missing'})
 train_identity[numericCols] = train_identity[numericCols].replace({ np.nan:-1} )
train.drop(['Province/State','Country/Region'],axis=1,inplace=True )<data_type_conversions>
train_identity.loc[train_identity['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
train_identity.loc[train_identity['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
train_identity.loc[train_identity['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
train_identity.loc[train_identity['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
train_identity.loc[train_identity['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'
train["Date"] = train["Date"].apply(lambda x: x.replace("-",""))
train['Date']=train['Date'].astype(int)
 train.info()<data_type_conversions>
train_identity['device_name'] = train_identity['DeviceInfo'].str.split('/', expand=True)[0]
train_identity.loc[train_identity['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
train_identity.loc[train_identity['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
train_identity.loc[train_identity['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
train_identity.loc[train_identity['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
train_identity.loc[train_identity['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
train_identity.loc[train_identity['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
train_identity.loc[train_identity['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
train_identity.loc[train_identity['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
train_identity.loc[train_identity['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
train_identity.loc[train_identity['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
train_identity.loc[train_identity['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
train_identity.loc[train_identity['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
train_identity.loc[train_identity['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
train_identity.loc[train_identity['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
train_identity.loc[train_identity['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
train_identity.loc[train_identity['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
train_identity.loc[train_identity['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
train_identity.loc[train_identity.device_name.isin(train_identity.device_name.value_counts() [train_identity.device_name.value_counts() < 200].index), 'device_name'] = "Others"
test["Date"] = test["Date"].apply(lambda x: x.replace("-",""))
test["Date"] = test["Date"].astype(int)
 test.info()<prepare_x_and_y>
raw_train_data = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
X_train=train.drop(['Id','ConfirmedCases','Fatalities'],axis=1)
 y_confrm=train[['ConfirmedCases']]
y_fat=train[['Fatalities']] 
<drop_column>
del train_identity,train_transaction
X_test=test.drop('ForecastId',axis=1)
 X_test.head()<predict_on_test>
raw_train_data[categoricalCols] = raw_train_data[categoricalCols].replace({ np.nan:'missing'})
 raw_train_data[numericCols] = raw_train_data[numericCols].replace({ np.nan:-1} )
rand_reg = RandomForestRegressor(random_state=42)
 rand_reg.fit(X_train,y_confrm)
 pred_grid1 = rand_reg.predict(X_test)
 pred_grid1 = pd.DataFrame(pred_grid1 ).round()
pred_grid1.columns = ["ConfirmedCases_prediction"]
pred_grid1.head()<predict_on_test>
def memory_usage_mb(df, *args, **kwargs):

 return df.memory_usage(*args, **kwargs ).sum() / 1024**2
def reduce_memory_usage(df, deep=True, verbose=True):
numeric2reduce = ["int16", "int32", "int64", "float64"]
start_mem = 0
if verbose:
start_mem = memory_usage_mb(df, deep=deep)
 for col, col_type in df.dtypes.iteritems() :
best_type = None
if col_type in numeric2reduce:
downcast = "integer" if "int" in str(col_type)else "float"
df[col] = pd.to_numeric(df[col], downcast=downcast)
 best_type = df[col].dtype.name
if verbose and best_type is not None and best_type != str(col_type):
print(f"Column '{col}' converted from {col_type} to {best_type}")
 if verbose:
end_mem = memory_usage_mb(df, deep=deep)
 diff_mem = start_mem - end_mem
percent_mem = 100 * diff_mem / start_mem
print(f"Memory usage decreased from"
f" {start_mem:.2f}MB to {end_mem:.2f}MB"
f"({diff_mem:.2f}MB, {percent_mem:.2f}% reduction)")
 return df
rand_reg.fit(X_train,y_fat)
 pred_grid2 = rand_reg.predict(X_test)
 pred_grid2 = pd.DataFrame(pred_grid2 ).round()
pred_grid2.columns = ["Fatality_prediction"]
pred_grid2.head()<load_from_csv>
raw_train_data = reduce_memory_usage(raw_train_data, deep=True, verbose=True )
sample=pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv' )<train_model>
na_vals = np.sum(raw_train_data.loc[:,variables]==-1)/ raw_train_data.shape[0]
goodNumericVars = []
for i_var in variables:
if na_vals[i_var] < 0.85:
goodNumericVars.append(i_var)
 goodNumericVars.remove('TransactionDT')
 goodNumericVars.remove('TransactionID')
 corrThresh = 0.9
upper = correlationMatrix.where(np.triu(np.ones(correlationMatrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > corrThresh)]
tree_reg=DecisionTreeRegressor(random_state=42)
 tree_reg.fit(X_train,y_confrm )<predict_on_test>
for i_var in to_drop:
if i_var in goodNumericVars:
goodNumericVars.remove(i_var)
y_tree_conf=tree_reg.predict(X_test)
 y_tree_conf=pd.DataFrame(y_tree_conf)
 y_tree_conf.columns=['Confrmed_prediction']
y_tree_conf.head()<predict_on_test>
del to_drop,corrThresh,upper,correlationMatrix,na_vals
tree_reg=DecisionTreeRegressor(random_state=42)
 tree_reg.fit(X_train,y_fat)
 y_tree_fat=tree_reg.predict(X_test)
 y_tree_fat=pd.DataFrame(y_tree_fat ).round()
y_tree_fat.columns=['fatality_prediction']
y_tree_fat.head()<concatenate>
variables = list(categoricalCols)
 na_vals = np.sum(raw_train_data.loc[:,variables] == 'missing')/ raw_train_data.shape[0]
goodCategoricalVars = []
for i_var in variables:
if na_vals[i_var] < 0.85:
goodCategoricalVars.append(i_var)
final_sub_tree=pd.concat([submission,y_tree_conf,y_tree_fat],axis=1)
 final_sub_tree.head()<rename_columns>
featureToUse = goodNumericVars + goodCategoricalVars
train_data = raw_train_data.loc[:,featureToUse]
target_data = raw_train_data['isFraud']
final_sub_tree.columns=[['ForecastId','ConfirmedCases', 'Fatalities']]
final_sub_tree.head()<save_to_csv>
train_data['TransactionAmt_to_mean_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_ProductCD'] = train_data['TransactionAmt'] / train_data.groupby(['ProductCD'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_dow'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_hour'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('mean')
 train_data['card1_card2'] = train_data['card1'].astype(str)+ '_' + train_data['card2'].astype(str)
 train_data['addr1_dist1'] = train_data['addr1'].astype(str)+ '_' + train_data['dist1'].astype(str)
 train_data['card1_addr1'] = train_data['card1'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card1_addr2'] = train_data['card1'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card2_addr1'] = train_data['card2'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card2_addr2'] = train_data['card2'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card4_addr1'] = train_data['card4'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card4_addr2'] = train_data['card4'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['DeviceInfo_P_emaildomain'] = train_data['DeviceInfo'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['P_emaildomain_addr1'] = train_data['P_emaildomain'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['id01_addr1'] = train_data['id_01'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['TransactionAmt_to_std_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_ProductCD'] = train_data['TransactionAmt'] / train_data.groupby(['ProductCD'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_dow'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_hour'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_decimal'] =(( train_data['TransactionAmt'] - train_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )
final_sub_tree.to_csv("submission.csv",index=False )<define_variables>
train_data['TransactionAmt_to_mean_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C13'] = train_data['TransactionAmt'] / train_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_D15'] = train_data['TransactionAmt'] / train_data.groupby(['D15'])['TransactionAmt'].transform('mean')
PATH_WEEK1='/kaggle/input/covid19-global-forecasting-week-1'<load_from_csv>
train_data['TransactionAmt_to_std_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_D15'] = train_data['TransactionAmt'] / train_data.groupby(['D15'])['TransactionAmt'].transform('std' )
df_train = pd.read_csv(f'{PATH_WEEK1}/train.csv' )<load_from_csv>
train_data['TransactionAmt'] = np.log(train_data['TransactionAmt'])
df_test = pd.read_csv(f'{PATH_WEEK1}/test.csv' )<rename_columns>
scale_pos_weight = np.sqrt(len(target_data.loc[target_data==0])/len(target_data.loc[target_data==1]))
del raw_train_data
df_train.rename(columns={'Country/Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country/Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province/State':'State'}, inplace=True)
 df_test.rename(columns={'Province/State':'State'}, inplace=True )<data_type_conversions>
test_identity_data = pd.read_csv(".. /input/ieee-fraud-detection/test_identity.csv")
 test_transaction_data = pd.read_csv(".. /input/ieee-fraud-detection/test_transaction.csv" )
df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )<remove_duplicates>
na_columns = test_transaction_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0] / test_transaction_data.shape[0] )
pxdf = px.data.gapminder()
country_isoAlpha = pxdf[['country', 'iso_alpha']].drop_duplicates()
country_isoAlpha.rename(columns = {'country':'Country'}, inplace=True)
 country_isoAlpha.set_index('Country', inplace=True)
 country_map = country_isoAlpha.to_dict('index' )<categorify>
test_transaction_data['Transaction_dow'] = np.floor(( test_transaction_data['TransactionDT'] /(3600 * 24)- 1)% 7)
 test_transaction_data['Transaction_hour'] = np.floor(test_transaction_data['TransactionDT'] / 3600)% 24
transaction_data_columns = test_transaction_data.columns
numericCols = test_transaction_data._get_numeric_data().columns
categoricalCols = list(set(transaction_data_columns)- set(numericCols))
test_transaction_data[categoricalCols] = test_transaction_data[categoricalCols].replace({ np.nan:'missing'})
 test_transaction_data[numericCols] = test_transaction_data[numericCols].replace({ np.nan:-1} )
def getCountryIsoAlpha(country):
try:
return country_map[country]['iso_alpha']
except:
return country<feature_engineering>
identity_data_columns = test_identity_data.columns
numericCols = test_identity_data._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
test_identity_data[categoricalCols] = test_identity_data[categoricalCols].replace({ np.nan:'missing'})
 test_identity_data[numericCols] = test_identity_data[numericCols].replace({ np.nan:-1})
 test_identity_data['id_31'] = test_identity_data['id_31'].str.replace('\d+', '')
 test_identity_data.loc[test_identity_data['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
test_identity_data.loc[test_identity_data['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'
test_identity_data['device_name'] = test_identity_data['DeviceInfo'].str.split('/', expand=True)[0]
test_identity_data.loc[test_identity_data['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
test_identity_data.loc[test_identity_data['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
test_identity_data.loc[test_identity_data['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
test_identity_data.loc[test_identity_data.device_name.isin(test_identity_data.device_name.value_counts() [test_identity_data.device_name.value_counts() < 200].index), 'device_name'] = "Others"
df_train['iso_alpha'] = df_train['Country'].apply(getCountryIsoAlpha)
 df_train.info()<count_missing_values>
raw_test_data = pd.merge(test_transaction_data, test_identity_data, on='TransactionID', how='left')
 transactionID = raw_test_data.loc[:,'TransactionID']
del test_identity_data,test_transaction_data
df_train.isna().sum()<drop_column>
raw_test_data_columns = raw_test_data.columns
numericCols = raw_test_data._get_numeric_data().columns
categoricalCols = list(set(raw_test_data_columns)- set(numericCols))
print('The categorical columns in training data are: ',categoricalCols)
 raw_test_data[categoricalCols] = raw_test_data[categoricalCols].replace({ np.nan:'missing'})
 raw_test_data[numericCols] = raw_test_data[numericCols].replace({ np.nan:-1})
 raw_test_data = reduce_memory_usage(raw_test_data, deep=True, verbose=True )
df_train.drop(columns='iso_alpha', inplace=True)
 df_train<data_type_conversions>
test_data = raw_test_data.loc[:,featureToUse]
X_Train = df_train.copy()
X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%Y%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.drop(columns=['ConfirmedCases', 'Fatalities'], inplace=True)
 X_Train.drop(columns=['Id', 'State', 'Country', 'Lat'], inplace=True)
 X_Train.head()<prepare_x_and_y>
test_data['TransactionAmt_to_mean_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_ProductCD'] = test_data['TransactionAmt'] / test_data.groupby(['ProductCD'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_dow'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_hour'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('mean')
 test_data['card1_card2'] = test_data['card1'].astype(str)+ '_' + test_data['card2'].astype(str)
 test_data['addr1_dist1'] = test_data['addr1'].astype(str)+ '_' + test_data['dist1'].astype(str)
 test_data['card1_addr1'] = test_data['card1'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card1_addr2'] = test_data['card1'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card2_addr1'] = test_data['card2'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card2_addr2'] = test_data['card2'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card4_addr1'] = test_data['card4'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card4_addr2'] = test_data['card4'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['DeviceInfo_P_emaildomain'] = test_data['DeviceInfo'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['P_emaildomain_addr1'] = test_data['P_emaildomain'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['id01_addr1'] = test_data['id_01'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['TransactionAmt_to_std_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_ProductCD'] = test_data['TransactionAmt'] / test_data.groupby(['ProductCD'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_dow'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_hour'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_decimal'] =(( test_data['TransactionAmt'] - test_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )
y1_Train = df_train.ConfirmedCases<set_options>
test_data['TransactionAmt_to_mean_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C13'] = test_data['TransactionAmt'] / test_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_D15'] = test_data['TransactionAmt'] / test_data.groupby(['D15'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_std_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_D15'] = test_data['TransactionAmt'] / test_data.groupby(['D15'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt'] = np.log(test_data['TransactionAmt'] )
filterwarnings('ignore' )<import_modules>
for i_cat in goodCategoricalVars:
le = LabelEncoder()
curData = pd.concat([train_data.loc[:,i_cat],test_data.loc[:,i_cat]],axis = 0)
 le.fit(curData)
 train_data.loc[:,i_cat] = le.transform(train_data.loc[:,i_cat])
 test_data.loc[:,i_cat] = le.transform(test_data.loc[:,i_cat] )
MODELS = {"Linear_Reg": LinearRegression() , "KNN_Reg": KNeighborsRegressor() , "LinearSVR_Reg": LinearSVR() , "DecisionTree_Reg": DecisionTreeRegressor() , "DecisionTree_Class": DecisionTreeClassifier() , "ExtraTree_Reg": ExtraTreeRegressor() , "RandomForest_Reg": RandomForestRegressor() , "GB_Reg": GradientBoostingRegressor() , "XGB_Reg": XGBRegressor() } 
<choose_model_class>
cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True )
model = XGBRegressor()<define_search_space>
def compute_roc_auc(clf,index):
y_predict = clf.predict_proba(train_data.iloc[index])[:,1]
fpr, tpr, thresholds = roc_curve(target_data.iloc[index], y_predict)
 auc_score = auc(fpr, tpr)
 return fpr, tpr, auc_score
param_grid = {'n_estimators': [1250]}<train_on_grid>
params = {'bagging_fraction': 0.7982116702024386,
'feature_fraction': 0.1785051643813966,
'max_depth': int(49.17611603427576),
'min_child_weight': 3.2852905549011155,
'min_data_in_leaf': int(31.03480802715621),
'n_estimators': int(1491.3676131788188),
'num_leaves': int(52.851307790411965),
'reg_alpha': 0.45963319421692145,
'reg_lambda': 0.6591286807489907,
'metric':'auc',
'boosting_type': 'dart',
'colsample_bytree':.8,
'subsample':.9,
'min_split_gain':.01,
'max_bin':127,
'bagging_freq':5,
'learning_rate':0.1 ,
'early_stopping_rounds':500 }
start = time.time()
grid_cv = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error')
 grid_cv.fit(X_Train, y1_Train)
 print(f'{type(model ).__name__} Hyper Paramter Tuning took a Time: {time.time() - start}' )<find_best_params>
fprs_lgb, tprs_lgb, scores_lgb = [], [], []
feature_importances = pd.DataFrame()
feature_importances['feature'] = train_data.columns
predictions = np.zeros(len(test_data))
for(train, test), i in zip(cv.split(train_data, target_data), range(5)) :
lgb_best = LGBMClassifier(boosting = params['boosting_type'],n_estimators = params['n_estimators'],
learning_rate = params['learning_rate'],num_leaves = params['num_leaves'],
colsample_bytree = params['colsample_bytree'],subsample = params['subsample'],
max_depth = params['max_depth'],reg_alpha = params['reg_alpha'],
reg_lambda = params['reg_lambda'],min_split_gain = params['min_split_gain'],
min_child_weight = params['min_child_weight'],max_bin = params['max_bin'],
bagging_freq = params['bagging_freq'],feature_fraction = params['feature_fraction'],
bagging_fraction = params['bagging_fraction'],min_data_in_leaf = params['min_data_in_leaf'],
early_stopping_rounds = params['early_stopping_rounds'])
 lgb_best.fit(train_data.iloc[train,:], target_data.iloc[train],
eval_set = [(train_data.iloc[train,:], target_data.iloc[train]), 
(train_data.iloc[test,:], target_data.iloc[test])],eval_metric='auc',verbose = 200)
 feature_importances['fold_{}'.format(i + 1)] = lgb_best.feature_importances_
_, _, auc_score_train = compute_roc_auc(lgb_best,train)
 fpr, tpr, auc_score = compute_roc_auc(lgb_best,test)
 scores_lgb.append(( auc_score_train, auc_score))
fprs_lgb.append(fpr)
 tprs_lgb.append(tpr)
 predictions += lgb_best.predict_proba(test_data, num_iteration=lgb_best.best_iteration_)[:,1]/cv.n_splits
print("Mean Squared Error: {}".format(grid_cv.best_score_))
print("Best Hyperparameters:
{}".format(grid_cv.best_params_)) 
<data_type_conversions>
print('Mean AUC:', np.mean(scores_lgb,axis = 1))
<predict_on_test><EOS> 
data = {'TransactionID':transactionID,'isFraud':predictions}
submissionDF = pd.DataFrame(data)
 submissionDF.to_csv('sample_submission2.csv',index=False )