Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<set_options>
sns.set(style="darkgrid")
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
pd.set_option('display.max_columns', 500)
 train_transaction.head(5 )<feature_engineering>
data = pd.read_csv("/kaggle/input/titanic/train.csv")
 data_test = pd.read_csv("/kaggle/input/titanic/test.csv" )
train_transaction['Transaction_dow'] = np.floor(( train_transaction['TransactionDT'] /(3600 * 24)- 1)% 7)
 train_transaction['Transaction_hour'] = np.floor(train_transaction['TransactionDT'] / 3600)% 24<count_missing_values>
y = data["Survived"]
X = data.copy()
X_test = data_test.copy()
X_full = pd.concat([X, X_test])
 n = len(X)
 n_test = len(X_test)
 n_full = n + n_test
na_columns = train_identity.isna().sum()
na_columns[na_columns==0]<count_missing_values>
( X_full.drop(["Survived"], axis = 1 ).isna().sum() / n_full)* 100
na_columns = train_transaction.isna().sum()
na_columns[na_columns==0]<define_variables>
def fill_na_age(row):
row["Age"] = df_median_age_class[row["Sex"]][row["Pclass"]][0] if pd.isnull(row["Age"])else row["Age"]
return row
X_full = X_full.apply(fill_na_age, axis = 1)
 X_full.info()
categoricalCols = list(set(transaction_columns)- set(numericCols))<feature_engineering>
print(X_full[X_full["Fare"].isnull() ] )
train_transaction[categoricalCols] = train_transaction[categoricalCols].replace({np.nan:'missing'})
 train_transaction[numericCols] = train_transaction[numericCols].replace({np.nan:-1})
<feature_engineering>
def fill_na_fare(row):
row["Fare"] = df_median_fare_class[row["Pclass"]][0] if pd.isnull(row["Fare"])else row["Fare"]
return row
X_full = X_full.apply(fill_na_fare, axis = 1)
 X_full.info()
print(X_full[X_full["PassengerId"] == 1044] )
false_Fraud_Amt = np.mean(train_transaction.loc[train_transaction["isFraud"]==0]['TransactionAmt'])
 true_Fraud_Amt = np.mean(train_transaction.loc[train_transaction["isFraud"]==1]['TransactionAmt'])
 print(false_Fraud_Amt)
 print(true_Fraud_Amt)
<define_variables>
X_full[X_full["Embarked"].isnull() ]
false_Fraud_Amt_max = np.max(train_transaction.loc[train_transaction["isFraud"]==0]['TransactionAmt'])
 true_Fraud_Amt_max = np.max(train_transaction.loc[train_transaction["isFraud"]==1]['TransactionAmt'])
 print(false_Fraud_Amt_max)
 print(true_Fraud_Amt_max )<filter>
embarked_survived = X.groupby(["Embarked", "Survived"])["PassengerId"].count()
embarked_survived.head(10 )
false_Fraud_Amt_min = np.min(train_transaction.loc[train_transaction["isFraud"]==0]['TransactionAmt'])
 true_Fraud_Amt_min = np.min(train_transaction.loc[train_transaction["isFraud"]==1]['TransactionAmt'])
 print(false_Fraud_Amt_min)
 print(true_Fraud_Amt_min )<feature_engineering>
def display_embarked_prob() :
surv_counts = {
"C": {},
"Q": {},
"S": {}
}
classes = set(embarked_survived.index.get_level_values(0))
for c in classes:
for survive in range(0, 2):
surv_counts[str(c)][survive] = embarked_survived[c][survive]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
df_survived_percentages = pd.DataFrame(surv_percentages ).transpose()
class_names =("C", "Q", "S")
 bar_count = np.arange(len(class_names))
bar_width = 0.5
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(10, 6))
plt.bar(bar_count, not_survived, color='
plt.bar(bar_count, survived, bottom=not_survived, color='
plt.xlabel('Embarked', size=15, labelpad=20)
 plt.ylabel('Survival Percentage', size=15, labelpad=20)
 plt.xticks(bar_count, class_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})
 plt.title('Survival Percentage by Embarked', size=18, y=1.05)
 plt.show()
display_embarked_prob()
maxCardData = {}
minCardData = {}
meanCardData = {}
for i,i_card in enumerate(cardTypes):
cardData = eval('train_transaction.loc[train_transaction["card4"]=="'+i_card+'" ]')
 maxCardData[i_card] = np.max(cardData['TransactionAmt'])
 minCardData[i_card] = np.min(cardData['TransactionAmt'])
 meanCardData[i_card] = np.mean(cardData['TransactionAmt'] )<drop_column>
embarked_class_passengers = X.groupby("Pclass")["Embarked"].value_counts()
print(embarked_class_passengers )
del fraudDataTransaction,normalDataTransaction,productData,cardData<feature_engineering>
X_full["CabinId"] = X_full["Cabin"].map(lambda cabin: cabin if pd.isnull(cabin)else cabin[0])
 X["CabinId"] = X["Cabin"].map(lambda cabin: cabin if pd.isnull(cabin)else cabin[0])
 X["CabinId"].sample(10)
identity_data_columns = train_identity.columns
numericCols = train_identity._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
print('The categorical columns in identity data are: ',categoricalCols)
 train_identity[categoricalCols] = train_identity[categoricalCols].replace({ np.nan:'missing'})
 train_identity[numericCols] = train_identity[numericCols].replace({ np.nan:-1} )<feature_engineering>
X.groupby(["CabinId", "Pclass"])["PassengerId"].count()
train_identity.loc[train_identity['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
train_identity.loc[train_identity['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
train_identity.loc[train_identity['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
train_identity.loc[train_identity['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
train_identity.loc[train_identity['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'<feature_engineering>
class_cabin_passengers = X.groupby(["Pclass", "CabinId"])["PassengerId"].count()
print(class_cabin_passengers )
train_identity['device_name'] = train_identity['DeviceInfo'].str.split('/', expand=True)[0]
train_identity.loc[train_identity['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
train_identity.loc[train_identity['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
train_identity.loc[train_identity['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
train_identity.loc[train_identity['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
train_identity.loc[train_identity['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
train_identity.loc[train_identity['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
train_identity.loc[train_identity['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
train_identity.loc[train_identity['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
train_identity.loc[train_identity['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
train_identity.loc[train_identity['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
train_identity.loc[train_identity['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
train_identity.loc[train_identity['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
train_identity.loc[train_identity['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
train_identity.loc[train_identity['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
train_identity.loc[train_identity['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
train_identity.loc[train_identity['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
train_identity.loc[train_identity['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
train_identity.loc[train_identity.device_name.isin(train_identity.device_name.value_counts() [train_identity.device_name.value_counts() < 200].index), 'device_name'] = "Others"<merge>
survival_by_gender = X.groupby("Sex" ).apply(lambda df: sum(df["Survived"] == 1))
survival_by_gender
raw_train_data = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
<drop_column>
survival_by_gender["female"] /(survival_by_gender["female"] + survival_by_gender["male"] )
del train_identity,train_transaction<categorify>
survival_by_class = X.groupby("Pclass")["Survived"].apply(lambda x: {"Survived": sum(x == 1), "Not Survived": sum(x == 0)})
 survival_by_class
raw_train_data[categoricalCols] = raw_train_data[categoricalCols].replace({ np.nan:'missing'})
 raw_train_data[numericCols] = raw_train_data[numericCols].replace({ np.nan:-1} )<categorify>
gender_by_class = X.groupby("Pclass")["Sex"].apply(lambda x: {"Male": sum(x == "male"), "Female": sum(x == "female")})
 gender_by_class
def memory_usage_mb(df, *args, **kwargs):

 return df.memory_usage(*args, **kwargs ).sum() / 1024**2
def reduce_memory_usage(df, deep=True, verbose=True):
numeric2reduce = ["int16", "int32", "int64", "float64"]
start_mem = 0
if verbose:
start_mem = memory_usage_mb(df, deep=deep)
 for col, col_type in df.dtypes.iteritems() :
best_type = None
if col_type in numeric2reduce:
downcast = "integer" if "int" in str(col_type)else "float"
df[col] = pd.to_numeric(df[col], downcast=downcast)
 best_type = df[col].dtype.name
if verbose and best_type is not None and best_type != str(col_type):
print(f"Column '{col}' converted from {col_type} to {best_type}")
 if verbose:
end_mem = memory_usage_mb(df, deep=deep)
 diff_mem = start_mem - end_mem
percent_mem = 100 * diff_mem / start_mem
print(f"Memory usage decreased from"
f" {start_mem:.2f}MB to {end_mem:.2f}MB"
f"({diff_mem:.2f}MB, {percent_mem:.2f}% reduction)")
 return df<load_pretrained>
df_class_survived = X.groupby(['Pclass', "Survived", "Sex"] ).count().drop(columns=[
'Age',
'SibSp',
'Parch',
'Fare',
'Embarked',
'Cabin',
'PassengerId',
'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
df_class_survived.head(15 )
raw_train_data = reduce_memory_usage(raw_train_data, deep=True, verbose=True )<define_variables>
def display_gender_by_class_prob(gender):
surv_counts = {
"1": {},
"2": {},
"3": {}
}
classes = df_class_survived.columns.levels[0]
for c in classes:
for survive in range(0, 2):
surv_counts[str(c)][survive] = df_class_survived[c][survive][gender][0]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
df_survived_percentages = pd.DataFrame(surv_percentages ).transpose()
class_names =("1", "2", "3")
 bar_count = np.arange(len(class_names))
bar_width = 0.5
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(10, 6))
plt.bar(bar_count, not_survived, color='
plt.bar(bar_count, survived, bottom=not_survived, color='
plt.xlabel('Pclass', size=15, labelpad=20)
 plt.ylabel('Survival Percentage', size=15, labelpad=20)
 plt.xticks(bar_count, class_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})
 plt.title('Survival Percentage by Class for gender {}'.format(gender), size=18, y=1.05)
 plt.show()
display_gender_by_class_prob("female")
 display_gender_by_class_prob("male" )
na_vals = np.sum(raw_train_data.loc[:,variables]==-1)/ raw_train_data.shape[0]
goodNumericVars = []
for i_var in variables:
if na_vals[i_var] < 0.85:
goodNumericVars.append(i_var)
 goodNumericVars.remove('TransactionDT')
 goodNumericVars.remove('TransactionID')
 corrThresh = 0.9
upper = correlationMatrix.where(np.triu(np.ones(correlationMatrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > corrThresh)]<drop_column>
numerical_columns = ["Age", "SibSp", "Parch", "Fare"]
categorical_columns = ["Pclass", "Embarked", "Sex"]
feature_columns = numerical_columns + categorical_columns
X = X[feature_columns]
X_test = X_test[feature_columns]
X.head()
for i_var in to_drop:
if i_var in goodNumericVars:
goodNumericVars.remove(i_var)
<drop_column>
X_train, X_valid, y_train, y_valid = train_test_split(X, y,
train_size=0.7, test_size=0.3,
random_state=0 )
del to_drop,corrThresh,upper,correlationMatrix,na_vals<define_variables>
X_train.isna().sum()
variables = list(categoricalCols)
 na_vals = np.sum(raw_train_data.loc[:,variables] == 'missing')/ raw_train_data.shape[0]
goodCategoricalVars = []
for i_var in variables:
if na_vals[i_var] < 0.85:
goodCategoricalVars.append(i_var)
<prepare_x_and_y>
X_train.isna().sum()
featureToUse = goodNumericVars + goodCategoricalVars
train_data = raw_train_data.loc[:,featureToUse]
target_data = raw_train_data['isFraud']<categorify>
numerical_transformer = SimpleImputer(strategy='mean')
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='constant')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
] )
train_data['TransactionAmt_to_mean_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_ProductCD'] = train_data['TransactionAmt'] / train_data.groupby(['ProductCD'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_dow'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_hour'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('mean')
 train_data['card1_card2'] = train_data['card1'].astype(str)+ '_' + train_data['card2'].astype(str)
 train_data['addr1_dist1'] = train_data['addr1'].astype(str)+ '_' + train_data['dist1'].astype(str)
 train_data['card1_addr1'] = train_data['card1'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card1_addr2'] = train_data['card1'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card2_addr1'] = train_data['card2'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card2_addr2'] = train_data['card2'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card4_addr1'] = train_data['card4'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card4_addr2'] = train_data['card4'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['DeviceInfo_P_emaildomain'] = train_data['DeviceInfo'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['P_emaildomain_addr1'] = train_data['P_emaildomain'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['id01_addr1'] = train_data['id_01'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['TransactionAmt_to_std_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_ProductCD'] = train_data['TransactionAmt'] / train_data.groupby(['ProductCD'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_dow'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_hour'] = train_data['TransactionAmt'] / train_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_decimal'] =(( train_data['TransactionAmt'] - train_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )<categorify>
preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_columns), 
('cat', categorical_transformer, categorical_columns)
 ] )
train_data['TransactionAmt_to_mean_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C13'] = train_data['TransactionAmt'] / train_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_D15'] = train_data['TransactionAmt'] / train_data.groupby(['D15'])['TransactionAmt'].transform('mean')
<categorify>
model = RandomForestClassifier(n_estimators = 150, max_depth = 5, random_state=0 )
train_data['TransactionAmt_to_std_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_D15'] = train_data['TransactionAmt'] / train_data.groupby(['D15'])['TransactionAmt'].transform('std' )<feature_engineering>
pipeline = Pipeline(steps=[ 
('preprocessor', preprocessor), 
('model', model)
 ] )
train_data['TransactionAmt'] = np.log(train_data['TransactionAmt'])
<define_variables>
pipeline.fit(X_train, y_train )
scale_pos_weight = np.sqrt(len(target_data.loc[target_data==0])/len(target_data.loc[target_data==1]))
del raw_train_data<load_from_csv>
preds = pipeline.predict(X_valid)
 preds
test_identity_data = pd.read_csv(".. /input/ieee-fraud-detection/test_identity.csv")
 test_transaction_data = pd.read_csv(".. /input/ieee-fraud-detection/test_transaction.csv" )<count_missing_values>
accuracy_score(y_valid, preds )
na_columns = test_transaction_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0] / test_transaction_data.shape[0] )<feature_engineering>
recall_score(y_valid, preds )
test_transaction_data['Transaction_dow'] = np.floor(( test_transaction_data['TransactionDT'] /(3600 * 24)- 1)% 7)
 test_transaction_data['Transaction_hour'] = np.floor(test_transaction_data['TransactionDT'] / 3600)% 24
transaction_data_columns = test_transaction_data.columns
numericCols = test_transaction_data._get_numeric_data().columns
categoricalCols = list(set(transaction_data_columns)- set(numericCols))
test_transaction_data[categoricalCols] = test_transaction_data[categoricalCols].replace({ np.nan:'missing'})
 test_transaction_data[numericCols] = test_transaction_data[numericCols].replace({ np.nan:-1} )<feature_engineering>
preds_test = pipeline.predict(X_test)
 preds_test.shape
identity_data_columns = test_identity_data.columns
numericCols = test_identity_data._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
test_identity_data[categoricalCols] = test_identity_data[categoricalCols].replace({ np.nan:'missing'})
 test_identity_data[numericCols] = test_identity_data[numericCols].replace({ np.nan:-1})
 test_identity_data['id_31'] = test_identity_data['id_31'].str.replace('\d+', '')
 test_identity_data.loc[test_identity_data['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
test_identity_data.loc[test_identity_data['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'
test_identity_data['device_name'] = test_identity_data['DeviceInfo'].str.split('/', expand=True)[0]
test_identity_data.loc[test_identity_data['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
test_identity_data.loc[test_identity_data['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
test_identity_data.loc[test_identity_data['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
test_identity_data.loc[test_identity_data.device_name.isin(test_identity_data.device_name.value_counts() [test_identity_data.device_name.value_counts() < 200].index), 'device_name'] = "Others"<merge>
output = pd.DataFrame({"PassengerId": data_test["PassengerId"],
"Survived": preds_test})
 output.head()
raw_test_data = pd.merge(test_transaction_data, test_identity_data, on='TransactionID', how='left')
 transactionID = raw_test_data.loc[:,'TransactionID']
del test_identity_data,test_transaction_data<feature_engineering>
output.to_csv('submission.csv', index=False )
raw_test_data_columns = raw_test_data.columns
numericCols = raw_test_data._get_numeric_data().columns
categoricalCols = list(set(raw_test_data_columns)- set(numericCols))
print('The categorical columns in training data are: ',categoricalCols)
 raw_test_data[categoricalCols] = raw_test_data[categoricalCols].replace({ np.nan:'missing'})
 raw_test_data[numericCols] = raw_test_data[numericCols].replace({ np.nan:-1})
 raw_test_data = reduce_memory_usage(raw_test_data, deep=True, verbose=True )<filter>
lr_model = LogisticRegression(random_state=0)
 lr_pipeline = Pipeline(steps = [ 
('preprocessor', preprocessor), 
('model', lr_model)
 ])
 lr_pipeline.fit(X_train, y_train)
 lr_preds = lr_pipeline.predict(X_valid)
 accuracy_score(y_valid, lr_preds)
test_data = raw_test_data.loc[:,featureToUse]<categorify>
lr_preds_test = lr_pipeline.predict(X_test)
 lr_output = pd.DataFrame({"PassengerId": data_test["PassengerId"],
"Survived": lr_preds_test})
 lr_output.to_csv('lr_submission.csv', index=False )
test_data['TransactionAmt_to_mean_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_ProductCD'] = test_data['TransactionAmt'] / test_data.groupby(['ProductCD'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_dow'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_hour'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('mean')
 test_data['card1_card2'] = test_data['card1'].astype(str)+ '_' + test_data['card2'].astype(str)
 test_data['addr1_dist1'] = test_data['addr1'].astype(str)+ '_' + test_data['dist1'].astype(str)
 test_data['card1_addr1'] = test_data['card1'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card1_addr2'] = test_data['card1'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card2_addr1'] = test_data['card2'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card2_addr2'] = test_data['card2'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card4_addr1'] = test_data['card4'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card4_addr2'] = test_data['card4'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['DeviceInfo_P_emaildomain'] = test_data['DeviceInfo'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['P_emaildomain_addr1'] = test_data['P_emaildomain'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['id01_addr1'] = test_data['id_01'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['TransactionAmt_to_std_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_ProductCD'] = test_data['TransactionAmt'] / test_data.groupby(['ProductCD'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_dow'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_dow'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_hour'] = test_data['TransactionAmt'] / test_data.groupby(['Transaction_hour'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_decimal'] =(( test_data['TransactionAmt'] - test_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )<categorify>
xgb_model = XGBClassifier(n_estimators = 1000, learning_rate = 0.1, max_depth = 5, random_state=0)
 xgb_pipeline = Pipeline(steps = [ 
('preprocessor', preprocessor), 
('model', xgb_model)
 ])
 xgb_pipeline.fit(X_train, y_train)
 xgb_preds = xgb_pipeline.predict(X_valid)
 accuracy_score(y_valid, xgb_preds )
test_data['TransactionAmt_to_mean_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C13'] = test_data['TransactionAmt'] / test_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_D15'] = test_data['TransactionAmt'] / test_data.groupby(['D15'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_std_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_D15'] = test_data['TransactionAmt'] / test_data.groupby(['D15'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt'] = np.log(test_data['TransactionAmt'] )<categorify>
xgb_preds_test = xgb_pipeline.predict(X_test)
 xgb_output = pd.DataFrame({"PassengerId": data_test["PassengerId"],
"Survived": xgb_preds_test})
 xgb_output.to_csv('xgb_submission.csv', index=False )
for i_cat in goodCategoricalVars:
le = LabelEncoder()
curData = pd.concat([train_data.loc[:,i_cat],test_data.loc[:,i_cat]],axis = 0)
 le.fit(curData)
 train_data.loc[:,i_cat] = le.transform(train_data.loc[:,i_cat])
 test_data.loc[:,i_cat] = le.transform(test_data.loc[:,i_cat] )<choose_model_class>
svm_model = LinearSVC(dual = False, random_state=0)
 svm_pipeline = Pipeline(steps = [ 
('preprocessor', preprocessor), 
('model', svm_model)
 ])
 svm_pipeline.fit(X_train, y_train)
 svm_preds = svm_pipeline.predict(X_valid)
 accuracy_score(y_valid, svm_preds )
cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True )<compute_train_metric>
svm_preds_test = svm_pipeline.predict(X_test)
 svm_output = pd.DataFrame({"PassengerId": data_test["PassengerId"],
"Survived": svm_preds_test})
 svm_output.to_csv('svm_submission.csv', index=False )