Kaggle competition: Titanic - Machine Learning from Disaster
print(train[train['userId'].duplicated() ]['userId'].unique().shape[0])
 print(test[test['userId'].duplicated() ]['userId'].unique().shape[0] )<define_variables>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
ddc = train[['date_collected', 'date']]<count_duplicates>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
print(ddc.shape[0])
 print(ddc[ddc.duplicated() ].shape[0] )<drop_column>
train_data = train_data.set_index("PassengerId")
 test_data = test_data.set_index("PassengerId" )
train.drop(columns=['date'], inplace=True)
 test.drop(columns=['date'], inplace=True )<count_values>
num_attribs = ["Age", "SibSp", "Parch", "Fare", ]
cat_attribs = ["Pclass", "Sex", "Embarked", ]
train['userId'].value_counts()<count_duplicates>
num_pipeline = Pipeline([ 
("imputer", SimpleImputer(strategy = "median")) , 
("scaler", StandardScaler()),
] )
duplicate = train[(train.duplicated(subset=['userId', 'date_collected'], keep=False)) ]
null_row = duplicate[(duplicate['max_recharge_trx'].isnull())|(duplicate['average_topup_trx'].isnull())|(duplicate['total_transaction'].isnull())]
count = 0
for i, el in null_row.iterrows() :
temp_df = duplicate[(duplicate['userId'] == el['userId'])&(duplicate['date_collected'] == el['date_collected'])]
idx_recharge, idx_topup, idx_transaction = -1, -1, -1
for j, row in temp_df.iterrows() :
if(~np.isnan(row['max_recharge_trx'])) :
idx_recharge = j
if(~np.isnan(row['average_topup_trx'])) :
idx_topup = j
if(~np.isnan(row['total_transaction'])) :
idx_transaction = j
if(idx_recharge != -1):
train.at[i, 'max_recharge_trx'] = train.at[idx_recharge, 'max_recharge_trx']
if(idx_topup != -1):
train.at[i, 'average_topup_trx'] = train.at[idx_topup, 'average_topup_trx']
if(idx_transaction != -1):
train.at[i, 'total_transaction'] = train.at[idx_transaction, 'total_transaction']
count += 1
if(count % 1000 == 0):
print(count )<feature_engineering>
cat_pipeline = Pipeline([ 
("imputer", SimpleImputer(strategy = "most_frequent")) , 
("cat_encoder", OneHotEncoder(sparse = False)) ,
] )
def update_max_recharge_trx(row):
if(np.isnan(row['max_recharge_trx'])) :
row['max_recharge_trx'] = row['average_recharge_trx'] * 2 - row['min_recharge_trx']
return row
train = train.apply(update_max_recharge_trx, axis=1 )<feature_engineering>
preprocess_pipeline = ColumnTransformer([ 
("num", num_pipeline, num_attribs), 
("cat", cat_pipeline, cat_attribs),
] )
def update_average_topup_trx(row):
if(np.isnan(row['average_topup_trx'])) :
if(row['num_topup_trx'] < 2):
row['average_topup_trx'] = row['min_topup_trx']
else:
row['average_topup_trx'] =(row['min_topup_trx'] + row['max_topup_trx'])/2
return row
train = train.apply(update_average_topup_trx, axis=1 )<feature_engineering>
X_train = preprocess_pipeline.fit_transform(train_data[num_attribs + cat_attribs])
 X_train
def update_num_transfer(x):
return x['num_transaction'] - x['num_topup_trx'] - x['num_recharge_trx']<feature_engineering>
y_train = train_data["Survived"]
train['num_transfer_trx'] = train.apply(update_num_transfer, axis=1)
 test['num_transfer_trx'] = test.apply(update_num_transfer, axis=1 )<groupby>
forest_clf = RandomForestClassifier(n_estimators = 100, random_state = 42)
 forest_clf.fit(X_train, y_train )
def update_average_trx(x):
if(x['num_transfer_trx'] == 0):
return 0
if(np.isnan(x['total_transaction'])) :
return np.nan
topup = x['average_topup_trx'] * x['num_topup_trx']
recharge = x['average_recharge_trx'] * x['num_recharge_trx']
total = x['total_transaction']
return(total-topup-recharge)/x['num_transfer_trx']<feature_engineering>
X_test = preprocess_pipeline.fit_transform(test_data[num_attribs + cat_attribs])
 y_pred = forest_clf.predict(X_test )
train['average_transfer_trx'] = train.apply(update_average_trx, axis=1)
 test['average_transfer_trx'] = test.apply(update_average_trx, axis=1 )<categorify>
forest_scores = cross_val_score(forest_clf, X_train, y_train, cv = 10)
 forest_scores.mean()
def fill_total_transaction(x):
if(np.isnan(x['total_transaction'])) :
if(np.isnan(x['average_transfer_trx'])) :
return x
x['total_transaction'] =(
(x['average_topup_trx'] * x['num_topup_trx'])+ 
(x['average_recharge_trx'] * x['num_recharge_trx'])+ 
(x['average_transfer_trx'] * x['num_transfer_trx'])
)
 return x
train = train.apply(fill_total_transaction, axis=1 )<filter>
svm_clf = SVC(gamma = "auto")
 svm_scores = cross_val_score(svm_clf, X_train, y_train, cv = 10)
 svm_scores.mean()
train[train['isActive'].isnull() ]<drop_column>
svm_clf.fit(X_train, y_train)
 y_pred = svm_clf.predict(X_test )
train.drop(train[train['isActive'].isnull() ].index, inplace=True )<count_values>
output = pd.DataFrame({'PassengerId': test_data.index, 'Survived': y_pred})
 output.to_csv('submission.csv', index = False)
 print("Your submission was successfully saved!" )