Kaggle competition: Titanic - Machine Learning from Disaster
all_data.drop('Neighborhood',axis = 1,inplace=True )<create_dataframe>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
all_data_na1 = pd.DataFrame(( all_data.isnull().sum() / len(all_data)) * 100)
 all_data_na1 = all_data_na1[all_data_na1[0] > 0]<groupby>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
all_data['FireplaceQu_E'] = all_data.groupby('KitchenQual_E')['FireplaceQu_E'].apply(lambda x: x.fillna(x.median()))
all_data['Fence_E'] = all_data.groupby('KitchenQual_E')['Fence_E'].apply(lambda x: x.fillna(x.median()))<define_variables>
train['Title'] = train['Name'].str.extract(pat = '([a-zA-Z]+)\.', expand = False)
 train.Title.value_counts()
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in all_data.columns:
if all_data[i].dtype in numeric_dtypes:
numerics2.append(i )<concatenate>
train['Title'] = train['Title'].replace(['Mlle', 'Ms'], 'Miss')
 train['Title'] = train['Title'].replace('Mme', 'Mrs')
 train['Title'] = train['Title'].replace(['Mlle', 'Mme', 'Ms', 'Don', 'Rev', 'Lady', 'Sir', 'Major', 'Col', 'Capt', 'Countess', 'Jonkheer', 'Dr'], 'Other' )
num = list(set(quantitative ).difference(['YearBuilt','YearRemodAdd', 'MoSold', 'YrSold']))<feature_engineering>
test['Title'] = test['Name'].str.extract(pat = '([a-zA-Z]+)\.', expand = False)
 test.Title.value_counts()
skew_features = all_data.apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[abs(skew_features)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(high_skew.shape[0]))
skew_index = high_skew.index
for i in skew_index:
all_data[i] = boxcox1p(all_data[i], boxcox_normmax(all_data[i] + 1))<split>
test['Title'] = test['Title'].replace('Ms', 'Miss')
 test['Title'] = test['Title'].replace(['Rev', 'Col', 'Dona', 'Dr'], 'Other' )
train = all_data[:ntrain]
test = all_data[ntrain:]<compute_train_metric>
train['Cabin_Code'] = train.Cabin.str[0]
train.Cabin_Code.value_counts()
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=train):
rmse = np.sqrt(-cross_val_score(model, X, y_train, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
test['Cabin_Code'] = test.Cabin.str[0]
test.Cabin_Code.value_counts()
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]<choose_model_class>
train['Ticket_Numeric'] = train.Ticket.str.isnumeric().astype('uint8')
 test['Ticket_Numeric'] = test.Ticket.str.isnumeric().astype('uint8' )
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))<choose_model_class>
pd.pivot_table(train,index='Survived',columns='Ticket_Numeric',values='Name', aggfunc='count', fill_value=0 )
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1,
)<choose_model_class>
train['Ticket_Alphabets'] = train.Ticket.str.extract('([A-Za-z./0-9]+\)', expand = False )
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )<choose_model_class>
train.Ticket_Alphabets.value_counts()
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )<choose_model_class>
train['Ticket_Alphabets'] = train.Ticket_Alphabets.str.lower().str.extract('([a-z.]+)')
 train['Ticket_Alphabets'] = train.Ticket_Alphabets.str.replace('.', '', regex = True )
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )<compute_test_metric>
train.Ticket_Alphabets.value_counts()
score = cv_rmse(ridge)
 print("RIDGE: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("LASSO: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )<train_model>
train.loc[(train['Title']=='Mr')&(train['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Mr'), 'Age'].mean(skipna = True)
 train.loc[(train['Title']=='Mrs')&(train['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Mrs'), 'Age'].mean(skipna = True)
 train.loc[(train['Title']=='Miss')&(train['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Miss'), 'Age'].mean(skipna = True)
 train.loc[(train['Title']=='Master')&(train['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Master'), 'Age'].mean(skipna = True)
 train.loc[(train['Title']=='Other')&(train['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Other'), 'Age'].mean(skipna = True )
print('START Fit')
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(train), np.array(y_train))
print('elasticnet')
 elastic_model_full_data = elasticnet.fit(train, y_train)
 print('Lasso')
 lasso_model_full_data = lasso.fit(train, y_train)
 print('Ridge')
 ridge_model_full_data = ridge.fit(train, y_train)
 print('Svr')
 svr_model_full_data = svr.fit(train, y_train)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(train, y_train)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(train, y_train)
 print('lightgbm')
 lgb_model_full_data = lightgbm.fit(train, y_train )<predict_on_test>
train['Embarked'] = train['Embarked'].fillna(train['Embarked'].mode().iloc[0] )
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.1 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
test.loc[(test['Title']=='Mr')&(test['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Mr'), 'Age'].mean(skipna = True)
 test.loc[(test['Title']=='Mrs')&(test['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Mrs'), 'Age'].mean(skipna = True)
 test.loc[(test['Title']=='Miss')&(test['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Miss'), 'Age'].mean(skipna = True)
 test.loc[(test['Title']=='Master')&(test['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Master'), 'Age'].mean(skipna = True)
 test.loc[(test['Title']=='Other')&(test['Age'].isna()), 'Age'] = train.loc[(train['Title']=='Other'), 'Age'].mean(skipna = True )
print('RMSLE score on train data:')
 print(rmsle(y_train, np.expm1(blend_models_predict(train)) )<save_to_csv>
test['Fare'] = test['Fare'].fillna(train['Fare'].median() )
test_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 output = pd.DataFrame({'Id': test_data.Id, 'SalePrice': np.floor(np.expm1(blend_models_predict(test)))})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
train = pd.get_dummies(data = train, columns = ['Sex', 'Embarked'], drop_first = True)
 train = pd.get_dummies(data = train, columns=['Cabin_Code'] )
warnings.filterwarnings('ignore')
<load_from_csv>
test = pd.get_dummies(data = test, columns = ['Sex', 'Embarked'], drop_first = True)
 test = pd.get_dummies(data = test, columns=['Cabin_Code'])
 test['Cabin_Code_T'] = 0
train = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )<create_dataframe>
sc = StandardScaler()
train[['Age', 'Fare']] = pd.DataFrame(sc.fit_transform(train[['Age', 'Fare']]))
test[['Age', 'Fare']] = pd.DataFrame(sc.transform(test[['Age', 'Fare']]))
def func(df):
a = df.isnull().sum()
b = df.count()
c =(a/b)* 100
d = pd.DataFrame(a, columns = ['Missingvalue%'])
 return d['Missingvalue%'].sum()<train_model>
train = train.drop(columns = ['Cabin', 'Ticket', 'PassengerId', 'Name', 'Title', 'Ticket_Alphabets'] )
print('missing values in train data:', func(train))
print('missing values in test data:', func(test))<count_values>
test = test.drop(columns = ['Cabin', 'Ticket', 'PassengerId', 'Name', 'Title'] )
print('% of 1 in train data:',(train.target.value_counts() [1]/train.shape[0])* 100 )<count_unique_values>
X = train.drop(columns = ['Survived'])
 y = train['Survived']
features = train.columns.values[2:202]
unique_max_train = []
unique_max_test = []
for feature in features:
values = train[feature].value_counts()
unique_max_train.append([feature, values.max() , values.idxmax() ])
 values = test[feature].value_counts()
unique_max_test.append([feature, values.max() , values.idxmax() ] )<feature_engineering>
classifier_lr = LogisticRegression(penalty = 'l2', random_state=0 )
idx = features = train.columns.values[2:202]
for df in [test, train]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )<define_variables>
classifier_nb = GaussianNB()
features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']<init_hyperparams>
classifier_svm = SVC(C=1, kernel='rbf', gamma = 'auto' )
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
} 
<train_model>
classifier_rf = RandomForestClassifier(n_estimators=100, criterion = 'entropy', min_samples_split=10, random_state=0 )
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=44000)
 oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
lr = cross_val_score(estimator = classifier_lr, X = X, y = y, cv = 10)
 print(lr )
Prtd = pd.DataFrame({"ID_code":test["ID_code"].values})
 Prtd["target"] = predictions
Prtd.to_csv("submission.csv", index=False )<install_modules>
nb = cross_val_score(estimator = classifier_nb, X = X, y = y, cv = 10)
 print(nb )
!pip install snapml<import_modules>
svm = cross_val_score(estimator = classifier_svm, X = X, y = y, cv = 10)
 print(svm )
import numpy as np
import pandas as pd
from sklearn import decomposition
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import StratifiedKFold
from sklearn.utils.class_weight import compute_sample_weight
import sklearn.ensemble as skl
from os import path
import time
from scipy.stats import normaltest
from scipy.special import logit
import seaborn as sns
from matplotlib import pyplot as plt
import snapml
import multiprocessing
from typing import List, Dict, Union
from tqdm import tqdm<define_variables>
rf = cross_val_score(estimator = classifier_rf, X = X, y = y, cv = 10)
 print(rf )
np.random.seed(130720 )<load_from_csv>
model = ['Logistic Regression', 'Naive Bayes', 'SVM', 'Random Forest']
accuracy_mean = [np.mean(lr), np.mean(nb), np.mean(svm), np.mean(rf)]
accuracy_std = [np.std(lr), np.std(nb), np.std(svm), np.std(rf)]
path_to_folder = '/kaggle/input/santander-customer-transaction-prediction'
df = pd.read_csv(path.join(path_to_folder, 'train.csv'))
df_test = pd.read_csv(path.join(path_to_folder, 'test.csv'))
id_codes = df_test['ID_code']
df_test.drop('ID_code', axis=1, inplace=True)
 df.drop('ID_code', axis=1, inplace=True)
 features = df.drop('target', axis=1)
 columns = features.columns
target = df.target<categorify>
K_Fold = pd.DataFrame({'Model' : model, 'Accuracy_Mean' : accuracy_mean, 'Accuracy_Std' : accuracy_std})
 print(K_Fold )
def time_decorator(function):

 def timed(*args, **kwargs):
ts = time.time_ns()
result = function(*args, **kwargs)
 te = time.time_ns()
return result,(te - ts)* 1e-9
return timed<count_values>
classifier_svm.fit(X, y)
 y_pred = classifier_svm.predict(test )
<create_dataframe><EOS> 
predictions = pd.DataFrame({"PassengerId" : list(range(892, 892 + len(y_pred))),"Survived" : y_pred})
 predictions.to_csv('predictions.csv', index = False )