Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_on_grid>
traindf = pd.read_csv('.. /input/titanic/train.csv' ).set_index('PassengerId')
 testdf = pd.read_csv('.. /input/titanic/test.csv' ).set_index('PassengerId')
 df = pd.concat([traindf, testdf], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrBoy'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['WomanOrBoyCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).count())
 df['WomanOrBoyCount'] = df.mask(df.IsWomanOrBoy, df.WomanOrBoyCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrBoy, df.FamilySurvivedCount - df.Survived.fillna(0), axis=0)
 df['WomanOrBoySurvived'] = df.FamilySurvivedCount / df.WomanOrBoyCount.replace(0, np.nan)
 df.WomanOrBoyCount = df.WomanOrBoyCount.replace(np.nan, 0)
 df['Alone'] =(df.WomanOrBoyCount == 0)
 train_y = df.Survived.loc[traindf.index]
df2 = pd.concat([df.WomanOrBoySurvived.fillna(0), df.Alone, df.Sex.replace({'female': 0, 'male': 1})], axis=1)
 test_x = df2.loc[testdf.index]
y_pred_top3 =(((test_x.WomanOrBoySurvived <= 0.238)&(test_x.Sex < 0.5)&(test_x.Alone > 0.5)) | \ 
(( test_x.WomanOrBoySurvived > 0.238)& \
~(( test_x.WomanOrBoySurvived > 0.55)&(test_x.WomanOrBoySurvived <= 0.633)))).astype(int)
 pd.DataFrame({'Survived': y_pred_top3}, \
index=testdf.index ).reset_index().to_csv('survived_top3.csv', index=False)
 print('Mean =', y_pred_top3.mean() , ' Std =', y_pred_top3.std() )
<categorify>
print(__doc__)
 warnings.filterwarnings("ignore")
 np.random.seed(0 )
<create_dataframe>
df['Title'] = df['Title'].replace('Ms','Miss')
 df['Title'] = df['Title'].replace('Mlle','Miss')
 df['Title'] = df['Title'].replace('Mme','Mrs' )
<categorify>
df['Embarked'] = df['Embarked'].fillna('S' )
<train_on_grid>
med_fare = df.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df['Fare'] = df['Fare'].fillna(med_fare )
<install_modules>
df['Deck'] = df['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 df.loc[(df['Deck'] == 'T'), 'Deck'] = 'A'
!pip install catboost<choose_model_class>
df['Age'] = df.groupby(['Sex', 'Pclass', 'Title'])['Age'].apply(lambda x: x.fillna(x.median()))
cat=CatBoostRegressor(random_state=123,cat_features= cat_columns)
<import_modules>
df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
from skopt.space import Real, Categorical, Integer
from skopt import BayesSearchCV<choose_model_class>
pd.set_option('max_columns',100)
 traindf.head(3 )
search_spaces = {'iterations': Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'random_strength': Real(1e-9, 10, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 40)}
tuned_cat=BayesSearchCV(cat,search_spaces,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_cat.fit(X_transformed,y )<find_best_params>
df.WomanOrBoySurvived = df.WomanOrBoySurvived.fillna(0)
 df.WomanOrBoyCount = df.WomanOrBoyCount.fillna(0)
 df.FamilySurvivedCount = df.FamilySurvivedCount.fillna(0)
 df.Alone = df.Alone.fillna(0 )
tuned_cat.best_params_<train_model>
train_y = df.Survived.loc[traindf.index]
embeded_cat_selector = SelectFromModel(tuned_cat.best_estimator_, max_features=X_transformed.shape[1])
 embeded_cat_selector.fit(X_transformed, y )<features_selection>
cols_to_drop = ['Name','Ticket','Cabin','Survived']
df = df.drop(cols_to_drop, axis=1 )
embeded_cat_support = embeded_cat_selector.get_support()
embeded_cat_feature = X_transformed.loc[:,embeded_cat_support].columns.tolist()
print(str(len(embeded_cat_feature)) , 'selected features' )<define_variables>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = df.columns.values.tolist()
for col in features:
if df[col].dtype in numerics: continue
categorical_columns.append(col)
 categorical_columns
reduced_features_catboost=embeded_cat_feature<create_dataframe>
for col in categorical_columns:
if col in df.columns:
le = LabelEncoder()
le.fit(list(df[col].astype(str ).values))
df[col] = le.transform(list(df[col].astype(str ).values))
cat_importance_features=pd.DataFrame(zip(tuned_cat.best_estimator_.feature_importances_,X_transformed.columns), columns=['Value','Feature'])
 cat_importance_features[cat_importance_features['Value']>0].shape[0]<filter>
train_x_all, test_x_all = df.loc[traindf.index], df.loc[testdf.index]
train_x_all.head(3 )
X_catboost_reduced=X_transformed[embeded_cat_feature]<choose_model_class>
limit_opt = 0.7
cat=CatBoostRegressor(random_state=123,cat_features= catboost_cat_columns)
 search_spaces = {'iterations': Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'random_strength': Real(1e-9, 10, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 30)}
tuned_cat=BayesSearchCV(cat,search_spaces,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_cat.fit(X_catboost_reduced,y )<find_best_score>
n_clusters_opt = 3
default_base = {'quantile':.2,
'eps':.3,
'damping':.9,
'preference': -200,
'n_neighbors': 10,
'n_clusters': n_clusters_opt,
'min_samples': 3,
'xi': 0.05,
'min_cluster_size': 0.05}
tuned_cat.best_score_<find_best_params>
feature_first = 'WomanOrBoySurvived'
clustered_features = ['Pclass', 'Sex', 'Age', 'Fare', 'Embarked', 'Title', 'WomanOrBoyCount', 'Alone', 'Deck', 'Family_Size']
tuned_cat.best_params_<install_modules>
def generate_data(x1,x2,df,t):
X = pd.concat([df[x1], df[x2]], axis=1 ).values
if t==1:
y = train_y.values.astype(int)
 return(X, y)
 else:
return X
!pip install lightgbm 
<choose_model_class>
datasets = []
for i in range(len(clustered_features)) :
datasets.append(( generate_data(feature_first,clustered_features[i],train_x_all,1),{}))
gbm = lgb.LGBMRegressor(random_state=123 )<data_type_conversions>
datasets_test = []
rez = pd.DataFrame(index = test_x.index)
 for i in range(len(clustered_features)) :
datasets_test.append(generate_data(feature_first,clustered_features[i],test_x_all,0))
X_transformed_copy=X_transformed.copy()
for col in cat_columns:
X_transformed_copy[col] = X_transformed_copy[col].astype('category')
<define_search_space>
def generate_clustering_algorithms(Z,n_clusters):
bandwidth = cluster.estimate_bandwidth(df, quantile=params['quantile'])
 connectivity = kneighbors_graph(
 Z, n_neighbors=params['n_neighbors'], include_self=False)
 connectivity = 0.5 *(connectivity + connectivity.T)
 ms = cluster.MeanShift(bandwidth=bandwidth, bin_seeding=True)
 kmeans = cluster.KMeans(n_clusters=n_clusters, random_state = 1000)
 two_means = cluster.MiniBatchKMeans(n_clusters=n_clusters)
 ward = cluster.AgglomerativeClustering(n_clusters=n_clusters, linkage='ward',
connectivity=connectivity)
 spectral = cluster.SpectralClustering(n_clusters=n_clusters, eigen_solver='arpack',
affinity="nearest_neighbors")
 dbscan = cluster.DBSCAN(eps=params['eps'])
 optics = cluster.OPTICS(min_samples=params['min_samples'],
xi=params['xi'],
min_cluster_size=params['min_cluster_size'])
 affinity_propagation = cluster.AffinityPropagation(damping=params['damping'])
 average_linkage = cluster.AgglomerativeClustering(linkage="average", affinity="cityblock",
n_clusters=params['n_clusters'], connectivity=connectivity)
 birch = cluster.Birch(n_clusters=params['n_clusters'])
 gmm = mixture.GaussianMixture(n_components=n_clusters, covariance_type='full')
 clustering_algorithms =(
('MiniBatchKMeans', two_means), 
('KMeans', kmeans), 
('AffinityPropagation', affinity_propagation), 
('MeanShift', ms), 
('SpectralClustering', spectral), 
('Ward', ward), 
('AgglomerativeClustering', average_linkage), 
('DBSCAN', dbscan), 
('OPTICS', optics), 
('Birch', birch), 
('GaussianMixture', gmm)
)
 return clustering_algorithms
param_test ={'num_leaves': Integer(6, 50),
'min_child_samples': Integer(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': Real(0.2,0.8),
'colsample_bytree': Real(0.4, 0.6),
'n_estimators':Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 30)}
tuned_gbm=BayesSearchCV(gbm,param_test,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_gbm.fit(X_transformed_copy,y)
<normalization>
train_x = pd.concat([train_x_all.WomanOrBoySurvived.fillna(0),
train_x_all.Alone,
train_x_all.Sex,
], axis=1)
 test_x = pd.concat([test_x_all.WomanOrBoySurvived.fillna(0),
test_x_all.Alone,
test_x_all.Sex,
], axis=1 )
train_sizes, train_scores, test_scores, fit_times, _ = learning_curve(tuned_gbm.best_estimator_, X_transformed_copy, y, cv=3,
return_times=True,random_state=123,
scoring='neg_mean_squared_error')
 train_scores_mean = np.mean(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)
 fig = go.Figure()
fig.add_trace(go.Scatter(x=train_sizes, y=train_scores_mean,
mode='lines+markers',
name='Training score'))
fig.add_trace(go.Scatter(x=train_sizes, y=test_scores_mean,
mode='lines+markers',
name='Cross Validation score'))
fig.show() 
<train_model>
train_x = train_x.join(train_x_all[rez_col], how='left', lsuffix="_rez")
 train_x.head(2 )
embeded_gbm_selector = SelectFromModel(tuned_gbm.best_estimator_, max_features=X_transformed_copy.shape[1])
 embeded_gbm_selector.fit(X_transformed_copy, y )<features_selection>
test_x = test_x.join(rez[rez_col], how='left', lsuffix="_rez")
 test_x.head(2 )
embeded_gbm_support = embeded_gbm_selector.get_support()
embeded_gbm_feature = X_transformed_copy.loc[:,embeded_gbm_support].columns.tolist()
print(str(len(embeded_gbm_feature)) , 'selected features' )<filter>
parameters = {'max_depth' : np.arange(2, 9, dtype=int),
'min_samples_leaf' : np.arange(1, 4, dtype=int)}
classifier = DecisionTreeClassifier(random_state=1000)
 model = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1)
 model.fit(train_x, train_y)
 best_parameters = model.best_params_
print(best_parameters )
X_gbm_reduced=X_transformed_copy[embeded_gbm_feature]<define_search_space>
model=DecisionTreeClassifier(max_depth = best_parameters['max_depth'],
random_state = 1118)
 model.fit(train_x, train_y )
param_test ={'num_leaves': Integer(6, 50),
'min_child_samples': Integer(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': Real(0.2,0.8),
'colsample_bytree': Real(0.4, 0.6),
'n_estimators':Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 30)}
tuned_gbm=BayesSearchCV(gbm,param_test,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_gbm.fit(X_gbm_reduced,y)
<find_best_score>
dot_data = export_graphviz(model, out_file=None, feature_names=train_x.columns, class_names=['0', '1'],
filled=True, rounded=False,special_characters=True, precision=7)
 graph = graphviz.Source(dot_data)
 graph
tuned_gbm.best_score_<train_on_grid>
y_pred = model.predict(test_x ).astype(int)
 print('Mean =', y_pred.mean() , ' Std =', y_pred.std())
train_sizes, train_scores, test_scores, fit_times, _ = learning_curve(tuned_gbm.best_estimator_, X_gbm_reduced, y, cv=3,
return_times=True,random_state=123,
scoring='neg_mean_squared_error')
 train_scores_mean = np.mean(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)
 fig = go.Figure()
fig.add_trace(go.Scatter(x=train_sizes, y=train_scores_mean,
mode='lines+markers',
name='Training score'))
fig.add_trace(go.Scatter(x=train_sizes, y=test_scores_mean,
mode='lines+markers',
name='Cross Validation score'))
fig.show() 
<load_from_csv>
train_y_pred = model.predict(train_x ).astype(int)
 diff = sum(abs(train_y-train_y_pred)) *100/len(train_y)
 diff
test_df_copy=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<create_dataframe>
pd.DataFrame({'Survived': y_pred}, index=testdf.index ).reset_index().to_csv('survived_new.csv', index=False )
test_df=test_df_copy.copy()<data_type_conversions>
train_x_all['Pred'] = train_y_pred
train_x_all['Survived'] = train_y
cat_columns=test_df.select_dtypes(include=['O','object'] ).columns
for cols in cat_columns:
test_df[cols].fillna(test_df[cols].mode() [0],inplace=True )<define_variables>
pd.set_option('max_columns',100)
 pd.set_option('max_rows',100 )
outlier_cols=test_df.skew().index[test_df.skew().values>1]<data_type_conversions>
train_x_all[train_x_all['Survived'] != train_x_all['Pred']].sort_values(by=['Survived'] )
num_columns=test_df.select_dtypes(exclude=['O','object'] ).columns
for cols in num_columns:
if cols in outlier_cols:
test_df[cols].fillna(test_df[cols].median() ,inplace=True)
 else:
test_df[cols].fillna(test_df[cols].mean() ,inplace=True )<count_missing_values>
diff_nrow = len(train_x_all[train_x_all['Survived'] != train_x_all['Pred']])
 diff_nrow