Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_missing_values>
\
%matplotlib inline
train_data['GrLivArea'].isnull().sum()<filter>
train_df = pd.read_csv('/kaggle/input/train.csv')
 test_df = pd.read_csv('/kaggle/input/test.csv')
 combine = [train_df, test_df]
Q1 = train_data['GrLivArea'].quantile(0.25)
 Q3 = train_data['GrLivArea'].quantile(0.75)
 IQR = Q3 - Q1
outliers = train_data.loc[(train_data['GrLivArea'] >(Q3 + 1.75 * IQR)) |(train_data['GrLivArea'] <(Q1 - 1.75 * IQR)) , 'GrLivArea']
print("Percent of Outliers: ", outliers.count() / train_data['GrLivArea'].count() * 100 )<count_missing_values>
train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data['FullBath'].isnull().sum()<count_missing_values>
train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data['TotRmsAbvGrd'].isnull().sum()<count_missing_values>
train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data['GarageCars'].isnull().sum()<count_missing_values>
train_df[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data['GarageArea'].isnull().sum()<filter>
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )
Q1 = train_data['GarageArea'].quantile(0.25)
 Q3 = train_data['GarageArea'].quantile(0.75)
 IQR = Q3 - Q1
outliers = train_data.loc[(train_data['GarageArea'] >(Q3 + 1.75 * IQR)) |(train_data['GarageArea'] <(Q1 - 1.75 * IQR)) , 'GarageArea']
print("Percent of Outliers: ", outliers.count() / train_data['GarageArea'].count() * 100 )<drop_column>
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
train_data.drop(train_data.loc[(train_data['GarageArea'] >(Q3 + 1.75 * IQR)) |(train_data['GarageArea'] <(Q1 - 1.75 * IQR)) ].index, inplace=True)
 train_data.shape<train_on_grid>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df.head()
<choose_model_class>
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape
hp_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0.5, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.02, max_delta_step=0, max_depth=4,
min_child_weight=1, monotone_constraints='() ',
n_estimators=1000, n_jobs=0, num_parallel_tree=1, random_state=42,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.8,
tree_method='exact', validate_parameters=1, verbosity=None)
 hp_clf = Pipeline(steps=[('preprocessor', preprocessor), 
('hp_model', hp_model)
 ])
 hp_clf.fit(X_train, y_train, hp_model__verbose=False)
 hp_preds = hp_clf.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, hp_preds))<prepare_x_and_y>
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()
X = train_data.drop(['SalePrice'], axis=1)
 y = train_data.SalePrice<data_type_conversions>
guess_ages = np.zeros(( 2,3))
guess_ages
X.columns.to_list()<feature_engineering>
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train_df.head()
X_feat_eng = X.copy()
X_feat_eng['years_since_update'] = X_feat_eng['YearRemodAdd'] - X_feat_eng['YearBuilt']
X_feat_eng['geometry'] = X_feat_eng['LotArea'] / X_feat_eng['LotFrontage']
X_feat_eng['land_topology'] = X_feat_eng['LandSlope'] + '_' + X_feat_eng['LandContour']
X_feat_eng['value_proposition'] = X_feat_eng['YearBuilt'] * X_feat_eng['OverallQual']
X_feat_eng['finished_basement'] = X_feat_eng['BsmtFinSF1'] > 0
X_feat_eng['garage_value'] = X_feat_eng['YearBuilt'] * X_feat_eng['GarageCars']
X_feat_eng['misc_value'] = X_feat_eng['Fireplaces'] + X_feat_eng['OverallQual']
X_feat_eng = X_feat_eng.drop(columns=['GarageCars'])
 X_feat_eng['Test'] = X_feat_eng['Street'] + '_' + X_feat_eng['Alley']
feature_numerical_cols = [cname for cname in X_feat_eng.columns if
X_feat_eng[cname].dtype in ['int64', 'float64']]
feature_categorical_cols = [cname for cname in X_feat_eng.columns if X_feat_eng[cname].nunique() < 50
and X_feat_eng[cname].dtype in ['object', 'bool']]
feature_numerical_transformer = SimpleImputer(strategy='constant')
 feature_categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 feature_preprocessor = ColumnTransformer(
 transformers=[ 
('num', feature_numerical_transformer, feature_numerical_cols), 
('cat', feature_categorical_transformer, feature_categorical_cols)
 ])
 feature_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0.0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.02, max_delta_step=0, max_depth=4,
min_child_weight=0.0, monotone_constraints='() ',
n_estimators=1250, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.8,
tree_method='exact', validate_parameters=1, verbosity=None)
 feature_clf = Pipeline(steps=[('feature_preprocessor', feature_preprocessor), 
('feature_model', feature_model)
 ])
 feature_X_train, feature_X_valid, feature_y_train, feature_y_valid = train_test_split(X_feat_eng, y, random_state=0)
 feature_clf.fit(feature_X_train, feature_y_train, feature_model__verbose=False)
 feature_preds = feature_clf.predict(feature_X_valid)
 print('RMSLE:', root_mean_squared_log_error(feature_y_valid, feature_preds))<load_from_csv>
for dataset in combine:
dataset.loc[ dataset['Age'] <= 24, 'Age'] = 0
dataset.loc[(dataset['Age'] > 24)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 80), 'Age'] = 2
train_df.head()
X_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<feature_engineering>
train_df = train_df.drop(['AgeBand'], axis=1)
 combine = [train_df, test_df]
train_df.head()
X_test['years_since_update'] = X_test['YearRemodAdd'] - X_test['YearBuilt']
X_test['geometry'] = X_test['LotArea'] / X_test['LotFrontage']
X_test['land_topology'] = X_test['LandSlope'] + '_' + X_test['LandContour']
X_test['value_proposition'] = X_test['YearBuilt'] * X_test['OverallQual']
X_test['finished_basement'] = X_test['BsmtFinSF1'] > 0
X_test['garage_value'] = X_test['YearBuilt'] * X_test['GarageCars']
X_test['misc_value'] = X_test['Fireplaces'] + X_test['OverallQual']
X_test['Test'] = X_test['Street'] + '_' + X_test['Alley']
X_test = X_test.drop(columns=['GarageCars'])
 feature_clf.fit(X_feat_eng, y, feature_model__verbose=False )<save_to_csv>
for i in combine:
i['Fam_Size'] = np.where(( i['SibSp']+i['Parch'])== 0 , 0,
np.where(( i['SibSp']+i['Parch'])<= 3,1,2))
del i['SibSp']
del i['Parch']
preds = feature_clf.predict(X_test)
 output = pd.DataFrame({'Id': X_test.Id,
'SalePrice': preds})
 output.to_csv('submission.csv', index=False )<load_from_csv>
train = pd.read_csv('.. /input/ames-housing-dataset/AmesHousing.csv')
 train.drop(['PID'], axis=1, inplace=True)
 origin = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.columns = origin.columns
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 print('Train:{} Test:{}'.format(train.shape,test.shape))<drop_column>
train_df.head()
missing = test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 test.dropna(axis=1, inplace=True)
 test.drop(['Electrical'], axis=1, inplace=True )<feature_engineering>
l_test = tqdm(range(0, len(test)) , desc='Matching')
 for i in l_test:
for j in range(0, len(train)) :
for k in range(1, len(test.columns)) :
if test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
l_test.close()<save_to_csv>
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port
submission.to_csv('result-with-best.csv', index=False )<import_modules>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import OneHotEncoder, LabelEncoder, StandardScaler, MinMaxScaler, RobustScaler
from sklearn.neighbors import KNeighborsRegressor
from sklearn.feature_selection import RFE, SelectPercentile, f_regression
from sklearn.model_selection import GridSearchCV, KFold, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.linear_model import Ridge, Lasso, ElasticNet
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
import catboost
from xgboost import XGBRegressor
import lightgbm as lgb
from mlxtend.regressor import StackingCVRegressor
from scipy import stats<define_variables>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()
random_state = 831<load_from_csv>
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()
data_train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv" ).set_index("Id")
 data_test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" ).set_index("Id")
 data = pd.concat([data_train, data_test] )<feature_engineering>
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 128, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 128)&(dataset['Fare'] <= 256.2), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 265.2)&(dataset['Fare'] <= 384.3), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 384.3, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 combine = [train_df, test_df]
train_df.head(10 )
def expand_categorical(data, features):
d = data[features]
values = np.unique(d.values.flatten() ).tolist()
result = pd.DataFrame(index=data.index, columns=values)
 for i, row in d.iterrows() :
v = np.unique(row.values.flatten() ).tolist()
result.loc[i,v] = 1
result.fillna(0, inplace=True)
 result.columns = [features[0]+"_"+col for col in result.columns]
data.drop(features, axis=1, inplace=True)
 return pd.concat([data, result], axis=1 )<data_type_conversions>
train_df = train_df.drop(['Embarked', 'Title'], axis=1)
 test_df = test_df.drop(['Embarked', 'Title'], axis=1)
 combine = [train_df, test_df]
train_df.head()
analyse_numeric("SalePrice" )<feature_engineering>
train_df = train_df.drop(['Pclass'], axis=1)
 test_df = test_df.drop(['Pclass'], axis=1)
 combine = [train_df, test_df]
train_df.head()
data["SalePrice"] = np.log(data["SalePrice"] )<data_type_conversions>
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape
data["MSSubClass"] = data["MSSubClass"].astype(str )<feature_engineering>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log
data["LotGeometry"] = data["LotArea"] / data["LotFrontage"]
analyse_numeric("LotGeometry" )<feature_engineering>
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc
data['LotGeometry'].fillna(0, inplace=True)
 data['LotGeometry'] = make_quantile_bins('LotGeometry', 6 )<data_type_conversions>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn
analyse_numeric("LotFrontage" )<feature_engineering>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian
data['LotFrontage'].fillna(0, inplace=True)
 data['LotFrontage'] = make_quantile_bins('LotFrontage', 6 )<data_type_conversions>
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron
analyse_numeric("LotArea" )<drop_column>
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2)
 acc_linear_svc
data.drop("Utilities", axis=1, inplace=True )<drop_column>
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_sgd
data["LandTopology"] = data["LandContour"] + data["LandSlope"]
data.drop("LandSlope", axis=1, inplace=True)
 analyse_categorical("LandTopology" )<concatenate>
decision_tree = DecisionTreeClassifier(random_state = 0, criterion="gini", max_depth = 13)
 decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree
data = expand_categorical(data, ["Condition1", "Condition2"] )<feature_engineering>
random_forest = RandomForestClassifier(criterion='gini',
n_estimators=500,
max_depth=10,
min_samples_split=5,
min_samples_leaf=1,
max_features='auto',
oob_score=True,
random_state=2,
n_jobs=-1)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest
data["YearsLastRemod"] = data["YearRemodAdd"] - data["YearBuilt"]
data["YearsLastRemod"] = make_quantile_bins("YearsLastRemod", 3)
 analyse_categorical("YearsLastRemod" )<data_type_conversions>
XGBClassifier
xg = XGBClassifier(n_estimators=300, max_depth=13, random_state=2)
 xg.fit(X_train, Y_train)
 Y_pred_xg = xg.predict(X_test)
 xg.score(X_train, Y_train)
 acc_xg = round(xg.score(X_train, Y_train)* 100, 2)
 acc_xg
analyse_numeric("YearBuilt" )<feature_engineering>
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent', 'Linear SVC',
'Decision Tree'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_linear_svc, acc_decision_tree]})
 models.sort_values(by='Score', ascending=False )
<data_type_conversions><EOS> 
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submission.csv', index=False )