Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_values>
%matplotlib inline
plt.style.use('ggplot' )
for col in ['Neighborhood', 'Exterior1st', 'Exterior2nd']:
print(df[col].value_counts() )<create_dataframe>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def find_correlation(dataframe, corr_limit=0.60):
high_correlations = []
low_correlations = []
for col in num_cols:
if col == "SalePrice":
pass
else:
correlation = dataframe[[col, "SalePrice"]].corr().loc[col, "SalePrice"]
print(col, correlation)
 if abs(correlation)> corr_limit:
high_correlations.append(col + ": " + str(correlation))
else:
low_correlations.append(col + ": " + str(correlation))
return low_correlations, high_correlations
low_corrs, high_corrs = find_correlation(df )<count_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def rare_analyser(dataframe, target, rare_perc):
rare_columns = [col for col in df.columns if len(df[col].value_counts())<= 20
and(df[col].value_counts() / len(df)< rare_perc ).any(axis=None)]
for var in rare_columns:
print(var, ":", len(dataframe[var].value_counts()))
print(pd.DataFrame({"COUNT": dataframe[var].value_counts() ,
"RATIO": dataframe[var].value_counts() / len(dataframe),
"TARGET_MEDIAN": dataframe.groupby(var)[target].median() }), end="
")
 rare_analyser(df, "SalePrice", 0.01)
 def rare_encoder(dataframe, rare_perc):
temp_df = dataframe.copy()
rare_columns = [col for col in temp_df.columns if temp_df[col].dtypes == 'O'
and(temp_df[col].value_counts() / len(temp_df)< rare_perc ).any(axis=None)]
for var in rare_columns:
tmp = temp_df[var].value_counts() / len(temp_df)
 rare_labels = tmp[tmp < rare_perc].index
temp_df[var] = np.where(temp_df[var].isin(rare_labels), 'Rare', temp_df[var])
 return temp_df
df = rare_encoder(df, 0.01)
 rare_analyser(df, "SalePrice", 0.01 )<drop_column>
train.drop(['PassengerId'],axis=1,inplace=True )
drop_list = ["Street", "Utilities", "LandSlope", "PoolQC", "MiscFeature"]
cat_cols = [col for col in df.columns if df[col].dtypes == 'O'
and col not in drop_list]
for col in drop_list:
df.drop(col, axis=1, inplace=True)
 rare_analyser(df, "SalePrice", 0.01 )<categorify>
pd.isnull(train ).sum()
def one_hot_encoder(dataframe, categorical_cols, nan_as_category=True):
original_columns = list(dataframe.columns)
 dataframe = pd.get_dummies(dataframe, columns=categorical_cols, dummy_na=nan_as_category, drop_first=True)
 new_columns = [c for c in dataframe.columns if c not in original_columns]
return dataframe, new_columns
df, new_cols_ohe = one_hot_encoder(df, cat_cols)
 cat_summary(df, new_cols_ohe, "SalePrice" )<count_missing_values>
pd.isnull(test ).sum()
def missing_values_table(dataframe):
variables_with_na = [col for col in dataframe.columns if dataframe[col].isnull().sum() > 0]
n_miss = dataframe[variables_with_na].isnull().sum().sort_values(ascending=False)
 ratio =(dataframe[variables_with_na].isnull().sum() / dataframe.shape[0] * 100 ).sort_values(ascending=False)
 missing_df = pd.concat([n_miss, np.round(ratio, 2)], axis=1, keys=['n_miss', 'ratio'])
 print(missing_df)
 return variables_with_na
missing_values_table(df)
<count_missing_values>
for i in train.select_dtypes(include=['object']):
print('Column ',i,' has ',train[i].nunique() ,' unique values' )
df = df.apply(lambda x: x.fillna(x.median()), axis=0)
 missing_values_table(df )<create_dataframe>
for i in test.select_dtypes(include=['object']):
print('Column ',i,' has ',test[i].nunique() ,' unique values' )
def outlier_thresholds(dataframe, variable):
quartile1 = dataframe[variable].quantile(0.05)
 quartile3 = dataframe[variable].quantile(0.95)
 interquantile_range = quartile3 - quartile1
up_limit = quartile3 + 1.5 * interquantile_range
low_limit = quartile1 - 1.5 * interquantile_range
return low_limit, up_limit
def has_outliers(dataframe, num_col_names, plot=False):
variable_names = []
for col in num_col_names:
low_limit, up_limit = outlier_thresholds(dataframe, col)
 if dataframe[(dataframe[col] > up_limit)|(dataframe[col] < low_limit)].any(axis=None):
number_of_outliers = dataframe[(dataframe[col] > up_limit)|(dataframe[col] < low_limit)].shape[0]
print(col, ":", number_of_outliers)
 variable_names.append(col)
 if plot:
sns.boxplot(x=dataframe[col])
 plt.show()
return variable_names
has_outliers(df, num_cols )<feature_engineering>
target = train.Survived
train_df = train.drop('Survived',axis=1)
 test_df = test.drop('PassengerId',axis=1)
 train_df['is_train'] = 1
test_df['is_train'] = 0
train_test = pd.concat([train_df,test_df],axis=0 )
def replace_with_thresholds(dataframe, variable):
low_limit, up_limit = outlier_thresholds(dataframe, variable)
 dataframe.loc[(dataframe[variable] < low_limit), variable] = low_limit
dataframe.loc[(dataframe[variable] > up_limit), variable] = up_limit
for col in num_cols:
replace_with_thresholds(df, col)
 has_outliers(df, num_cols )<feature_engineering>
train['Initial']=0
for i in train:
train['Initial']=train.Name.str.extract('([A-Za-z]+)\.')
 pd.crosstab(train.Initial,train.Sex ).T.style.background_gradient(cmap='summer_r' )
df['TotalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df["OverallGrade"] = df["OverallQual"] * df["OverallCond"]
df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df.loc[df["OverallQual"] < 2, "OverallQual"] = 2
df.loc[df["GarageCars"] > 4, "GarageCars"] = 4
df.loc[(df["OverallQual"] == 2)|(df["OverallQual"] == 3), "OverallQual"] = 3
df.loc[df["GarageYrBlt"] == 2207, "GarageYrBlt"] = 2005
df.loc[df["Fireplaces"] == 4, "Fireplaces"] = 0
df.loc[df["Fireplaces"] == 3, "Fireplaces"] = 2
df["new_area"] = df["GrLivArea"] + df["GarageArea"]
df["new_home"] = df["YearBuilt"]
df.loc[df["new_home"] == df["YearRemodAdd"], "new_home"] = 0
df.loc[df["new_home"] != df["YearRemodAdd"], "new_home"] = 1
df["new_bath"] = df["FullBath"] +(df["HalfBath"] * 0.5)
 df.head()<count_missing_values>
train_test['Initial'].replace(['Capt','Col','Countess','Don','Dona','Dr','Jonkheer','Lady','Major','Master','Mlle','Mme','Ms','Rev','Sir'],
['Special_male','Other_male','Special','Special_male','Special_female','Other','Special_male','Special','Special_male','Other_male','Special','Special','Special','Other_male','Special'],inplace=True )
missing_values_table(df)
 has_outliers(df, num_cols )<find_best_model_class>
train_test.groupby(['Initial','Pclass'])['Age'].aggregate(['mean','count'] )
train_df = df[df['SalePrice'].notnull() ]
test_df = df[df['SalePrice'].isnull() ]
X = train_df.drop('SalePrice', axis=1)
 y = train_df[["SalePrice"]]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=46)
 models = [('LinearRegression', LinearRegression()), 
('Ridge', Ridge()), 
('Lasso', Lasso()), 
('ElasticNet', ElasticNet())]
results = []
names = []
for name, model in models:
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 result = np.sqrt(mean_squared_error(y_test, y_pred))
results.append(result)
 names.append(name)
 msg = "%s: %f" %(name, result)
 print(msg)
 df["SalePrice"].mean()<find_best_model_class>
train_test.loc[(train_test.Age.isnull())]['Initial'].value_counts()
X = df.loc[:1459, :].drop(["SalePrice", "Id"], axis=1)
 y = df.loc[:1459, "SalePrice"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=46)
 catboost_model = CatBoostRegressor()
catboost_model = catboost_model.fit(X_train, y_train)
 y_pred = catboost_model.predict(X_test)
 print(np.sqrt(mean_squared_error(y_pred, y_test)))
 df1 = pd.DataFrame({"Id": df.loc[1460:, "Id"],
"SalePrice": catboost_model.predict(df.loc[1460:, :].drop(["SalePrice", "Id"], axis=1)) })
 df1.to_csv('submission_catboost.csv', index=False )<install_modules>
train_test.loc[(train_test.Age.isnull())]['Pclass'].value_counts()
pip install shap<import_modules>
train_test.loc[(train_test.Age.isnull())].groupby(['Initial','Pclass'])['Name'].aggregate('count' )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
 temp_train_test = pd.get_dummies(train_test[['Initial','Pclass','Age']])
 train_test_age_filled = imp.fit_transform(temp_train_test )
data_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 data_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 data_test = data_test.drop(['Id'],1)
 data = data_train.append(data_test)
 data = data.drop(['SalePrice'],1)
 data = data.drop(['Id'],1)
<compute_test_metric>
train_test_age_filled = pd.DataFrame(train_test_age_filled,columns=temp_train_test.columns.tolist() )
print("p-value for shapiro-wilk test is ",shapiro(target)[1] )<compute_test_metric>
train_test.Age = train_test_age_filled.Age
corr_cols = ['OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF',
'FullBath', 'YearBuilt']
print("Correlations")
 for i in corr_cols:
print(i,"and target",np.corrcoef(target,data_train[i])[0][1])
<sort_values>
train_test[np.isnan(train_test['Fare'])]
nas_ratio = data.isnull().sum() / data.shape[0] * 100
nas_ratio = nas_ratio.drop(nas_ratio[nas_ratio==0].index ).sort_values(ascending=False)
 nas_ratio<data_type_conversions>
train_test.groupby(['Pclass','Parch','SibSp'])['Fare'].mean()
data["PoolQC"] = data["PoolQC"].fillna("None")
 data["MiscFeature"] = data["MiscFeature"].fillna("None")
 data["Alley"] = data["Alley"].fillna("None")
 data["Fence"] = data["Fence"].fillna("None")
 data["FireplaceQu"] = data["FireplaceQu"].fillna("None")
 for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
data[col] = data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data[col] = data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
data[col] = data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None')
 data["MasVnrType"] = data["MasVnrType"].fillna("None")
 data["MasVnrArea"] = data["MasVnrArea"].fillna(0)
 data["Functional"] = data["Functional"].fillna("Typ")
 data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0])
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0])
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0])
 data['MSSubClass'] = data['MSSubClass'].fillna("None")
 data["LotFrontage"] = data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0])
 data = data.drop(['Utilities'], axis=1)
 data = data.drop(['Street'], axis=1)
 data = data.drop(['Condition2'], axis=1)
 data = data.drop(['RoofMatl'], axis=1)
 data = data.drop(['PoolArea'], axis=1)
 data = data.drop(['3SsnPorch'], axis=1)
 data = data.drop(['LandSlope'], axis=1)
 data = data.drop(['LowQualFinSF'], axis=1)
 data = data.drop(['Electrical'], axis=1)
 data = data.drop(['MiscFeature'], axis=1)
 label_enc = LabelEncoder()
data["YearBuilt"] = label_enc.fit_transform(data["YearBuilt"])
 data["YearRemodAdd"] = label_enc.fit_transform(data["YearRemodAdd"])
 data["GarageYrBlt"] = label_enc.fit_transform(data["GarageYrBlt"])
 data['MoSold'] = label_enc.fit_transform(data['MoSold'])
 data['OverallCond'] = label_enc.fit_transform(data['OverallCond'])
 data['MSSubClass'] = label_enc.fit_transform(data['MSSubClass'])
 data['YrSold'] = label_enc.fit_transform(data['YrSold'])
 data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
print("Now there are",data.isnull().sum().sum() ,"NA values now" )<feature_engineering>
train_test.loc[(train_test.Fare.isnull()),'Fare']=9
print(data.skew().sort_values(ascending=False ).head(10))
data["LotArea"], _ = stats.boxcox(data["LotArea"].values)
 print("skew after:",data["LotArea"].skew() )<data_type_conversions>
train_test.loc[train_test.Embarked.isnull() ]
to_std = ['LotFrontage', 'LotArea',
'YearBuilt','YearRemodAdd',
'MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF',
'1stFlrSF','BsmtFullBath','GarageYrBlt','GarageArea',
'WoodDeckSF','MoSold','YrSold','TotalSF']
for i in range(len(to_std)) :
data[to_std[i]] = data[to_std[i]].astype("float64" )<normalization>
train_test.groupby(['Embarked','Pclass'])['Fare'].mean()
for c in to_std:
std = StandardScaler()
std.fit(list(np.array(data[c] ).reshape(-1,1)))
 data[c] = std.transform(list(np.array(data[c] ).reshape(-1,1)) )<categorify>
train_test["Embarked"] = train_test["Embarked"].fillna("S" )
data = pd.get_dummies(data )<prepare_x_and_y>
train_temp = pd.get_dummies(train['Cabin'])
 res = dict(zip(train_temp.columns.tolist() ,
mutual_info_classif(train_temp, train['Survived'], discrete_features=True)
))
print(res )
X = data[:1460].values
y = data_train.SalePrice.values<split>
train_test.drop('Cabin',axis=1,inplace=True )
test = data[1460:]<compute_train_metric>
train_df = train_test[train_test.is_train == 1]
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True ).get_n_splits(X)
 rmse= np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse.mean())
<compute_train_metric>
train_df.drop(['is_train'],axis=1,inplace=True )
model_xgb = xgb.XGBRegressor(colsample_bytree=0.46, gamma=0.047,
learning_rate=0.05, max_depth=3,
min_child_weight=1.785, n_estimators=2200,
reg_alpha=0.465, reg_lambda=0.86,
subsample=0.522)
 print("xgb: ",rmsle_cv(model_xgb))<choose_model_class>
train_df['Survived'] = train['Survived']
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.82,
bagging_freq = 5, feature_fraction = 0.232,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 print("lgb:",rmsle_cv(model_lgb))<compute_train_metric>
train_df['Name_len'] = train_df.Name.apply(lambda x: len(x))
cat_boost = CatBoostRegressor(learning_rate=0.05,
n_estimators=800,depth=3,silent=True)
 print("catboost: ",rmsle_cv(cat_boost))<compute_test_metric>
train_df['FamilySize'] = train_df['SibSp'] + train_df['Parch']
Enet = ElasticNet(alpha=0.005,
l1_ratio=0.85,max_iter=15000)
 print("enet: ", rmsle_cv(Enet))<predict_on_test>
train_df.loc[(train_df['Pclass']==2)&(train_df['Fare'] == 0)]
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<choose_model_class>
train_test['Last_Name'] = train_test.Name.apply(lambda x: x.split('(')[0].split('"')[0].split(' ')[-1] )
models_forAverage = AveragingModels(models=(cat_boost,model_xgb,model_lgb))
print(rmsle_cv(models_forAverage))<train_model>
train_test['NameLen'] = train_test.Name.apply(lambda x : len(x))
cat_bst = CatBoostRegressor(iterations=1000,learning_rate=0.1,silent=True)
 cat_bst.fit(X,y )<compute_test_metric>
train_test['Sex'] = train_test['Sex'].map({'female': 1, 'male': 0} ).astype(int )
feature_interactions = cat_bst.get_feature_importance(catboost.Pool(X,y),type="Interaction" )<define_variables>
train_test['FamilySize'] = train_test['Parch'] + train_test['SibSp']
fi_new = []
for k, item in enumerate(feature_interactions):
first = data.dtypes.index[feature_interactions[k][0]]
second = data.dtypes.index[feature_interactions[k][1]]
if first != second:
fi_new.append([first+ " " + second, feature_interactions[k][2]] )<create_dataframe>
train_test['Alone']=0
train_test.loc[(train_test.FamilySize==0),'Alone']=1
interaction_score = pd.DataFrame(fi_new,columns=['Feature-Pair','Interaction Score'])
 interaction_score = interaction_score.sort_values(by="Interaction Score",ascending=False,
inplace=False,na_position='last' )<choose_model_class>
train_test['Small_Family']=0
train_test.loc[(train_test.FamilySize<=3)&(train_test.FamilySize!=0),'Small_Family']=1
extended_catboost = CatBoostRegressor(per_float_feature_quantization=['3:border_count=1024', '33:border_count=1024'],
depth=8,
learning_rate=0.025,
n_estimators=2000,silent=True)
 print(rmsle_cv(extended_catboost))<compute_train_metric>
train_test['Big_Family']=0
train_test.loc[(train_test.FamilySize>=4),'Big_Family']=1
models_forAverage2 = AveragingModels(models=(extended_catboost,model_xgb))
print(rmsle_cv(models_forAverage2))<load_from_csv>
train_test.drop(['Name','SibSp','Parch','Ticket','FamilySize'],axis=1,inplace=True )
data_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 IDS = data_test['Id']<train_model>
train_test = pd.get_dummies(train_test )
model_xgb2 = xgb.XGBRegressor(colsample_bytree=0.46, gamma=0.047,
learning_rate=0.05, max_depth=3,
min_child_weight=1.785, n_estimators=2200,
reg_alpha=0.465, reg_lambda=0.86,
subsample=0.522)
 model_xgb2.fit(data[:1460],y)
 extended_catboost2 = CatBoostRegressor(per_float_feature_quantization=['3:border_count=1024', '33:border_count=1024'],
depth=8,
learning_rate=0.025,
n_estimators=2000,silent=True)
 extended_catboost2.fit(data[:1460],y)
<save_to_csv>
train = train_test[train_test.is_train == 1].drop(['is_train'],axis=1)
 test = train_test[train_test.is_train == 0].drop(['is_train'],axis=1 )
sub = pd.DataFrame()
sub['Id'] = IDS
sub['SalePrice'] =(model_xgb2.predict(data[1460:])*0.5)+(extended_catboost2.predict(data[1460:])*0.5)
 sub.to_csv('submission.csv',index=False )<set_options>
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import LinearSVC
from sklearn import metrics
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
%matplotlib inline 
<load_from_csv>
X_train, X_test, Y_train, Y_test=train_test_split(train,train_df['Survived'],test_size=0.33,random_state=3 )
df_train=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<concatenate>
model = LogisticRegression(random_state=51)
 model.fit(X_train, Y_train)
 prediction=model.predict(X_test)
 print('Accuracy for rbf LogisticRegression is ',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )
df=pd.concat([df_train,df_test] ).reset_index(drop=True)
 df<sort_values>
model = RandomForestClassifier(n_estimators=30,random_state=51)
 model.fit(X_train, Y_train)
 prediction=model.predict(X_test)
 print('Accuracy for rbf RandomForestClassifier is ',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )
total=(df.isnull().sum() ).sort_values(ascending=False)
 percentage=(( df.isnull().sum() /df.isnull().count())*100 ).sort_values(ascending=False)
 missing_values=pd.concat([total,percentage],keys=['total','percentage'],axis=1)
 missing_values.head(40 )<data_type_conversions>
model=svm.SVC(kernel='linear',C=0.1,gamma=0.1,random_state=51)
 model.fit(X_train,Y_train)
 prediction=model.predict(X_test)
 print('Accuracy for linear SVM is',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )
df['PoolQC']=df['PoolQC'].fillna('None' )<feature_engineering>
model=LinearSVC(random_state=51)
 model.fit(X_train, Y_train)
 prediction=model.predict(X_test)
 print('The accuracy of the NaiveBayes is',np.mean(cross_val_score(model, train, train_df['Survived'], cv=3)) )
<data_type_conversions><EOS> 
model=RandomForestClassifier(n_estimators=30)
 model.fit(X_train, Y_train)
 prediction = model.predict(test)
 submission = pd.read_csv('.. /input/gender_submission.csv')
 submission.Survived = prediction
submission.to_csv('submission.csv',index=False )