Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
%matplotlib inline
plt.style.use('seaborn')
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import Perceptron
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import confusion_matrix
from sklearn.ensemble import VotingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import seaborn as sns
from pandas.tools.plotting import scatter_matrix<load_from_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_view = pd.read_csv(".. /input/train.csv" )<categorify>
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )
train_view['Age'].loc[train_view['Age'] <= 15] = 1
train_view['Age'].loc[(train_view['Age'] > 15)&(train_view['Age'] <= 25)] = 2
train_view['Age'].loc[(train_view['Age'] > 25)&(train_view['Age'] <= 35)] = 3
train_view['Age'].loc[(train_view['Age'] > 35)&(train_view['Age'] <= 48)] = 4
train_view['Age'].loc[(train_view['Age'] > 48)&(train_view['Age'] <= 65)] = 5
train_view['Age'].loc[(train_view['Age'] > 65)&(train_view['Age'] <= 75)] = 6
train_view['Age'].loc[train_view['Age'] > 75] = 7<categorify>
df = pd.concat([train.SalePrice, np.log(train.SalePrice + 1 ).rename('LogSalePrice')], axis=1, names=['SalePrice', 'LogSalePrice'])
 df.head()
train_view['Fare'].loc[train_view['Fare'] <= 14] = 1
train_view['Fare'].loc[(train_view['Fare'] > 14)&(train_view['Fare'] <= 20)] = 2
train_view['Fare'].loc[(train_view['Fare'] > 20)&(train_view['Fare'] <= 40)] = 3
train_view['Fare'].loc[(train_view['Fare'] > 40)&(train_view['Fare'] <= 60)] = 4
train_view['Fare'].loc[(train_view['Fare'] > 60)&(train_view['Fare'] <= 80)] = 5
train_view['Fare'].loc[(train_view['Fare'] > 80)&(train_view['Fare'] <= 100)] = 6
train_view['Fare'].loc[train_view['Fare'] > 100] = 7<categorify>
train.SalePrice = np.log1p(train.SalePrice )
train_view["Title"] = train_view["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train_view["Title"] = train_view["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train_view["Title"] = train_view["Title"].astype(int )<concatenate>
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
data_df = train_df.append(test_df )<categorify>
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 features.shape
data_df['Title'] = data_df['Name']
for name_string in data_df['Name']:
data_df['Title'] = data_df['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_df.replace({'Title': mapping}, inplace=True)
 titles = ['Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Rev']
for title in titles:
age_to_impute = data_df.groupby('Title')['Age'].median() [titles.index(title)]
data_df.loc[(data_df['Age'].isnull())&(data_df['Title'] == title), 'Age'] = age_to_impute
train_df['Age'] = data_df['Age'][:891]
test_df['Age'] = data_df['Age'][891:]
data_df.drop('Title', axis = 1, inplace = True )<categorify>
nulls = np.sum(features.isnull())
 nullcols = nulls.loc[(nulls != 0)]
dtypes = features.dtypes
dtypes2 = dtypes.loc[(nulls != 0)]
info = pd.concat([nullcols, dtypes2], axis=1 ).sort_values(by=0, ascending=False)
 print(info)
 print("There are", len(nullcols), "columns with missing values" )
data_df['Fare'].fillna(data_df['Fare'].median() , inplace = True)
 data_df['FareBin'] = pd.qcut(data_df['Fare'], 5)
 label = LabelEncoder()
data_df['FareBin_Code'] = label.fit_transform(data_df['FareBin'])
 train_df['FareBin_Code'] = data_df['FareBin_Code'][:891]
test_df['FareBin_Code'] = data_df['FareBin_Code'][891:]
train_df.drop(['Fare'], 1, inplace=True)
 test_df.drop(['Fare'], 1, inplace=True )<categorify>
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0] )
data_df['AgeBin'] = pd.qcut(data_df['Age'], 4)
 label = LabelEncoder()
data_df['AgeBin_Code'] = label.fit_transform(data_df['AgeBin'])
 train_df['AgeBin_Code'] = data_df['AgeBin_Code'][:891]
test_df['AgeBin_Code'] = data_df['AgeBin_Code'][891:]
train_df.drop(['Age'], 1, inplace=True)
 test_df.drop(['Age'], 1, inplace=True )<feature_engineering>
pd.set_option('max_columns', None)
 features[features['PoolArea'] > 0 & features['PoolQC'].isnull() ]
data_df['Family_Size'] = data_df['Parch'] + data_df['SibSp']
train_df['Family_Size'] = data_df['Family_Size'][:891]
test_df['Family_Size'] = data_df['Family_Size'][891:]<feature_engineering>
features.loc[2418, 'PoolQC'] = 'Fa'
features.loc[2501, 'PoolQC'] = 'Gd'
features.loc[2597, 'PoolQC'] = 'Fa'
data_df["Alone"] = np.where(data_df['SibSp'] + data_df['Parch'] + 1 == 1, 1,0)
 data_df["Alone"] = np.where(data_df['SibSp'] + data_df['Parch'] + 1 == 1, 1,0)
 train_df['Alone'] = data_df['Alone'][:891]
test_df['Alone'] = data_df['Alone'][891:]<feature_engineering>
pd.set_option('max_columns', None)
 features[(features['GarageType'] == 'Detchd')& features['GarageYrBlt'].isnull() ]
data_df['Last_Name'] = data_df['Name'].apply(lambda x: str.split(x, ",")[0])
 data_df['Fare'].fillna(data_df['Fare'].mean() , inplace=True)
 DEFAULT_SURVIVAL_VALUE = 0.5
data_df['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data_df[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data_df.loc[data_df['Family_Survival']!=0.5].shape[0] )<groupby>
features.loc[2124, 'GarageYrBlt'] = features['GarageYrBlt'].median()
features.loc[2574, 'GarageYrBlt'] = features['GarageYrBlt'].median()
features.loc[2124, 'GarageFinish'] = features['GarageFinish'].mode() [0]
features.loc[2574, 'GarageFinish'] = features['GarageFinish'].mode() [0]
features.loc[2574, 'GarageCars'] = features['GarageCars'].median()
features.loc[2124, 'GarageArea'] = features['GarageArea'].median()
features.loc[2574, 'GarageArea'] = features['GarageArea'].median()
features.loc[2124, 'GarageQual'] = features['GarageQual'].mode() [0]
features.loc[2574, 'GarageQual'] = features['GarageQual'].mode() [0]
features.loc[2124, 'GarageCond'] = features['GarageCond'].mode() [0]
features.loc[2574, 'GarageCond'] = features['GarageCond'].mode() [0]
for _, grp_df in data_df.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data_df[data_df['Family_Survival']!=0.5].shape[0]))
train_df['Family_Survival'] = data_df['Family_Survival'][:891]
test_df['Family_Survival'] = data_df['Family_Survival'][891:]<categorify>
basement_columns = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'TotalBsmtSF']
tempdf = features[basement_columns]
tempdfnulls = tempdf[tempdf.isnull().any(axis=1)]
label = LabelEncoder()
data_df['Embarked'] = data_df['Embarked'].fillna('None')
 data_df['Embarked_code'] = label.fit_transform(data_df['Embarked'])
 train_df['Embarked_code'] = data_df['Embarked_code'][:891]
test_df['Embarked_code'] = data_df['Embarked_code'][891:]<categorify>
tempdfnulls[(tempdfnulls.isnull() ).sum(axis=1)< 5]
train_df['Sex'].replace(['male','female'],[0,1],inplace=True)
 test_df['Sex'].replace(['male','female'],[0,1],inplace=True )<drop_column>
features.loc[332, 'BsmtFinType2'] = 'ALQ'
features.loc[947, 'BsmtExposure'] = 'No'
features.loc[1485, 'BsmtExposure'] = 'No'
features.loc[2038, 'BsmtCond'] = 'TA'
features.loc[2183, 'BsmtCond'] = 'TA'
features.loc[2215, 'BsmtQual'] = 'Po'
features.loc[2216, 'BsmtQual'] = 'Fa'
features.loc[2346, 'BsmtExposure'] = 'No'
features.loc[2522, 'BsmtCond'] = 'Gd'
train_df = train_df.drop(columns=['PassengerId','Pclass','Name','SibSp','Parch','Ticket','Cabin','Embarked'])
 test_df = test_df.drop(columns=['PassengerId','Pclass','Name','SibSp','Parch','Ticket','Cabin','Embarked'] )<create_dataframe>
subclass_group = features.groupby('MSSubClass')
 Zoning_modes = subclass_group['MSZoning'].apply(lambda x : x.mode() [0])
 Zoning_modes
train_df = train_df.replace(np.inf, np.nan)
 train_df_na =(train_df.isnull().sum() / len(train_df))
train_df_na = train_df_na.drop(train_df_na[train_df_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :train_df_na})
 missing_data.head()<create_dataframe>
features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
test_df = test_df.replace(np.inf, np.nan)
 test_df_na =(test_df.isnull().sum() / len(test_df))
test_df_na = test_df_na.drop(test_df_na[test_df_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :test_df_na})
 missing_data.head()<prepare_x_and_y>
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
nulls = np.sum(features.isnull())
 nullcols = nulls.loc[(nulls != 0)]
dtypes = features.dtypes
dtypes2 = dtypes.loc[(nulls != 0)]
info = pd.concat([nullcols, dtypes2], axis=1 ).sort_values(by=0, ascending=False)
 print(info)
 print("There are", len(nullcols), "columns with missing values" )
X_train = train_df.drop('Survived', 1)
 Y_train = train_df['Survived']
X_test = test_df.copy()<normalization>
neighborhood_group = features.groupby('Neighborhood')
 lot_medians = neighborhood_group['LotFrontage'].median()
lot_medians
std_scaler = StandardScaler()
X_train = std_scaler.fit_transform(X_train)
 X_test = std_scaler.transform(X_test )<import_modules>
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from xgboost import XGBClassifier<choose_model_class>
pd.set_option('max_columns', None)
 features[(features['GarageYrBlt'].isnull())& features['GarageArea'] > 0]
xgb = XGBClassifier(booster='gbtree',
gamma=0.1,
learning_rate = 0.01,
max_depth = 6,
min_child_weight= 1,
n_estimators= 100,
subsample= 0.5 )<choose_model_class>
pd.set_option('max_columns', None)
 features[(features['MasVnrArea'].isnull())]
rf=RandomForestClassifier(random_state=42,
min_samples_split=2,
max_leaf_nodes=10,
max_features='auto',
n_estimators= 500,
max_depth=5,
criterion='gini' )<choose_model_class>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
nulls = np.sum(features.isnull())
 nullcols = nulls.loc[(nulls != 0)]
dtypes = features.dtypes
dtypes2 = dtypes.loc[(nulls != 0)]
info = pd.concat([nullcols, dtypes2], axis=1 ).sort_values(by=0, ascending=False)
 print(info)
 print("There are", len(nullcols), "columns with missing values" )
ada = AdaBoostClassifier(random_state=40,
algorithm = 'SAMME.R',
learning_rate= 0.01,
n_estimators = 200
)<choose_model_class>
features[features['GarageYrBlt'] == 2207]
dt = DecisionTreeClassifier(random_state=40,
criterion = 'gini',
max_depth= 5,
max_features= 'auto',
max_leaf_nodes= 12,
min_samples_split= 2 )<choose_model_class>
features.loc[2590, 'GarageYrBlt'] = 2007
knn = KNeighborsClassifier(algorithm='auto',
leaf_size=26,
metric='minkowski',
metric_params=None,
n_jobs=-1,
n_neighbors=18,
p=2,
weights='uniform' )<choose_model_class>
factors = ['MSSubClass']
for i in factors:
features.update(features[i].astype('str'))
optimal = VotingClassifier(estimators=[('knn',knn),('ada',ada),('rf',rf),('xgb',xgb),('dt',dt)],voting='hard')
 optimal<predict_on_test>
objects3 = []
for i in features.columns:
if features[i].dtype == object:
objects3.append(i )
optimal.fit(X_train, Y_train)
 Y_pred=optimal.predict(X_test )<save_to_csv>
print("Training Set incomplete cases")
 sums_features = features[objects3].apply(lambda x: len(np.unique(x)))
 sums_features.sort_values(ascending=False )
sub = pd.DataFrame(pd.read_csv(".. /input/test.csv")['PassengerId'])
 sub['Survived'] = Y_pred
sub.to_csv(".. /working/submission.csv", index = False )<define_variables>
print(features['Street'].value_counts())
 print('-----')
 print(features['Utilities'].value_counts())
 print('-----')
 print(features['CentralAir'].value_counts())
 print('-----')
 print(features['PavedDrive'].value_counts() )
pd.options.mode.chained_assignment = None
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 TRAIN_FILE = ".. /input/train.csv"
TEST_FILE = ".. /input/test.csv"
SUBMISSION_FILE = ".. /input/gender_submission.csv"<load_from_csv>
features = features.drop(['Utilities', 'Street'], axis=1 )
train_data = pd.read_csv(TRAIN_FILE)
 train_data.info()<save_to_csv>
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5*features['HalfBath'])+
features['BsmtFullBath'] +(0.5*features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
train_data = pd.read_csv(TRAIN_FILE)
 print("
print(train_data[1:6].to_csv(index=False))
print("
train_data.info()<save_to_csv>
final_features = pd.get_dummies(features ).reset_index(drop=True)
 final_features.shape
test_data = pd.read_csv(TEST_FILE)
 print("
print(test_data[1:6].to_csv(index=False))
print("
test_data.info()<save_to_csv>
outliers = list(test2[test2<1e-3].index)
 outliers
submission_data = pd.read_csv(SUBMISSION_FILE)
 print("
print(submission_data[1:6].to_csv(index=False))<import_modules>
outliers = [30, 88, 462, 631, 1322]
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import LabelBinarizer
from sklearn import preprocessing<feature_engineering>
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )
def create_useless_column(data):
data["UselessColumn"] = 0
data_new = data[["UselessColumn"]]
return data_new
def extract_survived(data):
return data["Survived"]
def apply_model(data, data_label):
model = KNeighborsClassifier(n_neighbors=2)
 scores = cross_val_score(model, data, data_label, cv=5, verbose=1, scoring='accuracy')
 print(scores.mean())
 data_label = extract_survived(train_data)
 data = create_useless_column(train_data)
 apply_model(data, data_label )<load_from_csv>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 >99.94:
overfit.append(i )
def drop_survived(data):
return data.drop("Survived", axis=1, errors="ignore")
 def drop_passenger_id(data):
return data.drop("PassengerId", axis=1, errors="ignore")
 train_data = pd.read_csv(TRAIN_FILE)
 data = train_data
data = drop_survived(data)
 data = drop_passenger_id(data)[["Fare"]]
apply_model(data, data_label )<data_type_conversions>
overfit = list(overfit)
 overfit
print("Train: {} null(on {})".format(test_data["Pclass"].isnull().sum() , len(test_data)))
 print("Test: {} null(on {})".format(train_data["Pclass"].isnull().sum() , len(train_data)))
 print("")
 print("Correlation:")
 pclass = train_data[["Pclass", "Survived"]]
pclass["Class1"] =(pclass["Pclass"] == 1 ).astype(int)
 pclass["Class2"] =(pclass["Pclass"] == 2 ).astype(int)
 pclass["Class3"] =(pclass["Pclass"] == 3 ).astype(int)
 pclass = pclass.drop("Pclass", axis=1)
 corr = pclass.corr()
print(corr)
 sns.heatmap(corr,
xticklabels=corr.columns.values,
yticklabels=corr.columns.values, center=0 )<count_values>
overfit.append('MSZoning_C(all)' )
data = train_data.copy()
data["Pclass"] = data["Pclass"] - 1
data["Pclass"] = preprocessing.maxabs_scale(data["Pclass"])
 print(data["Pclass"].value_counts() )<feature_engineering>
overfit
def handle_pclass(data):
new_data = data
new_data["Pclass"] = new_data["Pclass"] -1
new_data["Pclass"] = preprocessing.maxabs_scale(data["Pclass"])
 return new_data
data = train_data.copy()
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)[["Pclass"]]
apply_model(data, data_label )<train_model>
X.drop(overfit,axis=1,inplace=True)
 testing_features.drop(overfit,axis=1,inplace=True )
names = train_data["Name"]
print(names[1:10])
 print("Train: {} null(on {})".format(test_data["Name"].isnull().sum() , len(test_data)))
 print("Test: {} null(on {})".format(train_data["Name"].isnull().sum() , len(train_data)))
 print("" )<feature_engineering>
lm = LinearRegression()
kfolds = KFold(n_splits=10, shuffle=True, random_state=23)
 def cv_rmse(model):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv = kfolds))
return(rmse)
 def cv_rmsle(model):
rmsle = np.sqrt(np.log(-cross_val_score(model, X, y,
scoring = 'neg_mean_squared_error',
cv=kfolds)))
 return(rmsle )
data = train_data.copy()
data["Name"] = data["Name"].str.replace(".",";")
 data["Name"] = data["Name"].str.replace(",",";")
 data["Name"] = data["Name"].str.split(';', expand=True)[1]
unique1 = data["Name"].unique()
print(unique1)
 data2 = test_data.copy()
data2["Name"] = data2["Name"].str.replace(".",";")
 data2["Name"] = data2["Name"].str.replace(",",";")
 data2["Name"] = data2["Name"].str.split(';', expand=True)[1]
unique2 = data2["Name"].unique()
print(unique2)
 for i in unique2:
if not unique1.__contains__(i):
print("Missing:" + i)
 corr = data[["Name","Survived"]].corr()
print(corr)
 dumies = pd.get_dummies(data["Name"])
 dumies["Survived"] = data["Survived"]
sns.heatmap(corr,
xticklabels=corr.columns.values,
yticklabels=corr.columns.values, center=0)
 data[["Survived"]].groupby([data["Name"], data["Pclass"], data["Sex"], data["Survived"]] ).count()<feature_engineering>
benchmark_model = make_pipeline(RobustScaler() ,
lm ).fit(X=X, y=y)
 cv_rmse(benchmark_model ).mean()
data = test_data.copy()
data["Name"] = test_data["Name"].str.replace(".",";")
 data["Name"] = data["Name"].str.replace(",",";")
 data["Name"] = data["Name"].str.split(';', expand=True)[1]
data["Name"] = data["Name"].str.replace("Capt","Mr")
 data["Name"] = data["Name"].str.replace("Col","Mr")
 data["Name"] = data["Name"].str.replace("Don","Mr")
 data["Name"] = data["Name"].str.replace("Dr","Mr")
 data["Name"] = data["Name"].str.replace("Jonkheer","Mr")
 data["Name"] = data["Name"].str.replace("Rev","Mr")
 data["Name"] = data["Name"].str.replace("Sir","Mr")
 data["Name"] = data["Name"].str.replace("Mme","Mrs")
 data["Name"] = data["Name"].str.replace("Mlle","Miss")
 data["Name"] = data["Name"].str.replace("the Countess","Mme")
 data["Name"] = data["Name"].str.replace("Mme","Mrs")
 data["Name"] = data["Name"].str.replace("Ms","Mrs")
 data["Name"] = data["Name"].str.replace("Major","Mr")
 data["Name"] = data["Name"].str.replace("Master","Mr")
 data["Name"] = data["Name"].str.replace("Lady","Miss")
 groups = data.groupby("Name" ).size()
groups.plot.bar()<feature_engineering>
coeffs = pd.DataFrame(list(zip(X.columns, benchmark_model.steps[1][1].coef_)) , columns=['Predictors', 'Coefficients'])
 coeffs.sort_values(by='Coefficients', ascending=False )
def handle_name(data):
new_data = data
new_data["Name"] = data["Name"]
new_data["Name"] = new_data["Name"].str.replace(".",";")
 new_data["Name"] = new_data["Name"].str.replace(",",";")
 new_data["Name"] = new_data["Name"].str.split(';', expand=True)[1]
new_data["Name"] = new_data["Name"].str.replace("Capt","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Col","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Don","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Dr","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Jonkheer","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Rev","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Sir","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Mme","Mrs")
 new_data["Name"] = new_data["Name"].str.replace("Mlle","Miss")
 new_data["Name"] = new_data["Name"].str.replace("the Countess","Mme")
 new_data["Name"] = new_data["Name"].str.replace("Mme","Mrs")
 new_data["Name"] = new_data["Name"].str.replace("Ms","Mrs")
 new_data["Name"] = new_data["Name"].str.replace("Major","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Master","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Lady","Miss")
 new_data["Miss"] = new_data["Name"].str.contains("Miss" ).astype(int)
 new_data["Mr"] = new_data["Name"].str.contains("Mr" ).astype(int)
 new_data["Mrs"] = new_data["Name"].str.contains("Mrs" ).astype(int)
 new_data = new_data.drop("Name", axis=1)
 return new_data
data = train_data
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data= handle_name(data)[["Pclass", "Miss", "Mr", "Mrs"]]
apply_model(data, data_label )<drop_column>
def ridge_selector(k):
ridge_model = make_pipeline(RobustScaler() ,
RidgeCV(alphas = [k],
cv=kfolds)).fit(X, y)
 ridge_rmse = cv_rmse(ridge_model ).mean()
return(ridge_rmse )
data = train_data
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data= handle_name(data)
 data= handle_sex(data)[["Pclass", "Sex", "Miss", "Mr", "Mrs"]]
apply_model(data, data_label )<drop_column>
r_alphas = [.0001,.0003,.0005,.0007,.0009, 
.01, 0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 20, 30, 50, 60, 70, 80]
ridge_scores = []
for alpha in r_alphas:
score = ridge_selector(alpha)
 ridge_scores.append(score )
data = train_data.copy()
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data= handle_name(data)
 data= handle_sex(data)
 data= handle_age(data)[["Pclass", "Miss", "Mr", "Mrs", "Sex", "Age"]]
apply_model(data, data_label )<normalization>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
ridge_model2 = make_pipeline(RobustScaler() ,
RidgeCV(alphas = alphas_alt,
cv=kfolds)).fit(X, y)
 cv_rmse(ridge_model2 ).mean()
def handle_sibsp(data):
new_data = data
new_data["SibSp"] = preprocessing.maxabs_scale(data["SibSp"])
 return new_data
def handle_parch(data):
new_data = data
new_data["Parch"] = preprocessing.maxabs_scale(data["Parch"])
 return new_data
data = train_data.copy()
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data = handle_name(data)
 data = handle_sex(data)
 data = handle_age(data)
 data = handle_sibsp(data)
 data = handle_parch(data)
 data = data[["Pclass", "Miss", "Mr", "Mrs", "Sex", "Age", "SibSp", "Parch"]]
apply_model(data, data_label )<prepare_output>
ridge_model2.steps[1][1].alpha_
data = train_data.copy()
print(data["Ticket"].head(10))<drop_column>
alphas = [0.00005, 0.0001, 0.0003, 0.0005, 0.0007,
0.0009, 0.01]
alphas2 = [0.00005, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005,
0.0006, 0.0007, 0.0008]
lasso_model2 = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7,
alphas = alphas2,
random_state = 42)).fit(X, y )
def drop_ticket(data):
return data.drop(["Ticket"], axis=1)
 data = train_data
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data = handle_name(data)
 data = handle_sex(data)
 data = handle_age(data)
 data = handle_sibsp(data)
 data = handle_parch(data)
 data = drop_ticket(data)
 data = data[["Pclass", "Miss", "Mr", "Mrs", "Sex", "Age", "SibSp", "Parch"]]
apply_model(data, data_label )<data_type_conversions>
lasso_model2.steps[1][1].alpha_
def handle_fare(data):
new_data = data
new_data["Fare"] = new_data["Fare"].fillna(new_data["Fare"].mean())
 new_data["Fare"] = new_data["Fare"]/ 20
new_data["Fare"] = new_data["Fare"].astype(int)
 new_data["Fare"] = preprocessing.maxabs_scale(data["Fare"])
 return new_data
test_data = pd.read_csv(TRAIN_FILE)
 groups = handle_fare(test_data ).groupby("Fare" ).size()
groups.plot.bar()<drop_column>
cv_rmse(lasso_model2 ).mean()
data = train_data
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data = handle_name(data)
 data = handle_sex(data)
 data = handle_age(data)
 data = handle_sibsp(data)
 data = handle_parch(data)
 data = drop_ticket(data)
 data = handle_fare(data)
 data = data[["Pclass", "Mr", "Mrs", "Miss", "Sex", "Age", "SibSp", "Parch", "Fare"]]
apply_model(data, data_label )<prepare_output>
coeffs = pd.DataFrame(list(zip(X.columns, lasso_model2.steps[1][1].coef_)) , columns=['Predictors', 'Coefficients'] )
data = train_data.copy()
print(data["Cabin"].head(15)) 
<data_type_conversions>
used_coeffs = coeffs[coeffs['Coefficients'] != 0].sort_values(by='Coefficients', ascending=False)
 print(used_coeffs.shape)
 print(used_coeffs )
def handle_cabin(data):
new_data = data
new_data["Cabin"] = new_data["Cabin"].isna().astype(int)
 return new_data
data = train_data
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data = handle_name(data)
 data = handle_sex(data)
 data = handle_age(data)
 data = handle_sibsp(data)
 data = handle_parch(data)
 data = drop_ticket(data)
 data = handle_fare(data)
 data = handle_cabin(data)
 data = data[["Pclass", "Mr", "Mrs", "Miss", "Sex", "Age", "SibSp", "Parch","Fare", "Cabin"]]
apply_model(data, data_label )<data_type_conversions>
overfit_test2 = []
for i in used_coeffs_values.columns:
counts2 = used_coeffs_values[i].value_counts()
zeros2 = counts2.iloc[0]
if zeros2 / len(used_coeffs_values)* 100 > 99.5:
overfit_test2.append(i)
 overfit_test2
print("Train: {} null(on {})".format(train_data["Embarked"].isnull().sum() , len(train_data)))
 print("Test: {} null(on {})".format(test_data["Embarked"].isnull().sum() , len(test_data)))
 print("")
 print("Correlation:")
 embarked = train_data[["Embarked", "Survived"]]
embarked["NotEmbarked"] = embarked["Embarked"].isna().astype(int)
 embarked["Embarked"] = embarked["Embarked"].fillna("")
 embarked['Southampton'] = embarked["Embarked"].str.contains("S" ).astype(int)
 embarked['Queenstown'] = embarked["Embarked"].str.contains("Q" ).astype(int)
 embarked['Cherbourg'] = embarked["Embarked"].str.contains("C" ).astype(int)
 embarked = embarked.drop("Embarked", axis=1)
 corr = embarked.corr()
print(corr)
 sns.heatmap(corr,
xticklabels=corr.columns.values,
yticklabels=corr.columns.values )<data_type_conversions>
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elastic_cv = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
elastic_model3 = elastic_cv.fit(X, y )
def handle_embarked(data):
new_data = data
new_data["NotEmbarked"] = new_data["Embarked"].isna().astype(int)
 new_data["Embarked"] = new_data["Embarked"].fillna("")
 new_data['Southampton'] = new_data["Embarked"].str.contains("S" ).astype(int)
 new_data['Queenstown'] = new_data["Embarked"].str.contains("Q" ).astype(int)
 new_data['Cherbourg'] = new_data["Embarked"].str.contains("C" ).astype(int)
 new_data = new_data.drop("Embarked", axis=1)
 return new_data
print(handle_embarked(train_data)[["NotEmbarked", "Southampton", "Queenstown", "Cherbourg"]].head() )<categorify>
print(elastic_model3.steps[1][1].l1_ratio_)
 print(elastic_model3.steps[1][1].alpha_ )
def process_data(data):
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data = handle_name(data)
 data = handle_sex(data)
 data = handle_age(data)
 data = handle_sibsp(data)
 data = handle_parch(data)
 data = drop_ticket(data)
 data = handle_fare(data)
 data = handle_cabin(data)
 data = handle_embarked(data)
 return data
data = train_data.copy()
data = process_data(data)
 print(data.head())
 apply_model(data, data_label )<save_to_csv>
rcParams['figure.figsize'] = 12, 4
%matplotlib inline
model = KNeighborsClassifier(n_neighbors=2)
 X_train = pd.read_csv(TRAIN_FILE)
 y_train = X_train["Survived"]
X_train = process_data(X_train)
 X_test = pd.read_csv(TEST_FILE)
 test_labels = X_test[["PassengerId"]]
X_test = process_data(X_test)
 model.fit(X_train, y_train)
 result = model.predict(X_test)
 print(len(result))
df = pd.DataFrame()
df['PassengerId'] = test_labels.astype(int)
 df['Survived'] = result.astype(int)
 print(df.head())
 df.to_csv("submission.csv", index=False)
 print("Done")
<load_from_csv>
def modelfit(alg, dtrain, target, useTrainCV=True,
cv_folds=5, early_stopping_rounds=50):
if useTrainCV:
xgb_param = alg.get_xgb_params()
xgtrain = xgb.DMatrix(dtrain.values,
label=y.values)
 print("
Getting Cross-validation result.. ")
 cvresult = xgb.cv(xgb_param, xgtrain,
num_boost_round=alg.get_params() ['n_estimators'],
nfold=cv_folds,metrics='rmse',
early_stopping_rounds=early_stopping_rounds,
verbose_eval = True)
 alg.set_params(n_estimators=cvresult.shape[0])
 print("
Fitting algorithm to data...")
 alg.fit(dtrain, target, eval_metric='rmse')
 print("
Predicting from training data...")
 dtrain_predictions = alg.predict(dtrain)
 print("
Model Report")
 print("RMSE : %.4g" % np.sqrt(mean_squared_error(target.values,
dtrain_predictions)) )
pd.options.mode.chained_assignment = None
TRAIN_FILE = ".. /input/train.csv"
TEST_FILE = ".. /input/test.csv"
train_data = pd.read_csv(TRAIN_FILE)
 test_data = pd.read_csv(TEST_FILE)
 def drop_survived(data):
return data.drop("Survived", axis=1, errors="ignore")
 def drop_passenger_id(data):
return data.drop("PassengerId", axis=1, errors="ignore")
 def apply_model(data, data_label):
model = KNeighborsClassifier(n_neighbors=2)
 scores = cross_val_score(model, data, data_label, cv=5, verbose=1, scoring='accuracy')
 print(scores.mean())
 def extract_survived(data):
return data["Survived"]
def handle_pclass(data):
new_data = data
new_data["Pclass"] = new_data["Pclass"] -1
new_data["Pclass"] = preprocessing.maxabs_scale(data["Pclass"])
 return new_data
def handle_name(data):
new_data = data
new_data["Name"] = data["Name"]
new_data["Name"] = new_data["Name"].str.replace(".",";")
 new_data["Name"] = new_data["Name"].str.replace(",",";")
 new_data["Name"] = new_data["Name"].str.split(';', expand=True)[1]
new_data["Name"] = new_data["Name"].str.replace("Capt","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Col","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Don","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Dr","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Jonkheer","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Rev","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Sir","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Mme","Mrs")
 new_data["Name"] = new_data["Name"].str.replace("Mlle","Miss")
 new_data["Name"] = new_data["Name"].str.replace("the Countess","Mme")
 new_data["Name"] = new_data["Name"].str.replace("Mme","Mrs")
 new_data["Name"] = new_data["Name"].str.replace("Ms","Mrs")
 new_data["Name"] = new_data["Name"].str.replace("Major","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Master","Mr")
 new_data["Name"] = new_data["Name"].str.replace("Lady","Miss")
 new_data["Miss"] = new_data["Name"].str.contains("Miss" ).astype(int)
 new_data["Mr"] = new_data["Name"].str.contains("Mr" ).astype(int)
 new_data["Mrs"] = new_data["Name"].str.contains("Mrs" ).astype(int)
 new_data = new_data.drop("Name", axis=1)
 return new_data
def handle_sex(data):
new_data = data
new_data["Sex"] = data["Sex"].str.contains("female" ).astype(int)
 return new_data
def handle_age(data):
new_data = data
new_data["Age"] = new_data["Age"].fillna(new_data["Age"].mean())
 new_data["Age"] = new_data["Age"]/15
new_data["Age"] = new_data["Age"].astype(int)
 new_data["Age"] = preprocessing.maxabs_scale(data["Age"])
 return new_data
def handle_sibsp(data):
new_data = data
new_data["SibSp"] = preprocessing.maxabs_scale(data["SibSp"])
 return new_data
def handle_parch(data):
new_data = data
new_data["Parch"] = preprocessing.maxabs_scale(data["Parch"])
 return new_data
def drop_ticket(data):
return data.drop(["Ticket"], axis=1)
 def handle_fare(data):
new_data = data
new_data["Fare"] = new_data["Fare"].fillna(new_data["Fare"].mean())
 new_data["Fare"] = new_data["Fare"]/ 20
new_data["Fare"] = new_data["Fare"].astype(int)
 new_data["Fare"] = preprocessing.maxabs_scale(data["Fare"])
 return new_data
def handle_cabin(data):
new_data = data
new_data["Cabin"] = new_data["Cabin"].isna().astype(int)
 return new_data
def handle_embarked(data):
new_data = data
new_data["NotEmbarked"] = new_data["Embarked"].isna().astype(int)
 new_data["Embarked"] = new_data["Embarked"].fillna("")
 new_data['Southampton'] = new_data["Embarked"].str.contains("S" ).astype(int)
 new_data['Queenstown'] = new_data["Embarked"].str.contains("Q" ).astype(int)
 new_data['Cherbourg'] = new_data["Embarked"].str.contains("C" ).astype(int)
 new_data = new_data.drop("Embarked", axis=1)
 return new_data
def process_data(data):
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data = handle_name(data)
 data = handle_sex(data)
 data = handle_age(data)
 data = handle_sibsp(data)
 data = handle_parch(data)
 data = drop_ticket(data)
 data = handle_fare(data)
 data = handle_cabin(data)
 data = handle_embarked(data)
 return data
data = train_data.copy()
data = process_data(data)
 apply_model(data, data_label)
 model = KNeighborsClassifier(n_neighbors=2)
 X_train = pd.read_csv(TRAIN_FILE)
 y_train = X_train["Survived"]
X_train = process_data(X_train)
 X_test = pd.read_csv(TEST_FILE)
 test_labels = X_test[["PassengerId"]]
X_test = process_data(X_test)
 model.fit(X_train, y_train)
 result = model.predict(X_test)
 print(len(result))
df = pd.DataFrame()
df['PassengerId'] = test_labels.astype(int)
 df['Survived'] = result.astype(int)
 print(df.head())
 df.to_csv("submission.csv", index=False)
<load_from_csv>
xgb3 = XGBRegressor(learning_rate =0.01, n_estimators=3460, max_depth=3,
min_child_weight=0 ,gamma=0, subsample=0.7,
colsample_bytree=0.7,objective= 'reg:linear',
nthread=4,scale_pos_weight=1,seed=27, reg_alpha=0.00006)
 xgb_fit = xgb3.fit(X, y )
warnings.filterwarnings("ignore")
 pd.options.mode.chained_assignment = None
TRAIN_FILE = ".. /input/train.csv"
TEST_FILE = ".. /input/test.csv"
train_data = pd.read_csv(TRAIN_FILE)
 test_data = pd.read_csv(TEST_FILE)
 def extract_survived(data):
return data["Survived"]
def drop_survived(data):
return data.drop("Survived", axis=1, errors="ignore")
 def drop_passenger_id(data):
return data.drop("PassengerId", axis=1, errors="ignore")
 def handle_pclass(data):
new_data = data
new_data["Pclass"] = new_data["Pclass"] -1
new_data["Pclass"] = preprocessing.maxabs_scale(data["Pclass"])
 return new_data
def replace_multi(string, separators, new_separator):
for s in separators:
string = string.replace(s, new_separator)
 def filter_data_contains(data, column, contain, target_column=None):
new_data = data
if target_column == None:
new_data[column] = new_data[column].str.contains(contain ).astype(int)
 else:
new_data[target_column] = new_data[column].str.contains(contain ).astype(int)
 return new_data
def extract_title(x):
return x.replace(".",";" ).replace(",",";" ).split(";")[1]
def handle_name(data):
new_data = data
new_data["Name"] = new_data["Name"].apply(extract_title)
 to_replace = {
"Mr": ["Capt", "Col", "Don", "Dr", "Jonkheer", "Rev", "Sir", "Major", "Master" ],
"Miss": ["Mlle", "Lady" ],
"Mrs" : ["Mme", "the Countess", "Ms" ]
}
for title in to_replace.keys() :
for t in to_replace[title]:
new_data["Name"] = new_data["Name"].str.replace(t, title)
 new_data = filter_data_contains(new_data, "Name", title, title)
 new_data = new_data.drop("Name", axis=1)
 return new_data
def fill_na_with_mean(data, column):
new_data = data
new_data[column] = new_data[column].fillna(new_data[column].mean())
 return new_data
def fill_na_with_mean(data, column):
new_data = data
new_data[column] = new_data[column].fillna(new_data[column].mean())
 return new_data
def is_na(data, column):
new_data = data
new_data[column] = new_data[column].isna().astype(int)
 return new_data
def handle_sex(data):
return filter_data_contains(data, "Sex", "female")
 def handle_age(data):
new_data = data
new_data = fill_na_with_mean(new_data, "Age")
 new_data["Age"] =new_data["Age"] / 15
new_data["Age"] = preprocessing.maxabs_scale(data["Age"])
 return new_data
def handle_sibsp(data):
new_data = data
new_data["SibSp"] = preprocessing.maxabs_scale(data["SibSp"])
 return new_data
def handle_parch(data):
new_data = data
new_data["Parch"] = preprocessing.maxabs_scale(data["Parch"])
 return new_data
def drop_ticket(data):
return data.drop(["Ticket"], axis=1)
 def handle_fare(data):
new_data = data
new_data["Fare"] = fill_na_with_mean(new_data, "Fare")
 new_data["Fare"] = new_data["Fare"]/ 20
new_data["Fare"] = preprocessing.maxabs_scale(data["Fare"])
 return new_data
def handle_cabin(data):
new_data = data
new_data = is_na(new_data,"Cabin")
 return new_data
def handle_embarked(data):
new_data = data
new_data["NotEmbarked"] = new_data["Embarked"].isna().astype(int)
 new_data["Embarked"] = new_data["Embarked"].fillna("")
 new_data = filter_data_contains(new_data, "Embarked", "S", "Southampton")
 new_data = filter_data_contains(new_data, "Embarked", "Q", "Queenstown")
 new_data = filter_data_contains(new_data, "Embarked", "C", "Cherbourg")
 new_data = new_data.drop("Embarked", axis=1)
 return new_data
def process_data(data):
data = drop_survived(data)
 data = drop_passenger_id(data)
 data = handle_pclass(data)
 data = handle_name(data)
 data = handle_sex(data)
 data = handle_age(data)
 data = handle_sibsp(data)
 data = handle_parch(data)
 data = drop_ticket(data)
 data = handle_fare(data)
 data = handle_cabin(data)
 data = handle_embarked(data)
 return data
data = train_data.copy()
data_label = extract_survived(data)
 data = process_data(data)
 classifiers = {
"Nearest Neighbors" : KNeighborsClassifier(3),
"LinearRegression": linear_model.LinearRegression() ,
"Ridge": linear_model.Ridge(alpha =.5),
"Lasso": linear_model.Lasso(alpha = 0.1),
"ElasticNet": linear_model.ElasticNet(random_state=0),
"Lars": linear_model.Lars(n_nonzero_coefs=1),
"LassoLars": linear_model.LassoLars(alpha=.1),
"Omp": linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=1),
"BayesianRidge":linear_model.BayesianRidge() ,
"ARDRegression":linear_model.ARDRegression() ,
"LogisitcRegression":linear_model.LogisticRegression() ,
"SGDClassifier":linear_model.SGDClassifier() ,
"Perceptron": linear_model.Perceptron() ,
"PassiveAggressiveClassifier": linear_model.PassiveAggressiveClassifier() ,
"Theil-Sen": linear_model.TheilSenRegressor(random_state=42),
"RANSAC": linear_model.RANSACRegressor(random_state=42),
"Huber": linear_model.HuberRegressor() ,
"SVC linear": SVC(kernel="linear", C=0.025),
"SVC": SVC(gamma=2, C=1, probability=True),
"GuassianProcess":GaussianProcessClassifier(1.0 * RBF(1.0)) ,
"DecisionTree":DecisionTreeClassifier(max_depth=5),
"RandomForest":RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
"NeutraNet":MLPClassifier(alpha=1),
"ADABoost":AdaBoostClassifier() ,
"GaussianNB":GaussianNB() ,
"QDA":QuadraticDiscriminantAnalysis()
}
best_model_names = {}
for model_name in classifiers.keys() :
try:
model = classifiers[model_name]
scores = cross_val_score(model, data, data_label, cv=5, verbose=1, scoring='accuracy')
 score = scores.mean()
if score >.8:
best_model_names[model_name] = scores.mean()
print("{} {}".format(model_name, scores.mean()))
except:
pass
print(best_model_names)
 res = pd.DataFrame()
X_train = pd.read_csv(TRAIN_FILE)
 y_train = X_train["Survived"]
X_train = process_data(X_train)
 X_test = pd.read_csv(TEST_FILE)
 test_labels = X_test[["PassengerId"]]
X_test = process_data(X_test)
 res["PassengerId"] = test_labels["PassengerId"]
for model_name in best_model_names.keys() :
model = classifiers[model_name]
model.fit(X_train, y_train)
 result = model.predict_proba(X_test)[:,1]
print("{}: {} rows".format(model_name, len(result)))
 res[model_name] = result
models_list = list(best_model_names.keys())
 res['ProbaMin'] = res[models_list].min(axis=1)
 res['ProbaMax'] = res[models_list].max(axis=1)
 res['Accurate'] =(res['ProbaMin']<.20)|(res['ProbaMax']>.80)
 res['Survived'] =(res['ProbaMax']-0.5)>(0.5-res['ProbaMin'])
 res['Survived'] = res['Survived'].astype(int)
 res.to_csv("submission_detail.csv", index=False)
 res_filtered = res[["PassengerId","Survived"]]
res_filtered.to_csv("submission.csv", index=False)
 res.head(20 )<categorify>
svr_opt = svm.SVR(C = 100000, gamma = 1e-08)
 svr_fit = svr_opt.fit(X, y )
%matplotlib inline
label=LabelEncoder() 
<load_from_csv>
lgbm_model = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df.head()<drop_column>
cv_rmse(lgbm_model ).mean()
test_df.Fare.fillna(test_df.Fare.mean() , inplace=True)
 data_df = train_df.append(test_df)
 passenger_id=test_df['PassengerId']
train_df.drop(['PassengerId'], axis=1, inplace=True)
 test_df.drop(['PassengerId'], axis=1, inplace=True)
 test_df.shape<filter>
lgbm_fit = lgbm_model.fit(X, y )
train_df=train_df[train_df['Fare']<400]<feature_engineering>
Image(url = "http://i.imgur.com/QBuDOjs.jpg" )
train_df['Sex'] = train_df.Sex.apply(lambda x: 0 if x == "female" else 1)
 test_df['Sex'] = test_df.Sex.apply(lambda x: 0 if x == "female" else 1 )<train_model>
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas = alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas = alphas2,
random_state = 42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
lightgbm = make_pipeline(RobustScaler() ,
LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6,
min_sum_hessian_in_leaf = 11))
xgboost = make_pipeline(RobustScaler() ,
XGBRegressor(learning_rate =0.01, n_estimators=3460,
max_depth=3,min_child_weight=0 ,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective= 'reg:linear',nthread=4,
scale_pos_weight=1,seed=27,
reg_alpha=0.00006))
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 stackX = np.array(X)
 stacky = np.array(y )
pd.options.display.max_columns = 99
test_df['Fare'].fillna(test_df['Fare'].mean() ,inplace=True)
 train_df.head() 
<feature_engineering>
for name_string in data_df['Name']:
data_df['Title']=data_df['Name'].str.extract('([A-Za-z]+)\.',expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_df.replace({'Title': mapping}, inplace=True )<categorify>
stack_gen_model = stack_gen.fit(stackX, stacky )
for name_string in data_df['Name']:
data_df['Title']=data_df['Name'].str.extract('([A-Za-z]+)\.',expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_df.replace({'Title': mapping}, inplace=True)
 data_df['Title'].value_counts()
train_df['Title']=data_df['Title'][:891]
test_df['Title']=data_df['Title'][891:]
titles=['Mr','Miss','Mrs','Master','Rev','Dr']
for title in titles:
age_to_impute = data_df.groupby('Title')['Age'].mean() [titles.index(title)]
data_df.loc[(data_df['Age'].isnull())&(data_df['Title'] == title), 'Age'] = age_to_impute
data_df.isnull().sum()
train_df['Age']=data_df['Age'][:891]
test_df['Age']=data_df['Age'][891:]
test_df.isnull().sum()<groupby>
em_preds = elastic_model3.predict(testing_features)
 lasso_preds = lasso_model2.predict(testing_features)
 ridge_preds = ridge_model2.predict(testing_features)
 stack_gen_preds = stack_gen_model.predict(testing_features)
 xgb_preds = xgb_fit.predict(testing_features)
 svr_preds = svr_fit.predict(testing_features)
 lgbm_preds = lgbm_fit.predict(testing_features )
train_df.groupby('Survived' ).mean()<groupby>
stack_preds =(( 0.2*em_preds)+(0.1*lasso_preds)+(0.1*ridge_preds)+ 
(0.2*xgb_preds)+(0.1*lgbm_preds)+(0.3*stack_gen_preds))
train_df.groupby('Sex' ).mean()<feature_engineering>
submission = pd.read_csv(".. /input/sample_submission.csv" )
train_df['family_size'] = train_df.SibSp + train_df.Parch+1
test_df['family_size'] = test_df.SibSp + test_df.Parch+1 
<categorify>
submission.iloc[:,1] = np.expm1(stack_preds )
<feature_engineering><EOS> 
submission.to_csv("final_submission.csv", index=False )