Kaggle competition: Personality Profile Prediction
<SOS> metric: meancolumnwiselogloss Kaggle data source: personality-profile-prediction<train_on_grid>
sns.set_style('whitegrid')
parameters = {
'model__n_estimators': [560, 580, 600, 620, 640],
}
grid_search = GridSearchCV(pipeline, parameters, scoring = 'neg_mean_absolute_error', n_jobs= 1, cv=3)
 grid_search.fit(X_full, y )<find_best_params>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
print(grid_search.best_params_)
 print(-1 * grid_search.best_score_ )<train_model>
ntrain = train.shape[0]
ntest = test.shape[0]
all_data = pd.concat(( train, test), sort=False ).reset_index(drop=True)
 all_data.drop("id", axis=1, inplace=True)
model = RandomForestClassifier(random_state=2, n_estimators=580)
 pipeline = Pipeline(steps=[('preprocessor', preprocessor)
 ,('model', model)
 ])
 pipeline.fit(X_full.drop(['Survived'], axis=1), y)
 predict = pipeline.predict(X_test_full.drop(['PassengerId'], axis=1))<save_to_csv>
grouped_wordclouds = all_data.groupby('type' ).sum()
grouped_wordclouds
out = pd.DataFrame({'PassengerId': X_test_full['PassengerId'], 'Survived': predict})
 out['Survived'] = out.apply(lambda x : int(x['Survived']), axis=1)
 out.to_csv('titanicOut.csv', index=False )<load_from_csv>
out = pd.read_csv('titanicOut.csv')
 out<set_options>
print("There are {} words in the combination of all posts.".format(len(grouped_wordclouds['posts'].sum())) )
warnings.filterwarnings('ignore')
 %matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams[ 'figure.figsize' ] = 8 , 6
py.init_notebook_mode(connected=True)
 train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 full_data = [train_df, test_df]
train_df.head()<count_missing_values>
pattern_url = r'http[s]?://(?:[A-Za-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9A-Fa-f][0-9A-Fa-f])) +'
subs_url = r'url-web'
all_data['posts'] = all_data['posts'].replace(to_replace=pattern_url, value=subs_url, regex=True)
 all_data['posts'] = all_data['posts'].str.lower()
def check_missing_data(df):
flag=df.isna().sum().any()
if flag==True:
total = df.isnull().sum()
percent =(df.isnull().sum())/(df.isnull().count() *100)
 output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 data_type = []
for col in df.columns:
dtype = str(df[col].dtype)
 data_type.append(dtype)
 output['Types'] = data_type
return(np.transpose(output))
else:
return(False)
 train_df = train_df.fillna(np.nan)
 test_df = test_df.fillna(np.nan)
 check_missing_data(train_df)
<sort_values>
def remove_punctuation_numbers(post):

 punc_numbers = string.punctuation + '0123456789'
return ''.join([l for l in post if l not in punc_numbers])
 all_data['posts'] = all_data['posts'].apply(remove_punctuation_numbers)
train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
<feature_engineering>
tokeniser = TreebankWordTokenizer()
all_data['tokens'] = all_data['posts'].apply(tokeniser.tokenize)
print("Before", train_df.shape, test_df.shape, full_data[0].shape, full_data[1].shape)
 train_df = train_df.drop(['Ticket', 'Cabin', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Ticket', 'Cabin'], axis=1)
 full_data = [train_df, test_df]
print("After", train_df.shape, test_df.shape, full_data[0].shape, full_data[1].shape)
 for dataset in full_data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in full_data:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df = train_df.drop(['Name'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 full_data = [train_df, test_df]
train_df.shape, test_df.shape
for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
<categorify>
stemmer = SnowballStemmer('english')
 def train_stemmer(words, stemmer):

 return [stemmer.stem(word)for word in words]
all_data['stem'] = all_data['tokens'].apply(train_stemmer, args=(stemmer,))
def get_person(passenger):
age,sex = passenger
return 'child' if age < 16 else sex
train_df['Person'] = train_df[['Age','Sex']].apply(get_person,axis=1)
 test_df['Person'] = test_df[['Age','Sex']].apply(get_person,axis=1)
 person_dummies_titanic = pd.get_dummies(train_df['Person'])
 person_dummies_titanic.columns = ['Child','Female','Male']
person_dummies_titanic.drop(['Male'], axis=1, inplace=True)
 person_dummies_test = pd.get_dummies(test_df['Person'])
 person_dummies_test.columns = ['Child','Female','Male']
person_dummies_test.drop(['Male'], axis=1, inplace=True)
 train_df = train_df.join(person_dummies_titanic)
 test_df = test_df.join(person_dummies_test)
 fig,(axis1,axis2)= plt.subplots(1,2,figsize=(10,5))
sns.countplot(x='Person', data=train_df, ax=axis1)
 person_perc = train_df[["Person", "Survived"]].groupby(['Person'],as_index=False ).mean()
sns.barplot(x='Person', y='Survived', data=person_perc, ax=axis2, order=['male','female','child'])
 train_df.drop(['Person'],axis=1,inplace=True)
 test_df.drop(['Person'],axis=1,inplace=True)
 full_data = [train_df, test_df]<feature_engineering>
lemmatizer = WordNetLemmatizer()
def train_lemma(words, lemmatizer):

 return [lemmatizer.lemmatize(word)for word in words]
all_data['lemma'] = all_data['tokens'].apply(train_lemma, args=(lemmatizer,))
for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 for dataset in full_data:
dataset[ 'Family_Single' ] = dataset[ 'FamilySize' ].map(lambda s : 1 if s == 1 else 0)
 dataset[ 'Family_Small' ] = dataset[ 'FamilySize' ].map(lambda s : 1 if 2 <= s <= 4 else 0)
 dataset[ 'Family_Large' ] = dataset[ 'FamilySize' ].map(lambda s : 1 if 5 <= s else 0)
 train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 full_data = [train_df, test_df]
train_df.head()<categorify>
detokenizer = TreebankWordDetokenizer()
all_data['detoken'] = all_data['lemma'].apply(lambda x: detokenizer.detokenize(x))
freq_port = train_df.Embarked.dropna().mode() [0]
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<feature_engineering>
train = all_data[:ntrain]
test = all_data[ntrain:]
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True)
 for dataset in full_data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 full_data = [train_df, test_df]
train_df.head()<categorify>
X = train.detoken
y = train.type
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 X = pd.concat([X_train, y_train], axis=1)
 ESTP = X[X.type == 'ESTP']
ESTJ = X[X.type == 'ESTJ']
ESFP = X[X.type == 'ESFP']
ESFJ = X[X.type == 'ESFJ']
ENTP = X[X.type == 'ENTP']
ENTJ = X[X.type == 'ENTJ']
ENFP = X[X.type == 'ENFP']
ENFJ = X[X.type == 'ENFJ']
INFP = X[X.type == 'INFP']
mbti_upsampledESTP = resample(ESTP,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledESFP = resample(ESFP,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledESFJ = resample(ESFJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledESTJ = resample(ESTJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledENFJ = resample(ENFJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledENTJ = resample(ENTJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledENTP = resample(ENTP,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledENFP = resample(ENFP,
replace=True,
n_samples=len(INFP),
random_state=27)
 upsampled_E = pd.concat([INFP, mbti_upsampledESTP,mbti_upsampledESTJ,mbti_upsampledESFP,mbti_upsampledESFJ,mbti_upsampledENFJ,mbti_upsampledENTJ,mbti_upsampledENTP,mbti_upsampledENFP])
 upsampled_E.type.value_counts()
<split>
ISTP = X[X.type == 'ISTP']
ISTJ = X[X.type == 'ISTJ']
ISFP = X[X.type == 'ISFP']
ISFJ = X[X.type == 'ISFJ']
INTP = X[X.type == 'INTP']
INTJ = X[X.type == 'INTJ']
INFJ = X[X.type == 'INFJ']
INFP = X[X.type == 'INFP']
mbti_upsampledISTP = resample(ISTP,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledISFP = resample(ISFP,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledISFJ = resample(ISFJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledISTJ = resample(ISTJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledINFJ = resample(INFJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledINTJ = resample(INTJ,
replace=True,
n_samples=len(INFP),
random_state=27)
 mbti_upsampledINTP = resample(INTP,
replace=True,
n_samples=len(INFP),
random_state=27)
 upsampled_I = pd.concat([mbti_upsampledISTP, mbti_upsampledISTJ, mbti_upsampledISFP, mbti_upsampledISFJ, mbti_upsampledINFJ, mbti_upsampledINTJ, mbti_upsampledINTP])
 upsampled_I.type.value_counts()
train, test = train_test_split(train_df, test_size = 0.3, random_state = 0)
 split_train_X = train.drop("Survived", axis=1)
 split_train_Y = train["Survived"]
split_test_X = test.drop("Survived", axis=1)
 split_test_Y = test["Survived"]<train_on_grid>
upsampled_data = pd.concat([upsampled_E,upsampled_I])
 upsampled_data.head()
logreg = LogisticRegression()
logreg.fit(split_train_X, split_train_Y)
 logreg.score(split_test_X, split_test_Y )<train_model>
X_train = upsampled_data.detoken
y_train = upsampled_data.type
gaussian = GaussianNB()
gaussian.fit(split_train_X, split_train_Y)
 gaussian.score(split_test_X, split_test_Y )<train_model>
clf = DummyClassifier(strategy='most_frequent' ).fit(X_train, y_train)
 y_pred = clf.predict(X_test)
 print(' Our baseline accuracy is %s' % accuracy_score(y_pred, y_test))
perceptron = Perceptron()
perceptron.fit(split_train_X, split_train_Y)
 perceptron.score(split_test_X, split_test_Y )<train_model>
nb = Pipeline([('vect', CountVectorizer(stop_words='english', ngram_range=(1, 2))), 
('tfidf', TfidfTransformer(sublinear_tf=True, norm='l2')) , 
('clf', MultinomialNB())])
 nb.fit(X_train, y_train)
decision_tree = DecisionTreeClassifier()
decision_tree.fit(split_train_X, split_train_Y)
 decision_tree.score(split_test_X, split_test_Y )<choose_model_class>
y_predNB = nb.predict(X_test)
 print('accuracy %s' % accuracy_score(y_predNB, y_test))
random_forest = RandomForestClassifier(n_estimators=100)
 parameters = {'n_estimators': [4, 6, 9],
'max_features': ['log2', 'sqrt','auto'],
'criterion': ['entropy', 'gini'],
'max_depth': [2, 3, 5, 10],
'min_samples_split': [2, 3, 5],
'min_samples_leaf': [1,5,8]
}
acc_scorer = make_scorer(accuracy_score)
 kfold = StratifiedKFold(n_splits=10)
 grid_obj = GridSearchCV(random_forest, parameters, cv=kfold, scoring=acc_scorer)
 grid_obj.fit(X, Y)
 random_forest = grid_obj.best_estimator_
grid_obj.best_score_<prepare_x_and_y>
y_probsNB = nb.predict_proba(X_test)
 print("The log loss error for our model is: ", log_loss(y_test, y_probsNB))
train_df_X = train_df.drop("Survived", axis=1)
 train_df_Y = train_df["Survived"]
test_df_X = test_df.drop("PassengerId", axis=1 ).copy()
train_df_X.shape, train_df_Y.shape, test_df_X.shape
train_X = train_df_X.values
train_Y = train_df_Y.values
test_X = test_df_X.values
classifiers = [
KNeighborsClassifier(3),
SVC(probability=True),
DecisionTreeClassifier() ,
RandomForestClassifier() ,
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
GaussianNB() ,
LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ,
LogisticRegression() ]
kfold = StratifiedKFold(n_splits=10, random_state=0)
 cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, train_X, y = train_Y, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","AdaBoost",
"RandomForest","ExtraTrees","GradientBoosting","MultipleLayerPerceptron","KNeighboors","LogisticRegression","LinearDiscriminantAnalysis"]})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores")
<choose_model_class>
logreg = Pipeline([('vect', CountVectorizer(stop_words='english', ngram_range=(1, 2))), 
('tfidf', TfidfTransformer(norm='l2')) , 
('clf', LogisticRegression(n_jobs=1, C=1e5)) ])
 logreg.fit(X_train, y_train)
ensemble_lin_rbf = VotingClassifier(estimators=[('KNN', KNeighborsClassifier(n_neighbors=10)) , 
('RBF', SVC(probability=True,kernel='rbf',C=0.5,gamma=0.1)) , 
('RFor', RandomForestClassifier(n_estimators=500,random_state=0)) , 
('LR', LogisticRegression(C=0.05)) , 
('DT', DecisionTreeClassifier(random_state=0)) , 
('NB', GaussianNB()), 
('svm', SVC(kernel='linear',probability=True))
],
voting='soft' ).fit(split_train_X, split_train_Y)
 print('The accuracy for ensembled model is:', ensemble_lin_rbf.score(split_test_X, split_test_Y))
cross = cross_val_score(ensemble_lin_rbf, X, Y, cv = 10, scoring = "accuracy")
 print('The cross validated score is', cross.mean())
 model = BaggingClassifier(base_estimator = DecisionTreeClassifier() , random_state = 0, n_estimators = 100)
 model.fit(split_train_X, split_train_Y)
 prediction = model.predict(split_test_X)
 print('The accuracy for bagged Decision Tree is:', accuracy_score(prediction, split_test_Y))
result = cross_val_score(model, X, Y, cv=10, scoring='accuracy')
 print('The cross validated score for bagged Decision Tree is:', result.mean())
 ada = AdaBoostClassifier(n_estimators = 200, random_state = 0, learning_rate = 0.1)
 result = cross_val_score(ada,X, Y, cv = 10, scoring='accuracy')
 print('The cross validated score for AdaBoost is:', result.mean())
 ada = AdaBoostClassifier(n_estimators = 200, random_state = 0, learning_rate = 0.05)
 result = cross_val_predict(ada, X, Y, cv = 10)
 sns.heatmap(confusion_matrix(Y,result), cmap='winter', annot=True, fmt='2.0f')
 plt.show()<train_model>
y_predLogRegTrain = logreg.predict(X_test)
ntrain = train_X.shape[0]
ntest = test_X.shape[0]
SEED = 0
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state = SEED)
 class SklearnHelper(object):
def __init__(self, clf, seed=0, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y)
 def feature_importances(self,x,y):
print(self.clf.fit(x,y ).feature_importances_)
 def get_oof(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
fold = 0
for(train_index, test_index)in kf.split(x_train, y_train):
x_tr = x_train[train_index]
y_tr = y_train[train_index]
x_te = x_train[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[fold, :] = clf.predict(x_test)
 fold += 1
oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)
<init_hyperparams>
y_predLogReg = logreg.predict(test.posts)
rf_params = {
'n_jobs': -1,
'n_estimators': 500,
'warm_start': True,
'max_depth': 6,
'min_samples_leaf': 2,
'max_features' : 'sqrt',
'verbose': 0
}
et_params = {
'n_jobs': -1,
'n_estimators':500,
'max_depth': 8,
'min_samples_leaf': 2,
'verbose': 0
}
ada_params = {
'n_estimators': 500,
'learning_rate' : 0.75
}
gb_params = {
'n_estimators': 500,
'max_depth': 5,
'min_samples_leaf': 2,
'verbose': 0
}
svc_params = {
'kernel' : 'linear',
'C' : 0.025
}
rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_params)
 et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)
 ada = SklearnHelper(clf=AdaBoostClassifier, seed=SEED, params=ada_params)
 gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)
 svc = SklearnHelper(clf=SVC, seed=SEED, params=svc_params)
 et_oof_train, et_oof_test = get_oof(et, train_X, train_Y, test_X)
 rf_oof_train, rf_oof_test = get_oof(rf, train_X, train_Y, test_X)
 ada_oof_train, ada_oof_test = get_oof(ada, train_X, train_Y, test_X)
 gb_oof_train, gb_oof_test = get_oof(gb, train_X, train_Y, test_X)
 svc_oof_train, svc_oof_test = get_oof(svc, train_X, train_Y, test_X)
 print("Training is complete")
 rf_feature = rf.feature_importances(train_X, train_Y)
 et_feature = et.feature_importances(train_X, train_Y)
 ada_feature = ada.feature_importances(train_X, train_Y)
 gb_feature = gb.feature_importances(train_X, train_Y )<concatenate>
print('accuracy %s' % accuracy_score(y_predLogRegTrain, y_test))
x_train = np.concatenate(( et_oof_train, rf_oof_train, ada_oof_train, gb_oof_train, svc_oof_train), axis=1)
 x_test = np.concatenate(( et_oof_test, rf_oof_test, ada_oof_test, gb_oof_test, svc_oof_test), axis=1 )<save_to_csv>
y_probslogreg = logreg.predict_proba(X_test)
 print("The log loss error for our model is: ", log_loss(y_test, y_probslogreg))
gbm = xgb.XGBClassifier(
 n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
scale_pos_weight=1 ).fit(train_X, train_Y)
 predictions = gbm.predict(test_X)
 submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": predictions
})
 submission.to_csv('submission.csv', index=False )<set_options>
print(classification_report(y_test, y_predLogRegTrain))
plt.style.use('seaborn')
 sns.set(font_scale=2.5)
 warnings.filterwarnings('ignore')
 %matplotlib inline 
<load_from_csv>
DecisionTree = Pipeline([('vect', CountVectorizer(stop_words='english', ngram_range=(1, 2))), 
('tfidf', TfidfTransformer()), 
('clf', DecisionTreeClassifier())])
 clf = DecisionTree.fit(X_train, y_train)
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
<define_variables>
y_predTree = clf.predict(X_test)
 print('accuracy %s' % accuracy_score(y_predTree, y_test))
def detect_outliers(df, n, features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col], 75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(df_train, 2, ["Age", "SibSp", "Parch", "Fare"] )<filter>
y_probsTree = clf.predict_proba(X_test)
 print("The log loss error for our model is: ", log_loss(y_test, y_probsTree))
df_train.loc[Outliers_to_drop]<drop_column>
print(classification_report(y_test, y_predTree))
df_train = df_train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<count_missing_values>
clf = Pipeline([('vect', CountVectorizer(stop_words='english')) , 
('tfidf', TfidfTransformer()), 
('clf', SVC(kernel='polynomial')) ])
 clf.fit(X_train, y_train)
 y_predSVC = clf.predict(X_test)
 print('accuracy %s' % accuracy_score(y_predSVC, y_test))
for col in df_train.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(df_train[col].isnull().sum() / df_train[col].shape[0]))
print(msg)
<count_missing_values>
sample = pd.read_csv('.. /input/random_example.csv' )
for col in df_test.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(df_test[col].isnull().sum() / df_test[col].shape[0]))
print(msg )<groupby>
submission = pd.DataFrame(data = sample['id'], columns= ['id'])
 submission['Type'] = y_predLogReg
df_train[['Pclass', 'Survived']].groupby(['Pclass'], as_index = True ).count() 
<feature_engineering>
submission['mind'] = submission['Type'].apply(lambda x: x[0] == 'E' ).astype('int')
 submission['energy'] = submission['Type'].apply(lambda x: x[1] == 'N' ).astype('int')
 submission['nature'] = submission['Type'].apply(lambda x: x[2] == 'T' ).astype('int')
 submission['tactics'] = submission['Type'].apply(lambda x: x[3] == 'J' ).astype('int')
 submission = submission.drop(['Type'],axis=1 )
df_train["FamilySize"] = df_train["SibSp"] + df_train["Parch"]+1
df_test["FamilySize"] = df_test["SibSp"] + df_test["Parch"]+1 
<feature_engineering>
submission.to_csv('submit_2.csv', index=False )