Kaggle competition: Spooky Author Identification
parameters = {
'max_features': [0.7,0.8, 0.9],
'min_samples_leaf' :[0.6,0.7,0.9],
'colsample_bytree' :[0.6,0.8,0.9],
'learning_rate' : [0.01,0.03,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 7)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x,y['Survived'])
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<choose_model_class>
from IPython.display import HTML
from matplotlib.pyplot import *
from matplotlib import cm
from matplotlib import animation
import io
import base64
parameters = {
'max_features': [0.7,0.8, 0.9],
'min_samples_leaf' :[0.6,0.7,0.9],
'colsample_bytree' :[0.6,0.8,0.9],
'learning_rate' : [0.01,0.03,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 6)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x_valid,y_valid)
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<compute_train_metric>
def unique_word_fraction(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
unique_count = list(set(text_splited)).__len__()
return(unique_count/word_count)
 eng_stopwords = set(stopwords.words("english"))
def stopwords_count(row):

 text = row['text'].lower()
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
stopwords_count = len([w for w in text_splited if w in eng_stopwords])
 return(stopwords_count/word_count)
 def punctuations_fraction(row):

 text = row['text']
char_count = len(text)
 punctuation_count = len([c for c in text if c in string.punctuation])
 return(punctuation_count/char_count)
 def char_count(row):

 return len(row['text'])
 def fraction_noun(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
pos_list = nltk.pos_tag(text_splited)
 noun_count = len([w for w in pos_list if w[1] in('NN','NNP','NNPS','NNS')])
 return(noun_count/word_count)
 def fraction_adj(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
pos_list = nltk.pos_tag(text_splited)
 adj_count = len([w for w in pos_list if w[1] in('JJ','JJR','JJS')])
 return(adj_count/word_count)
 def fraction_verbs(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
word_count = text_splited.__len__()
pos_list = nltk.pos_tag(text_splited)
 verbs_count = len([w for w in pos_list if w[1] in('VB','VBD','VBG','VBN','VBP','VBZ')])
 return(verbs_count/word_count)
svc=make_pipeline(StandardScaler() ,SVC(random_state=1))
r=[0.0001,0.001,0.1,1,10,50,100]
PSVM=[{'svc__C':r, 'svc__kernel':['linear']},
{'svc__C':r, 'svc__gamma':r, 'svc__kernel':['rbf']}]
GSSVM=GridSearchCV(estimator=svc, param_grid=PSVM, scoring='accuracy', cv=2)
 scores_svm=cross_val_score(GSSVM, x_valid, y_valid,scoring='accuracy', cv=5)
 np.mean(scores_svm )<save_to_csv>
train_df['unique_word_fraction'] = train_df.apply(lambda row: unique_word_fraction(row), axis =1)
 train_df['stopwords_count'] = train_df.apply(lambda row: stopwords_count(row), axis =1)
 train_df['punctuations_fraction'] = train_df.apply(lambda row: punctuations_fraction(row), axis =1)
 train_df['char_count'] = train_df.apply(lambda row: char_count(row), axis =1)
 train_df['fraction_noun'] = train_df.apply(lambda row: fraction_noun(row), axis =1)
 train_df['fraction_adj'] = train_df.apply(lambda row: fraction_adj(row), axis =1)
 train_df['fraction_verbs'] = train_df.apply(lambda row: fraction_verbs(row), axis =1)
 train_df.head()
Survived = grid_search2.best_estimator_.predict(x_valid)
 submission['Survived'] = Survived
submission['Survived'] = submission['Survived'].astype(int)
 submission.to_csv('submission.csv' , index=False )<import_modules>
test_df['unique_word_fraction'] = test_df.apply(lambda row: unique_word_fraction(row), axis =1)
 test_df['stopwords_count'] = test_df.apply(lambda row: stopwords_count(row), axis =1)
 test_df['punctuations_fraction'] = test_df.apply(lambda row: punctuations_fraction(row), axis =1)
 test_df['char_count'] = test_df.apply(lambda row: char_count(row), axis =1)
 test_df['fraction_noun'] = test_df.apply(lambda row: fraction_noun(row), axis =1)
 test_df['fraction_adj'] = test_df.apply(lambda row: fraction_adj(row), axis =1)
 test_df['fraction_verbs'] = test_df.apply(lambda row: fraction_verbs(row), axis =1)
 test_df.head()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import xgboost as xgb
import lightgbm as lgb
import re
import shap
from sklearn import preprocessing
from category_encoders.one_hot import OneHotEncoder
from sklearn.preprocessing import StandardScaler,LabelEncoder
from sklearn.model_selection import cross_val_score, KFold,GridSearchCV
from sklearn.feature_selection import SelectKBest,SelectFromModel
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline,make_pipeline
from sklearn.ensemble import RandomForestClassifier,VotingClassifier,BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn import linear_model
from sklearn.linear_model import LogisticRegression,Perceptron,SGDClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB<load_from_csv>
Surprize_words = ['finally', 'suddenly', 'mystery', 'fortune', 'sweet', 'horror', 'chance', '']
test = pd.read_csv(".. /input/titanic/test.csv")
 test['Survived'] = np.nan
train = pd.read_csv(".. /input/titanic/train.csv")
 df = pd.concat([test,train],sort=False)
 df.head()<feature_engineering>
start = time.time()
tfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1,3))
full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 end = time.time()
print("Time taken in tf-idf is {}.".format(end-start))
cols_to_drop = ['id', 'text']
train_X = train_df.drop(cols_to_drop+['author'], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1)
 author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}
train_y = train_df['author'].map(author_mapping_dict )
print(df.corr() ['Age'].sort_values(ascending=False))
df['Age'] = df.groupby("Pclass")['Age'].transform(lambda x: x.fillna(x.median()))<feature_engineering>
def runMNB(train_X, train_y, test_X, test_y, test_X2):
model = naive_bayes.MultinomialNB()
model.fit(train_X, train_y)
 pred_test_y = model.predict_proba(test_X)
 pred_test_y2 = model.predict_proba(test_X2)
 return pred_test_y, pred_test_y2, model
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2]
print(df.corr() ['Fare'].sort_values(ascending=False))
df['Fare'] = df.groupby("Pclass")['Fare'].transform(lambda x: x.fillna(x.median()))<feature_engineering>
tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,3))
tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2]
df['Embarked'].fillna(df['Embarked'].mode() [0], inplace = True)
 df["Fsize"] = df["SibSp"] + df["Parch"] + 1
df['IsAlone'] = np.where(df["Fsize"]==1,1,0)
 df['Age_Class']= df['Age']* df['Pclass']
df['Age_cut'] = pd.qcut(df['Age'],5,duplicates='drop')
 df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr','Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df['Title'] = df['Title'].replace('Mlle', 'Miss')
 df['Title'] = df['Title'].replace('Ms', 'Miss')
 df['Title'] = df['Title'].replace('Mme', 'Mrs')
 df['Title']= df['Title'].fillna(0)
 df['Fare_Per_Person'] = df['Fare']/(df["Fsize"]+1)
 df['Fare_Per_Person'] = df['Fare_Per_Person'].astype(int)
 df['Fare_cut'] = pd.qcut(df['Fare'],5,duplicates='drop')
 df['Cabin_len'] = df['Cabin'].astype(str ).apply(lambda x : len(x))
df['Cabin'] = df['Cabin'].fillna("M")
 df['Cabin'] = df['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 df['Ticket']=df.Ticket.apply(lambda x : len(x))
df = df.drop(['Name'], axis = 1)
 df.head()<normalization>
start = time.time()
n_comp = 400
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
end = time.time()
print("Time taken in tf-idf is {}.".format(end-start))
scaler = preprocessing.StandardScaler().fit(df[['Age','Fare']])
 df[['Age','Fare']] = scaler.transform(df[['Age','Fare']] )<categorify>
start = time.time()
tfidf_vec = TfidfVectorizer(ngram_range=(1,5), analyzer='char')
 full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 n_comp = 40
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
end = time.time()
print("Time taken in tf-idf is {}.".format(end-start))
df['Age_cut'] = LabelEncoder().fit_transform(df['Age_cut'])
 df['Fare_cut'] = LabelEncoder().fit_transform(df['Fare_cut'])
 titles = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
df['Title']= df['Title'].map(titles)
 groups = {"M": 1, "C": 2, "B": 3, "D": 4, "E": 5, "A": 6, "F": 7, "G": 8,'T':9}
df['Cabin'] = df['Cabin'].map(groups)
 ports = {"S": 0, "C": 1, "Q": 2}
df['Embarked'] = df['Embarked'].map(ports)
 df['Sex_female'] = np.where(df['Sex'] == 'female',1,0)
 df['Sex_male'] = np.where(df['Sex'] == 'male',1,0)
 df = df.drop(['Sex'],axis=1 )<categorify>
class Dictogram(dict):
def __init__(self, iterable=None):

 super(Dictogram, self ).__init__()
self.types = 0
self.tokens = 0
if iterable:
self.update(iterable)
 def update(self, iterable):

 for item in iterable:
if item in self:
self[item] += 1
self.tokens += 1
else:
self[item] = 1
self.types += 1
self.tokens += 1
def count(self, item):

 if item in self:
return self[item]
return 0
def return_random_word(self):
random_key = random.sample(self, 1)
 return random_key[0]
def return_weighted_random_word(self):
random_int = random.randint(0, self.tokens-1)
 index = 0
list_of_keys = self.keys()
for i in range(0, self.types):
index += self[list_of_keys[i]]
if(index > random_int):
return list_of_keys[i]
dummies = OneHotEncoder(cols= ['Pclass','Age_cut','Fare_cut','Embarked','Title'],use_cat_names=True)
 dummies.fit(df)
 df = dummies.transform(df )<data_type_conversions>
def make_higher_order_markov_model(order, data):
markov_model = dict()
for i in range(0, len(data)-order):
window = tuple(data[i: i+order])
 if window in markov_model:
markov_model[window].update([data[i+order]])
 else:
markov_model[window] = Dictogram([data[i+order]])
 return markov_model
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
upper.head()<prepare_x_and_y>
print(trn_df.loc[train_df['author']=='EAP'].shape[0])
 def tokenixed_list(row):

 text = row['text']
text_splited = text.split(' ')
 text_splited = [''.join(c for c in s if c not in string.punctuation)for s in text_splited]
text_splited = [s for s in text_splited if s]
return(text_splited)
 trn_df['splited_text'] = trn_df.apply(lambda row: tokenixed_list(row), axis =1 )
x_valid = df[df['Survived'].isna() ].drop(['Survived'], axis = 1)
 submission = x_valid['PassengerId'].to_frame()
x1 = df[df['Survived']==0]
x2 = df[df['Survived']==1]
x= pd.concat([x1,x2])
 y =x.loc[:,['Survived','PassengerId']]
x =x.drop(['Survived','PassengerId'], axis = 1 )<prepare_x_and_y>
ms = trn_df.loc[trn_df['author']=='MWS']['splited_text'].values
flat_ms = []
ms_flat = [flat_ms.extend(x)for x in ms]
flat_ms.__len__()
ms_MM = make_higher_order_markov_model(3, flat_ms )
teste = pd.read_csv(".. /input/testey/full-tita.csv",sep=";")
 teste = teste.drop(['x'],axis=1)
 teste
x_valid = x_valid.merge(teste)
 x_valid = x_valid.drop(['Survived'],axis = 1)
 y_valid = teste['Survived'] 
<choose_model_class>
def spit_out_prob(tupl , word , mm1= eap_MM, mm2 = hpl_MM, mm3 = ms_MM):


try:
a = mm1[tupl][word]
except:
a = 0
try:
b = mm2[tupl][word]
except:
b = 0
try:
c = mm3[tupl][word]
except:
c = 0
try:
prob_word = a/(a+b+c)
 return(prob_word)
 except:
return(0)
 def make_tupl(sentence =trn_df.loc[train_df['author']=='EAP']['splited_text'][0] , n=3):

 list_of_tuple = []
word_1 = []
for i in list(range(sentence.__len__() -3)) :
tuple_1 =(sentence[i], sentence[i+1], sentence[i+2])
 list_of_tuple.append(tuple_1)
 word_1.append(sentence[i+3])
 return(list_of_tuple, word_1)
 def sent_to_prob_eap(sentence, mm1 = eap_MM, p_eap =1/3, mm2 = hpl_MM, mm3 = ms_MM):

 list_of_tuples = make_tupl(sentence, n =3)[0]
words = make_tupl(sentence, n =3)[1]
p=0
for i in list(range(words.__len__())) :
p = p+spit_out_prob(list_of_tuples[i], words[i], mm1 =eap_MM, mm2 =hpl_MM, mm3 =ms_MM)
 return(p)
 def sent_to_prob_hpl(sentence, mm2 = hpl_MM, p_hpl =1/3, mm1 = eap_MM, mm3 = ms_MM):

 list_of_tuples = make_tupl(sentence, n =3)[0]
words = make_tupl(sentence, n =3)[1]
p = 0
for i in list(range(words.__len__())) :
p = p+spit_out_prob(list_of_tuples[i], words[i], mm1 =hpl_MM, mm2=eap_MM, mm3=ms_MM)
 return(p)
 def sent_to_prob_ms(sentence, mm3 = ms_MM, p_ms =1/3, mm2 = hpl_MM, mm1 = eap_MM):

 list_of_tuples = make_tupl(sentence, n =3)[0]
words = make_tupl(sentence, n =3)[1]
p = 0
for i in list(range(words.__len__())) :
p = p+spit_out_prob(list_of_tuples[i], words[i], ms_MM, hpl_MM, eap_MM)
 return(p)
kf = KFold(n_splits=5, shuffle=True, random_state=42 )<train_on_grid>
trn_df['EAP_markov_3'] = trn_df['splited_text'].apply(lambda row: sent_to_prob_eap(sentence =row))
trn_df['HPL_markov_3'] = trn_df['splited_text'].apply(lambda row: sent_to_prob_hpl(sentence = row))
trn_df['MWS_markov_3'] = trn_df['splited_text'].apply(lambda row: sent_to_prob_ms(sentence = row))
trn_df[['MWS_markov_3', 'EAP_markov_3', 'HPL_markov_3', 'author']].head()
del trn_df['splited_text']
lr = LogisticRegression(class_weight = 'balanced', solver = 'liblinear',penalty="l2")
 lr.fit(x,y['Survived'])
 print(cross_val_score(lr, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
val_df['splited_text'] = val_df.apply(lambda row: tokenixed_list(row), axis =1)
 val_df['EAP_markov_3'] = val_df['splited_text'].apply(lambda row: sent_to_prob_eap(sentence =row))
val_df['HPL_markov_3'] = val_df['splited_text'].apply(lambda row: sent_to_prob_hpl(sentence = row))
val_df['MWS_markov_3'] = val_df['splited_text'].apply(lambda row: sent_to_prob_ms(sentence = row))
val_df1 = val_df[['MWS_markov_3', 'EAP_markov_3', 'HPL_markov_3','author']].copy()
del val_df['splited_text']
lr = LogisticRegression(class_weight = 'balanced', solver = 'liblinear',penalty="l1")
 lr.fit(x,y['Survived'])
 print(cross_val_score(lr, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
test_df['splited_text'] = test_df.apply(lambda row: tokenixed_list(row), axis =1)
 test_df['EAP_markov_3'] = test_df['splited_text'].apply(lambda row: sent_to_prob_eap(sentence =row))
test_df['HPL_markov_3'] = test_df['splited_text'].apply(lambda row: sent_to_prob_hpl(sentence = row))
test_df['MWS_markov_3'] = test_df['splited_text'].apply(lambda row: sent_to_prob_ms(sentence = row))
test_df1 = test_df[['MWS_markov_3', 'EAP_markov_3', 'HPL_markov_3']].copy()
del test_df['splited_text']
svm = SVC(gamma='auto',random_state=42)
 svm.fit(x,y['Survived'])
 print(cross_val_score(svm, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}
trn_y = trn_df['author'].map(author_mapping_dict)
 val_y = val_df['author'].map(author_mapping_dict)
 cols_drop = ['id', 'text','MWS_markov_3', 'EAP_markov_3', 'HPL_markov_3']
trn_X = trn_df.drop(cols_drop+['author'], axis=1)
 val_X = val_df.drop(cols_drop+['author'], axis=1)
 test_X = test_df.drop(cols_drop, axis=1 )
svm = SVC(gamma='auto',random_state=42)
 svm.fit(x,y['Survived'])
 print(cross_val_score(svm, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
Xtr = trn_X.values
Xv = val_X.values
ytr = trn_y
yv = val_y
dtrain = xgb.DMatrix(Xtr, label=ytr)
 dvalid = xgb.DMatrix(Xv, label=yv)
 dtest = xgb.DMatrix(test_X.values)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
start = time.time()
xgb_par = {'min_child_weight': 1, 'eta': 0.1, 'colsample_bytree': 0.7, 'max_depth': 3,
'subsample': 0.8, 'lambda': 2.0, 'nthread': -1, 'silent': 1,
'eval_metric': "mlogloss", 'objective': 'multi:softprob', 'num_class': 3}
model_1 = xgb.train(xgb_par, dtrain, 1000, watchlist, early_stopping_rounds=50, maximize=False, verbose_eval=40)
 print('Modeling RMSLE %.5f' % model_1.best_score)
 end = time.time()
print("Time taken in training is {}.".format(end - start))
sgd = linear_model.SGDClassifier(max_iter=5, tol=None)
 sgd.fit(x,y['Survived'])
 print(cross_val_score(sgd, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
start = time.time()
yvalid = model_1.predict(dvalid)
 ytest = model_1.predict(dtest)
 end = time.time()
print("Time taken in prediction is {}.".format(end - start))
knn = KNeighborsClassifier(n_neighbors = 5)
 knn.fit(x,y['Survived'])
 print(cross_val_score(knn, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
val_df1.head()
val_df2 = pd.DataFrame(yvalid)
 val_df2.head()
val_df2.columns = ['EAP', 'HPL', 'MWS']
val_df2.head()
val_df1.reset_index(inplace = True, drop =True)
 val_df1.head()
knn = KNeighborsClassifier(n_neighbors = 5)
 knn.fit(x,y['Survived'])
 print(cross_val_score(knn, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
val = pd.concat([val_df1, val_df2], axis =1)
 val.head()
gaussian = GaussianNB()
gaussian.fit(x,y['Survived'])
 print(cross_val_score(gaussian, x, y['Survived'], cv=kf ).mean() )<train_model>
start = time.time()
if test_df.shape[0] == ytest.shape[0]:
print('Test shape OK.')
 test_id = test_df['id'].values
out_df = pd.DataFrame(ytest)
 out_df.columns = ['EAP', 'HPL', 'MWS']
out_df.insert(0, 'id', test_id)
 out_df.to_csv("Mahesh_xgb_test.csv", index=False)
 end = time.time()
print("Time taken in training is {}.".format(end - start))
rf_with_standardScaler=make_pipeline(StandardScaler() ,RandomForestClassifier())
 rf_with_standardScaler.fit(x,y['Survived'])
 print(cross_val_score(rf_with_standardScaler, x, y['Survived'], cv=kf ).mean() )<train_model>
out_df.head()
test_df2 = out_df[['EAP','HPL', 'MWS']]
test_df2.head()
test_df1.head()
t = pd.concat([test_df1, test_df2], axis =1)
 t.head()
rf_with_standardScaler=make_pipeline(StandardScaler() ,RandomForestClassifier())
 rf_with_standardScaler.fit(x,y['Survived'])
 print(cross_val_score(rf_with_standardScaler, x_valid, y_valid, cv=kf ).mean() )<compute_train_metric>
author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}
trn_y = val['author'].map(author_mapping_dict)
 Xtr = val[['MWS_markov_3','EAP_markov_3','HPL_markov_3','EAP','HPL','MWS']].values
ytr = trn_y
dtrain = xgb.DMatrix(Xtr, label=ytr)
 dtest = xgb.DMatrix(t.values)
 watchlist = [(dtrain, 'train')]
start = time.time()
xgb_par = {'min_child_weight': 1, 'eta': 0.1, 'colsample_bytree': 0.7, 'max_depth': 3,
'subsample': 0.8, 'lambda': 2.0, 'nthread': -1, 'silent': 1,
'eval_metric': "mlogloss", 'objective': 'multi:softprob', 'num_class': 3}
model_2 = xgb.train(xgb_par, dtrain, 2000, watchlist, early_stopping_rounds=50, maximize=False, verbose_eval=100)
 print('Modeling RMSLE %.5f' % model_2.best_score)
 end = time.time()
print("Time taken in training is {}.".format(end - start))
rf=RandomForestClassifier(n_estimators=100, oob_score = True)
 rf.fit(x,y['Survived'])
 print(cross_val_score(rf, x, y['Survived'], cv=kf ).mean())
 parametros = pd.DataFrame({'feature':x.columns,'Parameters':np.round(rf.feature_importances_,3)})
 parametros = parametros.sort_values('Parameters',ascending=False ).set_index('feature')
 parametros<train_model>
start = time.time()
ytest = model_2.predict(dtest)
 end = time.time()
print("Time taken in prediction is {}.".format(end - start))
rf=RandomForestClassifier(n_estimators=100)
 rf.fit(x,y['Survived'])
 print(cross_val_score(rf, x_valid, y_valid, cv=kf ).mean())
<train_on_grid>
start = time.time()
if test_df.shape[0] == ytest.shape[0]:
print('Test shape OK.')
 test_id = test_df['id'].values
out_df = pd.DataFrame(ytest)
 out_df.columns = ['EAP', 'HPL', 'MWS']
out_df.insert(0, 'id', test_id)
 out_df.to_csv("Mahesh_xgb_test_lc.csv", index=False)
 end = time.time()
print("Time taken in training is {}.".format(end - start))