Kaggle competition: Spooky Author Identification
bagging = BaggingClassifier(bootstrap=True,n_jobs = -1,n_estimators=100)
 bagging.fit(x,y['Survived'])
 print(cross_val_score(bagging, x, y['Survived'], cv=kf ).mean() )<train_model>
def warn(*args, **kwargs):
pass
warnings.warn = warn
bagging = BaggingClassifier(bootstrap=True,n_jobs = -1,n_estimators=100)
 bagging.fit(x,y['Survived'])
 print(cross_val_score(bagging, x_valid, y_valid, cv=kf ).mean() )<choose_model_class>
np.set_printoptions(suppress=True)
 author_data = pd.read_csv('.. /input/spooky-author-identification/train.zip')
 author_data.head()
xgboost = xgb.XGBClassifier(objective ='reg:logistic'
, colsample_bytree = 0.7
, learning_rate = 0.01
,max_depth = 6
, n_estimators = 100
,random_state=42
,max_features= 0.8
,min_samples_leaf =0.5
,min_child_weight= 3)
 xgboost.fit(x,y['Survived'])
 print(cross_val_score(xgboost, x, y['Survived'], cv=kf ).mean() )<train_model>
print(author_data['author'].value_counts())
 author_data.info()
xgboost = xgb.XGBClassifier(colsample_bytree = 0.8
, learning_rate = 0.05
,max_depth = 6
, n_estimators = 100
,random_state=42 
)
 xgboost.fit(x,y['Survived'])
 print(cross_val_score(xgboost, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
author = author_data.copy()
author['text_length'] = author['text'].apply(lambda text: len(text))
author.head()
lgbm=lgb.LGBMClassifier()
lgbm.fit(x,y['Survived'])
 print(cross_val_score(lgbm, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
author.groupby('author' ).mean()
lgbm=lgb.LGBMClassifier(colsample_bytree = 0.8
, learning_rate = 0.015
,max_depth = 5
, n_estimators = 100
,random_state=42)
 lgbm.fit(x,y['Survived'])
 print(cross_val_score(lgbm, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
X = author_data.drop(['author'], axis=1)
 y = author_data.copy() ['author']
stacking = VotingClassifier(estimators=[ 
('lr',lr),('svm',svm),('rf', rf)], voting='hard')
 stacking.fit(x,y['Survived'])
 print(cross_val_score(stacking, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y ,test_size=0.2, random_state=42 )
stacking = VotingClassifier(estimators=[ 
('lr',rf),('svm',knn),('xgboost', xgboost)], voting='hard')
 stacking.fit(x,y['Survived'])
 print(cross_val_score(stacking, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
nltk.word_tokenize('We don't do that here' )
parameters = {
'max_features': [0.8, 0.9],
'min_samples_leaf' :[0.5,0.9],
'colsample_bytree' :[0.8,0.9],
'learning_rate' : [0.01,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 6)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x,y['Survived'])
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<create_dataframe>
stemmer = nltk.PorterStemmer()
for word in('computing','computed','compulsive'):
print(word,'=>',stemmer.stem(word))
parametros = pd.DataFrame({'feature':x.columns,'Parameters':np.round(xgboost.feature_importances_,3)})
 parametros = parametros.sort_values('Parameters',ascending=False ).set_index('feature')
 drop = list(parametros.tail(30 ).index)
 print(drop)
 x = x.drop(drop,axis=1)
 x_valid = x_valid.drop(drop,axis=1 )<choose_model_class>
stop_words.ENGLISH_STOP_WORDS
parameters = {
'max_features': [0.7,0.8, 0.9],
'min_samples_leaf' :[0.6,0.7,0.9],
'colsample_bytree' :[0.6,0.8,0.9],
'learning_rate' : [0.01,0.03,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 7)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x,y['Survived'])
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<choose_model_class>
text_list = 'i myself believe it was beautiful'.split()
less_than_zero = list(filter(lambda x: not x in stop_words.ENGLISH_STOP_WORDS, text_list))
print(less_than_zero )
parameters = {
'max_features': [0.7,0.8, 0.9],
'min_samples_leaf' :[0.6,0.7,0.9],
'colsample_bytree' :[0.6,0.8,0.9],
'learning_rate' : [0.01,0.03,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 6)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x_valid,y_valid)
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<compute_train_metric>
sentence = 'I love real madrid for real but I dont like real betis'
sentence = re.sub(r'\W+', ' ', sentence, flags=re.M)
 c = Counter(sentence.split())
 print(c.most_common() )
svc=make_pipeline(StandardScaler() ,SVC(random_state=1))
r=[0.0001,0.001,0.1,1,10,50,100]
PSVM=[{'svc__C':r, 'svc__kernel':['linear']},
{'svc__C':r, 'svc__gamma':r, 'svc__kernel':['rbf']}]
GSSVM=GridSearchCV(estimator=svc, param_grid=PSVM, scoring='accuracy', cv=2)
 scores_svm=cross_val_score(GSSVM, x_valid, y_valid,scoring='accuracy', cv=5)
 np.mean(scores_svm )<save_to_csv>
c['love'] += 3
print(c.most_common() )
Survived = grid_search2.best_estimator_.predict(x_valid)
 submission['Survived'] = Survived
submission['Survived'] = submission['Survived'].astype(int)
 submission.to_csv('submission.csv' , index=False )<import_modules>
def derive_new_features(data):
data = data[['text']].copy()
data['unique_words_count'] = data.text.apply(lambda x: len(set(str(x ).split())))
 data['punctuation_count'] = data.text.apply(lambda x: len([x for x in x.lower().split() if x in string.punctuation]))
data['uppercase_words_count'] = data.text.apply(lambda x: sum([x.isupper() for x in x.split() ]))
data['title_words_count'] = data.text.apply(lambda x: sum([x.istitle() for x in x.split() ]))
return data.drop(['text'], axis=1 )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import xgboost as xgb
import lightgbm as lgb
import re
import shap
from sklearn import preprocessing
from category_encoders.one_hot import OneHotEncoder
from sklearn.preprocessing import StandardScaler,LabelEncoder
from sklearn.model_selection import cross_val_score, KFold,GridSearchCV
from sklearn.feature_selection import SelectKBest,SelectFromModel
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline,make_pipeline
from sklearn.ensemble import RandomForestClassifier,VotingClassifier,BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn import linear_model
from sklearn.linear_model import LogisticRegression,Perceptron,SGDClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB<load_from_csv>
class TextToWordCounterTransformer(BaseEstimator, TransformerMixin):
def __init__(self, remove_punctuation=True, lower_case=True, stemming=True, replace_numbers=True, remove_stopwords=True):
self.remove_punctuation = remove_punctuation
self.lower_case = lower_case
self.stemming = stemming
self.replace_numbers = replace_numbers
self.remove_stopwords = remove_stopwords
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
word_counters = []
for text in X['text']:
if self.lower_case:
text = text.lower()
if self.replace_numbers:
text = re.sub(r'\d+(?:\.\d*(?:[eE]\d+)) ?', 'NUMBER', text)
 if self.remove_punctuation:
text = re.sub(r'\W+', ' ', text, flags=re.M)
 if self.remove_stopwords:
words = [word for word in text.split() if not word in stop_words.ENGLISH_STOP_WORDS]
text = ' '.join(words)
 word_list = nltk.word_tokenize(text)
 word_count = Counter(word_list)
 if self.stemming:
stemmed_word_count = Counter()
for word in word_list:
stemmed_word = stemmer.stem(word)
 stemmed_word_count[stemmed_word] += 1
word_count = stemmed_word_count
word_counters.append(word_count)
 return np.array(word_counters )
test = pd.read_csv(".. /input/titanic/test.csv")
 test['Survived'] = np.nan
train = pd.read_csv(".. /input/titanic/train.csv")
 df = pd.concat([test,train],sort=False)
 df.head()<feature_engineering>
X_word_count = TextToWordCounterTransformer().fit_transform(X_train)
 X_word_count
print(df.corr() ['Age'].sort_values(ascending=False))
df['Age'] = df.groupby("Pclass")['Age'].transform(lambda x: x.fillna(x.median()))<feature_engineering>
class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):
def __init__(self, vocabulary_size=1000):
self.vocabulary_size = vocabulary_size
def fit(self, X, y=None):
total_count = Counter()
for word_count in X:
for word, count in word_count.items() :
total_count[word] += min(count, 10)
 most_common = total_count.most_common() [:self.vocabulary_size]
self.most_common_ = most_common
self.vocabulary_ = {word: index + 1 for index,(word, count)in enumerate(most_common)}
return self
def transform(self, X, y=None):
data = []
rows = []
cols = []
for row, word_count in enumerate(X):
for word, count in word_count.items() :
data.append(count)
 rows.append(row)
 cols.append(self.vocabulary_.get(word, 0))
return csr_matrix(( data,(rows, cols)) , shape=(len(X), self.vocabulary_size + 1))
print(df.corr() ['Fare'].sort_values(ascending=False))
df['Fare'] = df.groupby("Pclass")['Fare'].transform(lambda x: x.fillna(x.median()))<feature_engineering>
vectorizer = WordCounterToVectorTransformer(vocabulary_size=1000)
 vectorizer.fit_transform(X_word_count ).toarray()
df['Embarked'].fillna(df['Embarked'].mode() [0], inplace = True)
 df["Fsize"] = df["SibSp"] + df["Parch"] + 1
df['IsAlone'] = np.where(df["Fsize"]==1,1,0)
 df['Age_Class']= df['Age']* df['Pclass']
df['Age_cut'] = pd.qcut(df['Age'],5,duplicates='drop')
 df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr','Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df['Title'] = df['Title'].replace('Mlle', 'Miss')
 df['Title'] = df['Title'].replace('Ms', 'Miss')
 df['Title'] = df['Title'].replace('Mme', 'Mrs')
 df['Title']= df['Title'].fillna(0)
 df['Fare_Per_Person'] = df['Fare']/(df["Fsize"]+1)
 df['Fare_Per_Person'] = df['Fare_Per_Person'].astype(int)
 df['Fare_cut'] = pd.qcut(df['Fare'],5,duplicates='drop')
 df['Cabin_len'] = df['Cabin'].astype(str ).apply(lambda x : len(x))
df['Cabin'] = df['Cabin'].fillna("M")
 df['Cabin'] = df['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 df['Ticket']=df.Ticket.apply(lambda x : len(x))
df = df.drop(['Name'], axis = 1)
 df.head()<normalization>
preprocess_pipeline = Pipeline([ 
('text_to_word_count', TextToWordCounterTransformer()), 
('word_count_to_vector', WordCounterToVectorTransformer(vocabulary_size=14000)) ,
] )
scaler = preprocessing.StandardScaler().fit(df[['Age','Fare']])
 df[['Age','Fare']] = scaler.transform(df[['Age','Fare']] )<categorify>
class NewFeaturesAdderTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
return np.array(derive_new_features(X))
df['Age_cut'] = LabelEncoder().fit_transform(df['Age_cut'])
 df['Fare_cut'] = LabelEncoder().fit_transform(df['Fare_cut'])
 titles = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
df['Title']= df['Title'].map(titles)
 groups = {"M": 1, "C": 2, "B": 3, "D": 4, "E": 5, "A": 6, "F": 7, "G": 8,'T':9}
df['Cabin'] = df['Cabin'].map(groups)
 ports = {"S": 0, "C": 1, "Q": 2}
df['Embarked'] = df['Embarked'].map(ports)
 df['Sex_female'] = np.where(df['Sex'] == 'female',1,0)
 df['Sex_male'] = np.where(df['Sex'] == 'male',1,0)
 df = df.drop(['Sex'],axis=1 )<categorify>
NewFeaturesAdderTransformer().fit_transform(X_train )
dummies = OneHotEncoder(cols= ['Pclass','Age_cut','Fare_cut','Embarked','Title'],use_cat_names=True)
 dummies.fit(df)
 df = dummies.transform(df )<data_type_conversions>
full_pipeline = ColumnTransformer([ 
('feature_adder', NewFeaturesAdderTransformer() , ['text']), 
('text_pipeline', preprocess_pipeline, ['text']),
] )
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
upper.head()<prepare_x_and_y>
label_encoder = LabelEncoder()
x_valid = df[df['Survived'].isna() ].drop(['Survived'], axis = 1)
 submission = x_valid['PassengerId'].to_frame()
x1 = df[df['Survived']==0]
x2 = df[df['Survived']==1]
x= pd.concat([x1,x2])
 y =x.loc[:,['Survived','PassengerId']]
x =x.drop(['Survived','PassengerId'], axis = 1 )<prepare_x_and_y>
X_train_transformed = full_pipeline.fit_transform(X_train)
 y_train_transformed = label_encoder.fit_transform(y_train.values )
teste = pd.read_csv(".. /input/testey/full-tita.csv",sep=";")
 teste = teste.drop(['x'],axis=1)
 teste
x_valid = x_valid.merge(teste)
 x_valid = x_valid.drop(['Survived'],axis = 1)
 y_valid = teste['Survived'] 
<choose_model_class>
label_encoder.classes_
kf = KFold(n_splits=5, shuffle=True, random_state=42 )<train_on_grid>
sgd_clf = SGDClassifier(random_state=42, loss='hinge' )
lr = LogisticRegression(class_weight = 'balanced', solver = 'liblinear',penalty="l2")
 lr.fit(x,y['Survived'])
 print(cross_val_score(lr, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
cv_score = cross_val_score(
 sgd_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3, 
)
 print('mean score :', cv_score.mean() )
lr = LogisticRegression(class_weight = 'balanced', solver = 'liblinear',penalty="l1")
 lr.fit(x,y['Survived'])
 print(cross_val_score(lr, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
log_clf = LogisticRegression(random_state=42)
 cv_score = cross_val_score(
 log_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3, 
)
 print('mean score :', cv_score.mean() )
svm = SVC(gamma='auto',random_state=42)
 svm.fit(x,y['Survived'])
 print(cross_val_score(svm, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
mnb_clf = MultinomialNB()
cv_score = cross_val_score(
 mnb_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3, 
)
 print('mean score :', cv_score.mean() )
svm = SVC(gamma='auto',random_state=42)
 svm.fit(x,y['Survived'])
 print(cross_val_score(svm, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
def multiclass_logloss(actual, predicted, eps=1e-15):

 if len(actual.shape)== 1:
actual2 = np.zeros(( actual.shape[0], predicted.shape[1]))
for i, val in enumerate(actual):
actual2[i, val] = 1
actual = actual2
clip = np.clip(predicted, eps, 1 - eps)
 rows = actual.shape[0]
vsota = np.sum(actual * np.log(clip))
return -1.0 / rows * vsota
sgd = linear_model.SGDClassifier(max_iter=5, tol=None)
 sgd.fit(x,y['Survived'])
 print(cross_val_score(sgd, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
y_scores = cross_val_predict(
 log_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3,
method='predict_proba',
)
knn = KNeighborsClassifier(n_neighbors = 5)
 knn.fit(x,y['Survived'])
 print(cross_val_score(knn, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
pd.DataFrame(columns=label_encoder.classes_, data=y_scores ).head()
knn = KNeighborsClassifier(n_neighbors = 5)
 knn.fit(x,y['Survived'])
 print(cross_val_score(knn, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
multiclass_logloss(y_train_transformed, y_scores )
gaussian = GaussianNB()
gaussian.fit(x,y['Survived'])
 print(cross_val_score(gaussian, x, y['Survived'], cv=kf ).mean() )<train_model>
y_scores = cross_val_predict(
 mnb_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3,
method='predict_proba', 
)
 multiclass_logloss(y_train_transformed, y_scores )
rf_with_standardScaler=make_pipeline(StandardScaler() ,RandomForestClassifier())
 rf_with_standardScaler.fit(x,y['Survived'])
 print(cross_val_score(rf_with_standardScaler, x, y['Survived'], cv=kf ).mean() )<train_model>
stopword_params = [True, False]
vocabulary_params = [7000, 8000, 9000, 10000]
data_scores = []
for stopword, vocabulary in list(itertools.product(stopword_params, vocabulary_params)) :
full_pipeline.set_params(
 text_pipeline__text_to_word_count__remove_stopwords=stopword,
text_pipeline__word_count_to_vector__vocabulary_size=vocabulary, 
)
 X_train_processed = full_pipeline.fit_transform(X_train)
 y_scores = cross_val_predict(
 log_clf,
X_train_processed, y_train_transformed,
cv=3,
method='predict_proba', 
)
 data_scores += [(stopword, vocabulary, multiclass_logloss(y_train_transformed, y_scores)) ]
rf_with_standardScaler=make_pipeline(StandardScaler() ,RandomForestClassifier())
 rf_with_standardScaler.fit(x,y['Survived'])
 print(cross_val_score(rf_with_standardScaler, x_valid, y_valid, cv=kf ).mean() )<compute_train_metric>
for stopword, vocabulary, logloss in data_scores:
print('remove_stopwords:', stopword, ',', 'vocabulary_size:',vocabulary)
 print('logloss: ', logloss )
rf=RandomForestClassifier(n_estimators=100, oob_score = True)
 rf.fit(x,y['Survived'])
 print(cross_val_score(rf, x, y['Survived'], cv=kf ).mean())
 parametros = pd.DataFrame({'feature':x.columns,'Parameters':np.round(rf.feature_importances_,3)})
 parametros = parametros.sort_values('Parameters',ascending=False ).set_index('feature')
 parametros<train_model>
full_pipeline.set_params(
 text_pipeline__text_to_word_count__remove_stopwords=False,
text_pipeline__word_count_to_vector__vocabulary_size=8_000, 
)
 X_train_transformed = full_pipeline.fit_transform(X_train )
rf=RandomForestClassifier(n_estimators=100)
 rf.fit(x,y['Survived'])
 print(cross_val_score(rf, x_valid, y_valid, cv=kf ).mean())
<train_on_grid>
%%time
log_grid_params = {
'penalty': ['L1', 'l2'],
'dual': [False],
'tol':[1e-4, 1e-5],
'class_weight': ['balanced', None],
'solver': ['newton-cg', 'lbfgs', 'sag', 'saga'],
'multi_class': ['ovr', 'auto', 'multinomial'],
}
LogisticRegression
log_grid_search = GridSearchCV(
 estimator=LogisticRegression(random_state=42),
param_grid=log_grid_params,
scoring='neg_log_loss',
cv=3,
verbose=2, 
)
 log_grid_search.fit(X_train_transformed, y_train_transformed )
bagging = BaggingClassifier(bootstrap=True,n_jobs = -1,n_estimators=100)
 bagging.fit(x,y['Survived'])
 print(cross_val_score(bagging, x, y['Survived'], cv=kf ).mean() )<train_model>
print(log_grid_search.best_params_ )
bagging = BaggingClassifier(bootstrap=True,n_jobs = -1,n_estimators=100)
 bagging.fit(x,y['Survived'])
 print(cross_val_score(bagging, x_valid, y_valid, cv=kf ).mean() )<choose_model_class>
joblib.dump(log_grid_search, 'log_grid_best.pkl')
 log_clf_best = joblib.load('log_grid_best.pkl' ).best_estimator_
print(log_clf_best.get_params() )
xgboost = xgb.XGBClassifier(objective ='reg:logistic'
, colsample_bytree = 0.7
, learning_rate = 0.01
,max_depth = 6
, n_estimators = 100
,random_state=42
,max_features= 0.8
,min_samples_leaf =0.5
,min_child_weight= 3)
 xgboost.fit(x,y['Survived'])
 print(cross_val_score(xgboost, x, y['Survived'], cv=kf ).mean() )<train_model>
%%time
sgd_grid_params = {
'loss': ['log'],
'penalty' : ['l2'],
'eta0':[0.1],
'alpha': [1e-4, 1e-5],
'tol': [1e-3, 1e-4],
'epsilon': [0.3, 0.5, 1],
'learning_rate': ['adaptive'],
'class_weight': ['balanced', None],
'average':[True, False],
}
sgd_grid_search = GridSearchCV(
 estimator=SGDClassifier(random_state=42),
param_grid=sgd_grid_params,
scoring='neg_log_loss',
cv=3,
verbose=2, 
)
 sgd_grid_search.fit(X_train_transformed, y_train_transformed )
xgboost = xgb.XGBClassifier(colsample_bytree = 0.8
, learning_rate = 0.05
,max_depth = 6
, n_estimators = 100
,random_state=42 
)
 xgboost.fit(x,y['Survived'])
 print(cross_val_score(xgboost, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
print(sgd_grid_search.best_params_ )
lgbm=lgb.LGBMClassifier()
lgbm.fit(x,y['Survived'])
 print(cross_val_score(lgbm, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
joblib.dump(sgd_grid_search, 'sgd_grid_best.pkl')
 sgd_clf_best = joblib.load('sgd_grid_best.pkl' ).best_estimator_
print(sgd_clf_best.get_params() )
lgbm=lgb.LGBMClassifier(colsample_bytree = 0.8
, learning_rate = 0.015
,max_depth = 5
, n_estimators = 100
,random_state=42)
 lgbm.fit(x,y['Survived'])
 print(cross_val_score(lgbm, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
%%time
mnb_grid_params = {
'alpha': [0, 0.25, 0.5, 0.75, 1],
'fit_prior': [False, True],
}
GridSearchCV
mnb_grid_search = GridSearchCV(
 estimator=MultinomialNB() ,
param_grid=mnb_grid_params,
scoring='neg_log_loss',
cv=3,
verbose=2, 
)
 mnb_grid_search.fit(X_train_transformed, y_train_transformed )
stacking = VotingClassifier(estimators=[ 
('lr',lr),('svm',svm),('rf', rf)], voting='hard')
 stacking.fit(x,y['Survived'])
 print(cross_val_score(stacking, x, y['Survived'], cv=kf ).mean() )<train_on_grid>
print(mnb_grid_search.best_params_ )
stacking = VotingClassifier(estimators=[ 
('lr',rf),('svm',knn),('xgboost', xgboost)], voting='hard')
 stacking.fit(x,y['Survived'])
 print(cross_val_score(stacking, x_valid, y_valid, cv=kf ).mean() )<train_on_grid>
joblib.dump(mnb_grid_search, 'mnb_grid_best.pkl')
 mnb_clf_best = joblib.load('mnb_grid_best.pkl' ).best_estimator_
print(mnb_clf_best.get_params() )
parameters = {
'max_features': [0.8, 0.9],
'min_samples_leaf' :[0.5,0.9],
'colsample_bytree' :[0.8,0.9],
'learning_rate' : [0.01,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 6)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x,y['Survived'])
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<create_dataframe>
sgd_clf_best.set_params(loss='log')
 estimators=[('log_clf', log_clf_best),('sgd_clf',sgd_clf_best),('mnb_clf',mnb_clf_best)]
vot_clf = VotingClassifier(
 estimators=estimators,
voting='soft',
)
parametros = pd.DataFrame({'feature':x.columns,'Parameters':np.round(xgboost.feature_importances_,3)})
 parametros = parametros.sort_values('Parameters',ascending=False ).set_index('feature')
 drop = list(parametros.tail(30 ).index)
 print(drop)
 x = x.drop(drop,axis=1)
 x_valid = x_valid.drop(drop,axis=1 )<choose_model_class>
y_scores = cross_val_predict(
 vot_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3,
method='predict_proba', 
)
 multiclass_logloss(y_train_transformed, y_scores )
parameters = {
'max_features': [0.7,0.8, 0.9],
'min_samples_leaf' :[0.6,0.7,0.9],
'colsample_bytree' :[0.6,0.8,0.9],
'learning_rate' : [0.01,0.03,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 7)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x,y['Survived'])
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<choose_model_class>
sgd_clf_best.set_params(loss='hinge' )
parameters = {
'max_features': [0.7,0.8, 0.9],
'min_samples_leaf' :[0.6,0.7,0.9],
'colsample_bytree' :[0.6,0.8,0.9],
'learning_rate' : [0.01,0.03,0.1],
'min_child_weight': [3,5,7],
}
model2 = xgb.XGBClassifier(objective ='reg:logistic' ,random_state=42,n_estimators=100,max_depth = 6)
 grid_search2 = GridSearchCV(model2, parameters, cv=5,n_jobs=-1)
 grid_search2.fit(x_valid,y_valid)
 print(grid_search2.best_params_)
 print(grid_search2.best_score_ )<compute_train_metric>
estimators=[('sgd_clf',sgd_clf_best),('log_clf', log_clf_best),('mnb_clf',mnb_clf_best)]
stk_clf = StackingClassifier(
 estimators=estimators,
)
svc=make_pipeline(StandardScaler() ,SVC(random_state=1))
r=[0.0001,0.001,0.1,1,10,50,100]
PSVM=[{'svc__C':r, 'svc__kernel':['linear']},
{'svc__C':r, 'svc__gamma':r, 'svc__kernel':['rbf']}]
GSSVM=GridSearchCV(estimator=svc, param_grid=PSVM, scoring='accuracy', cv=2)
 scores_svm=cross_val_score(GSSVM, x_valid, y_valid,scoring='accuracy', cv=5)
 np.mean(scores_svm )<save_to_csv>
y_scores = cross_val_predict(
 stk_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3,
method='predict_proba', 
)
 multiclass_logloss(y_train_transformed, y_scores )
Survived = grid_search2.best_estimator_.predict(x_valid)
 submission['Survived'] = Survived
submission['Survived'] = submission['Survived'].astype(int)
 submission.to_csv('submission.csv' , index=False )<import_modules>
final_model = stk_clf
final_model.fit(X_train_transformed, y_train_transformed )
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import tree
import graphviz<import_modules>
X_test_transformed = full_pipeline.transform(X_test)
 y_test_transformed = label_encoder.transform(y_test)
 y_scores = final_model.predict_proba(X_test_transformed)
 y_scores
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
import seaborn as sns
from sklearn.metrics import confusion_matrix, roc_curve, auc<import_modules>
multiclass_logloss(y_test_transformed, y_scores )
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
from sklearn.metrics import recall_score
from xgboost import XGBClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import AdaBoostClassifier
from keras.models import Sequential,Model
from keras.layers import Dense, Activation, Embedding, concatenate, Flatten, Input
from keras.optimizers import Adam
from sklearn.preprocessing import LabelEncoder
from sklearn.svm import SVC<load_from_csv>
X_transformed = full_pipeline.transform(X)
 y_transformed = label_encoder.transform(y)
 final_model.fit(X_transformed, y_transformed )
Titanic_data = pd.read_csv(".. /input/train.csv", header=0)
 Titanic_data_test_pID = pd.read_csv(".. /input/test.csv", header=0)
 Titanic_data_test = pd.read_csv(".. /input/test.csv", header=0 )<feature_engineering>
test_data = pd.read_csv('.. /input/spooky-author-identification/test.zip')
 test_data.head()
for i in range(len(Titanic_data)) :
a = str(Titanic_data['Cabin'][i])
 if a.count(' ')>=1:
Titanic_data['Fare'][i] =(Titanic_data['Fare'][i]/(a.count(' ')+1)) 
<feature_engineering>
test_data_prepared = full_pipeline.transform(test_data)
 test_scores = final_model.predict_proba(test_data_prepared)
 test_scores
for i in range(len(Titanic_data_test)) :
a = str(Titanic_data_test['Cabin'][i])
 if a.count(' ')>=1:
Titanic_data_test['Fare'][i] =(Titanic_data_test['Fare'][i]/(a.count(' ')+1))<drop_column>
submission_file = pd.DataFrame({
'id': test_data['id'].values,
'EAP': test_scores[:,0],
'HPL': test_scores[:,1],
'MWS': test_scores[:,2],
})
 submission_file.head()
<define_variables><EOS> 
submission_file.to_csv('submission.csv', index=False )