Kaggle competition: Titanic - Machine Learning from Disaster
def remove_shortforms(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
def remove_special_char(text):
text = re.sub('[^A-Za-z0-9]+'," ",text)
 return text
def remove_wordswithnum(text):
text = re.sub("\S*\d\S*", "", text ).strip()
return text
def lowercase(text):
text = text.lower()
return text
def remove_stop_words(text):
text = remove_stopwords(text)
 return text
st = SnowballStemmer(language='english')
 def stemming(text):
r= []
for word in text :
a = st.stem(word)
 r.append(a)
 return r
def listToString(s):
str1 = " "
return(str1.join(s))
def remove_punctuations(text):
text = re.sub(r'[^\w\s]','',text)
 return text
def remove_links(text):
text = re.sub(r'http\S+', '', text)
 return text
lemmatizer = WordNetLemmatizer()
def lemmatize_words(text):
text = lemmatizer.lemmatize(text)
 return text
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text )<prepare_x_and_y>
discriminant_analysis, tree, gaussian_process, model_selection
Y = train['target']
train = train.drop('target',axis=1)
 data = pd.concat([train,test],axis=0 ).reset_index(drop=True)
 data.head()<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
for i in range(len(data['text'])) :
data['text'][i] = str(data['text'][i] )<feature_engineering>
passenger_id = test['PassengerId'].copy()
for i in range(len(data['text'])) :
data['text'][i] = remove_shortforms(data['text'][i])
 data['text'][i] = remove_special_char(data['text'][i])
 data['text'][i] = remove_wordswithnum(data['text'][i])
 data['text'][i] = lowercase(data['text'][i])
 data['text'][i] = remove_stop_words(data['text'][i])
 text = data['text'][i]
text = text.split()
data['text'][i] = stemming(text)
 s = data['text'][i]
data['text'][i] = listToString(s)
 data['text'][i] = lemmatize_words(data['text'][i] )<normalization>
traintest = pd.concat([train, test], axis=0, sort=False )
cv = CountVectorizer(ngram_range=(1,3))
text_bow = cv.fit_transform(data['text'])
 print(text_bow.shape )<split>
isnull = traintest.isnull().sum().reset_index()
isnull.columns = ['Feature', 'Total_null']
total_null = isnull[isnull['Total_null']>0]
total_null
train_text = text_bow[:train.shape[0]]
test_text = text_bow[train.shape[0]:]<split>
traintest['Title'] = traintest['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
replace = {
'Ms': 'Miss',
'Dona': 'Miss',
'Mlle': 'Miss',
'Mme': 'Miss',
'Don': 'Mr',
'Sir': 'Mr'
}
traintest.replace({'Title': replace}, inplace=True )
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )<compute_train_metric>
df_nan_ages = traintest['Age'].isnull()
title_ages = traintest[['Title', 'Age']].groupby('Title' ).mean().to_dict() ['Age']
traintest['Age'][df_nan_ages] = traintest[df_nan_ages]['Title'].apply(lambda x: title_ages[x])
 traintest.fillna({
'Fare': traintest['Fare'].mode() [0],
'Embarked': traintest['Embarked'].mode() [0]
}, inplace=True)
 traintest.drop(columns='Cabin', inplace=True)
 traintest.drop(columns='PassengerId', inplace=True )
lr = LogisticRegression(C=10,penalty='l2')
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report
:",classification_report(Y_test,pred))<prepare_output>
traintest['FamilySize'] = traintest.SibSp + traintest.Parch + 1
traintest['IsAlone'] =(traintest['FamilySize'] == 1)*1
traintest['AgeStage'] = traintest['Age']
traintest['AgeStage'][traintest['Age'] <= 11] = 'Child'
traintest['AgeStage'][(traintest['Age'] <= 20)&(traintest['Age'] > 11)] = 'Young'
traintest['AgeStage'][(traintest['Age'] <= 40)&(traintest['Age'] > 20)] = 'Adult'
traintest['AgeStage'][traintest['Age'] > 40] = 'Old'
traintest.drop(columns='Name', inplace=True )
lr = LogisticRegression(C=10,penalty='l2',max_iter=2000)
 lr.fit(train_text,Y)
 pred = lr.predict(test_text)
 submit = pd.DataFrame(test['id'],columns=['id'])
 print(len(pred))
submit.head()<save_to_csv>
isnull = traintest.isnull().sum().reset_index()
isnull.columns = ['Feature', 'Total_null']
total_null = isnull[isnull['Total_null']>0]
total_null
submit['target'] = pred
submit.to_csv("realnlp.csv",index=False )<categorify>

 categorical_vars = ["Pclass", "Sex", "Ticket", 'Embarked', 'Title', 'AgeStage']
numerical_vars = ['Fare', 'Age', 'FamilySize', 'Parch', 'SibSp']
traintest_set = []
traintest[categorical_vars] = traintest[categorical_vars].astype('category')
 traintest_labelEncoding = traintest.copy()
label_encoder = preprocessing.LabelEncoder()
for var in categorical_vars:
print(var)
 traintest_labelEncoding[var] = label_encoder.fit_transform(traintest_labelEncoding[var])
 traintest_set.append(traintest_labelEncoding)
 traintest_onehot = pd.get_dummies(traintest, columns=categorical_vars)
 traintest_onehot[numerical_vars] =(traintest_onehot[numerical_vars] - traintest_onehot[numerical_vars].mean())/(traintest_onehot[numerical_vars].max() - traintest_onehot[numerical_vars].min())
 traintest_set.append(traintest_onehot)
tfidf = TfidfVectorizer(ngram_range=(1,3))
text_tfidf = tfidf.fit_transform(data['text'])
 print(text_tfidf.shape )<split>
X_trains, y_train, X_tests = [], None,[]
for traintest in traintest_set:
train = traintest[traintest['Survived'].notnull() ]
y_train = train['Survived']
X_train = train.drop(columns='Survived')
 print("X train shape: ", X_train.shape)
 print("Y train shape: ", y_train.shape)
 X_trains.append(X_train)
 test = traintest[traintest['Survived'].isnull() ]
X_test = test.drop(columns='Survived')
 print("X test shape: ", X_test.shape)
 X_tests.append(X_test )
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )<compute_train_metric>
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
XGBClassifier()
]
lr = LogisticRegression(C=100,penalty='l2',max_iter=2000)
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report :",classification_report(Y_test,pred))<count_missing_values>
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.7, random_state = 0 )
print("Number of null values in data keywords column : ",data['keyword'].isnull().sum() )<data_type_conversions>
MLAs = [copy.deepcopy(MLA), copy.deepcopy(MLA)]
MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
result_table = []
for i in range(len(X_trains)) :
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = y_train.copy()
print("Y train shape: ", y_train.shape)
 row_index = 0
for alg in MLAs[i]:
target = y_train.copy()
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 print("Y train shape: ", y_train.shape)
 print(MLA_name)
 cv_results = model_selection.cross_validate(alg, X_trains[i], target, cv = cv_split)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(X_trains[i], y_train)
 MLA_predict[MLA_name] = alg.predict(X_trains[i])
 row_index+=1
result_table.append(( MLA_compare, MLA_predict))
data['keyword'] = data['keyword'].fillna("unknown")
 data.head()<feature_engineering>
MLA_compare = result_table[0][0]
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare
combined_text = [None] * len(data['text'])
 for i in range(len(data['text'])) :
if data['keyword'][i] == 'unknown':
combined_text[i] = data['text'][i]
else:
combined_text[i] = data['text'][i] + " " + data['keyword'][i] + " " + data['keyword'][i] + " " + data['keyword'][i]
data['combined_text'] = combined_text<feature_engineering>
MLA_compare = result_table[1][0]
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare
for i in range(len(data['combined_text'])) :
data['combined_text'][i] = str(data['combined_text'][i] )<feature_engineering>
prediction_df = pd.DataFrame()
alg_name = 'XGBClassifier'
feature_index = 0
alg_index = 19
prediction = MLAs[feature_index][alg_index].predict(X_tests[feature_index])
 temp = {'PassengerID': passenger_id, 'Survived': prediction.astype(int)}
result = pd.DataFrame(temp)
 result.to_csv('result_%s_feature%s.csv'%(alg_name, feature_index), index=False)
 prediction_df[alg_name] = prediction
alg_name = 'GradientBoostingClassifier'
feature_index = 0
alg_index = 3
prediction = MLAs[feature_index][alg_index].predict(X_tests[feature_index])
 temp = {'PassengerID': passenger_id, 'Survived': prediction.astype(int)}
result = pd.DataFrame(temp)
 result.to_csv('result_%s_feature%s.csv'%(alg_name, feature_index), index=False)
 prediction_df[alg_name] = prediction
alg_name = 'RidgeClassifierCV'
feature_index = 1
alg_index = 8
prediction = MLAs[feature_index][alg_index].predict(X_tests[feature_index])
 temp = {'PassengerID': passenger_id, 'Survived': prediction.astype(int)}
result = pd.DataFrame(temp)
 result.to_csv('result_%s_feature%s.csv'%(alg_name, feature_index), index=False)
 prediction_df[alg_name] = prediction
alg_name = 'LogisticRegressionCV'
feature_index = 1
alg_index = 6
prediction = MLAs[feature_index][alg_index].predict(X_tests[feature_index])
 temp = {'PassengerID': passenger_id, 'Survived': prediction.astype(int)}
result = pd.DataFrame(temp)
 result.to_csv('result_%s_feature%s.csv'%(alg_name, feature_index), index=False)
 prediction_df[alg_name] = prediction
for i in range(len(data['combined_text'])) :
data['combined_text'][i] = remove_shortforms(data['combined_text'][i])
 data['combined_text'][i] = remove_special_char(data['combined_text'][i])
 data['combined_text'][i] = remove_wordswithnum(data['combined_text'][i])
 data['combined_text'][i] = lowercase(data['combined_text'][i])
 data['combined_text'][i] = remove_stop_words(data['combined_text'][i])
 text = data['combined_text'][i]
text = text.split()
data['combined_text'][i] = stemming(text)
 s = data['combined_text'][i]
data['combined_text'][i] = listToString(s)
 data['combined_text'][i] = lemmatize_words(data['combined_text'][i] )<feature_engineering>
cosine_similarity(prediction_df.T )
cv = CountVectorizer(ngram_range=(1,3))
text_bow = cv.fit_transform(data['combined_text'])
 print(text_bow.shape )<split>
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.optimizers import Adam
from keras.regularizers import l2
from keras.callbacks import EarlyStopping
from sklearn import preprocessing
from keras import regularizers
train_text = text_bow[:train.shape[0]]
test_text = text_bow[train.shape[0]:]<split>
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )<compute_train_metric>
lr = LogisticRegression(C=1,penalty='l2',max_iter=2000)
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report :",classification_report(Y_test,pred))<categorify>
vote_est = [ 
('ada', ensemble.AdaBoostClassifier()), 
('bc', ensemble.BaggingClassifier()), 
('etc',ensemble.ExtraTreesClassifier()), 
('gbc', ensemble.GradientBoostingClassifier()), 
('rfc', ensemble.RandomForestClassifier()), 
('gpc', gaussian_process.GaussianProcessClassifier()), 
('lr', linear_model.LogisticRegressionCV()), 
('bnb', naive_bayes.BernoulliNB()), 
('gnb', naive_bayes.GaussianNB()), 
('knn', neighbors.KNeighborsClassifier()), 
('svc', svm.SVC(probability=True)) , 
('xgb', XGBClassifier())
 ]
tfidf = TfidfVectorizer(ngram_range=(1,3))
text_tfidf = tfidf.fit_transform(data['combined_text'])
 print(text_tfidf.shape )<split>
vote_ests = [vote_est, vote_est]
train_text = text_tfidf[:train.shape[0]]
test_text = text_tfidf[train.shape[0]:]<split>
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )<compute_train_metric>
lr = LogisticRegression(C=2,penalty='l2',max_iter=2000)
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report :",classification_report(Y_test,pred))<feature_engineering>
best_param = [[
[
{'learning_rate': 0.25, 'n_estimators': 300, 'random_state': 0}
],
[
{'max_samples': 0.5, 'n_estimators': 300, 'random_state': 0}
],
[
{'criterion': 'entropy', 'max_depth': 8, 'n_estimators': 50, 'random_state': 0}
],
[
{'learning_rate': 0.05, 'max_depth': 2, 'n_estimators': 300, 'random_state': 0}
],
[
{'criterion': 'entropy', 'max_depth': 8, 'n_estimators': 300, 'oob_score': True, 'random_state': 0}
],
[
{'max_iter_predict': 10, 'random_state': 0}
],
[
{'fit_intercept': True, 'random_state': 0, 'solver': 'newton-cg'}],
[{
'alpha': 0.1,
}],
[{}],
[{
'n_neighbors': 7,
'weights': 'distance',
'algorithm':'brute'
}],
[{
'C': 1,
'gamma': 0.1,
'decision_function_shape': 'ovo',
'probability': True,
'random_state': 0
}],
[
{'learning_rate': 0.05, 'max_depth': 4, 'n_estimators': 100, 'seed': 0}
]
],
[
[
{'learning_rate': 0.25, 'n_estimators': 300, 'random_state': 0}
],
[
{'max_samples': 1.0, 'n_estimators': 300, 'random_state': 0}
],
[
{'criterion': 'entropy', 'max_depth': None, 'n_estimators': 50, 'random_state': 0}
],
[
{'learning_rate': 0.05, 'max_depth': 6, 'n_estimators': 300, 'random_state': 0}
],
[
{'criterion': 'entropy', 'max_depth': None, 'n_estimators': 50, 'oob_score': True, 'random_state': 0}
],
[
{'max_iter_predict': 10, 'random_state': 0}
],
[
{'fit_intercept': True, 'random_state': 0, 'solver': 'lbfgs'}],
[{
'alpha': 0.1,
}],
[{}],
[{
'n_neighbors': 7,
'weights': 'distance',
'algorithm':'brute'
}],
[{
'C': 1,
'gamma': 0.1,
'decision_function_shape': 'ovo',
'probability': True,
'random_state': 0
}],
[
{'learning_rate': 0.05, 'max_depth': 4, 'n_estimators': 100, 'seed': 0}
]
]
]
print('Loading word vectors...')
 word2vec = {}
with open(os.path.join('.. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt'), encoding = "utf-8")as f:
for line in f:
values = line.split()
word = values[0]
vec = np.asarray(values[1:], dtype='float32')
 word2vec[word] = vec
print('Found %s word vectors.' % len(word2vec))<load_from_csv>
for i in range(len(vote_ests)) :
for clf, param in zip(vote_ests[i], best_param[i]):
print('The best parameter for {} is {}'.format(clf[1].__class__.__name__, param[0]))
clf[1].set_params(**param[0] )
train = pd.read_csv(r'/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv(r'/kaggle/input/nlp-getting-started/test.csv' )<prepare_x_and_y>
grid_hards = []
for i in range(len(vote_ests)) :
grid_hard = ensemble.VotingClassifier(estimators = vote_ests[i], voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_trains[i], y_train, cv = cv_split)
 grid_hard.fit(X_trains[i], y_train)
 grid_hards.append(grid_hard)
 print("Hard Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_hard_cv['test_score'].std() *100*3))
print('-'*10)
<train_model><EOS> 
alg_name = 'GridHardVoting'
feature_index = 0
prediction = grid_hards[feature_index].predict(X_tests[feature_index])
 temp = {'PassengerID': passenger_id, 'Survived': prediction.astype(int)}
result = pd.DataFrame(temp)
 result.to_csv('result_%s_feature%s.csv'%(alg_name, feature_index), index=False)
 prediction_df[alg_name] = prediction
alg_name = 'GridHardVoting'
feature_index = 1
prediction = grid_hards[feature_index].predict(X_tests[feature_index])
 temp = {'PassengerID': passenger_id, 'Survived': prediction.astype(int)}
result = pd.DataFrame(temp)
 result.to_csv('result_%s_feature%s.csv'%(alg_name, feature_index), index=False)
 prediction_df[alg_name] = prediction