Kaggle competition: Titanic - Machine Learning from Disaster
df_train2[tgt_col] = np.log1p(df_train2[tgt_col] )<prepare_x_and_y>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )
y_train = df_train2[tgt_col]
X_train = df_train2.drop([tgt_col], axis=1)
 X_test = df_test2<normalization>
data_size_train = train.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_train )
num_cols = []
for col in X_train.columns:
if X_train[col].dtype == 'int64' or X_train[col].dtype == 'float64':
num_cols.append(col)
 scaler = StandardScaler()
scaler.fit(X_train[num_cols])
 X_train[num_cols] = scaler.transform(X_train[num_cols])
 X_test[num_cols] = scaler.transform(X_test[num_cols] )<categorify>
data_size_test = test.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_test )
def student_enc(x):
if x == 'No':
return 3
elif x == 'Yes, part-time':
return 2
elif x == 'Yes, full-time':
return 1<feature_engineering>
train["NameTitle"] = train["Name"].str.split(", ", expand = True)[1].str.split(".", expand = True)[0]
test["NameTitle"] = test["Name"].str.split(", ", expand = True)[1].str.split(".", expand = True)[0]
train = train.drop("Name", axis = 1)
 test = test.drop("Name", axis = 1)
 print(train["NameTitle"].unique())
 print("------------------------" * 3)
 print(test["NameTitle"].unique() )
X_train['Student'] = X_train['Student'].apply(student_enc)
 X_test['Student'] = X_test['Student'].apply(student_enc )<categorify>
min_titles =(train["NameTitle"].value_counts() < 10)
 train["NameTitle"] = train["NameTitle"].apply(lambda x: "Misc" if min_titles.loc[x] == True else x)
 print(train["NameTitle"].unique() )
def company_size_enc(x):
if x == 'Fewer than 10 employees':
return 1
elif x == '10 to 19 employees':
return 2
elif x == '20 to 99 employees':
return 3
elif x == '100 to 499 employees':
return 4
elif x == '500 to 999 employees':
return 5
elif x == '1,000 to 4,999 employees':
return 6
elif x == '5,000 to 9,999 employees':
return 7
elif x == '10,000 or more employees':
return 8<feature_engineering>
min_titles =(test["NameTitle"].value_counts() < 10)
 test["NameTitle"] = test["NameTitle"].apply(lambda x: "Misc" if min_titles.loc[x] == True else x)
 print(test["NameTitle"].unique() )
X_train['CompanySize'] = X_train['CompanySize'].apply(company_size_enc)
 X_test['CompanySize'] = X_test['CompanySize'].apply(company_size_enc )<categorify>
train["FamilySize"] = train["SibSp"] + train["Parch"]
test["FamilySize"] = test["SibSp"] + test["Parch"]
def years_coding_enc(x):
if x == '0-2 years':
return 1
elif x == '3-5 years':
return 2
elif x == '6-8 years':
return 3
elif x == '9-11 years':
return 4
elif x == '12-14 years':
return 5
elif x == '15-17 years':
return 6
elif x == '18-20 years':
return 7
elif x == '21-23 years':
return 8
elif x == '24-26 years':
return 9
elif x == '27-29 years':
return 10
elif x == '30 or more years':
return 11<categorify>
train["AgeGroup"] = ""
train.loc[train["Age"] < 21, "AgeGroup"] = "under 21"
train.loc[train["Age"] >= 21, "AgeGroup"] = "21-65"
train.loc[train["Age"] > 65, "AgeGroup"] = "65+"
test["AgeGroup"] = ""
test.loc[test["Age"] < 21, "AgeGroup"] = "under 21"
test.loc[test["Age"] >= 21, "AgeGroup"] = "21-65"
test.loc[test["Age"] > 65, "AgeGroup"] = "65+"
train = train.drop("Age", axis = 1)
 test = test.drop("Age", axis = 1 )
X_train['YearsCoding'] = X_train['YearsCoding'].apply(years_coding_enc)
 X_test['YearsCoding'] = X_test['YearsCoding'].apply(years_coding_enc )<categorify>
train["FareGroup"] = ""
train.loc[train["Fare"] < 170, "FareGroup"] = "0-170"
train.loc[train["Fare"] >= 170, "FareGroup"] = "170-340"
train.loc[train["Fare"] > 340, "FareGroup"] = "340+"
test["FareGroup"] = ""
test.loc[test["Fare"] < 170, "FareGroup"] = "0-170"
test.loc[test["Fare"] >= 170, "FareGroup"] = "170-340"
test.loc[test["Fare"] > 340, "FareGroup"] = "340+"
train = train.drop("Fare", axis = 1)
 test = test.drop("Fare", axis = 1 )
def years_coding_prof_enc(x):
if x == '0-2 years':
return 1
elif x == '3-5 years':
return 2
elif x == '6-8 years':
return 3
elif x == '9-11 years':
return 4
elif x == '12-14 years':
return 5
elif x == '15-17 years':
return 6
elif x == '18-20 years':
return 7
elif x == '21-23 years':
return 8
elif x == '24-26 years':
return 9
elif x == '27-29 years':
return 10
elif x == '30 or more years':
return 11<feature_engineering>
train = train.drop(["SibSp", "Parch"], axis = 1)
 test = test.drop(["SibSp", "Parch"], axis = 1 )
X_train['YearsCodingProf'] = X_train['YearsCodingProf'].apply(years_coding_prof_enc)
 X_test['YearsCodingProf'] = X_test['YearsCodingProf'].apply(years_coding_prof_enc )<categorify>
train = pd.get_dummies(train, columns = ["Sex", "Embarked", "NameTitle", "AgeGroup", "FareGroup"])
 test = pd.get_dummies(test, columns = ["Sex", "Embarked", "NameTitle", "AgeGroup", "FareGroup"] )
def job_satisfaction_enc(x):
if x == 'Extremely dissatisfied':
return 1
elif x == 'Moderately dissatisfied':
return 2
elif x == 'Slightly dissatisfied':
return 3
elif x == 'Neither satisfied nor dissatisfied':
return 4
elif x == 'Slightly satisfied':
return 5
elif x == 'Moderately satisfied':
return 6
elif x == 'Extremely satisfied':
return 7<feature_engineering>
upper = corr.where(np.triu(np.ones(corr.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.90)]
print(to_drop )
X_train['JobSatisfaction'] = X_train['JobSatisfaction'].apply(job_satisfaction_enc)
 X_test['JobSatisfaction'] = X_test['JobSatisfaction'].apply(job_satisfaction_enc )<categorify>
X = train.drop(["PassengerId", "Survived"], axis = 1)
 Y = train["Survived"]
def career_satisfaction_enc(x):
if x == 'Extremely dissatisfied':
return 1
elif x == 'Moderately dissatisfied':
return 2
elif x == 'Slightly dissatisfied':
return 3
elif x == 'Neither satisfied nor dissatisfied':
return 4
elif x == 'Slightly satisfied':
return 5
elif x == 'Moderately satisfied':
return 6
elif x == 'Extremely satisfied':
return 7<feature_engineering>
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, BaggingClassifier, ExtraTreesClassifier, StackingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from catboost import CatBoostClassifier
from xgboost import XGBClassifier
import lightgbm as lgb
X_train['CareerSatisfaction'] = X_train['CareerSatisfaction'].apply(career_satisfaction_enc)
 X_test['CareerSatisfaction'] = X_test['CareerSatisfaction'].apply(career_satisfaction_enc )<categorify>
style.use("seaborn-whitegrid")
 %matplotlib inline
%config InlineBackend.figure_format = "retina"
def model_builder(X, Y, pipeline, params, kfolds, classifier_name, data_split = False):

 if data_split == True:
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
random_state = 0,
test_size = 0.25)
 cv = StratifiedKFold(n_splits = kfolds)
 start = time.time()
grid = GridSearchCV(pipeline, params, cv = cv, verbose = 0, n_jobs = -1)
 grid = grid.fit(X_train, Y_train)
 print("Best Params:")
 print("")
 print(grid.best_params_)
 grid = grid.best_estimator_
classifier_score = cross_val_score(grid, X_test, Y_test, cv = cv, n_jobs = -1)
 Y_predicted = grid.predict(X_test)
 stop = time.time()
print("")
 print("
print("Cross-Validated-Score: " + str(round(classifier_score.mean() , 6)))
 print("
print("")
 print("Training time: " + str(round(stop - start, 2)) + "s")
 train_size, train_score, test_score = learning_curve(grid, X_train, Y_train, cv = cv, n_jobs = -1)
 print("")
 print("Learning Curve:")
 plt.figure(figsize =(12,6))
plt.plot(train_size, np.mean(train_score, axis = 1), label = "Train scores")
 plt.plot(train_size, np.mean(test_score, axis = 1), label = "Test scores")
 plt.title(classifier_name)
 plt.legend()
plt.show()
return X_train, X_test, Y_train, Y_test, classifier_score, grid
if data_split == False:
cv = StratifiedKFold(n_splits = kfolds)
 start = time.time()
grid = GridSearchCV(pipeline, params, cv = cv, verbose = 0, n_jobs = -1)
 grid = grid.fit(X, Y)
 print("Best Params:")
 print("")
 print(grid.best_params_)
 grid = grid.best_estimator_
classifier_score = cross_val_score(grid, X, Y, cv = cv, n_jobs = -1)
 Y_predicted = grid.predict(X)
 stop = time.time()
print("")
 print("
print("Cross-Validated-Score: " + str(round(classifier_score.mean() , 6)))
 print("
print("")
 print("Training time: " + str(round(stop - start, 2)) + "s")
 train_size, train_score, test_score = learning_curve(grid, X, Y, cv = cv, n_jobs = -1)
 print("")
 print("Learning Curve:")
 plt.figure(figsize =(12,6))
plt.plot(train_size, np.mean(train_score, axis = 1), label = "Train scores")
 plt.plot(train_size, np.mean(test_score, axis = 1), label = "Test scores")
 plt.title(classifier_name)
 plt.legend()
plt.show()
return classifier_score, grid
def age_enc(x):
if x == 'Under 18 years old':
return 1
elif x == '18 - 24 years old':
return 2
elif x == '25 - 34 years old':
return 3
elif x == '35 - 44 years old':
return 4
elif x == '45 - 54 years old':
return 5
elif x == '55 - 64 years old':
return 6
elif x == '65 years or older':
return 7<feature_engineering>
pipe_xgb = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("xgb", XGBClassifier())])
 params_xgb = {
"pca__n_components" : [2, 6, 10, 18],
"xgb__n_estimators" : [300, 500, 700],
"xgb__learning_rate" : [0.005, 0.1],
"xgb__max_depth" : [5, 7],
"xgb__max_features" : [3, 5],
"xgb__gamma" : [0.5, 0.6, 0.7]}
xgb_score, grid_xgb = model_builder(X, Y, pipe_xgb, params_xgb, 10, "XGBoost")
 Y_predicted_xgb = grid_xgb.predict_proba(X)[:, 1]
X_train['Age'] = X_train['Age'].apply(age_enc)
 X_test['Age'] = X_test['Age'].apply(age_enc )<categorify>
pipe_boost = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("boost", GradientBoostingClassifier())])
 params_boost = {
"pca__n_components" : [2, 6, 10, 18],
"boost__n_estimators" : [300, 500, 700],
"boost__learning_rate" : [0.005, 0.1],
"boost__max_depth" : [3, 5],
"boost__max_features" : [3, 5]}
boost_score, grid_boost = model_builder(X, Y, pipe_boost, params_boost, 10, "Gradient Boosting")
 Y_predicted_boost = grid_boost.predict_proba(X)[:, 1]
def edu_enc(x):
if x == 'They never completed any formal education':
return 1
elif x == 'Primary/elementary school':
return 2
elif x == 'Secondary school(e.g.American high school, German Realschule or Gymnasium, etc.) ':
return 3
elif x == 'Some college/university study without earning a degree':
return 4
elif x == 'Bachelor’s degree(BA, BS, B.Eng., etc.) ':
return 6
elif x == 'Master’s degree(MA, MS, M.Eng., MBA, etc.) ':
return 7
elif x == 'Associate degree':
return 5
elif x == 'Other doctoral degree(Ph.D, Ed.D., etc.) ':
return 8
elif x == 'Professional degree(JD, MD, etc.) ':
return 9<count_unique_values>
pipe_xt = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("xt", ExtraTreesClassifier(criterion = "gini",
max_features = "auto")) ])
 params_xt = {
"pca__n_components" : [2, 6, 10, 18],
"xt__n_estimators" : [300, 500, 700],
"xt__max_depth" : [5, 7, 9]}
xt_score, grid_xt = model_builder(X, Y, pipe_xt, params_xt, 10, "Extra Trees")
 Y_predicted_xt = grid_xt.predict_proba(X)[:, 1]
cat_cols = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cat_cols.append(col)
<categorify>
pipe_knn = Pipeline([ 
("scaler", StandardScaler()), 
("knn", KNeighborsClassifier(algorithm = "auto")) ])
 params_knn = {
"knn__n_neighbors" : [2, 3, 5, 7, 9],
"knn__leaf_size" : [10, 20, 30, 40]}
knn_score, grid_knn = model_builder(X, Y, pipe_knn, params_knn, 10, "KNN")
 Y_predicted_knn = grid_knn.predict_proba(X)[:, 1]
for i in cat_cols:
summary = X_train[i].value_counts() / len(X_train)
 X_train[i] = X_train[i].map(summary)
 X_test[i] = X_test[i].map(summary )<feature_engineering>
all_results = [xgb_score, boost_score, log_score, rf_score,
svm_score, bag_score, xt_score, knn_score, cat_score, gbm_score]
result_names = ["XGBoost",
"Gradient Boosting",
"Logistic Regression",
"Random Forest",
"SVM",
"Bagging Classifier",
"Extra Trees Classifier",
"KNN",
"CatBoost",
"LightGBM"]
fig = plt.figure(figsize =(14,7))
fig.suptitle("Algorithm Comparison")
 ax = fig.add_subplot(111)
 plt.boxplot(all_results)
 ax.set_xticklabels(result_names)
 plt.show()
X_train['Country_LastNewJob'] = X_train['Country'] * X_train['LastNewJob']
X_train['Country_YearsCodingProf'] = X_train['Country'] * X_train['YearsCodingProf']
X_train['LastNewJob_YearsCodingProf'] = X_train['LastNewJob'] * X_train['YearsCodingProf']<feature_engineering>
%%capture
!pip install keras-tuner
X_test['Country_LastNewJob'] = X_test['Country'] * X_test['LastNewJob']
X_test['Country_YearsCodingProf'] = X_test['Country'] * X_test['YearsCodingProf']
X_test['LastNewJob_YearsCodingProf'] = X_test['LastNewJob'] * X_test['YearsCodingProf']<prepare_x_and_y>
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from kerastuner.tuners import RandomSearch
X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_train.median() , inplace=True )<split>
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
random_state = 0,
test_size = 0.25 )
scores = []
kf = KFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(kf.split(X_train, y_train))):
X_train_, y_train_ = X_train.values[train_ix], y_train.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train.values[test_ix]
model = lgb.LGBMRegressor(boosting_type='gbdt', class_weight=None, colsample_bytree=0.9,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=9999, n_jobs=-1, num_leaves=15, objective=None,
random_state=71, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
 model.fit(X_train_, y_train_, early_stopping_rounds=20, eval_metric='rmse', eval_set=[(X_val, y_val)])
 y_pred = model.predict(X_val)
 mse = mean_squared_error(y_val, y_pred)
 rmse = np.sqrt(mse)
 scores.append(rmse)
 print('CV Score of Fold_%d is %f' %(i, rmse))<train_model>
%%capture
def tune_model(hp):
nn = Sequential()
nn.add(Dense(hp.Choice("units_1", [512, 1024, 2048]), activation = "relu", input_shape =(18,)))
 nn.add(Dropout(hp.Choice("dropout_1", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_2", [128, 256, 512]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_2", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_3", [32, 64, 128]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_3", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_4", [8, 16, 32]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_4", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(1, activation = "relu"))
lr = hp.Choice("learning_rate", [0.0001, 0.001, 0.01, 0.1])
 mm = hp.Choice("momentum", [0.0, 0.2, 0.4, 0.6, 0.8])
 nn.compile(optimizer = keras.optimizers.RMSprop(learning_rate = lr, momentum = mm),
loss = "binary_crossentropy",
metrics = ["accuracy"])
 return nn
tuner = RandomSearch(tune_model,
objective = "val_accuracy",
max_trials = 3)
 tuner.search(x = X_train,
y = Y_train,
verbose = 3,
epochs = 400,
batch_size = 25,
validation_data =(X_test, Y_test))
model.fit(X_train, y_train)
 y_pred = model.predict(X_test )<save_to_csv>
print("Best Params: " + str(tuner.oracle.get_best_trials(num_trials = 1)[0].hyperparameters.values))
nn = tuner.get_best_models() [0]
print("")
 print("Test score: " + str(nn.evaluate(X_test, Y_test)))
 print("")
 print("Train score: " + str(nn.evaluate(X_train, Y_train)) )
submission = pd.read_csv('.. /input/exam-for-students20200129/sample_submission.csv', index_col=0)
 submission[tgt_col] = y_pred
submission[tgt_col] = np.exp(submission[tgt_col])- 1
submission.to_csv('submission.csv')
<import_modules>
Y_predicted_nn = nn.predict(test.drop(["PassengerId"], axis = 1))
Y_predicted_nn =(Y_predicted_nn.ravel() > 0.5 ).astype(int )
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords
from nltk import word_tokenize
import pandas as pd
import numpy as np
import re
from sklearn.model_selection import train_test_split
from collections import defaultdict
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score, classification_report
import json<load_from_disk>
from sklearn.metrics import roc_curve, roc_auc_score
with open('.. /input/train.json')as f:
raw_train = json.load(f)
 with open('.. /input/test.json')as f:
raw_test = json.load(f )<string_transform>
log_fpr, log_tpr, log_treshholds = roc_curve(Y, Y_predicted_log)
 boost_fpr, boost_tpr, boost_treshholds = roc_curve(Y, Y_predicted_boost)
 svm_fpr, svm_tpr, svm_treshholds = roc_curve(Y, Y_predicted_svm)
 rf_fpr, rf_tpr, rf_treshholds = roc_curve(Y, Y_predicted_rf)
 xgb_fpr, xgb_tpr, xgb_treshholds = roc_curve(Y, Y_predicted_xgb)
 bag_fpr, bag_tpr, bag_treshholds = roc_curve(Y, Y_predicted_bag)
 xt_fpr, xt_tpr, xt_treshholds = roc_curve(Y, Y_predicted_xt)
 knn_fpr, knn_tpr, knn_treshholds = roc_curve(Y, Y_predicted_knn)
 auc_score_log = roc_auc_score(Y, Y_predicted_log)
 auc_score_boost = roc_auc_score(Y, Y_predicted_boost)
 auc_score_svm = roc_auc_score(Y, Y_predicted_svm)
 auc_score_rf = roc_auc_score(Y, Y_predicted_rf)
 auc_score_xgb = roc_auc_score(Y, Y_predicted_xgb)
 auc_score_bag = roc_auc_score(Y, Y_predicted_bag)
 auc_score_xt = roc_auc_score(Y, Y_predicted_xt)
 auc_score_knn = roc_auc_score(Y, Y_predicted_knn)
 plt.figure(figsize =(14,7))
plt.plot([0,1], [0,1])
 plt.plot(log_fpr, log_tpr, label = "Logistic Regression(AUC-Score: " + str(round(auc_score_log, 2)) + ")")
 plt.plot(boost_fpr, boost_tpr, label = "Gradient Boosting(AUC-Score: " + str(round(auc_score_boost, 2)) + ")")
 plt.plot(svm_fpr, svm_tpr, label = "SVM(AUC-Score: " + str(round(auc_score_svm, 2)) + ")")
 plt.plot(rf_fpr, rf_tpr, label = "Random Forest(AUC-Score: " + str(round(auc_score_rf, 2)) + ")")
 plt.plot(xgb_fpr, xgb_tpr, label = "XGBoost(AUC-Score: " + str(round(auc_score_xgb, 2)) + ")")
 plt.plot(bag_fpr, bag_tpr, label = "Bagging Classifier(AUC-Score: " + str(round(auc_score_bag, 2)) + ")")
 plt.plot(xt_fpr, xt_tpr, label = "Extra Trees Clasifier(AUC-Score: " + str(round(auc_score_xt, 2)) + ")")
 plt.plot(knn_fpr, knn_tpr, label = "K-Nearest-Neighbor(AUC-Score: " + str(round(auc_score_knn, 2)) + ")")
 plt.title("ROC-Curve")
 plt.xlabel("FPR")
 plt.ylabel("TPR")
 plt.legend()
plt.show()
def ru_token(string):

 return [i for i in word_tokenize(string)if re.match(r'[\u0400-\u04ffа́]+$', i)]<init_hyperparams>
stack = StackingClassifier(estimators = [("XGBoost", grid_xgb), 
("GradientBoosting", grid_boost), 
("RandomForest", grid_rf), 
("Logistic Regression", grid_log), 
("SVM", grid_svm), 
("Extra Trees Classifier", grid_xt), 
("KNeighborsClassifier", grid_knn), 
("CatBoost", grid_cat), 
("LightGBM", grid_gbm)], n_jobs = -1)
 cv = StratifiedKFold(n_splits = 5)
 stack = stack.fit(X, Y)
 stack_score = cross_val_score(stack, X, Y, cv = cv, n_jobs = -1)
 print("
print("Cross-Validated-Score: " + str(round(stack_score.mean() , 6)))
 print("
Y_predicted_stack = stack.predict(test.drop(["PassengerId"], axis = 1))
params = {}
params['tokenizer'] = ru_token
params['stop_words'] = stopwords.words('russian')
 params['ngram_range'] =(1, 3)
 params['min_df'] = 3<feature_engineering>
submission = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_stack})
 submission.to_csv("submission_clf.csv", index = False )
tfidf = TfidfVectorizer(**params )<train_model>
submission_nn = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_nn})
 submission_nn.to_csv("submission_nn.csv", index = False )
tfidf.fit([i['text'] for i in raw_train + raw_test] )<split>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )
train = {}
val = {}
tmp = defaultdict(list)
 for e in raw_train:
tmp[e['sentiment']].append(e['text'])
 for l in tmp:
train[l], val[l] = train_test_split(tmp[l], test_size=0.2, random_state=2018 )<statistical_test>
data_size_train = train.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_train )
def upsampling_align(some_dict, random_state=2018):
rand = np.random.RandomState(random_state)
 upper = max([len(some_dict[l])for l in some_dict])
 print('upper bound: {}'.format(upper))
tmp = {}
for l in some_dict:
if len(some_dict[l])< upper:
repeat_time = int(upper/len(some_dict[l]))
remainder = upper % len(some_dict[l])
 _tmp = some_dict[l].copy()
rand.shuffle(_tmp)
 tmp[l] = some_dict[l] * repeat_time + _tmp[:remainder]
rand.shuffle(tmp[l])
 else:
tmp[l] = some_dict[l]
return tmp<concatenate>
data_size_test = test.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_test )
btrain = upsampling_align(train )<init_hyperparams>
train["NameTitle"] = train["Name"].str.split(", ", expand = True)[1].str.split(".", expand = True)[0]
test["NameTitle"] = test["Name"].str.split(", ", expand = True)[1].str.split(".", expand = True)[0]
train = train.drop("Name", axis = 1)
 test = test.drop("Name", axis = 1)
 print(train["NameTitle"].unique())
 print("------------------------" * 3)
 print(test["NameTitle"].unique() )
m_params = {}
m_params['solver'] = 'lbfgs'
m_params['multi_class'] = 'multinomial'<choose_model_class>
min_titles =(train["NameTitle"].value_counts() < 10)
 train["NameTitle"] = train["NameTitle"].apply(lambda x: "Misc" if min_titles.loc[x] == True else x)
 print(train["NameTitle"].unique() )
softmax = LogisticRegression(**m_params )<categorify>
min_titles =(test["NameTitle"].value_counts() < 10)
 test["NameTitle"] = test["NameTitle"].apply(lambda x: "Misc" if min_titles.loc[x] == True else x)
 print(test["NameTitle"].unique() )
train_x = [j for i in sorted(btrain.keys())for j in btrain[i]]
train_y = [i for i in sorted(btrain.keys())for j in btrain[i]]
softmax.fit(tfidf.transform(train_x), train_y )<define_variables>
train["FamilySize"] = train["SibSp"] + train["Parch"]
test["FamilySize"] = test["SibSp"] + test["Parch"]
test_x = [j for i in sorted(val.keys())for j in val[i]]
true = [i for i in sorted(val.keys())for j in val[i]]<predict_on_test>
train["AgeGroup"] = ""
train.loc[train["Age"] < 21, "AgeGroup"] = "under 21"
train.loc[train["Age"] >= 21, "AgeGroup"] = "21-65"
train.loc[train["Age"] > 65, "AgeGroup"] = "65+"
test["AgeGroup"] = ""
test.loc[test["Age"] < 21, "AgeGroup"] = "under 21"
test.loc[test["Age"] >= 21, "AgeGroup"] = "21-65"
test.loc[test["Age"] > 65, "AgeGroup"] = "65+"
train = train.drop("Age", axis = 1)
 test = test.drop("Age", axis = 1 )
pred = softmax.predict(tfidf.transform(test_x))<compute_test_metric>
train["FareGroup"] = ""
train.loc[train["Fare"] < 170, "FareGroup"] = "0-170"
train.loc[train["Fare"] >= 170, "FareGroup"] = "170-340"
train.loc[train["Fare"] > 340, "FareGroup"] = "340+"
test["FareGroup"] = ""
test.loc[test["Fare"] < 170, "FareGroup"] = "0-170"
test.loc[test["Fare"] >= 170, "FareGroup"] = "170-340"
test.loc[test["Fare"] > 340, "FareGroup"] = "340+"
train = train.drop("Fare", axis = 1)
 test = test.drop("Fare", axis = 1 )
accuracy_score(true, pred )<categorify>
train = train.drop(["SibSp", "Parch"], axis = 1)
 test = test.drop(["SibSp", "Parch"], axis = 1 )
lab = LabelEncoder()
c_true = lab.fit_transform(true)
 c_pred = lab.transform(pred)
 print(classification_report(c_true, c_pred, target_names=lab.classes_, digits=5))<concatenate>
train = pd.get_dummies(train, columns = ["Sex", "Embarked", "NameTitle", "AgeGroup", "FareGroup"])
 test = pd.get_dummies(test, columns = ["Sex", "Embarked", "NameTitle", "AgeGroup", "FareGroup"] )
bval = upsampling_align(val )<categorify>
upper = corr.where(np.triu(np.ones(corr.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.90)]
print(to_drop )
b_test_x = [j for i in sorted(bval.keys())for j in bval[i]]
b_true = [i for i in sorted(bval.keys())for j in bval[i]]
b_pred = softmax.predict(tfidf.transform(b_test_x))
lab = LabelEncoder()
c_true = lab.fit_transform(b_true)
 c_pred = lab.transform(b_pred)
 print(classification_report(c_true, c_pred, target_names=lab.classes_, digits=5))<prepare_output>
X = train.drop(["PassengerId", "Survived"], axis = 1)
 Y = train["Survived"]
sub_pred = softmax.predict(tfidf.transform([i['text'] for i in raw_test]))
sub_df = pd.DataFrame()
sub_df['id'] = [i['id'] for i in raw_test]
sub_df['sentiment'] = sub_pred<save_to_csv>
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, BaggingClassifier, ExtraTreesClassifier, StackingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from catboost import CatBoostClassifier
from xgboost import XGBClassifier
import lightgbm as lgb
sub_df.to_csv('softmax_reg.csv', index=False )<load_from_csv>
style.use("seaborn-whitegrid")
 %matplotlib inline
%config InlineBackend.figure_format = "retina"
def model_builder(X, Y, pipeline, params, kfolds, classifier_name, data_split = False):

 if data_split == True:
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
random_state = 0,
test_size = 0.25)
 cv = StratifiedKFold(n_splits = kfolds)
 start = time.time()
grid = GridSearchCV(pipeline, params, cv = cv, verbose = 0, n_jobs = -1)
 grid = grid.fit(X_train, Y_train)
 print("Best Params:")
 print("")
 print(grid.best_params_)
 grid = grid.best_estimator_
classifier_score = cross_val_score(grid, X_test, Y_test, cv = cv, n_jobs = -1)
 Y_predicted = grid.predict(X_test)
 stop = time.time()
print("")
 print("
print("Cross-Validated-Score: " + str(round(classifier_score.mean() , 6)))
 print("
print("")
 print("Training time: " + str(round(stop - start, 2)) + "s")
 train_size, train_score, test_score = learning_curve(grid, X_train, Y_train, cv = cv, n_jobs = -1)
 print("")
 print("Learning Curve:")
 plt.figure(figsize =(12,6))
plt.plot(train_size, np.mean(train_score, axis = 1), label = "Train scores")
 plt.plot(train_size, np.mean(test_score, axis = 1), label = "Test scores")
 plt.title(classifier_name)
 plt.legend()
plt.show()
return X_train, X_test, Y_train, Y_test, classifier_score, grid
if data_split == False:
cv = StratifiedKFold(n_splits = kfolds)
 start = time.time()
grid = GridSearchCV(pipeline, params, cv = cv, verbose = 0, n_jobs = -1)
 grid = grid.fit(X, Y)
 print("Best Params:")
 print("")
 print(grid.best_params_)
 grid = grid.best_estimator_
classifier_score = cross_val_score(grid, X, Y, cv = cv, n_jobs = -1)
 Y_predicted = grid.predict(X)
 stop = time.time()
print("")
 print("
print("Cross-Validated-Score: " + str(round(classifier_score.mean() , 6)))
 print("
print("")
 print("Training time: " + str(round(stop - start, 2)) + "s")
 train_size, train_score, test_score = learning_curve(grid, X, Y, cv = cv, n_jobs = -1)
 print("")
 print("Learning Curve:")
 plt.figure(figsize =(12,6))
plt.plot(train_size, np.mean(train_score, axis = 1), label = "Train scores")
 plt.plot(train_size, np.mean(test_score, axis = 1), label = "Test scores")
 plt.title(classifier_name)
 plt.legend()
plt.show()
return classifier_score, grid
df_train = pd.read_csv('/kaggle/input/dma-fall19/yelp_train.csv')
 df_test = pd.read_csv('/kaggle/input/dma-fall19/yelp_test.csv' )<correct_missing_values>
pipe_xgb = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("xgb", XGBClassifier())])
 params_xgb = {
"pca__n_components" : [2, 6, 10, 18],
"xgb__n_estimators" : [300, 500, 700],
"xgb__learning_rate" : [0.005, 0.1],
"xgb__max_depth" : [5, 7],
"xgb__max_features" : [3, 5],
"xgb__gamma" : [0.5, 0.6, 0.7]}
xgb_score, grid_xgb = model_builder(X, Y, pipe_xgb, params_xgb, 10, "XGBoost")
 Y_predicted_xgb = grid_xgb.predict_proba(X)[:, 1]
df_train = df_train.dropna()<prepare_x_and_y>
pipe_boost = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("boost", GradientBoostingClassifier())])
 params_boost = {
"pca__n_components" : [2, 6, 10, 18],
"boost__n_estimators" : [300, 500, 700],
"boost__learning_rate" : [0.005, 0.1],
"boost__max_depth" : [3, 5],
"boost__max_features" : [3, 5]}
boost_score, grid_boost = model_builder(X, Y, pipe_boost, params_boost, 10, "Gradient Boosting")
 Y_predicted_boost = grid_boost.predict_proba(X)[:, 1]
x_train = df_train[['cool', 'funny', 'useful', 'user_average_stars',
'user_review_count', 'business_latitude', 'business_longitude',
'business_review_count', 'business_average_stars']]
x_test = df_test[['cool', 'funny', 'useful', 'user_average_stars',
'user_review_count', 'business_latitude', 'business_longitude',
'business_review_count', 'business_average_stars']]
y_train = df_train['is_good_rating']<find_best_model_class>
pipe_xt = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("xt", ExtraTreesClassifier(criterion = "gini",
max_features = "auto")) ])
 params_xt = {
"pca__n_components" : [2, 6, 10, 18],
"xt__n_estimators" : [300, 500, 700],
"xt__max_depth" : [5, 7, 9]}
xt_score, grid_xt = model_builder(X, Y, pipe_xt, params_xt, 10, "Extra Trees")
 Y_predicted_xt = grid_xt.predict_proba(X)[:, 1]
clf = DecisionTreeClassifier()
clf.fit(x_train, y_train)
 accuracy_score(clf.predict(x_train), y_train )<predict_on_test>
pipe_knn = Pipeline([ 
("scaler", StandardScaler()), 
("knn", KNeighborsClassifier(algorithm = "auto")) ])
 params_knn = {
"knn__n_neighbors" : [2, 3, 5, 7, 9],
"knn__leaf_size" : [10, 20, 30, 40]}
knn_score, grid_knn = model_builder(X, Y, pipe_knn, params_knn, 10, "KNN")
 Y_predicted_knn = grid_knn.predict_proba(X)[:, 1]
clf.predict(x_test )<predict_on_test>
all_results = [xgb_score, boost_score, log_score, rf_score,
svm_score, bag_score, xt_score, knn_score, cat_score, gbm_score]
result_names = ["XGBoost",
"Gradient Boosting",
"Logistic Regression",
"Random Forest",
"SVM",
"Bagging Classifier",
"Extra Trees Classifier",
"KNN",
"CatBoost",
"LightGBM"]
fig = plt.figure(figsize =(14,7))
fig.suptitle("Algorithm Comparison")
 ax = fig.add_subplot(111)
 plt.boxplot(all_results)
 ax.set_xticklabels(result_names)
 plt.show()
submission = pd.DataFrame(index=df_test.review_id)
 submission['is_good_rating'] = clf.predict(x_test )<save_to_csv>
%%capture
!pip install keras-tuner
submission.reset_index().to_csv('submission.csv', index=False )<load_from_csv>
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from kerastuner.tuners import RandomSearch
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<create_dataframe>
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
random_state = 0,
test_size = 0.25 )
submission_pd = pd.DataFrame(test_df.Id )<feature_engineering>
%%capture
def tune_model(hp):
nn = Sequential()
nn.add(Dense(hp.Choice("units_1", [512, 1024, 2048]), activation = "relu", input_shape =(18,)))
 nn.add(Dropout(hp.Choice("dropout_1", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_2", [128, 256, 512]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_2", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_3", [32, 64, 128]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_3", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_4", [8, 16, 32]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_4", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(1, activation = "relu"))
lr = hp.Choice("learning_rate", [0.0001, 0.001, 0.01, 0.1])
 mm = hp.Choice("momentum", [0.0, 0.2, 0.4, 0.6, 0.8])
 nn.compile(optimizer = keras.optimizers.RMSprop(learning_rate = lr, momentum = mm),
loss = "binary_crossentropy",
metrics = ["accuracy"])
 return nn
tuner = RandomSearch(tune_model,
objective = "val_accuracy",
max_trials = 3)
 tuner.search(x = X_train,
y = Y_train,
verbose = 3,
epochs = 400,
batch_size = 25,
validation_data =(X_test, Y_test))
submission_pd['SalePrice'] = ybar<save_to_csv>
print("Best Params: " + str(tuner.oracle.get_best_trials(num_trials = 1)[0].hyperparameters.values))
nn = tuner.get_best_models() [0]
print("")
 print("Test score: " + str(nn.evaluate(X_test, Y_test)))
 print("")
 print("Train score: " + str(nn.evaluate(X_train, Y_train)) )
submission_pd.to_csv('submission0.csv' )<save_to_csv>
Y_predicted_nn = nn.predict(test.drop(["PassengerId"], axis = 1))
Y_predicted_nn =(Y_predicted_nn.ravel() > 0.5 ).astype(int )
submission_pd.to_csv('submission.csv', index=False )<install_modules>
from sklearn.metrics import roc_curve, roc_auc_score
!pip install -q albumentations
!pip install -q pretrainedmodels
!pip install -q kekas
!pip install -q adabound<set_options>
log_fpr, log_tpr, log_treshholds = roc_curve(Y, Y_predicted_log)
 boost_fpr, boost_tpr, boost_treshholds = roc_curve(Y, Y_predicted_boost)
 svm_fpr, svm_tpr, svm_treshholds = roc_curve(Y, Y_predicted_svm)
 rf_fpr, rf_tpr, rf_treshholds = roc_curve(Y, Y_predicted_rf)
 xgb_fpr, xgb_tpr, xgb_treshholds = roc_curve(Y, Y_predicted_xgb)
 bag_fpr, bag_tpr, bag_treshholds = roc_curve(Y, Y_predicted_bag)
 xt_fpr, xt_tpr, xt_treshholds = roc_curve(Y, Y_predicted_xt)
 knn_fpr, knn_tpr, knn_treshholds = roc_curve(Y, Y_predicted_knn)
 auc_score_log = roc_auc_score(Y, Y_predicted_log)
 auc_score_boost = roc_auc_score(Y, Y_predicted_boost)
 auc_score_svm = roc_auc_score(Y, Y_predicted_svm)
 auc_score_rf = roc_auc_score(Y, Y_predicted_rf)
 auc_score_xgb = roc_auc_score(Y, Y_predicted_xgb)
 auc_score_bag = roc_auc_score(Y, Y_predicted_bag)
 auc_score_xt = roc_auc_score(Y, Y_predicted_xt)
 auc_score_knn = roc_auc_score(Y, Y_predicted_knn)
 plt.figure(figsize =(14,7))
plt.plot([0,1], [0,1])
 plt.plot(log_fpr, log_tpr, label = "Logistic Regression(AUC-Score: " + str(round(auc_score_log, 2)) + ")")
 plt.plot(boost_fpr, boost_tpr, label = "Gradient Boosting(AUC-Score: " + str(round(auc_score_boost, 2)) + ")")
 plt.plot(svm_fpr, svm_tpr, label = "SVM(AUC-Score: " + str(round(auc_score_svm, 2)) + ")")
 plt.plot(rf_fpr, rf_tpr, label = "Random Forest(AUC-Score: " + str(round(auc_score_rf, 2)) + ")")
 plt.plot(xgb_fpr, xgb_tpr, label = "XGBoost(AUC-Score: " + str(round(auc_score_xgb, 2)) + ")")
 plt.plot(bag_fpr, bag_tpr, label = "Bagging Classifier(AUC-Score: " + str(round(auc_score_bag, 2)) + ")")
 plt.plot(xt_fpr, xt_tpr, label = "Extra Trees Clasifier(AUC-Score: " + str(round(auc_score_xt, 2)) + ")")
 plt.plot(knn_fpr, knn_tpr, label = "K-Nearest-Neighbor(AUC-Score: " + str(round(auc_score_knn, 2)) + ")")
 plt.title("ROC-Curve")
 plt.xlabel("FPR")
 plt.ylabel("TPR")
 plt.legend()
plt.show()
%matplotlib inline
train_on_gpu = True
Blur, OpticalDistortion, GridDistortion, HueSaturationValue, Flip, VerticalFlip
print(os.listdir(".. /input/"))
sample_sub = pd.read_csv('.. /input/sample_submission.csv')
<prepare_output>
stack = StackingClassifier(estimators = [("XGBoost", grid_xgb), 
("GradientBoosting", grid_boost), 
("RandomForest", grid_rf), 
("Logistic Regression", grid_log), 
("SVM", grid_svm), 
("Extra Trees Classifier", grid_xt), 
("KNeighborsClassifier", grid_knn), 
("CatBoost", grid_cat), 
("LightGBM", grid_gbm)], n_jobs = -1)
 cv = StratifiedKFold(n_splits = 5)
 stack = stack.fit(X, Y)
 stack_score = cross_val_score(stack, X, Y, cv = cv, n_jobs = -1)
 print("
print("Cross-Validated-Score: " + str(round(stack_score.mean() , 6)))
 print("
Y_predicted_stack = stack.predict(test.drop(["PassengerId"], axis = 1))
train_folder = '.. /input/plates/plates/train/'
test_folder = '.. /input/plates/plates/test/'
cleaned = 'cleaned'
dirty = 'dirty'
train_clean = os.listdir(train_folder + cleaned)
 train_dirty = os.listdir(train_folder + dirty)
 test_files = os.listdir(test_folder)
 train_df = pd.concat([pd.DataFrame(train_clean, columns=['id']), pd.DataFrame(train_dirty, columns=['id'])])
 train_df.reset_index(drop=True, inplace=True)
 test_df = pd.DataFrame(test_files, columns=['id'])
 train_df['label'] = [1 if file_name.startswith('cleaned')else 0 for file_name in train_df['id']]
test_df['label'] = -1
print(train_df.head(3))
fig = plt.figure(figsize=(20,5))
for idx, img in enumerate(np.random.choice(train_df['id'], 10)) :
ax = fig.add_subplot(2, 10//2, idx+1, xticks=[], yticks=[])
 im = Image.open(train_folder + img.split('_')[0] + '/' + img)
 plt.imshow(im)
 lab = img
ax.set_title(f'{lab}' )<feature_engineering>
submission = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_stack})
 submission.to_csv("submission_clf.csv", index = False )
<init_hyperparams><EOS> 
submission_nn = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_nn})
 submission_nn.to_csv("submission_nn.csv", index = False )