Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
%matplotlib inline
rcParams['figure.figsize'] = 12,5
def fillNullNumbers(dfInput, dfMetadata, dfMissing, missing_default, label):
media_imp = SimpleImputer(missing_values=missing_default, strategy='mean')
 moda_imp = SimpleImputer(missing_values=missing_default, strategy='most_frequent')
 for index,row in dfMissing.iterrows() :
columnName = row['column_name']
columnType = dfMetadata.level[(dfMetadata.index == columnName)][0]
if(columnType == 'interval'):
imputerToUse = media_imp
imputerString = 'media_imp'
elif(columnType == 'ordinal'):
imputerToUse = moda_imp
imputerString = 'moda_imp'
else:
imputerToUse = None
imputerString = None
if(imputerToUse != None):
dfInput[columnName] = imputerToUse.fit_transform(dfInput[[columnName]] ).ravel()
print(f"{label} - Preenchida coluna {columnName}, cujo tipo é {columnType}, usando o Imputer {imputerString}")
 return dfInput<correct_missing_values>
df_train = pd.read_csv(".. /input/titanic/train.csv")
 df_test = pd.read_csv(".. /input/titanic/test.csv")
 submission = pd.read_csv(".. /input/titanic/gender_submission.csv", index_col='PassengerId' )
dfPreprocess = fillNullNumbers(dfPreprocess, meta_preprocess, missing, -1, 'Pré Processado' )<categorify>
def resumetable(df):
print(f"Dataset Shape: {df.shape}")
 summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First Value'] = df.loc[0].values
summary['Second Value'] = df.loc[1].values
summary['Third Value'] = df.loc[2].values
for name in summary['Name'].value_counts().index:
summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2)
 return summary
def performOneHotEncoding(dfInput, meta_generic, dist_limit):
v = meta_generic[(meta_generic.level == 'nominal')&(meta_generic.keep)].index
display(v)
 for f in v:
dist_values = dfInput[f].value_counts().shape[0]
print('Atributo {} tem {} valores distintos'.format(f, dist_values))
if(dist_values > dist_limit):
print('Atributo {} tem mais de {} valores distintos e por isso será ignorado'.format(f, dist_limit))
dfInput.drop([f], axis=1)
 v = v.drop([f])
 print('Antes do one-hot encoding tinha-se {} atributos'.format(dfInput.shape[1]))
dfInput = pd.get_dummies(dfInput, columns=v, drop_first=True)
 print('Depois do one-hot encoding tem-se {} atributos'.format(dfInput.shape[1]))
return dfInput<categorify>
resumetable(df_train )
dfPreprocess = performOneHotEncoding(dfPreprocess, meta_preprocess, 200 )<normalization>
df_train['Survived'].replace({0:'No', 1:'Yes'}, inplace=True )
min_max_scaler = MinMaxScaler()
dfPreprocess[dfPreprocess.columns] = min_max_scaler.fit_transform(dfPreprocess[dfPreprocess.columns] )<import_modules>
df_train["Embarked"] = df_train["Embarked"].fillna('S' )
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, cross_val_score, ShuffleSplit, KFold, train_test_split, StratifiedKFold
from sklearn.metrics import recall_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import log_loss
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_score
from sklearn.metrics import f1_score<groupby>
df_train['Fare'].quantile([.01,.1,.25,.5,.75,.9,.99] ).reset_index()
def showDistribution(val_classes, targetName):
nonUsed, used = pd.DataFrame(val_classes ).groupby(targetName ).size()
print('---')
 print(f'Das {pd.DataFrame(val_classes ).shape[0]} entradas no dataset, {nonUsed} foram de lances não convertidos e {used} foram de lances convertidos.')
 print(f'Temos assim {round(( used/len(val_classes)) * 100,6)}% de lances que foram convertidos em pontos.')
 print('---' )<compute_train_metric>
df_train['Fare_log'] = np.log(df_train['Fare'] + 1)
 df_test['Fare_log'] = np.log(df_test['Fare'] + 1 )
def logisticRegression(X_Train, y_Train, X_Val, y_Val):
model = LogisticRegression(solver='lbfgs')
 model.fit(X_Train, y_Train)
 y_pred_class = model.predict(X_Val)
 y_pred_proba = model.predict_proba(X_Val)
 recall = recall_score(y_Val, y_pred_class)
 accuracy = accuracy_score(y_Val, y_pred_class)
 logloss = log_loss(y_Val, y_pred_proba)
 precision = precision_score(y_Val, y_pred_class)
 f1 = f1_score(y_Val, y_pred_class)
 print(f'Baseline - Regressão Logistica')
 print('---')
 print(f'Acurácia: {round(accuracy, 6)}%')
 print(f'Recall: {round(recall, 6)}%')
 print(f'Precisão: {round(precision, 6)}%')
 print(f'Log Loss: {round(logloss, 6)}')
 print(f'F1 Score: {round(f1, 6)}')
 print('---')
 print('Matriz de Confusão')
 display(pd.DataFrame(confusion_matrix(y_Val, y_pred_class)))
 print('---')
 return model, 'Baseline - Regressão Logistica'<compute_train_metric>
df_train['Title'] = df_train.Name.apply(lambda x: re.search('([A-Z][a-z]+)\.', x ).group(1))
df_test['Title'] = df_test.Name.apply(lambda x: re.search('([A-Z][a-z]+)\.', x ).group(1)) 
(df_train['Title'].value_counts(normalize=True)* 100 ).head(5)
def xGBClassifier(X_Train, y_Train, X_Val, y_Val, modelName, modelParams):
if(modelParams == None):
clf = XGBClassifier()
else:
clf = XGBClassifier(**modelParams)
 modelName = modelName + ' - Parameters: ' + str(modelParams)
 clf.fit(X_Train, y_Train)
 y_pred_class = clf.predict(X_Val)
 y_pred_proba = clf.predict_proba(X_Val)
 recall = recall_score(y_Val, y_pred_class)
 accuracy = accuracy_score(y_Val, y_pred_class)
 logloss = log_loss(y_Val, y_pred_proba)
 precision = precision_score(y_Val, y_pred_class)
 f1 = f1_score(y_Val, y_pred_class)
 print(modelName)
 print('---')
 print(f'Acurácia: {round(accuracy, 6)}%')
 print(f'Recall: {round(recall, 6)}%')
 print(f'Precisão: {round(precision, 6)}%')
 print(f'Log Loss: {round(logloss, 6)}')
 print(f'F1 Score: {round(f1, 6)}')
 print('---')
 print('Matriz de Confusão')
 display(pd.DataFrame(confusion_matrix(y_Val, y_pred_class)))
 print('---')
 return clf, modelName<split>
Title_Dictionary = {
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Dr": "Officer",
"Rev": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"the Countess":"Royalty",
"Dona": "Royalty",
"Lady" : "Royalty",
"Mme": "Mrs",
"Ms": "Mrs",
"Mrs" : "Mrs",
"Mlle": "Miss",
"Miss" : "Miss",
"Mr" : "Mr",
"Master" : "Master"
}
df_train['Title'] = df_train.Title.map(Title_Dictionary)
 df_test['Title'] = df_test.Title.map(Title_Dictionary )
def xGB_KFold(X, y, kfoldAmount, modelName, modelParams):
if(modelParams == None):
clf = XGBClassifier()
else:
clf = XGBClassifier(**modelParams)
 modelName = modelName + ' - Parameters: ' + str(modelParams)
 clf_score = []
iterator = 1
for train_index, test_index in KFold(shuffle=True, n_splits=kfoldAmount, random_state=42 ).split(X):
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
y_train, y_test = y.iloc[train_index], y.iloc[test_index]
print(f'Processando Fold {iterator}/{kfoldAmount}')
 clf.fit(X_train, y_train)
 y_pred_class = clf.predict(X_test)
 y_pred_proba = clf.predict_proba(X_test)
 recall = recall_score(y_test, y_pred_class)
 accuracy = accuracy_score(y_test, y_pred_class)
 logloss = log_loss(y_test, y_pred_proba)
 precision = precision_score(y_test, y_pred_class)
 f1 = f1_score(y_test, y_pred_class)
 print(f'Fold {iterator}/{kfoldAmount} - Resultados')
 print('---')
 print(f'Acurácia: {round(accuracy, 6)}%')
 print(f'Recall: {round(recall, 6)}%')
 print(f'Precisão: {round(precision, 6)}%')
 print(f'Log Loss: {round(logloss, 6)}')
 print(f'F1 Score: {round(f1, 6)}')
 print('---')
 print('Matriz de Confusão')
 display(pd.DataFrame(confusion_matrix(y_test, y_pred_class)))
 print('---')
 clf_score.append(logloss)
 iterator += 1
print('Score Médio = ', round(np.array(clf_score ).mean() , 6))
return clf, modelName<compute_train_metric>
df_train.loc[df_train.Age.isnull() , 'Age'] = df_train.groupby(['Sex','Pclass','Title'] ).Age.transform('median')
 df_test.loc[df_train.Age.isnull() , 'Age'] = df_test.groupby(['Sex','Pclass','Title'] ).Age.transform('median')
 print(df_train["Age"].isnull().sum())
def decisionTreeClassifier(X_Train, y_Train, X_Val, y_Val):
clf = DecisionTreeClassifier()
clf.fit(X_Train, y_Train)
 y_pred_class = clf.predict(X_Val)
 y_pred_proba = clf.predict_proba(X_Val)
 recall = recall_score(y_Val, y_pred_class)
 accuracy = accuracy_score(y_Val, y_pred_class)
 logloss = log_loss(y_Val, y_pred_proba)
 precision = precision_score(y_Val, y_pred_class)
 f1 = f1_score(y_Val, y_pred_class)
 print(f'Decision Tree - Default Parameters')
 print('---')
 print(f'Acurácia: {round(accuracy, 6)}%')
 print(f'Recall: {round(recall, 6)}%')
 print(f'Precisão: {round(precision, 6)}%')
 print(f'Log Loss: {round(logloss, 6)}')
 print(f'F1 Score: {round(f1, 6)}')
 print('---')
 print('Matriz de Confusão')
 display(pd.DataFrame(confusion_matrix(y_Val, y_pred_class)))
 print('---')
 return clf, f'Decision Tree - Default Parameters'<train_on_grid>
interval =(0, 5, 12, 18, 25, 35, 60, 120)
 cats = ['babies', 'Children', 'Teen', 'Student', 'Young', 'Adult', 'Senior']
df_train["Age_cat"] = pd.cut(df_train.Age, interval, labels=cats)
 df_test["Age_cat"] = pd.cut(df_test.Age, interval, labels=cats)
 df_train["Age_cat"].unique()
def gridSearchKNN(X_Train, y_Train, X_Val, y_Val, k_range):
clf=KNeighborsClassifier()
param_grid=dict(n_neighbors=k_range)
 scores = ['neg_log_loss']
for sc in scores:
grid=GridSearchCV(clf,param_grid,cv=2,scoring=sc,n_jobs=-1)
 print("K-Nearest Neighbors - Tuning hyper-parameters for %s" % sc)
 grid.fit(X_Train,y_Train)
 print(grid.best_params_)
 print(np.round(grid.best_score_,3))
y_pred_class = grid.predict(X_Val)
 y_pred_proba = grid.predict_proba(X_Val)
 recall = recall_score(y_Val, y_pred_class)
 accuracy = accuracy_score(y_Val, y_pred_class)
 logloss = log_loss(y_Val, y_pred_proba)
 precision = precision_score(y_Val, y_pred_class)
 f1 = f1_score(y_Val, y_pred_class)
 print(f'KNN with recall-maxing hyperparameters - {grid.best_params_}')
 print('---')
 print(f'Acurácia: {round(accuracy, 6)}%')
 print(f'Recall: {round(recall, 6)}%')
 print(f'Precisão: {round(precision, 6)}%')
 print(f'Log Loss: {round(logloss, 6)}')
 print(f'F1 Score: {round(f1, 6)}')
 print('---')
 print('Matriz de Confusão')
 display(pd.DataFrame(confusion_matrix(y_Val, y_pred_class)))
 print('---')
 return grid, f'KNN with recall-maxing hyperparameters - {grid.best_params_}'<train_on_grid>
df_train["FSize"] = df_train["Parch"] + df_train["SibSp"] + 1
df_test["FSize"] = df_test["Parch"] + df_test["SibSp"] + 1
family_map = {1: 'Alone', 2: 'Small', 3: 'Small', 4: 'Small',
5: 'Medium', 6: 'Medium', 7: 'Large', 8: 'Large',
11: 'Large'}
df_train['FSize'] = df_train['FSize'].map(family_map)
 df_test['FSize'] = df_test['FSize'].map(family_map )
def gridSearchSVC(X_Train, y_Train, X_Val, y_Val):
svc=SVC()
param_grid = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4, 1e-5],'C': [1, 10, 100, 1000]},
{'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]
scores = ['neg_log_loss']
for sc in scores:
grid=GridSearchCV(svc,param_grid,cv=4,scoring=sc,n_jobs=-1)
 print("Support Vector Classifier - Tuning hyper-parameters for %s" % sc)
 grid.fit(X_Train,y_Train)
 print(grid.best_params_)
 print(np.round(grid.best_score_,3))
y_pred_class = grid.predict(X_Val)
 recall = recall_score(y_Val, y_pred_class)
 accuracy = accuracy_score(y_Val, y_pred_class)
 precision = precision_score(y_Val, y_pred_class)
 f1 = f1_score(y_Val, y_pred_class)
 print(f'SVC with recall-maxing hyperparameters - {grid.best_params_}')
 print('---')
 print(f'Acurácia: {round(accuracy, 6)}%')
 print(f'Recall: {round(recall, 6)}%')
 print(f'Precisão: {round(precision, 6)}%')
 print(f'F1 Score: {round(f1, 6)}')
 print('---')
 print('Matriz de Confusão')
 display(pd.DataFrame(confusion_matrix(y_Val, y_pred_class)))
 print('---')
 return grid, f'SVC with recall-maxing hyperparameters - {grid.best_params_}'<compute_train_metric>
df_train['Family'] = extract_surname(df_train['Name'])
 df_test['Family'] = extract_surname(df_test['Name'] )
def predictTestDataset(X_Test, y_Test, clfModel, clfName):
y_pred_class = clfModel.predict(X_Test)
 y_pred_proba = clfModel.predict_proba(X_Test)
 recall = recall_score(y_Test, y_pred_class)
 accuracy = accuracy_score(y_Test, y_pred_class)
 logloss = log_loss(y_Test, y_pred_proba)
 precision = precision_score(y_Test, y_pred_class)
 f1 = f1_score(y_Test, y_pred_class)
 print(clfName)
 print('---')
 print(f'Acurácia: {round(accuracy, 6)}%')
 print(f'Recall: {round(recall, 6)}%')
 print(f'Precisão: {round(precision, 6)}%')
 print(f'Log Loss: {round(logloss, 6)}')
 print(f'F1 Score: {round(f1, 6)}')
 print('---')
 print('Matriz de Confusão')
 display(pd.DataFrame(confusion_matrix(y_Test, y_pred_class)))
 print('---' )<predict_on_test>
df_train['Ticket'].value_counts() [:10]
def predictContestDataset(X_Test, clfModel, clfName):
print(clfName)
 print('---')
 y_pred_class = clfModel.predict(X_Test)
 y_pred_proba = clfModel.predict_proba(X_Test)
 pd_prediction = pd.DataFrame(y_pred_class)
 pd_prediction.columns = ['target']
showDistribution(pd_prediction, 'target')
 return y_pred_class, y_pred_proba<randomize_order>
df_train['Ticket_Frequency'] = df_train.groupby('Ticket')['Ticket'].transform('count')
 df_test['Ticket_Frequency'] = df_test.groupby('Ticket')['Ticket'].transform('count' )
def performSubSampling(sample_size_target, sample_size_non_target, dfInput, targetValue):
target_indices = dfInput[dfInput.target == targetValue].index
target_values = dfInput.loc[np.random.choice(activated_indices, sample_size, replace=False)]
non_target_indices = dfInput[dfInput.target != targetValue].index
non_target_values = dfInput.loc[np.random.choice(inactive_indices, sample_size_non_target, replace=False)]
subsampled = pd.concat([target_values, non_target_values])
 subsampled.sort_index(inplace=True)
 return subsampled<split>
def cabin_extract(df):
return df['Cabin'].apply(lambda x: str(x)[0] if(pd.notnull(x)) else str('M'))
df_train['Cabin'] = cabin_extract(df_train)
 df_test['Cabin'] = cabin_extract(df_test )
dfPredict = dfPreprocess[dfPreprocess['target'].isnull() ]
dfPreprocess = dfPreprocess.dropna()<prepare_x_and_y>
df_train['Cabin'] = df_train['Cabin'].replace(['A', 'B', 'C'], 'ABC')
 df_train['Cabin'] = df_train['Cabin'].replace(['D', 'E'], 'DE')
 df_train['Cabin'] = df_train['Cabin'].replace(['F', 'G'], 'FG')
 df_train.loc[df_train['Cabin'] == 'T', 'Cabin'] = 'A'
df_test['Cabin'] = df_test['Cabin'].replace(['A', 'B', 'C'], 'ABC')
 df_test['Cabin'] = df_test['Cabin'].replace(['D', 'E'], 'DE')
 df_test['Cabin'] = df_test['Cabin'].replace(['F', 'G'], 'FG')
 df_test.loc[df_test['Cabin'] == 'T', 'Cabin'] = 'A'
X = dfPreprocess.drop(['target'], axis=1)
 y = dfPreprocess['target']
y.columns = ['target']<prepare_x_and_y>
family_cats = CategoricalDtype(categories=['Alone', 'Small', 'Medium', 'Large'], ordered=True )
X_predict = dfPredict.drop(['target'], axis=1 )<split>
df_train.FSize = df_train.FSize.astype(family_cats)
 df_test.FSize = df_test.FSize.astype(family_cats )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.30, random_state=42, stratify=y )<compute_train_metric>
df_train.Age_cat = df_train.Age_cat.cat.codes
df_train.Fare_cat = df_train.Fare_cat.cat.codes
df_test.Age_cat = df_test.Age_cat.cat.codes
df_test.Fare_cat = df_test.Fare_cat.cat.codes
df_train.FSize = df_train.FSize.cat.codes
df_test.FSize = df_test.FSize.cat.codes
logRegModel, logRegName = logisticRegression(X_train, y_train, X_val, y_val )<choose_model_class>
df_train.drop([ 'Ticket', 'Name'], axis=1, inplace=True)
 df_test.drop(['Ticket', 'Name', ], axis=1, inplace=True)
xgbPureModel, xgbPureName = xGBClassifier(X_train, y_train, X_val, y_val, 'XGBoost - Base', None)
 xgbPresetModel, xgbPresetName = xGBClassifier(X_train, y_train, X_val, y_val, 'XGBoost - Preset', {'n_estimator':400, 'learning_rate' : 0.5,'random_state' : 0,'max_depth':70,'objective':"binary:logistic",'subsample':.8,'min_child_weig':6,'colsample_bytr':.8,'scale_pos_weight':1.6, 'gamma':10, 'reg_alph':8, 'reg_lambda':1})
 xgbHyperParametrizedModel, xgbHyperParametrizedName = xGBClassifier(X_train, y_train, X_val, y_val, 'XGBoost - Grid Search Outputted',{'colsample_bytree': 0.6, 'gamma': 9, 'learning_rate': 0.01, 'max_depth': 7, 'n_estimators': 500, 'subsample': 0.6, 'random_state': 42})
 decTreeModel, decTreeName = decisionTreeClassifier(X_train, y_train, X_val, y_val )<train_on_grid>
df_test['Survived'] = 'test'
df = pd.concat([df_train, df_test], axis=0, sort=False )
<train_on_grid>
le = LabelEncoder()
df['Family'] = le.fit_transform(df['Family'].astype(str))
showDistribution(y, 'target')
 xgbGSModel, xgbGSName = xGB_KFold(X, y, 10, 'XGBoost - KFolded',
{'colsample_bytree': 0.6,
'gamma': 9,
'learning_rate': 0.01,
'max_depth': 7,
'n_estimators': 500,
'subsample': 0.6,
'random_state': 42
} )<create_dataframe>
df = pd.get_dummies(df, columns=['Sex', 'Cabin', 'Embarked', 'Title'],\
prefix=['Sex', "Cabin", 'Emb', 'Title'], drop_first=True)
 df_train, df_test = df[df['Survived'] != 'test'], df[df['Survived'] == 'test'].drop('Survived', axis=1)
 del df
contest_prediction, contest_prediction_probability = predictContestDataset(X_predict, xgbGSModel, xgbGSName )<save_to_csv>
df_train['Survived'].replace({'Yes':1, 'No':0}, inplace=True )
sample = pd.read_csv('.. /input/kobe-bryant-shot-selection/sample_submission.csv', low_memory=False)
 sample.shot_made_flag = contest_prediction_probability
sample.shot_made_flag = 1 - sample.shot_made_flag
sample.to_csv("submission.csv", float_format='%.6f', index=False )<load_from_csv>
print(f'Train shape: {df_train.shape}')
 print(f'Train shape: {df_test.shape}' )
data=pd.read_csv('.. /input/data.csv' )<count_values>
df_train.drop(['Age', 'Fare','Fare_log','Family', 'SibSp', 'Parch'], axis=1, inplace=True)
 df_test.drop(['Age', 'Fare','Fare_log','Family', 'SibSp', 'Parch'], axis=1, inplace=True )
object_vars=[var for var in data if data[var].dtype=='object']
numerical_vars=[var for var in data if data[var].dtype=='float' or data[var].dtype=='int']
for var in object_vars:
print(data[var].value_counts() )<drop_column>
X_train = df_train.drop(["Survived","PassengerId"],axis=1)
 y_train = df_train["Survived"]
X_test = df_test.drop(["PassengerId"],axis=1 )
data=data.drop(['team_id','team_name'],axis=1)
<drop_column>
resumetable(X_train )
data['home']=data['matchup'].apply(lambda x: 1 if 'vs' in x else 0)
 data=data.drop('matchup',axis=1 )<drop_column>
warnings.filterwarnings("ignore")
data=data.drop(['lon','lat'],axis=1 )<drop_column>
clfs = []
seed = 3
clfs.append(( "LogReg",
Pipeline([("Scaler", StandardScaler()), 
("LogReg", LogisticRegression())])))
 clfs.append(( "XGBClassifier",
Pipeline([("Scaler", StandardScaler()), 
("XGB", XGBClassifier())])))
 clfs.append(( "KNN",
Pipeline([("Scaler", StandardScaler()), 
("KNN", KNeighborsClassifier())])))
 clfs.append(( "DecisionTreeClassifier",
Pipeline([("Scaler", StandardScaler()), 
("DecisionTrees", DecisionTreeClassifier())])))
 clfs.append(( "RandomForestClassifier",
Pipeline([("Scaler", StandardScaler()), 
("RandomForest", RandomForestClassifier(n_estimators=100)) ])))
 clfs.append(( "GradientBoostingClassifier",
Pipeline([("Scaler", StandardScaler()), 
("GradientBoosting", GradientBoostingClassifier(n_estimators=100)) ])))
 clfs.append(( "RidgeClassifier",
Pipeline([("Scaler", StandardScaler()), 
("RidgeClassifier", RidgeClassifier())])))
 clfs.append(( "BaggingRidgeClassifier",
Pipeline([("Scaler", StandardScaler()), 
("BaggingClassifier", BaggingClassifier())])))
 clfs.append(( "ExtraTreesClassifier",
Pipeline([("Scaler", StandardScaler()), 
("ExtraTrees", ExtraTreesClassifier())])))
 scoring = 'accuracy'
n_folds = 7
results, names = [], []
for name, model in clfs:
kfold = KFold(n_splits=n_folds, random_state=seed)
 cv_results = cross_val_score(model, X_train, y_train,
cv= 5, scoring=scoring,
n_jobs=-1)
 names.append(name)
 results.append(cv_results)
 msg = "%s: %f(+/- %f)" %(name, cv_results.mean() , cv_results.std())
 print(msg)
 fig = plt.figure(figsize=(15,6))
fig.suptitle('Classifier Algorithm Comparison', fontsize=22)
 ax = fig.add_subplot(111)
 sns.boxplot(x=names, y=results)
 ax.set_xticklabels(names)
 ax.set_xlabel("Algorithmn", fontsize=20)
 ax.set_ylabel("Accuracy of Models", fontsize=18)
 ax.set_xticklabels(ax.get_xticklabels() ,rotation=45)
 plt.show()
data['time_remaining_seconds']=data['minutes_remaining']*60+data['seconds_remaining']
data=data.drop(['minutes_remaining','seconds_remaining'],axis=1 )<feature_engineering>
def objective(params):
time1 = time.time()
params = {
'max_depth': params['max_depth'],
'max_features': params['max_features'],
'n_estimators': params['n_estimators'],
'min_samples_split': params['min_samples_split'],
'criterion': params['criterion']
}
print("
print(f"params = {params}")
 FOLDS = 10
count=1
skf = StratifiedKFold(n_splits=FOLDS, random_state=42, shuffle=True)
 kf = KFold(n_splits=FOLDS, shuffle=False, random_state=42)
 score_mean = 0
for tr_idx, val_idx in kf.split(X_train, y_train):
clf = RandomForestClassifier(
 random_state=4,
verbose=0, n_jobs=-1,
**params 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 score = make_scorer(accuracy_score )(clf, X_vl, y_vl)
 score_mean += score
print(f'{count} CV - score: {round(score, 4)}')
 count += 1
time2 = time.time() - time1
print(f"Total Time Run: {round(time2 / 60,2)}")
 gc.collect()
print(f'Mean ROC_AUC: {score_mean / FOLDS}')
 del X_tr, X_vl, y_tr, y_vl, clf, score
return -(score_mean / FOLDS)
 rf_space = {
'max_depth': hp.choice('max_depth', range(2,8)) ,
'max_features': hp.choice('max_features', range(1,X_train.shape[1])) ,
'n_estimators': hp.choice('n_estimators', range(100,500)) ,
'min_samples_split': hp.choice('min_samples_split', range(5,35)) ,
'criterion': hp.choice('criterion', ["gini", "entropy"])
 }
data['time_remaining_seconds']
data['last_3_seconds']=data.time_remaining_seconds.apply(lambda x: 1 if x<4 else 0 )<drop_column>
best = fmin(fn=objective,
space=rf_space,
algo=tpe.suggest,
max_evals=40,
)
data=data.drop('shot_id',axis=1 )<data_type_conversions>
best_params = space_eval(rf_space, best)
 best_params
data['game_date']=pd.to_datetime(data['game_date'])
 data['game_month']=data['game_date'].dt.month
data=data.drop('game_date',axis=1 )<drop_column>
clf = RandomForestClassifier(
 **best_params, random_state=4, 
)
 clf.fit(X_train, y_train)
 y_preds= clf.predict(X_test)
 submission['Survived'] = y_preds.astype(int)
 submission.to_csv('Titanic_rf_model_pred.csv' )
data=data.drop(['game_id','game_event_id'],axis=1 )<categorify>
class_weights = class_weight.compute_class_weight('balanced',
np.unique(y_train),
y_train )
categorical_vars=['action_type','combined_shot_type','season','opponent','shot_type','period','shot_zone_basic','shot_zone_area','shot_zone_range','game_month']
for var in categorical_vars:
data=pd.concat([data,pd.get_dummies(data[var],prefix=var)], 1)
 data=data.drop(var,1 )<prepare_x_and_y>
def objective_logreg(params):
time1 = time.time()
params = {
'tol': params['tol'],
'C': params['C'],
'solver': params['solver'],
}
print("
print(f"params = {params}")
 FOLDS = 10
count=1
skf = StratifiedKFold(n_splits=FOLDS, random_state=42, shuffle=True)
 kf = KFold(n_splits=FOLDS, shuffle=False, random_state=42)
 score_mean = 0
for tr_idx, val_idx in kf.split(X_train, y_train):
clf = LogisticRegression(
 random_state=4,
**params 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 score = make_scorer(accuracy_score )(clf, X_vl, y_vl)
 score_mean += score
print(f'{count} CV - score: {round(score, 4)}')
 count += 1
time2 = time.time() - time1
print(f"Total Time Run: {round(time2 / 60,2)}")
 gc.collect()
print(f'Mean ROC_AUC: {score_mean / FOLDS}')
 del X_tr, X_vl, y_tr, y_vl, clf, score
return -(score_mean / FOLDS)
 space_logreg = {
'tol' : hp.uniform('tol', 0.00001, 0.001),
'C' : hp.uniform('C', 0.001, 2),
'solver' : hp.choice('solver', ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']),
}
train=data[pd.notnull(data['shot_made_flag'])]
test=data[pd.isnull(data['shot_made_flag'])]
y_train=train['shot_made_flag']
train=train.drop('shot_made_flag',1)
 y_train=y_train.astype('int')
 test=test.drop('shot_made_flag',1 )<predict_on_test>
best = fmin(fn=objective_logreg,
space=space_logreg,
algo=tpe.suggest,
max_evals=45,
)
def log_scorer(estimator, X, y):
pred_probs = estimator.predict_proba(X)[:, 1]
return log_loss(y, pred_probs )<train_model>
best_params = space_eval(space_logreg, best)
 best_params
<compute_test_metric><EOS> 
clf = LogisticRegression(
 **best_params, random_state=4, 
)
 clf.fit(X_train, y_train)
 y_preds= clf.predict(X_test)
 submission['Survived'] = y_preds.astype(int)
 submission.to_csv('Titanic_logreg_model_pred.csv')