Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
GradientBoostingClassifier, ExtraTreesClassifier)
 INPUT_DIR = '.. /input'
N_FOLDS = 4
N_ITER = 50
SEED = 32
inv_d = dict(zip(unique_ingredients, np.arange(len(unique_ingredients))))<feature_engineering>

 df_train_raw = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))
df_test_raw = pd.read_csv(os.path.join(INPUT_DIR, 'test.csv'))
X = np.zeros(shape =(len(df), len(unique_ingredients)))
 for i in range(len(df)) :
if i % 10000 == 0:
print(i)
 sample = df.loc[i]
l_ingr = sample['ingredients']
for ingr in l_ingr:
X[i,inv_d[ingr]] = 1.0<prepare_x_and_y>

 def get_title(name):

 title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
df_full = [df_train_raw.copy() , df_test_raw.copy() ]
for dataset in df_full:
dataset['LastName'] = dataset['Name'].apply(lambda x: str.split(x, ",")[0])
 dataset['LastName'] = dataset['LastName'].astype('category' ).cat.codes
dataset['Namelength'] = dataset['Name'].apply(len)
 dataset['Title'] = dataset['Name'].apply(get_title)
 dataset['Title'] = dataset['Title'].replace(
 ['Lady', 'Countess', 'Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 dataset['Title'] = dataset['Title'].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5})
 dataset['Title'] = dataset['Title'].fillna(0)
 dataset['HasCabin'] = dataset["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
dataset['IsAlone'] = dataset['IsAlone'].astype('category')
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})
 dataset['Fare'] = dataset['Fare'].fillna(dataset['Fare'].median())
 dataset['CatFare'] = pd.qcut(dataset['Fare'], q=5, labels=False)
 dataset['Age'].fillna(dataset['Age'].median() , inplace=True)
 dataset['CatAge'] = pd.qcut(dataset['Age'], q=4, labels=False)
 dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1})
 drop_columns = ['PassengerId', 'Name', 'Cabin', 'Ticket']
dataset.drop(drop_columns, axis=1, inplace=True)
 df_train, df_test = df_full
df_train.describe(include='all' )
y = df['cuisine'].values<count_unique_values>

 def min_max_scale(train_data, test_data, numeric_cols):

 data = pd.concat([train_data, test_data])
 scaled_train_data, scaled_test_data = train_data.copy() , test_data.copy()
for feature_name in numeric_cols:
max_v = data[feature_name].max()
min_v = data[feature_name].min()
scaled_train_data[feature_name] =(train_data[feature_name] - min_v)/(max_v - min_v)
 scaled_test_data[feature_name] =(test_data[feature_name] - min_v)/(max_v - min_v)
 return scaled_train_data, scaled_test_data
def categorical_encode(train_data, test_data, categorical_cols):

 data = pd.concat([train_data, test_data])
 scaled_train_data, scaled_test_data = train_data.copy() , test_data.copy()
result = df.copy()
for feature_name in categorical_cols:
max_v = df[feature_name].max()
min_v = df[feature_name].min()
result[feature_name] =(df[feature_name] - min_v)/(max_v - min_v)
 return result
label_column = 'Survived'
cols = list(df_train.columns.values)
 num_cols = [e for e in df_train.select_dtypes(include=[np.number] ).columns.tolist() if e != label_column]
cat_cols = [e for e in cols if e not in num_cols and e != label_column]
print(num_cols, cat_cols)
 x_train, y_train = df_train.drop(label_column, axis=1), df_train[label_column].astype(int)
 x_test = df_test
x_train, x_test = min_max_scale(x_train, x_test, num_cols)
 x_train.describe(include = 'all' )
len(np.unique(y))<load_from_disk>

 MODELS = {
'lr': {
'model': LogisticRegression,
'params': {
'fit_intercept': [True, False],
'multi_class': ['ovr'],
'penalty': ['l2'],
'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'tol': [0.01, 0.05, 0.1, 0.5, 1, 5],
'random_state': [SEED],
},
'best_params': {'tol': 0.05, 'solver': 'newton-cg', 'random_state': 32, 'penalty': 'l2', 'multi_class': 'ovr', 'fit_intercept': True},
'best score': 0.813692480359147,
},
'mlp': {
'model': MLPClassifier,
'params': {
'activation' : ['identity', 'logistic', 'tanh', 'relu'],
'solver' : ['lbfgs', 'adam'],
'learning_rate' : ['constant', 'invscaling', 'adaptive'],
'learning_rate_init': [.01,.05,.1,.2,.5, 1, 2],
'random_state': [SEED],
},
'best_params': {'solver': 'lbfgs', 'random_state': 32, 'learning_rate_init': 2, 'learning_rate': 'adaptive', 'activation': 'identity'},
'best_score': 0.8092031425364759,
},
'dt': {
'model': DecisionTreeClassifier,
'params': {
'criterion': ['gini', 'entropy'],
'max_depth': range(6, 10),
'max_features': ['auto', 'sqrt', 'log2', None],
'min_samples_split': [2, 5, 10],
'min_samples_leaf': [1, 2, 4],
},
'best_params': {'min_samples_split': 2, 'min_samples_leaf': 4, 'max_features': None, 'max_depth': 6, 'criterion': 'gini'},
'best_score': 0.8181818181818182,
},
'svc': {
'model': SVC,
'params': {
'C': [0.1, 0.5, 1., 2., 4.],
'kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
'gamma': ['auto', 'scale'],
'degree': range(5),
'tol': [0.1, 0.5, 1, 5],
},
'best_params': {'tol': 1, 'shrinking': False, 'probability': False, 'kernel': 'rbf', 'gamma': 'scale', 'degree': 4, 'C': 2.0},
'best_score': 0.8428731762065096
},
'rf': {
'model': RandomForestClassifier,
'params': {
'n_estimators': range(10, 251, 20),
'max_features': ['auto', 'sqrt', 'log2', None],
'max_depth': range(5, 20),
'min_samples_split': range(2, 10),
'min_samples_leaf': range(1, 10),
'bootstrap': [True, False],
'random_state': [SEED],
},
'best_params': {'random_state': 32, 'n_jobs': -1, 'n_estimators': 70, 'min_samples_split': 10, 'min_samples_leaf': 2, 'max_features': None, 'max_depth': 17, 'bootstrap': True},
'best_score': 0.8417508417508418
},
'ada': {
'model': AdaBoostClassifier,
'params': {
'n_estimators': range(10, 251, 20),
'learning_rate': [.01,.05,.1,.2,.5, 1, 2],
'algorithm': ['SAMME', 'SAMME.R'],
'random_state': [SEED],
},
'best_params': {'random_state': 32, 'n_estimators': 170, 'learning_rate': 1, 'algorithm': 'SAMME.R'},
'best_score': 0.8237934904601572
},
'et': {
'model': ExtraTreesClassifier,
'params': {
'n_estimators': range(10, 251, 20),
'max_features': ['auto', 'sqrt', 'log2', None],
'max_depth': range(5, 20),
'min_samples_split': range(2, 10),
'min_samples_leaf': range(1, 10),
'bootstrap': [True, False],
'random_state': [SEED],
},
'best_params': {'random_state': 32, 'n_jobs': -1, 'n_estimators': 70, 'min_samples_split': 5, 'min_samples_leaf': 2, 'max_features': None, 'max_depth': 11, 'bootstrap': True},
'best_score': 0.8294051627384961
},
'gb': {
'model': GradientBoostingClassifier,
'params': {
'n_estimators': range(10, 251, 20),
'max_depth': range(5, 20),
'loss': ['deviance', 'exponential'],
'learning_rate': [.01,.05,.1,.2,.5, 1, 2],
'subsample': [.25,.5,.8, 1.],
'min_samples_split': range(2, 10),
'min_samples_leaf': range(1, 10),
'random_state': [SEED],
},
'best_params': {'subsample': 0.5, 'random_state': 32, 'n_estimators': 150, 'min_samples_split': 5, 'min_samples_leaf': 1, 'max_depth': 13, 'loss': 'exponential', 'learning_rate': 1},
'best_score': 0.8361391694725028
}
}
df_test=pd.read_json('.. /input/whats-cooking-kernels-only/test.json', orient='records', dtype={"id":int,"ingredients":list} )<prepare_x_and_y>

 FIT_FROM_SCRATCH = True
for name, model in MODELS.items() :
if 'best_score' in model and not FIT_FROM_SCRATCH:
print(f'Fitting {name}...')
 model['best_estimator'] = model['model'](**model['best_params'] ).fit(x_train, y_train)
 scores = cross_val_score(model['best_estimator'], x_train, y_train, cv=N_FOLDS)
 score = sum(scores)/ len(scores)
 diff = score - model['best_score']
if diff > 0:
print(f'Accuracy of model {name}: {score}(BIGGER for {diff})')
 elif diff < 0:
print(f'Accuracy of model {name}: {score}(SMALLER for {-diff})')
 else:
print(f'Accuracy of model {name}: {score}(SAME)')
 else:
searcher = RandomizedSearchCV(param_distributions=model['params'],
estimator=model['model']() , scoring="accuracy",
verbose=1, n_iter=N_ITER, cv=N_FOLDS)
 print(f'Fitting {name}...')
 searcher.fit(x_train, y_train)
 print(f'Best parameters found for {name}: {searcher.best_params_}')
 print(f'Best accuracy found {name}: {searcher.best_score_}')
 model['best_estimator'] = searcher.best_estimator_
model['best_params'] = searcher.best_params_
model['best_score'] = searcher.best_score_
X_test = np.zeros(shape =(len(df_test), len(unique_ingredients)))
 for i in range(len(df_test)) :
if i % 1000 == 0:
print(i)
 sample = df_test.loc[i]
l_ingr = sample['ingredients']
for ingr in l_ingr:
try:
X_test[i,inv_d[ingr]] = 1.0
except:
pass<import_modules>

 df = pd.DataFrame()
X_train, X_test = {}, {}
for name, model in MODELS.items() :
vtrain = MODELS[name]['best_estimator'].predict(x_train)
 vtest = MODELS[name]['best_estimator'].predict(x_test)
 df[name] = np.reshape(vtrain, [-1])
 X_train[name] = vtrain
X_test[name] = vtest
from sklearn.neural_network import MLPClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split<train_model>
pred = MODELS['svc']['best_estimator'].predict(x_test )
<predict_on_test><EOS> 
submission = pd.DataFrame({'PassengerId': df_test_raw['PassengerId'], 'Survived': pred})
 submission.to_csv('submission.csv', index=False )