Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<categorify>
%matplotlib inline
def flatten(d, parent_key='', sep='_'):

 items = []
for k, v in d.items() :
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(flatten(v, new_key, sep=sep ).items())
 else:
items.append(( new_key, v))
return dict(items)
 def showExecTime(startPoint, initialString = "", verbose=True):

 eex = time.time()
seconds = round(eex - startPoint, 2)
 minutes =(seconds/60)
 hours = int(minutes/60)
 minutes = int(minutes % 60)
 seconds = round(seconds % 60, 2)
 if verbose:
print("
- "+initialString+" Execution time: %sh %sm %ss -" %(hours, minutes, seconds))
return eex - startPoint 
<load_from_csv>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
test = pd.read_csv(".. /input/test.csv")
 train = pd.read_csv(".. /input/train.csv" )<define_variables>
target = train_df["target"]
train_df = train_df.drop(["target","id"],axis=1)
 test_id = test_df["id"]
test_df = test_df.drop(["id"],axis=1 )
colors = {
"Ghost" : "
"Ghoul" : "
"Goblin": "
}<feature_engineering>
train_df=reduce_mem_usage(train_df)
 test_df=reduce_mem_usage(test_df )
train['new_var'] = train['hair_length'] + train['has_soul'] + train['bone_length'] - train['rotting_flesh']
train['new_var'] = train['new_var'] - min(train['new_var'])
 train['new_var'] = train['new_var'] / max(train['new_var'])
 train.describe() ['new_var']<feature_engineering>
shuffle_split = StratifiedShuffleSplit(test_size=0.25,train_size=0.75,n_splits=35,random_state=87951)
 param_grid = {
'alpha' : [0.1,1,10,0.01],
'l1_ratio' : np.arange(0.40,1.00,0.10),
'tol' : [0.0001,0.001]
}
eNet = ElasticNet(max_iter=10000)
 grid_search = GridSearchCV(eNet,
param_grid,
scoring='roc_auc',
cv = shuffle_split,
return_train_score=True,
n_jobs = -1)
 grid_search.fit(train_df,target )
train['bone_hair'] = train['hair_length'] + train['bone_length']
train['bone_hair'] = train['bone_hair'] - min(train['bone_hair'])
 train['bone_hair'] = train['bone_hair'] / max(train['bone_hair'])
 train['hair_soul'] = train['hair_length'] + train['has_soul']
train['hair_soul'] = train['hair_soul'] - min(train['hair_soul'])
 train['hair_soul'] = train['hair_soul'] / max(train['hair_soul'])
 train['bone_soul'] = train['bone_length'] + train['has_soul']
train['bone_soul'] = train['bone_soul'] - min(train['bone_soul'])
 train['bone_soul'] = train['bone_soul'] / max(train['bone_soul'])
 train['flesh_soul'] = train['rotting_flesh'] + train['has_soul']
train['flesh_soul'] = train['flesh_soul'] - min(train['flesh_soul'])
 train['flesh_soul'] = train['flesh_soul'] / max(train['flesh_soul'] )<feature_engineering>
print("Best parameters : {}".format(grid_search.best_params_))
print("Best cross validation score: {:.2f}".format(grid_search.best_score_))
print("Best estimator: {}".format(grid_search.best_estimator_))
train['new_var_2'] =(train['hair_length'] * train['has_soul'] * train['bone_length'] *(1-train['rotting_flesh'])) **(1/4)
 train['bone_hair_2'] =(train['hair_length'] * train['bone_length'])**(1/2)
 train['flesh_soul_2'] =(train['rotting_flesh'] * train['has_soul'])**(1/2)
 train['hair_soul_2'] =(train['hair_length'] * train['has_soul'])**(1/2)
 train['bone_soul_2'] =(train['bone_length'] * train['has_soul'])**(1/2 )<define_variables>
results = pd.DataFrame(grid_search.cv_results_)
 results.sort_values(['mean_test_score'],ascending = False)[:10]
old_numerical = ['bone_length', 'rotting_flesh', 'hair_length', 'has_soul']
new_ones = ['new_var', 'bone_hair', 'flesh_soul', 'hair_soul', 'bone_soul']
new_ones_2 = [x+'_2' for x in new_ones]
new_ones, new_ones_2<feature_engineering>
scores_mean = np.array(results.mean_test_score ).reshape(-1)
 scores_std = np.array(results.std_test_score ).reshape(-1)
 print("mean CV scores for each fold {} ".format(scores_mean))
print("std CV scores for each fold {} ".format(scores_std))
try:
train.drop(['flesh_soul', 'flesh_soul_2'], axis=1, inplace=True)
 new_ones.remove('flesh_soul')
 new_ones_2.remove('flesh_soul_2')
 except:
pass
test['new_var'] = test['hair_length'] + test['has_soul'] + test['bone_length'] - test['rotting_flesh']
test['new_var'] = test['new_var'] - min(test['new_var'])
 test['new_var'] = test['new_var'] / max(test['new_var'])
 test['bone_hair'] = test['hair_length'] + test['bone_length']
test['bone_hair'] = test['bone_hair'] - min(test['bone_hair'])
 test['bone_hair'] = test['bone_hair'] / max(test['bone_hair'])
 test['hair_soul'] = test['hair_length'] + test['has_soul']
test['hair_soul'] = test['hair_soul'] - min(test['hair_soul'])
 test['hair_soul'] = test['hair_soul'] / max(test['hair_soul'])
 test['bone_soul'] = test['bone_length'] + test['has_soul']
test['bone_soul'] = test['bone_soul'] - min(test['bone_soul'])
 test['bone_soul'] = test['bone_soul'] / max(test['bone_soul'])
 test['new_var_2'] =(test['hair_length'] * test['has_soul'] * test['bone_length'] *(1-test['rotting_flesh'])) **(1/4)
 test['bone_hair_2'] =(test['hair_length'] * test['bone_length'])**(1/2)
 test['hair_soul_2'] =(test['hair_length'] * test['has_soul'])**(1/2)
 test['bone_soul_2'] =(test['bone_length'] * test['has_soul'])**(1/2)
 color = pd.get_dummies(test['color'])
 test_data = pd.concat([test, color[all_colors]], axis = 1)
 color = pd.get_dummies(train['color'])
 train_data = pd.concat([train, color[all_colors]], axis = 1)
 train_data = train_data.drop('id', axis=1, inplace=False)
 test_data = test_data.drop('id', axis=1, inplace=False)
 train_data.columns - test_data.columns == ['type']<choose_model_class>
clf = grid_search.best_estimator_
type(clf )
nb = {'name': 'Bernoulli NaiveBayes'}
nb['model'] = naive_bayes.BernoulliNB()
lr = {'name': 'Logistic Regression'}
lr['model'] = linear_model.LogisticRegression(solver='lbfgs')
 lrcv = {'name': 'Cross-Validated Logistic Regression'}
lrcv['model'] = linear_model.LogisticRegressionCV(Cs=100, solver='lbfgs', n_jobs=-1)
 svc = {'name': 'Support Vector Machine'}
svc['model'] = OneVsRestClassifier(svm.SVC(kernel='rbf', probability=True))
dtree = {'name': 'Decision Tree'}
dtree['model'] = tree.DecisionTreeClassifier(max_depth=20, min_samples_leaf=3)
 rf = {'name': 'Random Forest'}
rf['model'] = ensemble.RandomForestClassifier(n_estimators=1000, max_depth=20, min_samples_leaf=3, n_jobs=-2)
 k, w = 7, ['uniform', 'distance'][1]
knn = {'name': str(k)+'-Nearest Neighbors '+w}
knn['model'] = KNeighborsClassifier(n_neighbors=k, weights=w, algorithm='auto', n_jobs=-2)
 dummy_uni = {'name': 'Dummy MostFrequent'}
dummy_uni['model'] = dummy.DummyClassifier(strategy='most_frequent')
 dummy_str = {'name': 'Dummy Stratified'}
dummy_str['model'] = dummy.DummyClassifier(strategy='stratified' )<train_model>
models = [nb, lr, lrcv, svc, dtree, rf, knn, dummy_uni, dummy_str]<train_model>
pred_el = grid_search.predict(test_df )
<define_variables><EOS> 
sub_df = pd.DataFrame()
sub_df["id"] = test_id
sub_df["target"] = pred_el
sub_df.to_csv("baseline_el.csv", index=False )