Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<categorify>
warnings.filterwarnings("ignore")
 %matplotlib inline
print("Validating group 1")
 models1 = {'data': evaluate_models(copy.deepcopy(models), all_colors, False),
'attr': 'only colors'}
print("Validating group 2")
 models2 = {'data': evaluate_models(copy.deepcopy(models), old_numerical, False),
'attr': 'old numerical'}
print("Validating group 3")
 models3 = {'data': evaluate_models(copy.deepcopy(models), old_numerical + new_ones + new_ones_2, False),
'attr': 'all numerical'}
print("Validating group 4")
 models4 = {'data': evaluate_models(copy.deepcopy(models), all_features, False),
'attr': 'all features'}
print("Validating group 5")
 models5 = {'data': evaluate_models(copy.deepcopy(models), new_ones, False),
'attr': 'new ones'}
print("Validating group 6")
 models6 = {'data': evaluate_models(copy.deepcopy(models), new_ones_2, False),
'attr': 'new ones squared'}
print("Validating group 7")
 models7 = {'data': evaluate_models(copy.deepcopy(models), new_ones + new_ones_2, False),
'attr': 'all new ones'}<predict_on_test>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("{} observations and {} features in train set.".format(train.shape[0],train.shape[1]))
print("{} observations and {} features in test set.".format(test.shape[0],test.shape[1]))
model = lrcv['model']
features = new_ones_2
evaluate_models(copy.deepcopy([lrcv]), features)
 model.fit(train_data[features], train_data['type'])
 predicted_test = model.predict(test_data[features] )<save_to_csv>
train_test = pd.concat([train[train.columns[2:]], test[test.columns[1:]]] )
submission = pd.concat([test['id'], pd.DataFrame(predicted_test)], axis=1)
 submission.columns = ['id', 'type']
submission.to_csv("new_sub.csv", header=True, index=False )<load_from_csv>
AUCs = []
Ginis = []
for i in range(300):
AUC = roc_auc_score(train.target.values, train[str(i)].values)
 AUCs.append(AUC)
 Gini = 2*AUC - 1
Ginis.append(Gini)
train = pd.read_csv(".. /input/train.csv", index_col=0)
 test = pd.read_csv(".. /input/test.csv", index_col=0)
 y_train = train["type"].map({"Ghoul": 1, "Ghost": 2, "Goblin": 3})
 train.drop("type", inplace=True, axis=1)
 train_test = pd.concat([train, test], axis=0)
 train_test = train_test.drop('color', 1)
 X_train = train_test.iloc[:len(y_train)].as_matrix()
X_test = train_test.iloc[len(y_train):].as_matrix()
y_train = y_train.transpose().as_matrix()
kernel = 1.0 * RBF([1.0, 1.0, 1.0, 1.0])
 gpc = GaussianProcessClassifier(kernel)
 gpc.fit(X_train, y_train)
 y_pred = gpc.predict(X_test)
 with open('submission-GGGaussian.csv', 'w')as f:
f.write("id,type
")
 y_test_it = 0
for row in test.iterrows() :
f.write("{},{}
".format(row[0],y_pred[y_test_it]))
y_test_it += 1<set_options>
np.sort(np.abs(Ginis)) [::-1]
%matplotlib inline<load_from_csv>
np.argsort(np.abs(Ginis)) [::-1]
test = pd.read_csv(".. /input/test.csv")
 train = pd.read_csv(".. /input/train.csv")
 sample_submission = pd.read_csv(".. /input/sample_submission.csv" )<count_missing_values>
[Ginis[k] for k in np.argsort(np.abs(Ginis)) [::-1][:13]]
print(train.isnull().values.any() , test.isnull().values.any() )<categorify>
roc_auc_score(train.target.values,train['33'].values )
cat_columns = ["X0", "X1", "X2", "X3", "X4", "X5", "X6", "X8"]
le0 = LabelEncoder()
le0.fit(list(train.X0)+list(test.X0))
train.X0 = le0.transform(train.X0)
 test.X0 = le0.transform(test.X0)
 le1 = LabelEncoder()
le1.fit(list(train.X1)+list(test.X1))
train.X1 = le1.transform(train.X1)
 test.X1 = le1.transform(test.X1)
 le2 = LabelEncoder()
le2.fit(list(train.X2)+list(test.X2))
train.X2 = le2.transform(train.X2)
 test.X2 = le2.transform(test.X2)
 le3 = LabelEncoder()
le3.fit(list(train.X3)+list(test.X3))
train.X3 = le3.transform(train.X3)
 test.X3 = le3.transform(test.X3)
 le4 = LabelEncoder()
le4.fit(list(train.X4)+list(test.X4))
train.X4 = le4.transform(train.X4)
 test.X4 = le4.transform(test.X4)
 le5 = LabelEncoder()
le5.fit(list(train.X5)+list(test.X5))
train.X5 = le5.transform(train.X5)
 test.X5 = le5.transform(test.X5)
 le6 = LabelEncoder()
le6.fit(list(train.X6)+list(test.X6))
train.X6 = le6.transform(train.X6)
 test.X6 = le6.transform(test.X6)
 le8 = LabelEncoder()
le8.fit(list(train.X8)+list(test.X8))
train.X8 = le8.transform(train.X8)
 test.X8 = le8.transform(test.X8)
<drop_column>
roc_auc_score(train.target.values,train['65'].values )
train.drop(constant_columns, axis=1, inplace=True)
 test.drop(constant_columns, axis=1, inplace=True )<create_dataframe>
roc_auc_score(train.target.values,-train['217'].values )
train_y = pd.DataFrame(train["y"])
 train_x = train.drop(["y"], axis=1 )<count_unique_values>
roc_auc_score(train.target.values,-train['117'].values )
for col_name in cat_columns:
print(col_name, train[col_name].nunique() )<count_values>
roc_auc_score(train.target.values,-train['91'].values )
train_x.X4.value_counts()<import_modules>
roc_auc_score(train.target.values,-train['295'].values )
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor<save_to_csv>
roc_auc_score(train.target.values,train['24'].values )
XGBR = XGBRegressor()
XGBR.fit(train_x, train_y, verbose=False)
 predicts = XGBR.predict(test)
 submission = pd.DataFrame({'ID': test.ID, 'y': predicts})
 submission.to_csv('submission.csv', index=False )<set_options>
roc_auc_score(train.target.values,train['199'].values )
%matplotlib inline
InteractiveShell.ast_node_interactivity='all'<load_from_csv>
roc_auc_score(train.target.values,-train['80'].values )
train = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/train.csv', index_col='ID')
 test = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/test.csv', index_col='ID')
 sub = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/sample_submission.csv' )<prepare_x_and_y>
roc_auc_score(train.target.values,-train['73'].values )
y = train['y']
train.drop('y', axis=1, inplace=True )<count_missing_values>
roc_auc_score(train.target.values,-train['194'].values )
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<define_variables>
roc_auc_score(train.target.values,0.146*train['33'].values + 0.12*train['65'].values-0.06*train['217'].values-0.05*train['117'].values
-0.05*train['91'].values-0.05*train['295'].values+0.05*train['24'].values+0.05*train['199'].values-
0.05*train['80'].values- 0.05*train['73'].values-0.05*train['194'].values )
def get_num_cat_cols(df):

 cat_vars = [col for col in df.columns if df[col].dtype == 'object']
num_vars = [col for col in df.columns if df[col].dtype != 'object']
return cat_vars, num_vars<define_variables>
preds =(0.146*test['33'].values + 0.12*test['65'].values-0.06*test['217'].values-0.05*test['117'].values
-0.05*test['91'].values-0.05*test['295'].values+0.05*test['24'].values+0.05*test['199'].values-
0.05*test['80'].values- 0.05*test['73'].values-0.05*test['194'].values)
 preds = rankdata(preds)/preds.shape[0]
preds
cat_vars, num_vars = get_num_cat_cols(train)
 print('Categorical variables: ', cat_vars)
 print('Numeric variables: ', num_vars )<feature_engineering>
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 sample_submission['target'] = preds
sample_submission.to_csv('submission.csv', index=False )
train[num_vars].head()
train['1_count'] = train[num_vars].sum(axis=1)
 test['1_count'] = test[num_vars].sum(axis=1)
 train.head(3 )<feature_engineering>
pca = PCA(n_components=0.99)
 pca.fit(train_test.values )
train['1_count_percent'] =(train[num_vars].sum(axis=1)* 100)/ len(num_vars)
 test['1_count_percent'] =(test[num_vars].sum(axis=1)* 100)/ len(num_vars)
 train.head()<drop_column>
pca.n_components_
cols_with_less_data = [c for c in num_vars if train[c].nunique() <2]
train.drop(cols_with_less_data, axis=1, inplace=True)
 test.drop(cols_with_less_data, axis=1, inplace=True)
<set_options>
pca = PCA(n_components=0.9)
 pca.fit(train_test.values)
 pca.n_components_
gc.collect()<count_unique_values>
Sum_of_squared_distances = []
K = range(1,15)
 for k in K:
km = KMeans(n_clusters=k)
 km = km.fit(train_test)
 Sum_of_squared_distances.append(km.inertia_ )
[print('There are {1} unique values in {0} column'.format(c, train[c].nunique())) for c in cat_vars];<define_variables>
NN = 80
train_pred = 0
gini_list = [Ginis[k] for k in np.argsort(np.abs(Ginis)) [::-1][:NN]]
for i in range(NN):
if gini_list[i] > 0:
train_pred += train[str(np.argsort(np.abs(Ginis)) [::-1][i])].values
else:
train_pred -= train[str(np.argsort(np.abs(Ginis)) [::-1][i])].values
roc_auc_score(train.target.values, train_pred )
cat_vars, num_vars = get_num_cat_cols(train )<categorify>
test_pred = 0
gini_list = [Ginis[k] for k in np.argsort(np.abs(Ginis)) [::-1][:NN]]
for i in range(NN):
if gini_list[i] > 0:
test_pred += test[str(np.argsort(np.abs(Ginis)) [::-1][i])].values
else:
test_pred -= test[str(np.argsort(np.abs(Ginis)) [::-1][i])].values
def label_encode_columns(df):

 cat_cols, _ = get_num_cat_cols(df)
 le = LabelEncoder()
for c in cat_cols:
le.fit(df[c])
 df[c] = le.transform(df[c])
 return df 
<categorify>
test_pred = rankdata(test_pred)/test_pred.shape[0]
test_pred
<split><EOS> 
sample_submission['target'] = test_pred
sample_submission.to_csv('submission_80.csv', index=False )