Kaggle competition: PMR3508 - Tarefa 1 - 3508 Adult Dataset
<SOS> metric: categorizationaccuracy Kaggle data source: pmr3508-tarefa-1-3508-adult-dataset<normalization>
import pandas as pd
import numpy as np
import sklearn
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score
from sklearn.metrics import accuracy_score
from sklearn import preprocessing
X_train_rfecv = X_train [rfecv_features_list]
X_train_rfecv_scaled = scaler.fit(X_train_rfecv ).transform(X_train_rfecv)
 X_test_rfecv = X_test [rfecv_features_list]
X_test_rfecv_scaled = scaler.transform(X_test_rfecv )<choose_model_class>
adult = pd.read_csv(".. /input/adult-data/train_data.csv",
sep=r'\s*,\s*',
engine='python',
na_values="?")
 adult.head()
vanilla_input_size = 0
def VanillaRegressor(input_size = None):
model = Sequential()
model.add(Dense(5, input_dim=input_size, kernel_initializer='normal', activation='relu'))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mean_squared_error', optimizer='adam')
 return model
def BetaRegressor(input_size = None):
model = Sequential()
model.add(Dense(128, kernel_initializer='normal',input_dim = input_size, activation='relu'))
model.add(Dense(256, kernel_initializer='normal',activation='relu'))
model.add(Dense(256, kernel_initializer='normal',activation='relu'))
model.add(Dense(256, kernel_initializer='normal',activation='relu'))
model.add(Dense(1, kernel_initializer='normal',activation='linear'))
model.compile(loss='mean_squared_error', optimizer='adam')
 return model
train_sets = [
X_train,
X_train_xgb,
X_train_rfecv,
X_train_corr
]
other_models = [
ElasticNet() ,
Lasso() ,
BayesianRidge() ,
LassoLarsIC() ,
RandomForestRegressor() ,
GradientBoostingRegressor() ,
KernelRidge() ,
xgb.XGBRegressor() ,
xgb.XGBRFRegressor() ,
lgb.LGBMRegressor() ,
SVR()
]
other_models_names = [
'ElasticNet',
'Lasso',
'BayesianRidge',
'LassoLarsIC',
'RandomForestRegressor',
'GradientBoostingRegressor',
'KernelRidge',
'XGBRegressor',
'XGBRFRegressor',
'LGBMRegressor',
'SVR'
] 
<split>
adult = adult.dropna()
chosen_X_train = X_train_rfecv.copy()
chosen_X_test = X_test_rfecv.copy()<train_on_grid>
adult["native.country"].value_counts()
class grid() :
def __init__(self, model):
self.model = model
def grid_get(self, X, y, param_grid):
grid_search = GridSearchCV(self.model, param_grid, scoring='neg_mean_squared_error', verbose=99)
 grid_search.fit(X, y)
 print(grid_search.best_params_, np.sqrt(-grid_search.best_score_))
grid_search.cv_results_['mean_test_score'] = np.sqrt(-grid_search.cv_results_['mean_test_score'])
 print(pd.DataFrame(grid_search.cv_results_)[['params', 'mean_test_score', 'std_test_score']])
 make_search = False
max_iter = 15000
dataset = chosen_X_train 
<train_on_grid>
testadult = pd.read_csv(".. /input/adult-data/test_data.csv",
sep=r'\s*,\s*',
engine='python',
na_values="?")
 testadult.head()
if(make_search):
grid(lgb.LGBMRegressor() ).grid_get(dataset, y_train_transformed, {
'n_jobs' : [-1],
'boosting_type' : ['gbdt'],
'num_leaves' : np.linspace(1, 50, 11 ).astype(int),
'learning_rate' : np.linspace(0.01, 0.1, 11),
'n_estimators' : np.linspace(100, 700, 6 ).astype(int)
 })
<choose_model_class>
adult.loc[adult['sex']!="Male", 'sex'] = '0'
adult.loc[adult['sex']=="Male", 'sex'] = '1'
testadult.loc[testadult['sex']!="Male", 'sex'] = '0'
testadult.loc[testadult['sex']=="Male", 'sex'] = '1'
train_sets = [
X_train_rfecv,
]
br = BayesianRidge()
llic = LassoLarsIC()
gb = GradientBoostingRegressor(learning_rate=0.064, loss='huber', n_estimators=620)
 kr = KernelRidge()
lgbm = lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5)
 xgbr = xgb.XGBRegressor()
other_models = [
]
for j in range(len(other_models)) :
for i in range(len(train_sets)) :
score = rmse_cv(other_models[j], train_sets[i], y_train_transformed, cv=2)
 print("{} scored {:.4f}(+-{:.4f})with the set<define_search_model>
adult['sex'].value_counts()
def make_combinations(iterable):
my_combs = []
for item in iterable.copy() :
iterable.remove(item)
 for i in range(len(iterable)) :
for comb in combinations(iterable, i+1):
my_combs.append(( item, comb))
iterable.append(item)
 return my_combs
models = [
BayesianRidge() ,
make_pipeline(MaxAbsScaler() , PolynomialFeatures(2), LassoLarsIC()),
GradientBoostingRegressor(learning_rate=0.064, loss='huber', n_estimators=620),
KernelRidge() ,
xgb.XGBRegressor() ,
make_pipeline(MinMaxScaler() , PolynomialFeatures(2), lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5))
]
my_combs = make_combinations(models)
 print("I have {} combinations to test!".format(len(my_combs)) )<choose_model_class>
testadult['sex'].value_counts()
class CustomEnsemble(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models, meta_model, scaler=MaxAbsScaler() , feature_generator=None):
self.models = models
if scaler:
if feature_generator:
self.meta_model = make_pipeline(scaler, feature_generator, meta_model)
 else:
self.meta_model = make_pipeline(scaler, meta_model)
 else:
if feature_generator:
self.meta_model = make_pipeline(feature_generator, meta_model)
 else:
self.meta_model = meta_model
def fit(self,X,y):
predictions = np.zeros(( X.shape[0], len(self.models)))
 for i, model in enumerate(self.models):
model.fit(X, y)
 predictions[:,i] = model.predict(X)
 self.meta_model.fit(predictions, y)
 def predict(self,X):
predictions = np.zeros(( X.shape[0], len(self.models)))
 for i, model in enumerate(self.models):
predictions[:,i] = model.predict(X)
 return self.meta_model.predict(predictions)
 def __str__(self):
return "<CustomEnsemble(meta={}, models={})>".format(self.meta_model, self.models)
 def __repr__(self):
return self.__str__()
models = [
BayesianRidge() ,
KernelRidge() ,
make_pipeline(MinMaxScaler() , PolynomialFeatures(2), lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5))
]
my_combs = make_combinations(models)
 print("I chose {} combinations to test!".format(2*len(my_combs)))
 best_model = CustomEnsemble(
 models = [
make_pipeline(MinMaxScaler() , PolynomialFeatures(2), lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5)) ,
BayesianRidge()
],
meta_model = KernelRidge() ,
scaler = MaxAbsScaler() 
)
 best = 0.09228172938603223<train_model>
adult.loc[adult['race']!="White", 'race'] = '0'
adult.loc[adult['race']=="White", 'race'] = '1'
testadult.loc[testadult['race']!="White", 'race'] = '0'
testadult.loc[testadult['race']=="White", 'race'] = '1'
print("And the best model goes to...")
 print(best_model)
 print("Its score was {}".format(best))<save_to_csv>
adult['race'].value_counts()
best_model.fit(chosen_X_train, y_train_transformed)
 sub = pd.DataFrame()
sub['Id'] = df_test['Id']
sub['SalePrice'] = inv_boxcox1p(best_model.predict(chosen_X_test), bclambda)
 sub.to_csv('submission.csv',index=False )<import_modules>
testadult['race'].value_counts()
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
adult.loc[adult['native.country']!="United-States", 'native.country'] = '0'
adult.loc[adult['native.country']=="United-States", 'native.country'] = '1'
testadult.loc[testadult['native.country']!="United-States", 'native.country'] = '0'
testadult.loc[testadult['native.country']=="United-States", 'native.country'] = '1'
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )<count_values>
adult['native.country'].value_counts()
df_train['Survived'].value_counts()<groupby>
testadult['native.country'].value_counts()
df_train[['Pclass','Survived']].groupby(['Pclass'] ).mean()<feature_engineering>
Xadult = adult[['education.num','age','race','sex','capital.gain','capital.loss','hours.per.week']]
Yadult = adult.income
Xtestadult = testadult[['education.num','age','race','sex','capital.gain','capital.loss','hours.per.week']]
knn = KNeighborsClassifier(n_neighbors=25)
 knn.fit(Xadult,Yadult )
df_test['Survived'] = 0
train_test = df_train.append(df_test )<categorify>
cval = 10
scores = cross_val_score(knn, Xadult, Yadult, cv=cval)
 scores
train_test = pd.get_dummies(train_test,columns=['Pclass'])
 train_test.head()<feature_engineering>
total = 0
for i in scores:
total += i
acuracia_esperada = total/cval
acuracia_esperada
train_test['Sex'] = pd.factorize(train_test['Sex'])[0]
train_test.head()<categorify>
YtestPred = knn.predict(Xtestadult)
 YtestPred
train_test['SibSp_Parch'] = train_test['SibSp'] + train_test['Parch']
train_test = train_test.drop(['SibSp', 'Parch'], axis=1)
 train_test.head()<categorify>
maior_50 = 0
menor_50 = 0
for i in YtestPred:
if i == '<=50K':
menor_50 += 1
else:
maior_50 += 1
dicio = {'<=50K':menor_50, '>50K':maior_50}
train_test = pd.get_dummies(train_test,columns=["Embarked"])
 train_test.head()<categorify>
result = np.vstack(( testadult["Id"], YtestPred)).T
x = ["Id","income"]
Resultado = pd.DataFrame(columns = x, data = result)
 Resultado.to_csv("Resultado1.csv", index = False )
train_test.loc[train_test["Age"].isnull() ,"age_nan"] = 1
train_test.loc[train_test["Age"].notnull() ,"age_nan"] = 0
train_test = pd.get_dummies(train_test,columns=['age_nan'])
 train_test.head()<drop_column>
Xadult = adult[['education.num','age','sex','capital.gain','capital.loss','hours.per.week','native.country']]
Yadult = adult.income
Xtestadult = testadult[['education.num','age','sex','capital.gain','capital.loss','hours.per.week','native.country']]
knn = KNeighborsClassifier(n_neighbors=25)
 knn.fit(Xadult,Yadult )
train_test = train_test.drop(['Cabin', 'Ticket', 'Name', 'PassengerId'], axis=1)
 train_test.head()<data_type_conversions>
cval = 10
scores = cross_val_score(knn, Xadult, Yadult, cv=cval)
 scores
train_test = train_test.fillna(0 )<prepare_x_and_y>
total = 0
for i in scores:
total += i
acuracia_esperada = total/cval
acuracia_esperada
train_data = train_test[:891]
test_data = train_test[891:]
train_data_X = train_data.drop(['Survived'],axis=1)
 train_data_Y = train_data['Survived']
test_data_X = test_data.drop(['Survived'],axis=1 )<save_to_csv>
YtestPred = knn.predict(Xtestadult)
 YtestPred
clf = RandomForestClassifier(n_estimators=150,min_samples_leaf=2,max_depth=6,oob_score=True)
 clf.fit(train_data_X,train_data_Y)
 clf.oob_score_
df_test["Survived"] = clf.predict(test_data_X)
 result = df_test[['PassengerId','Survived']].set_index('PassengerId')
 result.to_csv('result1.csv' )<load_from_csv>
maior_50 = 0
menor_50 = 0
for i in YtestPred:
if i == '<=50K':
menor_50 += 1
else:
maior_50 += 1
dicio = {'<=50K':menor_50, '>50K':maior_50}
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
result = np.vstack(( testadult["Id"], YtestPred)).T
x = ["Id","income"]
Resultado = pd.DataFrame(columns = x, data = result)
 Resultado.to_csv("Resultado2.csv", index = False )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
Xadult = adult[['education.num','race','sex','capital.gain','capital.loss','hours.per.week','native.country']]
Yadult = adult.income
Xtestadult = testadult[['education.num','race','sex','capital.gain','capital.loss','hours.per.week','native.country']]
knn = KNeighborsClassifier(n_neighbors=25)
 knn.fit(Xadult,Yadult )
print("Check NaN values in Test set:")
 isnull = test.isnull().sum().reset_index()
isnull.columns = ['Feature', 'Total_null']
total_null = isnull[isnull['Total_null']>0]
total_null<count_missing_values>
cval = 10
scores = cross_val_score(knn, Xadult, Yadult, cv=cval)
 scores
print("Check NaN values in Train set:")
 isnull = train.isnull().sum().reset_index()
isnull.columns = ['Feature', 'Total_null']
total_null = isnull[isnull['Total_null']>0]
total_null<concatenate>
total = 0
for i in scores:
total += i
acuracia_esperada = total/cval
acuracia_esperada
traintest = pd.concat([train, test], axis=0, sort=False )<count_duplicates>
YtestPred = knn.predict(Xtestadult)
 YtestPred
traintest[traintest.duplicated() ].index<concatenate>
maior_50 = 0
menor_50 = 0
for i in YtestPred:
if i == '<=50K':
menor_50 += 1
else:
maior_50 += 1
dicio = {'<=50K':menor_50, '>50K':maior_50}
<categorify><EOS> 
result = np.vstack(( testadult["Id"], YtestPred)).T
x = ["Id","income"]
Resultado = pd.DataFrame(columns = x, data = result)
 Resultado.to_csv("Resultado3.csv", index = False )