Kaggle competition: PMR3508 - Tarefa 1 - 3508 Adult Dataset
<SOS> metric: categorizationaccuracy Kaggle data source: pmr3508-tarefa-1-3508-adult-dataset<choose_model_class>
%matplotlib inline
plt.style.use('seaborn' )
model=keras.Sequential([keras.layers.Dense(500,input_shape=[train_data.shape[1]])
 ,keras.layers.Dense(400,activation='relu')
 ,keras.layers.Dense(300,activation='relu')
 ,keras.layers.Dense(200,activation='relu')
 ,keras.layers.Dense(100,activation='relu')
 ,keras.layers.Dense(1,activation='linear')
 ] )<train_model>
print(check_output(["ls", ".. /input/adultb"] ).decode("utf8"))
df_train = pd.read_csv(".. /input/adultb/train_data.csv", na_values = '?')
 df_train.set_index('Id',inplace=True)
 df_train.head()
model.compile(optimizer='adam',loss='mean_squared_error',metrics=['mean_squared_error'])
 model.fit(train_data,y_train,batch_size=32,verbose=1,epochs=1000)
<predict_on_test>
total = df_train.isnull().sum().sort_values(ascending = False)
 percent =(( df_train.isnull().sum() /df_train.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', '%'])
 missing_data.head()
y_pre=model.predict(test_data)
 test_id=test_id.astype(np.int32 )<create_dataframe>
total = df_train.isnull().sum().sort_values(ascending = False)
 percent =(( df_train.isnull().sum() /df_train.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', '%'])
 missing_data.head()
cols = ['Id', 'SalePrice']
submit_df = pd.DataFrame(np.hstack(( test_id.reshape(-1,1),np.exp(y_pre.reshape(-1,1)))) ,columns=cols,dtype='Int32' )<save_to_csv>
male = df_train[df_train['sex'] == 'Male'].count() [0]
female = df_train.shape[0] - male
submit_df.to_csv('submission.csv', index=False )<import_modules>
male_income = []
temp = df_train[df_train['sex'] == 'Male']
male_income.append(temp[temp['income'] == '>50K'].count() [0])
 male_income.append(male-male_income[0])
 female_income = []
temp = df_train[df_train['sex'] == 'Female']
female_income.append(temp[temp['income'] == '>50K'].count() [0])
 female_income.append(female-female_income[0])
 male_over = male_income[0]/male
female_over = female_income[0]/female
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
female = df_train[df_train['sex'] == 'Female']
male = df_train[df_train['sex'] == 'Male']
df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df.head()<load_from_csv>
white = df_train[df_train['race'] == 'White'].count() [0]
black = df_train[df_train['race'] == 'Black'].count() [0]
amer = df_train[df_train['race'] == 'Amer-Indian-Eskimo'].count() [0]
other = df_train[df_train['race'] == 'Other'].count() [0]
asian = df_train[df_train['race'] == 'Asian-Pac-Islander'].count() [0]
white_income = []
temp = df_train[df_train['race'] == 'White']
white_income.append(temp[temp['income'] == '>50K'].count() [0])
 white_income.append(white-white_income[0])
 black_income = []
temp = df_train[df_train['race'] == 'Black']
black_income.append(temp[temp['income'] == '>50K'].count() [0])
 black_income.append(black-black_income[0])
 amer_income = []
temp = df_train[df_train['race'] == 'Amer-Indian-Eskimo']
amer_income.append(temp[temp['income'] == '>50K'].count() [0])
 amer_income.append(amer-amer_income[0])
 asian_income = []
temp = df_train[df_train['race'] == 'Asian-Pac-Islander']
asian_income.append(temp[temp['income'] == '>50K'].count() [0])
 asian_income.append(asian-asian_income[0])
 other_income = []
temp = df_train[df_train['race'] == 'Other']
other_income.append(temp[temp['income'] == '>50K'].count() [0])
 other_income.append(other-other_income[0] )
test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test_df.head()<count_missing_values>
white = df_train[df_train['race'] == 'White']
black = df_train[df_train['race'] == 'Black']
df.isnull().sum()<define_variables>
over_50k = df_train[df_train['income'] == '>50K']
under_50k = df_train[df_train['income'] == '<=50K']
columns=['MSZoning','Street','LotShape','LandContour','Utilities','LotConfig',
'Condition2','BldgType','Condition1','HouseStyle','SaleType',
'SaleCondition','ExterCond','ExterQual','Foundation','BsmtQual',
'BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','Heating','HeatingQC',
'CentralAir', 'Electrical','KitchenQual','Functional','GarageType',
'GarageFinish','GarageQual','GarageCond','PavedDrive','LandSlope','Neighborhood']
len(columns )<concatenate>
df_train['income'].value_counts()
final_df = pd.concat([df,test_df],axis=0 )<categorify>
base = df_train
final_df = pd.get_dummies(final_df, columns=columns, drop_first=True)
 final_df<count_duplicates>
quantitative_columns = ['age', 'fnlwgt', 'education.num', 'capital.gain', 'capital.loss', 'hours.per.week']
qualitative_columns = ['education', 'marital.status', 'occupation', 'relationship', 'race',
'sex', 'native.country', 'income']
final_df = final_df.loc[:,~final_df.columns.duplicated() ]<remove_duplicates>
def isprivate(value):
if value == 'Private':
return 1
return 0
def catg(value, categories, ordenation = None):
if ordenation is not None:
ordenation = np.arange(0, len(categories))
for pos in ordenation:
if value == categories[pos]:
return pos
return -1
def equals(value, x):
for v in x:
if v == value:
return 1
return 0
final_df.drop_duplicates(inplace=True )<split>
private = pd.DataFrame({'private': base['workclass'].apply(isprivate)} )
df=final_df.iloc[:1458,:]
test_df=final_df.iloc[1458:,:]
test_df.drop(['SalePrice'],axis=1,inplace=True )<filter>
df_train['native.country'].value_counts()
df[(np.abs(stats.zscore(df)) <3 ).all(axis=1)]<create_dataframe>
countries = ['Mexico', 'Philippines', 'Puerto-Rico', 'El-Salvador', 'India', 'Cuba', 'Jamaica',
'South', 'China', 'Dominican-Republic', 'Vietnam', 'Guatemala', 'Columbia', 'Taiwan',
'Haiti', 'Iran', 'Nicaragua', 'Peru', 'Ecuador', 'Trinadad&Tobago', 'Cambodia',
'Laos', 'Thailand', 'Yugoslavia', 'Outlying-US(Guam-USVI-etc)', 'Honduras']
sul_global = pd.DataFrame({'sul.global': base['native.country'].apply(equals, args = [countries])} )
def getCorrelatedFeature(corrdata, threshold):
feature = []
value = []
for i, index in enumerate(corrdata.index):
if abs(corrdata[index])> threshold:
feature.append(index)
 value.append(corrdata[index])
 corrdf = pd.DataFrame(data = value, index = feature, columns=['corr value'])
 return corrdf<feature_engineering>
edu_order = [15, 11, 5, 12, 10, 1, 14, 7, 2, 8, 4, 13, 0, 3, 6, 9]
args = [base['education'].unique() , edu_order]
education_classes = pd.DataFrame({'education.classes': base['education'].apply(catg, args = args)} )
corr_value = getCorrelatedFeature(corrmat['SalePrice'], threshold)
 corr_value<prepare_x_and_y>
median = np.median(base[base['capital.gain'] > 0]['capital.gain'])
 aux = pd.cut(base['capital.gain'],
bins = [-1, 0, median, base['capital.gain'].max() +1],
labels = [0, 1, 2])
 capital_gain_clusters = pd.DataFrame({'capital.gain.clusters': aux})
 capital_gain_clusters = capital_gain_clusters.astype(np.int)
 median = np.median(base[base['capital.loss'] > 0]['capital.loss'])
 aux = pd.cut(base['capital.loss'],
bins = [-1, 0, median, base['capital.loss'].max() +1],
labels = [0, 1, 2])
 capital_loss_clusters = pd.DataFrame({'capital.loss.clusters': aux})
 capital_loss_clusters = capital_loss_clusters.astype(np.int )
X = df.drop(labels=['SalePrice'], axis = 1)
 y = df['SalePrice']
y= np.log1p(y )<split>
new_data = pd.concat([sul_global, private, education_classes,
hours_per_week_clusters, capital_gain_clusters,
capital_loss_clusters], axis = 1 )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42 )<set_options>
aux = base['income'].apply(equals, args = [['>50K']])
 aux = pd.concat([new_data, pd.DataFrame({'income': aux})], axis = 1)
 new = aux.astype(np.int)
 aux.head()
warnings.filterwarnings('ignore', category=FutureWarning)
 warnings.filterwarnings('ignore', category=DeprecationWarning )<find_best_params>
base = base.drop(['fnlwgt', 'education', 'sex', 'native.country', 'workclass', 'marital.status'], axis = 1)
 base.columns
def print_results(results):
print('BEST PARAMS: {}\
'.format(results.best_params_))
means = results.cv_results_['mean_test_score']
stds = results.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, results.cv_results_['params']):
print('{}(+/-{})for {}'.format(round(mean, 3), round(std * 2, 3), params))<train_model>
base = pd.concat([new_data, base], axis = 1 )
lr = LinearRegression()
lr.fit(X_train, y_train)
 lr.score(X_train, y_train )<normalization>
names = ['occupation', 'relationship', 'race']
enc_x = []
for i in range(len(names)) :
enc_x.append(prep.LabelEncoder())
 enc_y = prep.LabelEncoder()
sc = StandardScaler()
sc.fit(X_train)
 X_scaled = sc.transform(X_train )<train_model>
i = 0
for name in names:
base[name] = enc_x[i].fit_transform(base[name])
 i += 1
base['income'] = enc_y.fit_transform(base['income'] )
ridge = Ridge()
ridge.fit(X_scaled, y_train)
 ridge.score(X_scaled, y_train )<train_on_grid>
unselected_columns = []
unselected_columns.append('capital.loss')
 unselected_columns.append('capital.gain')
 unselected_columns.append('sul.global')
 unselected_columns.append('private')
 unselected_columns.append('education.classes')
 unselected_columns.append('hours.per.week.clusters')
 base = base.drop(unselected_columns, axis = 1)
 base.head()
parameters = {
'solver': ['cholesky', 'auto'],
'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]
}
cv = GridSearchCV(ridge, parameters, cv=5)
 cv.fit(X_train, y_train)
 print_results(cv)
 print(cv.best_estimator_)
 joblib.dump(cv.best_estimator_, '.. /.. /.. /ridge_model.pkl' )<train_on_grid>
aux = base.astype(np.int )
lasso = Lasso()
parameters = {
'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]
}
cv = GridSearchCV(lasso, parameters, cv=5)
 cv.fit(X_train, y_train)
 print_results(cv)
 print('best estimator:', cv.best_estimator_)
 joblib.dump(cv.best_estimator_, '.. /.. /.. /lasso_model.pkl' )<train_on_grid>
from sklearn.preprocessing import StandardScaler
en = ElasticNet(l1_ratio=0.5)
 parameters = {
'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]
}
cv = GridSearchCV(en, parameters, cv=5)
 cv.fit(X_train, y_train)
 print_results(cv)
 print('best estimator:', cv.best_estimator_)
 joblib.dump(cv.best_estimator_, '.. /.. /.. /en_model.pkl' )<train_model>
X = base.drop(['income'], axis = 1)
 y = base['income']
dtr = DecisionTreeRegressor(max_depth = 2, min_samples_leaf=10)
 dtr.fit(X_scaled, y_train)
 dtr.score(X_scaled, y_train )<train_model>
scaler_x = StandardScaler()
X = scaler_x.fit_transform(X )
gbr = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=0.1)
 gbr.fit(X_train, y_train)
 gbr.score(X_train, y_train )<train_model>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score
gbrt = GradientBoostingRegressor(max_depth=3, n_estimators=120, random_state=42)
 gbrt.fit(X_train, y_train)
 print(gbrt.score(X_train, y_train))<train_model>
scores_mean = []
scores_std = []
k_lim_inf = 1
k_lim_sup = 30
folds = 5
k_max = None
max_std = 0
max_acc = 0
i = 0
print('Finding best k...')
 for k in range(k_lim_inf, k_lim_sup):
KNNclf = KNeighborsClassifier(n_neighbors=k, p = 1)
 score = cross_val_score(KNNclf, X, y, cv = folds)
 scores_mean.append(score.mean())
 scores_std.append(score.std())
 if scores_mean[i] > max_acc:
k_max = k
max_acc = scores_mean[i]
max_std = scores_std[i]
i += 1
if not(k%3):
print(' K = {0} | Best CV acc = {1:2.2f}% +/-{3:4.2f}%(best k = {2})'.format(k, max_acc*100, k_max, max_std*100))
print('
Best k: {}'.format(k_max))
xgb_reg = xgboost.XGBRegressor(n_estimators = 550, random_state=42)
 xgb_reg.fit(X_train, y_train)
 print(xgb_reg.score(X_train, y_train))<predict_on_test>
k = k_max
KNNclf = KNeighborsClassifier(n_neighbors=k, p = 1)
 KNNclf.fit(X, y )
y_pred=xgb_reg.predict(test_df)
 y_pred<prepare_output>
df_test = pd.read_csv('.. /input/adultb/test_data.csv', na_values='?')
 df_test.set_index('Id', inplace = True)
 df_test.head()
y_pred[y_pred<0] = 0
y_pred = np.expm1(y_pred)
 y_pred<save_to_csv>
features = ['age', 'education.num', 'occupation', 'relationship', 'race', 'hours.per.week']
base_test = pd.concat([new_data, df_test[features]], axis = 1 )
prediction = pd.DataFrame(y_pred)
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 prediction_df = pd.concat([submission['Id'], prediction], axis=1)
 prediction_df.columns=['Id','SalePrice']
prediction_df.to_csv('sample_submission.csv',index=False )<import_modules>
total = base_test.isnull().sum().sort_values(ascending = False)
 percent =(( base_test.isnull().sum() /base_test.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', '%'])
 missing_data.head()
import pandas as pd
from pathlib import Path
from pandas_profiling import ProfileReport
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import LabelEncoder
import datetime
from sklearn.model_selection import GridSearchCV
from sklearn import preprocessing
from sklearn.model_selection import cross_val_score<define_variables>
total = base_test.isnull().sum().sort_values(ascending = False)
 percent =(( base_test.isnull().sum() /base_test.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', '%'])
 missing_data.head()
base_dir='/kaggle/input/covid19-global-forecasting-week-3/'
train_file='train.csv'
test_file='test.csv'
submit_file='submission.csv'<load_from_csv>
names = ['occupation', 'relationship', 'race']
i = 0
for name in names:
base_test[name] = enc_x[i].transform(base_test[name])
 i += 1
train_df = pd.read_csv(base_dir+'train.csv')
 test_df = pd.read_csv(base_dir+'test.csv')
 submission = pd.read_csv(base_dir+'submission.csv' )<create_dataframe>
X_prev = scaler_x.transform(base_test.values )
train=train_df.copy()
test= test_df.copy()<filter>
temp = KNNclf.predict(X_prev)
 temp = enc_y.inverse_transform(temp)
 temp = {'Income': temp}
predictions = pd.DataFrame(temp )
<feature_engineering><EOS> 
predictions.to_csv("submission.csv", index = True, index_label = 'Id' )