Kaggle competition: Atividade_3_PMR3508
<SOS> metric: rmsle Kaggle data source: atividade_3_pmr3508<load_from_csv>
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import sklearn
from sklearn.neighbors import KNeighborsRegressor
from sklearn import linear_model
from sklearn.ensemble import RandomForestRegressor
import os
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 print(train.shape)
 train.head(5)
<prepare_x_and_y>
trainData = pd.read_csv(".. /input/databa/train.csv")
 testData = pd.read_csv(".. /input/databa/test.csv" )
y = train['Survived'].to_frame()
train.drop('Survived',axis=1,inplace=True)
 train['FamilySize'] = train['SibSp'] + train['Parch']
test['FamilySize'] = test['SibSp'] + test['Parch']<count_missing_values>
def calc_rmsle(Yreal, Ypred):
sum=0.0
n=len(Yreal)
 for x in range(n):
if Ypred[x]<0:
sum = sum +(0 - np.log(Yreal[x]+1)) **2
else:
sum = sum +(np.log(Ypred[x]+1)- np.log(Yreal[x]+1)) **2
return np.sqrt(( sum/n))
col_with_missing_values = [col for col in train.columns if(train[col].isnull().any()
or test[col].isnull().any())]
numeric_col_with_missing_val = [col for col in col_with_missing_values if
np.issubdtype(train[col].dtype, np.number)]
non_numeric_col_with_missing_val = list(set(col_with_missing_values)- set(numeric_col_with_missing_val))
print('Column with missing values :',col_with_missing_values)
 print('Numeric Column with missing values :', numeric_col_with_missing_val)
 print('Non Numeric Column with missing values :', non_numeric_col_with_missing_val )<feature_engineering>
Xtrain = trainData
Xtrain = Xtrain.drop('Id', axis=1)
 Xtrain = Xtrain.drop('median_house_value', axis=1)
 Ytrain = trainData['median_house_value']
Xtest = testData
Xtest = Xtest.drop('Id', axis=1 )
train['Cabin'].fillna('0',inplace=True)
 test['Cabin'].fillna('0', inplace=True)
 train['Cabin'] = train['Cabin'].str[0]
test['Cabin'] = test['Cabin'].str[0]
di = {'A':1,'B':2,'C':3,'D':4,'E':5,'F':6,'G':7,'T':8}
train['Cabin'].replace(di,inplace=True)
 test['Cabin'].replace(di,inplace=True)
 train['Cabin'].unique()<categorify>
best_rmsle = 1
for i in range(3, 30):
knn = KNeighborsRegressor(n_neighbors=i)
 knn.fit(Xtrain, Ytrain)
 Ypred = knn.predict(Xtrain)
 if calc_rmsle(Ytrain, Ypred)< best_rmsle:
best_n = i
best_rmsle = calc_rmsle(Ytrain, Ypred)
 print(best_n)
 print(best_rmsle )
for col in numeric_col_with_missing_val:
my_dict = train.groupby(['Pclass','Sex'])[col].agg(['median'] ).to_dict()
for(key,value)in my_dict.items() :
for(k1,v1)in value.items() :
train.loc[(( np.isnan(train[col])) |(train[col]==0)) &(train['Pclass']==k1[0])&(train['Sex']==k1[1]),col] = v1
test.loc[(( np.isnan(test[col])) |(test[col]==0)) &(test['Pclass']==k1[0])&(test['Sex']==k1[1]),col] = v1
m = train['Embarked'].mode() [0]
train['Embarked'].fillna(m,inplace=True)
 test['Embarked'].fillna(m,inplace=True )<define_variables>
best_rmsle = 1
for i in range(1, 10):
lasso = linear_model.Lasso(alpha = i*0.1)
 lasso.fit(Xtrain, Ytrain)
 Ypred = lasso.predict(Xtrain)
 if calc_rmsle(Ytrain, Ypred)<best_rmsle:
best_alpha = i*0.1
best_rmsle = calc_rmsle(Ytrain, Ypred)
 print(best_alpha)
 print(best_rmsle )
def get_outlier_limits(df,col):
q1 = df[col].quantile(0.25)
 q3 = df[col].quantile(0.75)
 iqr = q3 - q1
ul = q3 + 1.5 * iqr
ll = q1 - 1.5 * iqr
return(ul,ll)
 def get_outlier_index(df,col,ul,ll):
return df[(df[col]>ul)|(df[col]<ll)].index
outlier_indices = []
outlier_columns = ['Age','FamilySize','Fare']
for col in outlier_columns:
ul, ll = get_outlier_limits(train,col)
 outlier_indices.extend(get_outlier_index(train,col,ul,ll))
drop_index = [k for(k,v)in Counter(outlier_indices ).items() if v > 1]
print('Number of outliers found :',len(drop_index))
print('Dropping outliers')
 train.drop(drop_index,axis=0,inplace=True)
 y.drop(drop_index,axis=0,inplace=True )<drop_column>
best_rmsle = 1
for i in range(1,30):
forest = RandomForestRegressor(max_depth=i, random_state=0, n_estimators=50)
 forest.fit(Xtrain, Ytrain)
 Ypred = forest.predict(Xtrain)
 if calc_rmsle(Ytrain, Ypred)<best_rmsle:
best_d = i
best_rmsle = calc_rmsle(Ytrain, Ypred)
 print(best_d)
 print(best_rmsle )
<categorify><EOS> 
forest = RandomForestRegressor(max_depth=26, random_state=0, n_estimators=50)
 forest.fit(Xtrain, Ytrain)
 Ytest = forest.predict(Xtest)
 predictions = pd.DataFrame({"Id":testData.Id, "median_house_value":Ytest})
 predictions.to_csv("predictions.csv", index=False)
 predictions