Kaggle competition: Titanic - Machine Learning from Disaster
df['populacao']= df['populacao'].str.replace(',','', regex=False)
 df['populacao']= df['populacao'].str.replace('(2)','', regex=False)
 df['populacao']= df['populacao'].str.replace('(1)','', regex=False)
 df['populacao']= df['populacao'].str.replace('() ','', regex=False)
 df['populacao']= df.populacao.astype(float )<data_type_conversions>
print('All Good!' )
df['area']= df['area'].str.replace(',','', regex=False)
 df['area']= df.area.astype(float )<data_type_conversions>
def read_data(train = 'train', test = 'test'):

 train = pd.read_csv(".. /input/" + train + ".csv")
 test = pd.read_csv(".. /input/" + test + ".csv")
 return train, test
def write_df(df, filename):

 df.to_csv(filename + '.csv', index = False)
 print(filename, 'written to csv.')
 print('All Good!' )
df['densidade_dem']= df['densidade_dem'].str.replace(',','', regex=False)
 df['densidade_dem']= df.densidade_dem.astype(float )<data_type_conversions>
train, test = read_data()
print('All Good!' )
for col in df.columns:
if df[col].dtype == 'object':
df[col] = df[col].astype('category' ).cat.codes<count_missing_values>
print('Train data:', train.columns.values)
 print('
')
 print('Test data:', test.columns.values )
col_nan=df.isnull().any(axis=0)
 col_nan<feature_engineering>
predictor_cols = test.columns.values
data = pd.concat([train[predictor_cols], test])
 data.is_copy = False
print('All Good!' )
df['servidores']= np.where(df['servidores'].isna() , round(df['populacao']*df.servidores.mean() /df.populacao.mean() , 0),
df['servidores'] )<feature_engineering>
data['Travelling_alone'] = data['SibSp'] + data['Parch']
data['Travelling_alone'] = np.where(data['Travelling_alone'] > 0, 0, 1)
 data.drop(['SibSp', 'Parch'], axis=1, inplace=True)
 print('All Good!' )
df['gasto_pc_educacao'] = np.where(df['gasto_pc_educacao'].isna() , round(df['populacao']*df.gasto_pc_educacao.mean() /df.populacao.mean() , 2),
df['gasto_pc_educacao'] )<feature_engineering>
data.drop(['PassengerId', 'Ticket', 'Name'], axis=1, inplace=True)
 print('All Good!' )
df['comissionados_por_servidor']= df['comissionados']/df['servidores']<feature_engineering>
def na_count(data):

 na_cols = []
for i in data.columns:
if data[i].isnull().sum() != 0:
na_cols.append([i, data[i].isnull().sum() , round(data[i].isnull().sum() / len(data[i])* 100, 4)])
 return na_cols
print('All Good!' )
df['densidade_dem'] = np.where(df['densidade_dem'].isna() , round(df['populacao']/df['area'], 1),
df['densidade_dem'] )<feature_engineering>
na_cols = na_count(data)
 print('Missing Values:')
 for i in na_cols:
print(i )
df['hab_p_medico'] = np.where(df['hab_p_medico'].isna() , -1, df['hab_p_medico'])
<feature_engineering>
data['Age'].fillna(data['Age'].median(skipna=True), inplace=True)
 print('All Good!' )
df['perc_pop_econ_ativa'] = np.where(df['perc_pop_econ_ativa'].isna() , -1, df['perc_pop_econ_ativa'] )<feature_engineering>
data['Fare'].fillna(data['Fare'].median(skipna=True), inplace=True)
 print('All Good!' )
df['participacao_transf_receita'] = np.where(df['participacao_transf_receita'].isna() , -1, df['participacao_transf_receita'] )<feature_engineering>
data.drop('Cabin', axis=1, inplace = True)
 print('All Good!' )
df['gasto_pc_saude'] = np.where(df['gasto_pc_saude'].isna() , -1, df['gasto_pc_saude'] )<feature_engineering>
data['Embarked'].fillna('S', inplace = True)
 print('All Good!' )
df['exp_vida'] = np.where(df['exp_vida'].isna() , df['exp_vida'].mean() , df['exp_vida'] )<feature_engineering>
na_cols = na_count(data)
 print('Missing Values:')
 for i in na_cols:
print(i )
df['exp_anos_estudo'] = np.where(df['exp_anos_estudo'].isna() , df['exp_anos_estudo'].mean() , df['exp_anos_estudo'] )<count_missing_values>
data[data['Fare'] == max(data['Fare'])]
col_nan=df.isnull().any(axis=0)
 col_nan<import_modules>
data = pd.get_dummies(data, columns=['Pclass'])
 print('All Good!' )
import seaborn as sns<split>
data.drop('Pclass_3', axis=1, inplace=True)
 print('All Good!' )
test = df[df['nota_mat'].isnull() ]<filter>
data = pd.get_dummies(data, columns=['Sex'])
 print('All Good!' )
df = df[~df['nota_mat'].isnull() ]<split>
data.drop('Sex_female', axis=1, inplace=True)
 print('All Good!' )
train, valid = train_test_split(df, random_state=42 )<define_variables>
data = pd.get_dummies(data, columns=['Embarked'])
 print('All Good!' )
removed_cols = ['nota_mat', 'municipio', 'hab_p_medico', 'capital', 'comissionados_por_servidor']<define_variables>
data.drop('Embarked_Q', axis=1, inplace=True)
 print('All Good!' )
feats = [c for c in df.columns if c not in removed_cols]<import_modules>
data['is_minor'] = np.where(data['Age'] <= 16, 1, 0 )
from sklearn.ensemble import RandomForestClassifier<choose_model_class>
train_df = data.iloc[:891]
test_df = data.iloc[891:]
print('All Good!' )
rf= RandomForestClassifier(n_estimators=200, min_samples_split=5, max_depth=4, random_state=42 )<train_model>
X_train, X_test, Y_train, Y_test = train_test_split(train_df, train['Survived'], test_size = 0.33, random_state = 0)
 print('All Good!' )
rf.fit(train[feats], train['nota_mat'] )<predict_on_test>
DTC3 = tree.DecisionTreeClassifier(max_depth = 3)
 DTC3.fit(X_train, Y_train)
 Y_pred_DTC3 = DTC3.predict(X_test)
 print('All Good!' )
preds = rf.predict(valid[feats] )<import_modules>
DTC3_mae = mean_absolute_error(Y_test, Y_pred_DTC3)
 print('Decision Tree Classifier with max_depth = 3:
Mean Absolute Error: ', DTC3_mae )
from sklearn.metrics import mean_squared_error<import_modules>
DTC3_pred = DTC3.predict(test_df)
 print('All Good!' )
from sklearn.metrics import mean_squared_error<compute_test_metric>
DTC3_submission = pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':DTC3_pred})
 DTC3_submission.to_csv('DTC3_submission.csv', index=False)
 print('All Good!' )
mean_squared_error(valid['nota_mat'], preds )<compute_test_metric>
RFC = RandomForestClassifier(n_estimators = 100)
 RFC.fit(X_train, Y_train)
 Y_pred_RFC = RFC.predict(X_test)
 print('All Good!' )
accuracy_score(valid['nota_mat'], preds )<predict_on_test>
RFC_mae = mean_absolute_error(Y_test, Y_pred_RFC)
 print('Random Forest Classifier with max_depth = 3:
Mean Absolute Error: ', RFC_mae )
preds_test = rf.predict(test[feats] )<feature_engineering>
RFC_pred = RFC.predict(test_df)
 print('All Good!' )
test['nota_mat'] = preds_test<save_to_csv>
RFC_submission = pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':RFC_pred})
 RFC_submission.to_csv('RFC_submission.csv', index=False)
 print('All Good!' )
test[['codigo_mun', 'nota_mat']].to_csv('rf2.csv', index=False )<load_from_csv>
LogReg = LogisticRegression()
LogReg.fit(X_train, Y_train)
 Y_pred_LogReg = LogReg.predict(X_test)
 print('All Good!' )
for dirname, _, filenames in os.walk('/kaggle/input/'):
for filename in filenames:
print(os.path.join(dirname, filename))
Trainset = pd.read_csv("/kaggle/input/hackstat2k19/Trainset.csv", header = 0)
 Trainset = Trainset.dropna()
xset = pd.read_csv("/kaggle/input/hackstat2k19/xtest.csv")
 xset = xset.dropna()
Trainset = Trainset.replace(to_replace=['Returning_Visitor', 'New_Visitor'], value=[1, 2])
 Trainset = Trainset.replace(to_replace=['Jan', 'Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'], value=[1, 2,3,4,5,6,7,8,9,10,11,12])
 Trainset = Trainset.replace(to_replace=[False, True], value=[1, 2])
 xset = xset.replace(to_replace=['Returning_Visitor', 'New_Visitor'], value=[1, 2])
 xset = xset.replace(to_replace=['Jan', 'Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'], value=[1, 2,3,4,5,6,7,8,9,10,11,12])
 xset = xset.replace(to_replace=[False, True], value=[1, 2])
 print(Trainset.shape)
 print(list(Trainset.columns))
print(xset.shape)
 Trainset
xArray = Trainset.iloc[:, [0,1,2,3,4,5,6,7,8,9,11,12,13,14]].values
yArray = Trainset.iloc[:, 17].values
print(xArray)
 print(len(yArray))
m = 10466
M =[]
xsetArray = xset.iloc[:, [0,1,2,3,4,5,6,7,8,9,11,12,13,14]].values
alpha = 0.05
class LogisticRegression:
def gradient_decent(self, x, y):
w = np.ones(( np.shape(x)[1],1))
print(type(x))
wARR = np.mat(w)
 print(type(wARR))
print(type(y))
for i in range(m):
w = w - alpha * x.transpose() *(self.logistic(x * w)- y)
 return w
def classify(self, x, w):
prob = self.logistic(sum(x * w))
classification = 0
if prob > 0.5: classification = 1
return classification
def logistic(self,ws):
return 1.0/(1 + np.exp(-ws))
logisticRegression = LogisticRegression()
wArray = logisticRegression.gradient_decent(np.mat(xArray), np.mat(yArray ).transpose())
 print("close to window proceed")
 score = xsetArray
print(score)
 for p in range(1850):
M += [(p+1,logisticRegression.classify([float(score[p][0]), float(score[p][1]),float(score[p][2]), float(score[p][3]),float(score[p][4]), float(score[p][5]),float(score[p][6]),float(score[p][7]),float(score[p][8]),float(score[p][9]),float(score[p][10]), float(score[p][11]),float(score[p][12]),float(score[p][13])], wArray)) ]
print(M)
 dfObj = pd.DataFrame(M, columns = ["ID","Revenue"])
<compute_test_metric>
LogReg_mae = mean_absolute_error(Y_test, Y_pred_LogReg)
 print('Random Forest Classifier with max_depth = 3:
Mean Absolute Error: ', LogReg_mae )
def handle_command_line() :
flag = True
while(flag):
entry = input("input:")
 if(entry != "exit"):
score = entry.split()
p = logisticRegression.classify([float(score[0]), float(score[1]),float(score[2]), float(score[3]),float(score[4]), float(score[5]),float(score[6]), float(score[7],float(score[8]), float(score[9])], wArray)
 else:
flag = False<load_from_csv>
LogReg_pred = LogReg.predict(test_df)
 print('All Good!' )
<load_from_csv><EOS> 
LogReg_submission = pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':LogReg_pred})
 LogReg_submission.to_csv('LogReg_submission.csv', index=False)
 print('All Good!' )