Kaggle competition: PMR3508 - Tarefa 1 - 3508 Adult Dataset
<SOS> metric: categorizationaccuracy Kaggle data source: pmr3508-tarefa-1-3508-adult-dataset<prepare_output>
%matplotlib inline
data = np.array([np.array(test_data['PassengerId']), sclf.predict(x_test_imp)] ).swapaxes(0, 1)
 results = pd.DataFrame(data, columns=['PassengerId', 'Survived'])
 results.set_index('PassengerId', inplace=True)
 results.head()<save_to_csv>
adult = pd.read_csv(".. /input/mydata/train_data.csv",
names=[
"Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country", "Target"],
skiprows=1,
sep=r'\s*,\s*',
engine='python',
na_values="?" )
results.to_csv('predict.csv' )<data_type_conversions>
adult["Country"].value_counts()
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 print("*"*50)
 print(df_train.info())
 print("*"*50)
 print(df_test.info())
 print("*"*50 )<data_type_conversions>
nadult = adult.dropna()
NULL_VAL = "NULL_VAL"
def fillState(state, country):
if state == NULL_VAL:
return country
return state
X_Train = df_train.loc[:, ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
X_Train['State'].fillna(NULL_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Test = df_test.loc[:, ['State', 'Country', 'Date', 'ForecastId']]
X_Test['State'].fillna(NULL_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 print("*"*50)
 print(X_Train.head())
 print("*"*50)
 print(X_Test.head())
 print("*"*50 )<categorify>
testAdult = pd.read_csv(".. /input/mydata/test_data.csv",
names=[
"ID","Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country"],
skiprows=1,
index_col=0,
sep=r'\s*,\s*',
engine='python',
na_values="?" )
le = LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 print("*"*50)
 print(X_Train.head())
 print("*"*50)
 print(X_Test.head())
 print("*"*50 )<categorify>
Yadult = nadult.Target
le = LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
condition_train =(X_Train.Country == country)&(X_Train.State == state)
 X_Train_CS = X_Train.loc[condition_train, ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
condition_test =(X_Test.Country == country)&(X_Test.State == state)
 X_Test_CS = X_Test.loc[condition_test, ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
 df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()<save_to_csv>
from sklearn.neighbors import KNeighborsClassifier
df_out.to_csv('submission.csv', index=False )<load_from_csv>
from sklearn.neighbors import KNeighborsClassifier
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<load_from_csv>
from sklearn.neighbors import KNeighborsClassifier
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()<feature_engineering>
knn = KNeighborsClassifier(n_neighbors=3 )
train_data['IsMale'] = train_data['Sex'].apply(lambda x: 1 if x == 'male' else 0)
 train_data.head()<feature_engineering>
from sklearn.model_selection import cross_val_score
test_data['IsMale'] = test_data['Sex'].apply(lambda x: 1 if x == 'male' else 0)
 test_data.head()<prepare_x_and_y>
scores = cross_val_score(knn, Xadult, Yadult, cv=10 )
y = train_data["Survived"]
features = ["Pclass", "IsMale", "SibSp", "Parch", "Age", "Fare"]
X = train_data[features]
X.tail()<normalization>
knn.fit(Xadult,Yadult )
X_filled = KNN(k=3 ).fit_transform(X)
 X_filled[-5:]<prepare_output>
YtestPred = knn.predict(XtestAdult )
X_filled = pd.DataFrame(data=X_filled,columns=['Pclass','IsMale','SibSp','Parch','Age','Fare'])
 X_filled.head()<train_model>
from sklearn.metrics import accuracy_score
X_train, X_valid, y_train, y_valid = train_test_split(X_filled, y, random_state=1)
 model = XGBRFClassifier(n_estimators=1000, max_depth=5, random_state=1, learning_rate=0.5)
 model.fit(X_train, y_train,
early_stopping_rounds=5,
eval_set=[(X_valid, y_valid)],
verbose=False)
 predictions = model.predict(X_valid)
 print("Mean Absolute Error: " + str(mean_absolute_error(predictions, y_valid)) )<create_dataframe>
UCITest = pd.read_csv(".. /input/mydata/adult.test",
names=[
"Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country", "Target"],
sep=r'\s*,\s*',
engine='python',
na_values="?" )
X_test = test_data[features]
X_test_filled = KNN(k=3 ).fit_transform(X_test)
 X_test_filled = pd.DataFrame(data=X_test_filled,columns=['Pclass','IsMale','SibSp','Parch','Age','Fare'])
 X_test_filled<predict_on_test>
nUCITest = UCITest.dropna()
predictions_final = model.predict(X_test_filled)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions_final})
 output<save_to_csv>
XUCITest = nUCITest[["Age","Education-Num","Capital Gain", "Capital Loss", "Hours per week"]]
output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
YUCITest = nUCITest.Target
import numpy as np
import pandas as pd
import statsmodels.formula.api as sm
import sklearn.preprocessing as sk
from sklearn.metrics import mean_squared_error,mean_squared_log_error,r2_score
from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor,AdaBoostRegressor
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import weight_boosting
from sklearn.ensemble import BaggingRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from tensorflow import keras<load_from_csv>
YUCIPred = knn.predict(XUCITest)
 YUCIPred
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 y_train=train_data.pop('SalePrice')
 test_id=test_data['Id'].values.reshape(-1,1 )<feature_engineering>
accuracy_score(YUCIPred, YUCITest )
y_train = np.log(y_train)
<count_missing_values>
knn = KNeighborsClassifier(n_neighbors=30 )
train_data.isna().sum() [train_data.isna().sum() >0]<drop_column>
knn.fit(Xadult, Yadult )
train_data=train_data.drop(columns=['PoolQC','Fence','MiscFeature','Alley'],axis=0)
 test_data=test_data.drop(columns=['PoolQC','Fence','MiscFeature','Alley'],axis=0 )<define_variables>
knn.fit(Xadult, Yadult )
cat_list = list(train_data.dtypes[train_data.dtypes == 'object'].index )<count_missing_values>
scores = cross_val_score(knn, Xadult, Yadult, cv=10 )
train_data.isna().sum() [train_data.isna().sum() >0]<count_missing_values>
YUCIPred = knn.predict(XUCITest )
test_data.isna().sum() [test_data.isna().sum() >0]<count_missing_values>
accuracy_score(YUCITest, YUCIPred )
train_data.isna().sum() [train_data.isna().sum() >0]<count_missing_values>
accuracies = {}
for i in range(1, 100):
knn = KNeighborsClassifier(n_neighbors=i)
 knn.fit(Xadult, Yadult)
 scores = cross_val_score(knn, Xadult, Yadult, cv=10)
 Ypred = knn.predict(XUCITest)
 accuracy = accuracy_score(YUCITest,Ypred)
 accuracies[i] = accuracy
print('k={}, accuracy={}, CVmean={}'.format(i, accuracy, scores.mean()))
test_data.isna().sum() [train_data.isna().sum() >0]<create_dataframe>
knn = KNeighborsClassifier(n_neighbors=28 )
train_data=pd.DataFrame(data=train_data,columns=keys)
 test_data=pd.DataFrame(data=test_data,columns=keys)
<concatenate>
scores = cross_val_score(knn, Xadult, Yadult, cv=10)
 scores
total_data=pd.concat([train_data,test_data],axis=0, )<prepare_x_and_y>
knn.fit(Xadult, Yadult )
total_data=pd.get_dummies(data=total_data,drop_first=True)
 train_data=total_data.iloc[:1460,:]
test_data=total_data.iloc[1460:,:]
XX=train_data.copy()
y=y_train.copy()<create_dataframe>
knn.fit(Xadult, Yadult )
history=pd.DataFrame(columns=['itreation','will_deleted','pvalue','R_adjusted','rsquared'])
 for x in range(len(XX.columns)) :
regressor_osl=sm.OLS(endog=y,exog=XX ).fit()
pvalues=regressor_osl.pvalues
pvalues=pd.DataFrame(pvalues)
 p=pd.DataFrame(pvalues.reset_index())
 out=pd.DataFrame()
out=p.loc[p[0]== max(p[0])]
out=pd.Series.tolist(out)
 w1=out[0][0]
R=regressor_osl.rsquared_adj
r=regressor_osl.rsquared
XX.pop(w1)
 history=history.append([{'itreation':x+1,'will_deleted':out[0][0],'pvalue':out[0][1],'R_adjusted':R,'rsquared':r}] )<drop_column>
adult['Sex'] = adult['Sex'].transform(lambda x: 1 if x=='Male' else 0 if x==x else x )
drop=history[history['pvalue']>.05]['will_deleted']
drop=list(drop)
 drop<drop_column>
predictions = knn.predict(testAdult[["Age","Education-Num","Capital Gain", "Capital Loss", "Hours per week"]] )
train_data.drop(columns=drop,inplace=True)
 test_data.drop(columns=drop,inplace=True )<filter>
result = np.vstack(( testAdult.index.values, predictions)).T
x = ['Id','income']
resultado = pd.DataFrame(columns=x, data=result)
 resultado.set_index('Id', inplace=True )
<normalization><EOS> 
resultado.to_csv('mypredictions.csv' )