Kaggle competition: Atividade_3_PMR3508
<SOS> metric: rmsle Kaggle data source: atividade_3_pmr3508<feature_engineering>
import pandas as pd
import numpy as np
import sklearn
import matplotlib.pyplot as plt
from sklearn import ensemble, neighbors
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_log_error, make_scorer
from sklearn.feature_selection import f_regression
import os
Train['title&description'] = Train['title'].str[:] + ' ' + Train['description'].str[:]<feature_engineering>
test_d = pd.read_csv('.. /input/test.csv',index_col='Id')
 train_d = pd.read_csv('.. /input/train.csv', index_col='Id' )
Test['title&description'] = Test['title'].str[:] + ' ' + Test['description'].str[:]<drop_column>
test_d['median_house_value'] = np.nan
data = train_d.append(test_d, sort=False )
Train.drop(columns=['title', 'description'], inplace=True)
 Test.drop(columns=['title', 'description'], inplace=True )<set_options>
train_d['median_age_sum'] = train_d.groupby('median_age')['median_house_value'].transform('sum')
 ages = train_d['median_age'].value_counts()
for i, row in train_d.iterrows() :
train_d.loc[i,'median_age_freq'] = ages[train_d.loc[i,'median_age']]
gc.collect()<load_pretrained>
data['rooms_not_bedrooms'] = np.subtract(data['total_rooms'],data['total_bedrooms'])
 data['bedrooms_per_house'] = np.divide(data['total_bedrooms'],data['households'])
 data['rooms_not_bedrooms_per_house'] = np.divide(data['total_bedrooms'],data['households'])
 data['pop_per_house'] = np.divide(data['population'],data['households'])
 data['pop_per_bedroom'] = np.divide(data['population'],data['total_bedrooms'])
 data['pop_per_room'] = np.divide(data['population'],data['total_bedrooms'])
 data['income_per_pop'] = np.divide(data['median_income'],data['pop_per_house'] )
nltk.download('stopwords')
 ru_stopwords = list(stopwords.words("russian"))<import_modules>
ages_disc=[]
for i,row in data.iterrows() :
if row['median_age'] < 16:
ages_disc.append(0)
 elif row['median_age'] < 36:
ages_disc.append(1)
 elif row['median_age'] < 52:
ages_disc.append(2)
 else:
ages_disc.append(3)
 data['age_class'] = ages_disc
from sklearn.feature_extraction.text import TfidfVectorizer<train_model>
corr_superior = correlacao.where(np.triu(np.ones(correlacao.shape), k=1 ).astype(np.bool))
drop = [column for column in corr_superior.columns if any(abs(corr_superior[column])> 0.9)]
drop
tf_idf = TfidfVectorizer(stop_words=ru_stopwords)
 tf_idf.fit(Train['title&description'] )<import_modules>
correlacao.loc[correlacao['total_bedrooms'].abs() > 0.9, correlacao['total_bedrooms'].abs() > 0.9]
from sklearn.svm import LinearSVC<categorify>
correlacao.loc[correlacao['rooms_not_bedrooms'].abs() > 0.9, correlacao['rooms_not_bedrooms'].abs() > 0.9]
Train_tf_idf = tf_idf.transform(Train['title&description'])
 Test_tf_idf = tf_idf.transform(Test['title&description'] )<train_on_grid>
correlacao.loc[correlacao['rooms_not_bedrooms_per_house'].abs() > 0.9, correlacao['rooms_not_bedrooms_per_house'].abs() > 0.9]
clf = LinearSVC()
clf.fit(Train_tf_idf, Train['Category'] )<set_options>
correlacao.loc[correlacao['pop_per_bedroom'].abs() > 0.9, correlacao['pop_per_bedroom'].abs() > 0.9]
del Train, Train_tf_idf
gc.collect()<create_dataframe>
data.drop(['rooms_not_bedrooms_per_house','pop_per_room','pop_per_house','total_rooms','median_age'], axis = 1, inplace = True )
Answer = pd.DataFrame(columns=['Id', 'Category'])
 Answer['Id'] = Test['itemid']<predict_on_test>
normcols = list(data.columns)
 [normcols.remove(x)for x in ['latitude','longitude','median_house_value']]
fit_norm = data.dropna()
fit_norm = fit_norm[normcols]
means = {}
stds = {}
for c in normcols:
means[c] = fit_norm[c].mean()
stds[c] = fit_norm[c].std()
fit_norm = data.copy()
for c in normcols:
fit_norm.loc[:,c] = fit_norm.loc[:,c].subtract(means[c] ).divide(stds[c])
 fit_norm['latitude'] = data['latitude']
fit_norm['longitude'] = data['longitude']
fit_norm['median_house_value'] = data['median_house_value']
fit_norm.head()
Answer['Category'] = clf.predict(Test_tf_idf )<save_to_csv>
msle = make_scorer(mean_squared_log_error )
Answer.to_csv('my_submission_tfidfvect_nltk_stopwords_lsvc.csv', index=None )<set_options>
knn = KNeighborsRegressor(n_neighbors=10)
 scores = cross_val_score(knn,
fit_norm.dropna().drop('median_house_value', axis = 1),
fit_norm.dropna().loc[:,'median_house_value'],
cv=10,
scoring = msle)
 scores = np.sqrt(scores)
 display(scores)
 print(f'MSLE = {round(scores.mean() , 4)}, std = {round(scores.std() , 4)}' )
sns.set()
%matplotlib inline<load_from_csv>
scores_array = []
scores_var = []
for i in range(21):
knn = KNeighborsRegressor(n_neighbors=10, metric='wminkowski', p=2, metric_params={'w': np.divide([i/20]*2+[1-i/20]*9,15)})
 scores = cross_val_score(knn,
fit_norm.dropna().drop('median_house_value', axis = 1),
fit_norm.dropna().loc[:,'median_house_value'],
cv=10,
scoring = msle)
 scores = np.sqrt(scores)
 scores_array.append(scores.mean())
 scores_var.append(scores.std() )
DATA_DIR = '/kaggle/input/texts-classification-iad-hse-intro-2020/'
df_train = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))
df_test = pd.read_csv(os.path.join(DATA_DIR, 'test.csv'))
sub = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'))<data_type_conversions>
test_data = fit_norm.loc[fit_norm['median_house_value'].isnull() ].drop('median_house_value',axis=1 )
df_train.fillna('', inplace=True)
 df_test.fillna('', inplace=True )<count_values>
classifier = KNeighborsRegressor(n_neighbors=8, metric='wminkowski', p=2, metric_params={'w': np.divide([18/20]*2+[1-18/20]*9,15)})
 classifier.fit(fit_norm.dropna().drop('median_house_value', axis = 1),fit_norm.dropna().loc[:,'median_house_value'])
 testPred = classifier.predict(test_data)
 arq = open("prediction_knn.csv", "w")
 arq.write("Id,median_house_value
")
 for i, j in zip(test_data.index, testPred):
arq.write(str(i)+ "," + str(int(j)) +"
")
 arq.close()
df_train['Category'].value_counts()<count_unique_values>
regr = RandomForestRegressor(max_depth=20, random_state=0,n_estimators=400)
 regr.fit(fit_norm.dropna().drop('median_house_value', axis = 1),fit_norm.dropna().loc[:,'median_house_value'])
 testPred = regr.predict(test_data)
 arq = open("prediction_rfr.csv", "w")
 arq.write("Id,median_house_value
")
 for i, j in zip(test_data.index, testPred):
arq.write(str(i)+ "," + str(int(j)) +"
")
 arq.close()
<feature_engineering><EOS> 
boost = AdaBoostRegressor(RandomForestRegressor(max_depth=5, random_state=0, n_estimators=300),
n_estimators=100)
 boost.fit(fit_norm.dropna().drop('median_house_value', axis = 1),fit_norm.dropna().loc[:,'median_house_value'])
 testPred = boost.predict(test_data)
 arq = open("prediction_ada_rfr.csv", "w")
 arq.write("Id,median_house_value
")
 for i, j in zip(test_data.index, testPred):
arq.write(str(i)+ "," + str(int(j)) +"
")
 arq.close()