Kaggle competition: Atividade_3_PMR3508
<SOS> metric: rmsle Kaggle data source: atividade_3_pmr3508<split>
import sklearn
from sklearn.metrics import mean_squared_log_error, make_scorer
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import Lasso
import sklearn.naive_bayes as skNB
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestRegressor,AdaBoostRegressor
from sklearn.neighbors import KNeighborsRegressor
from IPython.display import Image
from IPython.core.display import HTML
X_train, X_val, y_train, y_val = train_test_split(df_train, df_train['Category'], random_state=13)
 X_train.shape, X_val.shape, y_train.shape, y_val.shape<feature_engineering>
train_raw = pd.read_csv(".. /input/atividade-3-pmr3508/train.csv", sep = ",")
 train = train_raw.copy()
target = train_raw["median_house_value"]
display(train_raw.shape,
train_raw.head() )
%%time
tfidf = TfidfVectorizer(max_features=10000)
 X_train_tfidf = tfidf.fit_transform(X_train['title&description'] )<train_model>
test_raw = pd.read_csv(".. /input/atividade-3-pmr3508/test.csv", sep = ",",header=0)
 test = test_raw.copy()
display(test_raw.shape,test_raw.head())
%%time
lr = LogisticRegression(random_state=13)
 lr.fit(X_train_tfidf, y_train )<categorify>
def New_features(data):
data.loc[:,'rooms_per_household'] = data.loc[:,'total_rooms']/data.loc[:,'households']
data.loc[:,'rooms_per_person'] = data.loc[:,'total_rooms']/data.loc[:,'population']
data.loc[:,'bedrooms_per_household'] = data.loc[:,'total_bedrooms']/data.loc[:,'households']
data.loc[:,'bedrooms_per_person'] = data.loc[:,'total_bedrooms']/data.loc[:,'households']
data.loc[:,'persons_per_household'] = data.loc[:,'population']/data.loc[:,'households']
data.loc[:, 'median_income_per_person'] = data.loc[:,'median_income']/data.loc[:,'persons_per_household']
New_features(train_raw)
 New_features(test)
 New_features(train)
 train.head()
%%time
X_val_tfidf = tfidf.transform(X_val['title&description'] )<predict_on_test>
arq = open(".. /input/us-citiestxt/US_cities.txt","r")
 new_table=[]
linhas=arq.readlines()
for i in range(87,len(linhas)) :
palavras=linhas[i].split(",")
 if palavras[1] == "5":
new_table.append([palavras[3][1:-1],palavras[5][:-1],palavras[4]])
 arq=open("US_cities.csv","w")
 for i in new_table:
arq.write(i[0]+","+i[1]+","+i[2]+"
" )
%%time
y_val_pred = lr.predict(X_val_tfidf)
 accuracy_score(y_val, y_val_pred )<drop_column>
coords={}
arq=open("US_cities.csv")
 for i in arq.readlines() :
lista=i.strip().split(",")
 coords[lista[0]]=(float(lista[1]),float(lista[2]))
coords
del X_train, y_train, X_val, y_val, X_train_tfidf, X_val_tfidf<feature_engineering>
dSF_LA=[]
a=np.array(coords["San Francisco"])
 b=np.array(coords["Los Angeles"])
 for i,c in train.iterrows() :
c=np.array(( c['longitude'],c['latitude']))
dSF_LA.append(min(np.linalg.norm(a-c),np.linalg.norm(b-c)))
 train = train.join(pd.Series(dSF_LA,name="dSF_LA"))
dSF_LA=[]
a=np.array(coords["San Francisco"])
 b=np.array(coords["Los Angeles"])
 for i,c in train.iterrows() :
c=np.array(( c['longitude'],c['latitude']))
dSF_LA.append(min(np.linalg.norm(a-c),np.linalg.norm(b-c)))
 test = test.join(pd.Series(dSF_LA,name="dSF_LA"))
df_test['title&description'] = df_test['title'].str[:] + ' ' + df_test['description'].str[:]<predict_on_test>
train.corr() ["median_house_value"].abs().sort_values(ascending=False )
%%time
X_test_tfidf = tfidf.transform(df_test['title&description'])
 y_test_pred = lr.predict(X_test_tfidf )<prepare_output>
msle = make_scorer(mean_squared_log_error)
 def score(nome,regressor):
pontos = cross_val_score(regressor,trainF,target,cv=10,scoring=msle)
 print(nome+ ': '+ str(round(np.sqrt(pontos ).mean() ,4)) + "
" )
sub['Category'] = y_test_pred
sub.head()<save_to_csv>
Lreg = Lasso(positive = True)
 score("Regressor Lasso",Lreg)
sub.to_csv('submission.csv', index=False )<save_to_csv>
RFreg = RandomForestRegressor(max_depth=20, random_state=0,n_estimators=400)
 score("Regressor Random Forest",RFreg )
sub.to_csv('submission.csv', index=False )<import_modules>
KNNreg = KNeighborsRegressor(n_neighbors=np.array(pontuacao ).argmin() +1 )
import os
from typing import Dict, List, Tuple
from joblib import Parallel, delayed
import pandas as pd
import numpy as np
from scipy.optimize.minpack import curve_fit
from scipy.optimize import least_squares
from xgboost import XGBRegressor<load_from_csv>
score("Regressor KNN",KNNreg )
def load_kaggle_csv(dataset: str, datadir: str)-> pd.DataFrame:

 df = pd.read_csv(
 f"{os.path.join(datadir,dataset)}.csv", parse_dates=["Date"] 
)
 df['country'] = df["Country_Region"]
if "Province_State" in df:
df["Country_Region"] = np.where(
 df["Province_State"].isnull() ,
df["Country_Region"],
df["Country_Region"] + "_" + df["Province_State"], 
)
 df.drop(columns="Province_State", inplace=True)
 if "ConfirmedCases" in df:
df["ConfirmedCases"] = df.groupby("Country_Region")[
"ConfirmedCases"
].cummax()
if "Fatalities" in df:
df["Fatalities"] = df.groupby("Country_Region")["Fatalities"].cummax()
if not "DayOfYear" in df:
df["DayOfYear"] = df["Date"].dt.dayofyear
df["Date"] = df["Date"].dt.date
return df
def RMSLE(actual: np.ndarray, prediction: np.ndarray)-> float:

 return np.sqrt(
 np.mean(
 np.power(np.log1p(np.maximum(0, prediction)) - np.log1p(actual), 2)
)
)
 def get_extra_features(df):
df['school_closure_status_daily'] = np.where(df['school_closure'] < df['Date'], 1, 0)
 df['school_closure_first_fatality'] = np.where(df['school_closure'] < df['first_1Fatalities'], 1, 0)
 df['school_closure_first_10cases'] = np.where(df['school_closure'] < df['first_10ConfirmedCases'], 1, 0)
 df['case_delta1_10'] =(df['first_10ConfirmedCases'] - df['first_1ConfirmedCases'] ).dt.days
df['case_death_delta1'] =(df['first_1Fatalities'] - df['first_1ConfirmedCases'] ).dt.days
df['case_delta1_100'] =(df['first_100ConfirmedCases'] - df['first_1ConfirmedCases'] ).dt.days
df['days_since'] = df['DayOfYear']-df['case1_DayOfYear']
df['weekday'] = pd.to_datetime(df['Date'] ).dt.weekday
col = df.isnull().mean()
rm_null_col = col[col > 0.2].index.tolist()
return df
def dateparse(x):
try:
return pd.datetime.strptime(x, '%Y-%m-%d')
 except:
return pd.NaT
def prepare_lat_long(df):
df["Country_Region"] = np.where(
 df["Province/State"].isnull() ,
df["Country/Region"],
df["Country/Region"] + "_" + df["Province/State"], 
)
 return df[['Country_Region', 'Lat', 'Long']].drop_duplicates()<load_from_csv>
Adareg = AdaBoostRegressor(RandomForestRegressor(max_depth=15,n_estimators=10),n_estimators=60 )
df_lat = prepare_lat_long(pd.read_csv("/kaggle/input/inputlat-long/lat_long.csv"))
train = load_kaggle_csv("train", "/kaggle/input/covid19-global-forecasting-week-3")
 country_health_indicators =(
(pd.read_csv("/kaggle/input/country-health-indicators/country_health_indicators_v3.csv",
parse_dates=['first_1ConfirmedCases', 'first_10ConfirmedCases',
'first_50ConfirmedCases', 'first_100ConfirmedCases',
'first_1Fatalities', 'school_closure'], date_parser=dateparse)).rename(
 columns ={'Country_Region':'country'}))
train =(pd.merge(train, country_health_indicators,
on="country",
how="left")).merge(df_lat, on='Country_Region', how='left')
 train = get_extra_features(train)
 train.head(3 )<load_from_csv>
Adareg.fit(trainF,target )
test = load_kaggle_csv("test", "/kaggle/input/covid19-global-forecasting-week-3")
 test =(pd.merge(
 test, country_health_indicators, on="country", how="left")).merge(
 df_lat, on ='Country_Region', how='left')
 test = get_extra_features(test)
 del country_health_indicators<train_on_grid>
R = Adareg.predict(testF)
 R
def logistic(x: np.ndarray, x0: float, L: float, k: float)-> np.ndarray:

 return L /(1 + np.exp(-k *(x - x0)))
 def fit_single_logistic(x: np.ndarray, y: np.ndarray, maxfev: float)-> Tuple:

 p0 = [np.median(x), y[-1], 0.1]
pn0 = p0 *(np.random.random(len(p0)) + [0.5, 1.0, 0.5])
 try:
params, pcov = curve_fit(
 logistic,
x,
y,
p0=pn0,
maxfev=maxfev,
sigma=np.maximum(1, np.sqrt(y)) *(0.1 + 0.9 * np.random.random()),
bounds=([0, y[-1], 0.01], [200, 1e6, 1.5]), 
)
 pcov = pcov[np.triu_indices_from(pcov)]
except(RuntimeError, ValueError):
params = p0
pcov = np.zeros(len(p0)*(len(p0)- 1))
y_hat = logistic(x, *params)
 rmsle = RMSLE(y_hat, y)
 return(params, pcov, rmsle, y_hat)
 def fit_logistic(
 df: pd.DataFrame,
n_jobs: int = 8,
n_samples: int = 80,
maxfev: int = 8000,
x_col: str = "DayOfYear",
y_cols: List[str] = ["ConfirmedCases", "Fatalities"], 
)-> pd.DataFrame:
def fit_one(df: pd.DataFrame, y_col: str)-> Dict:
best_rmsle = None
best_params = None
x = df[x_col].to_numpy()
y = df[y_col].to_numpy()
for(params, cov, rmsle, y_hat)in Parallel(n_jobs=n_jobs )(
 delayed(fit_single_logistic )(x, y, maxfev=maxfev)
 for i in range(n_samples)
):
if rmsle >=(best_rmsle or rmsle):
best_rmsle = rmsle
best_params = params
result = {f"{y_col}_rmsle": best_rmsle}
result.update({f"{y_col}_p_{i}": p for i, p in enumerate(best_params)})
 return result
result = {}
for y_col in y_cols:
result.update(fit_one(df, y_col))
return pd.DataFrame([result])
 def predict_logistic(
 df: pd.DataFrame,
x_col: str = "DayOfYear",
y_cols: List[str] = ["ConfirmedCases", "Fatalities"], 
):
def predict_one(col):
df[f"yhat_logistic_{col}"] = logistic(
 df[x_col].to_numpy() ,
df[f"{col}_p_0"].to_numpy() ,
df[f"{col}_p_1"].to_numpy() ,
df[f"{col}_p_2"].to_numpy() , 
)
 for y_col in y_cols:
predict_one(y_col )<merge>
arq=open("result1.csv","w")
 arq.write("Id,median_house_value
")
 for i,j in zip(list(R),list(Id)) :
arq.write(str(j)+","+str(i)+"
")
 arq.close()