Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<save_to_csv>
idx = pd.IndexSlice
test[['Id', 'Target']].to_csv('submission.csv', index=False )<define_variables>
df = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv', index_col=0 )
heads = train[train['parentesco1'] == 1]<feature_engineering>
df['Country_Region'].replace('Taiwan*', 'Taiwan', inplace=True)
 df = df[df['Country_Region'] != 'Diamond Princess']
heads['hsize-pc'] = heads['hhsize'] / heads['tamviv']
heads['phone-pc'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-pc'] = heads['v18q1'] / heads['tamviv']
heads['rooms-pc'] = heads['rooms'] / heads['tamviv']
heads['rent-pc'] = heads['v2a1'] / heads['tamviv']<choose_model_class>
df_codes =(pd.read_csv('.. /input/my-covid19-dataset/iso-country-codes.csv')
.rename(columns={'Country Name': 'Country_Region', 'Alpha-3 code': 'Country Code'})
.loc[:,['Country_Region', 'Country Code']])
 df =(pd.merge(df, df_codes, how='left', on='Country_Region'))
rf2 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced' )<train_model>
def location(state, country):
if type(state)==str and not state==country:
return country + ' - ' + state
else:
return country
df['Date'] = df['Date'].apply(lambda x:(dt.datetime.strptime(x, '%Y-%m-%d')))
 df['Location'] = df[['Province_State', 'Country_Region']].apply(lambda row: location(row[0],row[1]), axis=1)
 t_start = df['Date'].unique() [0]
t_end = df['Date'].unique() [-1]
print('Number of Locations: ' + str(len(df['Location'].unique())))
 print('Dates: from ' + np.datetime_as_string(t_start, unit='D')+ ' to ' +
np.datetime_as_string(t_end, unit='D')+ '
' )
rf2.fit(heads[feats], heads['Target'] )<predict_on_test>
lst_out = ['Cayman Islands', 'Curacao', 'Faroe Islands', 'French Guiana', 'French Polynesia', 'Guadeloupe',
'Mayotte', 'Martinique', 'Reunion', 'Saint Barthelemy', 'St Martin', 'Aruba', 'Channel Islands',
'Gibraltar', 'Montserrat', 'Diamond Princess', 'From Diamond Princess', 'Puerto Rico',
'Virgin Islands', 'Guam']
df_loc =(df.loc[[(( c not in lst_out)and(p not in lst_out))
for(c, p)in zip(df['Province_State'], df['Country_Region'])],
['Location','Province_State', 'Country_Region','Country Code']] 
.drop_duplicates() )
test['Target'] = rf2.predict(test[feats] ).astype(int )<count_values>
df_pop = pd.read_csv('.. /input/my-covid19-dataset/citypopulation-de/population.csv' )
test['Target'].value_counts(normalize=True )<train_model>
df_loc = pd.merge(df_loc, df_pop, how='left', on=['Province_State', 'Country_Region'] )
abc = AdaBoostClassifier(n_estimators=200, learning_rate=1.0, random_state=42)
 abc.fit(heads[feats], heads['Target'])
 accuracy_score(test['Target'], abc.predict(test[feats]))<train_model>
df_pop =(pd.read_csv('.. /input/my-covid19-dataset/un-org/population-2020.csv')
.rename(columns={'ISO 3166-1 alpha code': 'Country Code'}))
gbm = GradientBoostingClassifier(n_estimators=200, learning_rate=1.0, max_depth=1, random_state=42)
 gbm.fit(heads[feats], heads['Target'])
 accuracy_score(test['Target'], gbm.predict(test[feats]))<save_to_csv>
df_pop = df_pop.loc[~df_pop['Country Code'].isin(['CHN', 'USA', 'CAN', 'AUS']), ['Country Code', 'Population']]
test[['Id', 'Target']].to_csv('submission.csv', index=False )<compute_test_metric>
df_pop = pd.merge(df_loc, df_pop, how='left', on='Country Code', suffixes=('_P/S','_C/R'))
cm = confusion_matrix('Target', test['prediction'] )<load_from_csv>
df_pop['Population'] =(df_pop[['Population_P/S','Population_C/R']] 
.apply(lambda x: x[1] if np.isnan(x[0])else x[0], axis=1))
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<prepare_x_and_y>
df_pop = df_pop.loc[:,['Location','Population']].set_index('Location', verify_integrity=True )
x_train = df['Id']
x_test = test['Id']
y_train = df['Target']
df_all = df.append(test)
 df_all.shape<import_modules>
df = pd.merge(df, df_pop, how='left', on='Location' )
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split,cross_val_score, GridSearchCV
from sklearn.metrics import classification_report,multilabel_confusion_matrix 
<create_dataframe>
df['Fatalities per Million'] = df['Fatalities'] / df['Population'] * 1000
df['Confirmed Cases per Million'] = df['ConfirmedCases'] / df['Population'] * 1000
titulo = 'Parentesco'
variaveis = 'parentesco1','parentesco2','parentesco3','parentesco4','parentesco5','parentesco6','parentesco7','parentesco8','parentesco9','parentesco10','parentesco11','parentesco12'
eixoX = ['Chefe de família','Cônjugue','Filho','Divorciado','Genro/Nora','Neto','Pai','Sogro','Irmão','Cunhada','Outro Familiar','Outro Não Familiar']
grafico_barras(variaveis,eixoX,titulo )<define_variables>
df['Day Count Confirmed'] =(df['ConfirmedCases']>0 ).groupby(df['Location'] ).cumsum().astype('int')
 df['Day Count Fatalities'] =(df['Fatalities']>0 ).groupby(df['Location'] ).cumsum().astype('int' )
varNumericas = ['v2a1','rooms','v18q','v18q1','r4h1','r4h2','r4h3','r4m1','r4m2','r4m3','r4t1','r4t2','r4t3','tamhog','tamviv','escolari','rez_esc','hhsize','hogar_nin','hogar_adul','hogar_mayor','hogar_total','dependency','edjefe','edjefa','meaneduc','bedrooms','overcrowding','qmobilephone','age','SQBescolari','SQBage','SQBhogar_total','SQBedjefe','SQBhogar_nin','SQBovercrowding','SQBdependency','SQBmeaned','agesq']<define_variables>
df['Fatalities'] = df.loc[df['Day Count Fatalities']>0, 'Fatalities'].apply(lambda x: max(x,1))
df['Day Count Fatalities'] =(df['Fatalities']>0 ).groupby(df['Location'] ).cumsum().astype('int' )
naoUsar = ['idhogar','Id','Target']
naoUsarNumericas = ['tamhog','hogar_total','agesq','hhsize']<count_values>
df['New Cases'] = df['ConfirmedCases'].groupby(df['Location'] ).diff() / df['ConfirmedCases']
df['New Fatalities'] = df['Fatalities'].groupby(df['Location'] ).diff() / df['Fatalities']
df['Case Fatality Rate'] = df['Fatalities'] / df['ConfirmedCases']
df['Target'].value_counts().sort_values()<count_values>
df_all = df
df = df[(df['Day Count Fatalities']>0)& 
(( df['Fatalities per Million']>1)| 
(df['New Fatalities']>1/7)) ]
df_all['edjefa'].value_counts()<count_values>
df_testing =(pd.read_csv('.. /input/my-covid19-dataset/ourworldindata/tests/tests-vs-confirmed-cases-covid-19.csv')
.loc[:,['Entity', 'Total COVID-19 tests']] 
.rename(columns={'Entity':'Location', 'Total COVID-19 tests': 'Tests'}))
df_all['edjefe'].value_counts()<count_values>
df = pd.merge(df, df_testing, how='left', on='Location' )
df_all['dependency'].value_counts()<data_type_conversions>
df['Tests per Million'] = df['Tests'] / df['Population'] * 1000
df['Tests per Million'].fillna(df['Confirmed Cases per Million'], inplace=True)
 df.drop(columns=['Province_State', 'Country_Region', 'Tests', 'Population'], inplace=True)
 df['Confirmed Rate'] = df['Confirmed Cases per Million'] / df['Tests per Million']
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int)
 df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )<count_missing_values>
mask =(~df.set_index(['Location','Date'], verify_integrity=True)
.index.get_level_values(0)
.duplicated(keep='last'))
df_plt =(df[mask].drop(columns=['Date','Country Code',
'Day Count Confirmed','ConfirmedCases','Confirmed Cases per Million',
'Confirmed Rate','Fatalities'])
.set_index(['Location'], verify_integrity=True))
df_all.isnull().sum()<data_type_conversions>
df_plt.corr().style.background_gradient(cmap='Reds' ).set_precision(2 )
df_all['v2a1'].fillna(-1, inplace=True)
 df_all['v18q1'].fillna(0, inplace=True)
 df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )<count_missing_values>
df = df.set_index(['Location','Day Count Fatalities'], verify_integrity=True ).sort_index()
mask = ~df.index.get_level_values(0 ).duplicated(keep='last' )
df_all.isnull().sum()<define_variables>
df_cfr = df.loc[mask,:].reset_index(level=1, drop=True ).loc[:,['Country Code', 'Case Fatality Rate']]
df_cfr[['Case Fatality Rate']] = df_cfr[['Case Fatality Rate']].apply(lambda x:(x-x.min())/(x.max() -x.min()))
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<choose_model_class>
state = 'US - New York'
fperiod = 90
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42 )<train_model>
fatalities = df.loc[idx[state, :], 'Fatalities per Million'].reset_index(drop=True )
rf.fit(train[feats], train['Target'] )<predict_on_test>
init_alpha =.3
init_beta =.7
init_phi =.8
initial_level = fatalities[0]
initial_slope = fatalities[1] / fatalities[0]
start_params = [init_alpha, init_beta, initial_level, initial_slope, init_phi]
with warnings.catch_warnings() :
warnings.simplefilter('ignore')
 fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(start_params=start_params, remove_bias=True, use_basinhopping=True))
fcast = fmodel.forecast(fperiod )
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
df_params = pd.DataFrame(index=df.index.unique(level='Location'),
columns=['Alpha', 'Beta', 'Phi' ,'Forecast per Million'])
 keys = ['smoothing_level', 'smoothing_slope', 'damping_slope']
for state in df.index.get_level_values(level=0 ).unique() :
fatalities = df.loc[idx[state, :], 'Fatalities per Million'].reset_index(drop=True)
 if len(fatalities.dropna().index)< 2: continue
with warnings.catch_warnings() :
warnings.simplefilter('ignore')
 fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(remove_bias=True, smoothing_seasonal=0))
_, beta, phi = list(map(fmodel.model.params.get, keys))
if not(( beta==0)and(phi==0)) :
init_alpha =.3
init_beta =.7
init_phi =.8
initial_level = fatalities[0]
initial_slope = fatalities[1] / fatalities[0]
start_params = [init_alpha, init_beta, initial_level, initial_slope, init_phi]
fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(start_params=start_params, remove_bias=True, use_basinhopping=True))
if fcast.iloc[-1]>.02*1e3:
fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(damping_slope=.8, remove_bias=True, use_basinhopping=True))
df_params.loc[state, ['Alpha', 'Beta', 'Phi']] = list(map(fmodel.model.params.get, keys))
df_params.loc[state, 'Forecast per Million'] = fmodel.forecast(fperiod ).iloc[-1]
test['Target'].value_counts(normalize=True )<save_to_csv>
df_params = df_params.apply(pd.to_numeric, errors='ignore' )
test[['Id', 'Target']].to_csv('submission_1.csv', index=False )<define_variables>
mask =(
(pd.isnull(df_params['Phi'])) | 
(( df_params['Beta']==0)&(df_params['Phi']==1)) | 
(df_params['Phi']==0)
)
 idx_data = df_params.index[~mask]
varNaoUtilizadas = ['Id', 'idhogar', 'Target']
varNaoUtilizadasCat = ['female','area2']
varNaoUtilizadasNum = ['tamhog','hogar_total','hhsize']
varNaoUtilizadasSQ = ['SQBescolari','SQBage','SQBhogar_total','SQBedjefe','SQBhogar_nin','SQBovercrowding','SQBdependency','SQBmeaned','agesq']
varNaoUtilizadas = varNaoUtilizadas + varNaoUtilizadasCat + varNaoUtilizadasNum + varNaoUtilizadasSQ
varNaoUtilizadas<define_variables>
df_plt = df_params.dropna().reset_index()
s, x, y, z = zip(*df_plt[['Location', 'Alpha', 'Beta', 'Phi']].values )
feats = [c for c in df_all.columns if c not in varNaoUtilizadas]<train_model>
df_plt.loc[(df_plt['Beta']==0)&(df_plt['Phi']==0)]
rf.fit(train[feats], train['Target'] )<predict_on_test>
df_plt.loc[(df_plt['Beta']==0)&(df_plt['Phi']==1)]
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
df_plt.loc[df_plt['Forecast per Million']>.01*1e6]
test['Target'].value_counts(normalize=True )<save_to_csv>
df_plt.loc[[not(( a in [0,1])or 
(b in [0,1])or 
(abs(a-b)<1e-2)) for(a,b)in df_plt[['Alpha','Beta']].values]]
test[['Id', 'Target']].to_csv('submission.csv', index=False)
<define_variables>
df_plt = pd.merge(df_plt, df_pop, how='left', on='Location')
 df_plt['Forecast'] = df_plt['Forecast per Million'] * df_plt['Population'] / 1000
varNaoUtilizadas = ['Id', 'idhogar', 'Target']
varNaoUtilizadasCat = ['female','area2']
varNaoUtilizadasNum = ['tamhog','hogar_total','hhsize']
varNaoUtilizadasSQ = ['SQBescolari','SQBage','SQBhogar_total','SQBedjefe','SQBhogar_nin','SQBovercrowding','SQBdependency','SQBmeaned','agesq']
varPoucosRegistros = ['techoentrepiso','techocane','techootro','abastaguafuera','abastaguano','sanitario1','sanitario2', 'sanitario5','sanitario6','energcocinar1', 'energcocinar3','energcocinar4','elimbasu2','elimbasu3','elimbasu4','elimbasu6','tipovivi2','tipovivi3','tipovivi4','tipovivi5']
varNaoUtilizadas = varNaoUtilizadas + varNaoUtilizadasCat + varNaoUtilizadasNum + varNaoUtilizadasSQ + varPoucosRegistros
varNaoUtilizadas<define_variables>
df_r = pd.read_csv('.. /input/my-covid19-dataset/latlong.csv' ).set_index('Location', verify_integrity=True ).loc[:,['Lat', 'Long']]
feats = [c for c in df_all.columns if c not in varNaoUtilizadas]<train_model>
df_wdi =(pd.read_csv('.. /input/my-covid19-dataset/world-bank/world-development-indicators.csv')
.dropna(subset=['Country Code','Series Name'], how='all')
.set_index(['Country Code', 'Series Name'], verify_integrity=True)
.drop(columns=['Country Name', 'Series Code'])
.replace({'.. ': np.nan})
.dropna(how='all')
.astype(float))
rf.fit(train[feats], train['Target'] )<predict_on_test>
wdi_default = df_wdi.fillna(df_wdi.median() ).loc['WLD',:]
df_wdi = df_wdi.fillna(wdi_default ).reset_index()
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
new_row = df_wdi.loc[df_wdi['Country Code']=='SGP'].copy(deep=True)
 new_row.loc[:,'Country Code'] = 'TWN'
df_wdi = df_wdi.append(new_row, ignore_index=True )
test['Target'].value_counts(normalize=True )<save_to_csv>
df_ncd1 =(pd.read_csv('.. /input/my-covid19-dataset/ncd-risc/obesity/NCD_RisC_Lancet_2017_BMI_age_standardised_country.csv')
.loc[:,['ISO', 'Sex', 'Prevalence of BMI>=30 kg/m2(obesity)']] 
.rename(columns={'ISO': 'Country Code'})
.groupby('Country Code')
.mean() 
.reset_index() )
test[['Id', 'Target']].to_csv('submission.csv', index=False)
<choose_model_class>
df_ncd2 =(pd.read_csv('.. /input/my-covid19-dataset/ncd-risc/blood-pressure/NCD_RisC_Lancet_2016_BP_age_standardised_countries.csv')
.loc[:,['ISO', 'Sex', 'Prevalence of raised blood pressure']] 
.rename(columns={'ISO': 'Country Code'})
.groupby('Country Code')
.mean() 
.reset_index() )
rf = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced' )<define_variables>
df_ihme = pd.read_csv('.. /input/my-covid19-dataset/ihme-gdb/IHME-GBD_2017_DATA-8e93cebf-1.csv')
 mask = [cause_name in ['Neoplasms','Cardiovascular diseases',
'Chronic respiratory diseases','Diabetes and kidney diseases']
for cause_name in df_ihme['cause_name']]
df_ihme =(df_ihme.rename(columns={'Neoplasms': 'Cancer prevalence'})
.loc[mask, ['location_name','cause_name','val']] 
.pivot(index='location_name', columns='cause_name', values='val')
.reset_index() 
.rename(columns={'location_name': 'Location'})
.merge(df_codes, how='left', left_on='Location', right_on='Country_Region')
.drop(columns=['Location','Country_Region']))
feats = [c for c in df_all.columns if c not in varNaoUtilizadas]<train_model>
lst_df = [df_wdi, df_ncd1, df_ncd2, df_ihme]
df_feat = reduce(lambda df_left, df_right: pd.merge(df_left, df_right, how='left', on='Country Code'), lst_df )
rf.fit(train[feats], train['Target'] )<predict_on_test>
df_feat =(df_cfr.reset_index() 
.merge(df_feat, how='left', on='Country Code' ).drop(columns=['Country Code']))
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
df_feat = df_feat.set_index('Location', verify_integrity=True ).apply(lambda x:(x-x.min())/(x.max() -x.min()))
df_feat.fillna(df_feat.median() , inplace=True )
test['Target'].value_counts(normalize=True )<save_to_csv>
X_data =(df_params 
.loc[idx_data, ['Alpha', 'Beta', 'Phi']] 
.merge(df_feat, how='left', left_index=True, right_index=True))
idx_pred = [idx for idx in df_params.index if idx not in idx_data]
X_pred =(df_params 
.loc[idx_pred, ['Alpha', 'Beta', 'Phi']] 
.merge(df_feat, how='left', left_index=True, right_index=True))
X = X_data.iloc[:, 3:].values
y = X_data.iloc[:, :3].values
test[['Id', 'Target']].to_csv('submission.csv', index=False)
<define_search_space>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=99 )
param_grid = {'max_depth': [None,5,10],
'max_leaf_nodes': [None,2,6],
'min_impurity_decrease' : [1,1e-3],
'n_jobs': [-1],
'min_samples_leaf': [2,4],
'n_estimators': [100,300,700],
'class_weight' : [None,'balanced']}
grid = GridSearchCV(rf,param_grid=param_grid,cv=4,scoring='f1_macro' )<train_model>
n_neighbors = 10
knn = KNeighborsRegressor(n_neighbors=n_neighbors, weights='uniform', metric='minkowski', p=2 )
grid.fit(train[feats], train['Target'] )<create_dataframe>
radius = 1.0
rnn = RadiusNeighborsRegressor(radius=radius, weights='uniform', metric='minkowski', p=2 )
grid_df = pd.DataFrame(grid.cv_results_)
 grid_df<sort_values>
lnr = LinearRegression(copy_X=True, fit_intercept=False )
grid_df.sort_values('rank_test_score',ascending=True ).iloc[0,:]<find_best_params>
lnr.fit(X_train,y_train)
 y_pred = lnr.predict(X_test)
 score = r2_score(y_test, y_pred, multioutput='uniform_average')
 print('Coefficient of determination of the linear regression: {:.2%}'.format(score))
grid.best_params_<choose_model_class>
rdg = RidgeCV(alphas=[10**n for n in range(-4,4)] )
rf = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=100,max_leaf_nodes=None,
min_impurity_decrease=0.001, min_samples_leaf=4,
verbose=0, class_weight='balanced' )<define_variables>
rdg.fit(X_train,y_train)
 y_pred = rdg.predict(X_test)
 score = r2_score(y_test, y_pred, multioutput='uniform_average')
 print('Coefficient of determination of the linear regression: {:.2%}'.format(score))
feats = [c for c in df_all.columns if c not in varNaoUtilizadas]<train_model>
pln.fit(X_train,y_train)
 y_pred = pln.predict(X_test)
 score = r2_score(y_test, y_pred, multioutput='uniform_average')
 print('Coefficient of determination of the linear regression: {:.2%}'.format(score))
rf.fit(train[feats], train['Target'] )<predict_on_test>
rgr = knn
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
rgr.fit(X, y)
 y_pred = rgr.fit(X, y ).predict(X_pred.iloc[:, 3:].values)
 X_pred.loc[:,['Alpha', 'Beta','Phi']] = y_pred
test['Target'].value_counts(normalize=True )<save_to_csv>
df_RGR =(pd.concat([X_data, X_pred] ).loc[:,['Alpha','Beta','Phi']].reset_index() 
.merge(df_params.reset_index() , how='left', on='Location', suffixes=('_RGR','')) 
.drop_duplicates().set_index('Location', verify_integrity=True))
df_RGR.loc[idx_pred, ['Alpha','Beta','Phi']] =(df_RGR.loc[idx_pred, ['Alpha_RGR','Beta_RGR','Phi_RGR']] 
.apply(pd.to_numeric ).values)
 df_params = df_RGR.drop(columns=['Alpha_RGR','Beta_RGR','Phi_RGR'] )
test[['Id', 'Target']].to_csv('submission.csv', index=False)
<set_options>
fperiod = 90
df =(df_all 
.loc[df_all['Day Count Fatalities']>0] 
.set_index(['Location','Day Count Fatalities'], verify_integrity=True)
.sort_index(level=[0, 1], ascending=[1, 1]))
KMeansSMOTE,SMOTE,SVMSMOTE)
 EditedNearestNeighbours,
RepeatedEditedNearestNeighbours,
NeighbourhoodCleaningRule,AllKNN,TomekLinks)
<load_from_csv>
mask =(df.index.get_level_values(0)==state)&(df.index.get_level_values(1)>0)
 fatalities = df.loc[mask, 'Fatalities'].reset_index(drop=True)
 alpha, beta, phi = df_params.loc[idx[state], ['Alpha', 'Beta', 'Phi']].values
if len(fatalities)>2:
with warnings.catch_warnings() :
warnings.simplefilter('ignore')
 fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(smoothing_level=alpha, smoothing_slope=beta, damping_slope=phi,
remove_bias=True, use_basinhopping=True))
fcast = fmodel.forecast(fperiod )
df = pd.read_csv("/kaggle/input/costa-rican-household-poverty-prediction/train.csv" )<groupby>
lst_states = df.index.get_level_values(0 ).unique()
lst_params = df_params.index.unique()
for state in lst_states:
mask =(df.index.get_level_values(0)==state)&(df.index.get_level_values(1)>0)
 fatalities = df.loc[mask, 'Fatalities'].reset_index(drop=True)
 alpha = beta = phi = 0
if state in lst_params:
alpha, beta, phi = df_params.loc[idx[state], ['Alpha', 'Beta', 'Phi']].values
if len(fatalities)>1:
if(np.min(fatalities)==0)or(not fatalities.is_monotonic):
print('Inconsistent data identified for {}.Please check.'.format(state))
fatalities = fatalities.apply(lambda x: max(x,1))
with warnings.catch_warnings() :
warnings.simplefilter('ignore')
 fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(smoothing_level=alpha, smoothing_slope=beta, damping_slope=phi,
remove_bias=True, use_basinhopping=True))
fcast = fmodel.forecast(fperiod)
 date_start = df.loc[idx[state,:], 'Date'][-1] + dt.timedelta(days=1)
 rng_dt = pd.date_range(start=date_start, periods=fperiod)
 arrays = [[state]*len(fcast.index), fcast.index]
idx_fcast = pd.MultiIndex.from_arrays(arrays, names=('Location', 'Day Count Fatalities'))
df = df.append(pd.DataFrame(index=idx_fcast,
data={'Fatalities': fcast.values, 'Date': rng_dt}), sort=True )
g = df.columns.to_series().groupby(df.dtypes ).groups<feature_engineering>
df_train = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv', index_col=0)
 df_test = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv', index_col=0)
 df_submit = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv', index_col=0 )
def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
return df<feature_engineering>
df_train['Country_Region'].replace('Taiwan*', 'Taiwan', inplace=True)
 df_test['Country_Region'].replace('Taiwan*', 'Taiwan', inplace=True )
def trata_cat(df):
df['dependency'] = np.sqrt(df['SQBdependency'])
 df.loc[(df.v14a == 1)&(df.sanitario1 == 1)&(df.abastaguano == 0), "v14a"] = 0
df.loc[(df.v14a == 1)&(df.sanitario1 == 1)&(df.abastaguano == 0), "sanitario1"] = 0
df['num_over_18'] = 0
df['num_over_18'] = df[df.age >= 18].groupby('idhogar' ).transform("count")
 df['num_over_18'] = df.groupby("idhogar")["num_over_18"].transform("max")
 df['num_over_18'] = df['num_over_18'].fillna(0)
 cols = ['edjefe', 'edjefa']
df[cols] = df[cols].replace({'yes':1, 'no':0})
 df["dependency"] = df["dependency"].astype(float)
 df["edjefe"] = df["edjefe"].astype(int)
 df["edjefa"] = df["edjefa"].astype(int)
 df['edjef'] = np.max(df[['edjefa','edjefe']], axis=1)
 df = extract_features(df)
 return df<drop_column>
df_train['Date'] = df_train['Date'].apply(lambda x:(dt.datetime.strptime(x, '%Y-%m-%d')))
 df_train['Location'] =(df_train[['Province_State','Country_Region']] 
.apply(lambda row: location(row[0],row[1]), axis=1))
df_test['Date'] = df_test['Date'].apply(lambda x:(dt.datetime.strptime(x, '%Y-%m-%d')))
 df_test['Location'] =(df_test[['Province_State','Country_Region']] 
.apply(lambda row: location(row[0],row[1]), axis=1))
def dropa_trata(df):
df_limpo = df.drop(["Id"],axis=1 ).copy()
df_tratado = trata_cat(df_limpo)
 return df_tratado.drop(["idhogar"],axis=1 )<train_model>
df_test =(df_test.reset_index() 
.merge(df_train.reset_index().loc[:,['Location','Date','ConfirmedCases','Fatalities']],
how='left', on=['Location','Date'])
.merge(df.reset_index().loc[:,['Location','Date','Fatalities']],
how='left', on=['Location','Date'], suffixes=('_Train','_Forecast')) 
.set_index('ForecastId', verify_integrity=True))
def treino(df,model,train_submit=False):
if train_submit:
X, y = df.drop("Target",axis=1),df["Target"]
model.fit(X,y)
 return model
if "Target" in df.columns:
X, y = df.drop("Target",axis=1),df["Target"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,stratify=y, random_state=42)
 model.fit(X_train,y_train)
 predicted = model.predict(X_test)
 print('Classifcation report:
', classification_report(y_test, predicted))
print('Confusion matrix:
', confusion_matrix(y_test, predicted))
return model
else:
return model.predict(df )<train_model>
df_test['Fatalities'] = df_test.loc[:,['Fatalities_Train','Fatalities_Forecast']].apply(
 lambda x: x[0] if not pd.isnull(x[0])else x[1], axis=1)
 df_test.drop(columns=['Fatalities_Train','Fatalities_Forecast'], inplace=True )
def treino_sampling(df,model,train_submit=False):
if train_submit:
X, y = df.drop("Target",axis=1),df["Target"]
model.fit(X,y)
 return model
if "Target" in df.columns:
resampling = RandomOverSampler()
pipeline = Pipeline([('Resampling', resampling),('XGBClassifier', model)])
 X, y = df.drop("Target",axis=1),df["Target"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,stratify=y, random_state=42)
 pipeline.fit(X_train, y_train)
 predicted = model.predict(X_test)
 print('Classifcation report:
', classification_report(y_test, predicted))
print('Confusion matrix:
', confusion_matrix(y_test, predicted))
return model
else:
return model.predict(df )<find_best_model_class>
for state in lst_states:
mask =(df_test['Location']==state)&(df_test['Fatalities'].isnull())
 df_test.loc[mask, ['ConfirmedCases','Fatalities']] = df_flat.loc[idx[state],
['ConfirmedCases','Fatalities']].values
def sampling_predict(df,func):
resampling = func()
model = XGBClassifier()
pipeline = Pipeline([('Resampling', resampling),('XGBClassifier', model)])
 X=df.drop("Target",axis=1)
 y=df.Target
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=0)
 pipeline.fit(X_train, y_train)
 predicted = pipeline.predict(X_test)
 print('Classifcation report:
', classification_report(y_test, predicted))
print('Confusion matrix:
', confusion_matrix(y_test, predicted))<choose_model_class>
df_train['Case Fatality Rate'] =(df_train[['Fatalities','ConfirmedCases']] 
.apply(lambda x: 0 if(( x[0]==0)and(x[1]==0)) else x[0]/x[1], axis=1))
df_train = df_train.set_index(['Location','Date'], verify_integrity=True ).sort_index()
mask = ~df_train.index.get_level_values(0 ).duplicated(keep='last')
 df_cfr = df_train.loc[mask,:].reset_index(level=1, drop=True ).loc[:,['Case Fatality Rate']]
xgb=XGBClassifier()
modelo_final_sem_sampler = treino(df=dropa_trata(df),model=xgb )<categorify>
mask = df_test['ConfirmedCases'].isnull()
df_test.loc[mask,'ConfirmedCases'] =(df_test 
.reset_index() 
.merge(df_cfr.reset_index() , how='left', on='Location')
.set_index('ForecastId', verify_integrity=True)
.loc[mask,['Fatalities','Case Fatality Rate']] 
.apply(lambda x: 0 if x[1]==0 else x[0]/x[1], axis=1))
under = ["RandomUnderSampler","EditedNearestNeighbours",
"RepeatedEditedNearestNeighbours",
"NeighbourhoodCleaningRule","AllKNN","TomekLinks"]
over =("RandomOverSampler","ADASYN","BorderlineSMOTE","SMOTE","SVMSMOTE")
 df_d1 = df.fillna(0 )<predict_on_test>
df_test['ConfirmedCases'] = df_test['ConfirmedCases'].apply(lambda x: round(x, 0)).astype('int')
 df_test['Fatalities'] = df_test['Fatalities'].apply(lambda x: round(x, 0)).astype('int' )
for model in under:
print("
"+model+"
")
 sampling_predict(dropa_trata(df_d1),eval(model))<predict_on_test>
df_test = df_test.reset_index().loc[:,['ForecastId','ConfirmedCases','Fatalities']]
df_submit = df_submit.reset_index().drop(columns=['ConfirmedCases','Fatalities'] )
for model in over:
print("
"+model+"
")
 sampling_predict(dropa_trata(df_d1),eval(model))<define_search_model>
df_submit = df_submit.merge(df_test, how='left', on='ForecastId' ).set_index('ForecastId', verify_integrity=True )
modelo_final = treino_sampling(dropa_trata(df),model=xgb )<save_to_csv>
df_submit.to_csv('submission.csv', index=True )