Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
dftestefinal=pd.merge(df_test,dffat,on=['Local','Date'],how='left')
 dftestefinal['Fatalities']=dftestefinal['ConfirmedCases']*dftestefinal['Mortalidade']
dftestefinal.tail()<sort_values>
warnings.filterwarnings("ignore" )
dff=dftestefinal.sort_values(by='ConfirmedCases',ascending=False)
 dff.head(20 )<merge>
train_data = pd.read_csv('.. /input/train.csv')
 train_data.head()
dfm = dfm.melt('Local', var_name='Date', value_name='Fatalities')
 dfat=pd.merge(dftestefinal,dfm[['Local','Fatalities','Date']],on=['Local','Date'],how='left',suffixes=('_predicted','_real'))
dfat['Fatalities_real'].fillna('Vazio',inplace=True)
 dfat['Fatalities']=np.where(dfat['Fatalities_real']=='Vazio',dfat['Fatalities_predicted'],dfat['Fatalities_real'])
 dfat[dfat['Local']=='Brazil']<data_type_conversions>
test_data = pd.read_csv('.. /input/test.csv')
 test_data.head()
submission=dfat[['ForecastId','ConfirmedCases','Fatalities']]
submission['ForecastId']=submission['ForecastId'].astype('int32')
 submission['Fatalities']=submission['Fatalities'].astype('float')
 print(submission.dtypes)
 submission.sample(10)
<count_missing_values>
sns.set(style="white", palette="muted", color_codes=True )
df_test[df_test['ConfirmedCases'].isna() ]<create_dataframe>
def get_one_hot(array):
return np.array(( array['Pclass'] == 1, array['Pclass'] == 2,
array['Pclass'] == 3, array['Sex'] == 'male',
array['Sex'] == 'female', array['SibSp'],
array['Parch'], array['Fare'],
array['Embarked'] == 'C', array['Embarked'] == 'Q',
array['Embarked'] == 'S')).swapaxes(0, 1 ).astype('float32' )
dftpronto=dfat.copy()<save_to_csv>
x_train = train_data[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']]
x_train.head()
submission.to_csv('submission.csv',index=None)
 submission.sample(10 )<set_options>
x_train = get_one_hot(x_train)
 x_train[:10]
np.random.seed(1337)
 base_folder = '/kaggle/input/'
plt.rcParams['figure.figsize'] = [15, 7]
ext_cols = ['LaborForceTotal', 'LaborForcePerCapita', 'DeathRate', 'AirTrafficPassengersTotal',
'AirTrafficPassengersPerCapita', 'HospitalBedDensity', 'Obesity', 'OldPeople',
'PhysiciansDensity', 'AlcoholConsumptionPerCapita', 'CigaretteConsumptionPerCapita']
pop_cols = ['Yearly change', 'Density', 'Land Area', 'Med.Age', 'Urban Pop', 'Population']
add_cols = ['DayNum', 'PreviousDay-0ExposedDensity']
data_base = base_folder + 'covid19-global-forecasting-week-4/'
df = pd.read_csv(data_base + 'train.csv' ).drop(columns=['Id'])
 df.rename(columns={'Province_State': 'Province/State', 'Country_Region': 'Country/Region'}, inplace=True)
 df['Province/State'].fillna('entire country', inplace=True)
 df['ConfirmedCasesDelta'] = df.groupby(['Country/Region', 'Province/State'])['ConfirmedCases'].diff().fillna(0)
 df['FatalitiesDelta'] = df.groupby(['Country/Region', 'Province/State'])['Fatalities'].diff().fillna(0)
 df['Date'] = pd.to_datetime(df['Date'] ).dt.date
day_zero = min(df['Date'])
 df['DayNum'] =(df['Date'] - day_zero ).apply(lambda x: int(x.days))
df<feature_engineering>
y_train = np.array(train_data['Survived'])
 y_train[:10]
def add_extra_features_from_previous_days(data_fr, tail_size=5):
cols_tmp = []
col_prefix = 'PreviousDay'
for i in range(0, tail_size):
col_cc = '{}-{}ConfirmedCases'.format(col_prefix, i)
 col_f = '{}-{}Fatalities'.format(col_prefix, i)
 data_fr[col_cc] = data_fr.groupby(['Country/Region', 'Province/State'])['ConfirmedCases'].shift(periods=i+1, fill_value=0)
 data_fr[col_f] = data_fr.groupby(['Country/Region', 'Province/State'])['Fatalities'].shift(periods=i+1, fill_value=0)
 data_fr[col_cc + 'Delta'] = data_fr.groupby(['Country/Region', 'Province/State'])[col_cc].diff().fillna(0)
 data_fr[col_f + 'Delta'] = data_fr.groupby(['Country/Region', 'Province/State'])[col_f].diff().fillna(0)
 cols_tmp += [col_cc, col_f, col_cc + 'Delta', col_f + 'Delta']
return cols_tmp<define_variables>
x_test = get_one_hot(x_test)
 x_test[:10]
TAIL = 50
previous_days_cols = add_extra_features_from_previous_days(df, TAIL)
 df<feature_engineering>
from mlxtend.classifier import StackingCVClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.impute import SimpleImputer
def special_day_CC(org_df, number_of_cc):
print('calculating for every Country & Province days passed from the first day when ConfirmedCases >= {}'.format(number_of_cc))
col_final = 'Day_CC{}'.format(number_of_cc)
 col = 'Day_CC{}_zero'.format(number_of_cc)
 org_df[col] = org_df.where(
(org_df['ConfirmedCases'] >= number_of_cc)& 
(( org_df['PreviousDay-0ConfirmedCases'] < number_of_cc)|(org_df['Date'] == day_zero)) 
)['DayNum']
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].ffill()
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].bfill()
day_num = org_df['DayNum'] - org_df[col] + 1
org_df[col_final] =(day_num - day_num.where(day_num<0 ).fillna(0)).fillna(0)
 return col, col_final
def special_day_F(org_df, number_of_f):
print('calculating for every Country & Province days passed from the first day when Fatalities >= {}'.format(number_of_f))
col_final = 'Day_F{}'.format(number_of_f)
 col = 'Day_F{}_zero'.format(number_of_f)
 org_df[col] = org_df.where(
(org_df['Fatalities'] >= number_of_f)& 
(( org_df['PreviousDay-0Fatalities'] < number_of_f)|(org_df['Date'] == day_zero)) 
)['DayNum']
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].ffill()
org_df[col] = org_df.groupby(['Country/Region', 'Province/State'])[col].bfill()
day_num = org_df['DayNum'] - org_df[col] + 1
org_df[col_final] =(day_num - day_num.where(day_num<0 ).fillna(0)).fillna(0)
 return col, col_final 
<load_from_csv>
imp = SimpleImputer(missing_values=np.nan, strategy='mean')
 imp = imp.fit(x_train)
 x_train_imp = imp.transform(x_train )
df_population = pd.read_csv(base_folder + 'worldpopulaton-ver2/all_population.csv', delimiter=';', decimal=',', na_values='N.A.')
 df_population['Urban Pop'] = df_population['Urban Pop'].fillna(100.0)
 df_population = pd.get_dummies(df_population, columns=['Continent'])
 continent_columns = []
for c in df_population.columns:
if 'Continent_' in c:
continent_columns.append(c)
 df_population<define_variables>
clf1 = RandomForestClassifier()
clf2 = GradientBoostingClassifier()
lr = LogisticRegression()
sclf = StackingCVClassifier(classifiers=[clf1, clf2], meta_classifier=lr )
countries_to_replace = [ 
('Czech Republic', 'Czechia'), 
('United States of America', 'US'), 
('Côte d'Ivoire(Ivory Coast)', 'Côte d'Ivoire'), 
('Korea(South)', 'Korea, South'), 
('Swaziland', 'Eswatini'), 
('Myanmar(Burma)', 'Burma'), 
('East Timor', 'Timor-Leste'), 
('Macedonia', 'North Macedonia'), 
('Cape Verde', 'Cabo Verde'), 
('Congo(Republic)', 'Congo(Brazzaville)'), 
('Congo(Democratic Republic)', 'Congo(Kinshasa)'), 
('Palestinian Territories', 'West Bank and Gaza'), 
('United Kingdom of Great Britain and Northern Ireland', 'United Kingdom'), 
('Vatican City', 'Holy See'), 
('Sao Tome & Principe', 'Sao Tome and Principe')
 ]
country_state_pairs_to_replace = [ 
(( 'Greenland', 'entire country'),('Denmark', 'Greenland')) , 
(( 'Anguilla', 'entire country'),('United Kingdom', 'Anguilla')) , 
(( 'Bermuda', 'entire country'),('United Kingdom', 'Bermuda')) , 
(( 'British Virgin Islands', 'entire country'),('United Kingdom', 'British Virgin Islands')) , 
(( 'Isle of Man', 'entire country'),('United Kingdom', 'Isle of Man')) , 
(( 'Turks and Caicos Islands', 'entire country'),('United Kingdom', 'Turks and Caicos Islands')) , 
(( 'Sint Maarten', 'entire country'),('Netherlands', 'Sint Maarten')) , 
(( 'Saint Pierre & Miquelon', 'entire country'),('France', 'Saint Pierre and Miquelon')) , 
(( 'Falkland Islands', 'entire country'),('United Kingdom', 'Falkland Islands(Malvinas)'))
]
csv_dir = base_folder + 'worldbymap-ver2/'
files = [
'labor_force',
'death_rate',
'air_traffic_passengers',
'hospital_bed_density',
'obesity',
'old_people',
'physicians_density',
'cigarettes',
'alcohol'
]
wbm = {}
for f in files:
wbm[f] = pd.read_csv(csv_dir + f + '.csv', delimiter=';', decimal=',', na_values='N.A.')
 for ctr in countries_to_replace:
wbm[f]['Country'] = wbm[f]['Country'].replace(ctr[0], ctr[1])
 for pair in country_state_pairs_to_replace:
ind =(wbm[f]['Country'] == pair[0][0])&(wbm[f]['State'] == pair[0][1])
 wbm[f].loc[ind, 'Country'] = pair[1][0]
wbm[f].loc[ind, 'State'] = pair[1][1]
wbm[files[0]]<merge>
param_test = {'randomforestclassifier__n_estimators': [10, 120],
'randomforestclassifier__max_depth': [2, 15],
'gradientboostingclassifier__n_estimators': [10, 120],
'gradientboostingclassifier__max_depth': [2, 15],
'gradientboostingclassifier__learning_rate' : [0.01, 0.1],
'meta_classifier__C': [0.1, 10.0]}
df_add = pd.DataFrame()
for dataset in wbm.keys() :
if df_add.shape ==(0, 0):
df_add = wbm[dataset].copy()
else:
df_add = pd.merge(df_add, wbm[dataset], on=['Country', 'State'], how='left')
 df_add.rename(columns={"Country": "Country/Region", "State": "Province/State"}, inplace=True)
 df_add<merge>
sclf.fit(x_train_imp, y_train )
df_external = pd.merge(df_population, df_add, on=['Country/Region', 'Province/State'], how='left')
 def fill_missing_percapita_values(dfr, feature_total, feature_percapita):
cond =(dfr[feature_percapita].isna())&(df_external[feature_total].notna())&(df_external['Population'].notna())
 ind = df_external[cond].index
df_external.loc[ind, feature_percapita] = df_external.loc[ind, feature_total] / df_external.loc[ind, 'Population'] * 100.0
fill_missing_percapita_values(df_external, 'LaborForceTotal', 'LaborForcePerCapita')
 fill_missing_percapita_values(df_external, 'AirTrafficPassengersTotal', 'AirTrafficPassengersPerCapita')
 df_external[pop_cols+ext_cols] = df_external[pop_cols+ext_cols].apply(lambda x: x.fillna(x.mean()),axis=0)
 df_external<merge>
x_test_imp = imp.transform(x_test )
df_pop = pd.merge(df, df_external, on=['Country/Region', 'Province/State'], how='left')
 df_pop<define_variables>
data = np.array([np.array(test_data['PassengerId']), sclf.predict(x_test_imp)] ).swapaxes(0, 1)
 results = pd.DataFrame(data, columns=['PassengerId', 'Survived'])
 results.set_index('PassengerId', inplace=True)
 results.head()
cond_ctry = [ 
(( df['Country/Region']=='Germany')&(df['Province/State']=='entire country'), 'blue'), 
(( df['Country/Region']=='China')&(df['Province/State']=='Hubei'), 'green'), 
(( df['Country/Region']=='Italy')&(df['Province/State']=='entire country'), 'cyan'), 
(( df['Country/Region']=='Spain')&(df['Province/State']=='entire country'), 'magenta'), 
(( df['Country/Region']=='Korea, South')&(df['Province/State']=='entire country'), 'gray'),
]
start_day = 'Day_F25'
cond_day = df[start_day]>0
feature = 'Fatalities'
for i, cnd in enumerate(cond_ctry):
my_df = df_pop[(cnd[0])&(cond_day)]
my_x = my_df[start_day]
my_y = my_df[feature]/my_df['PreviousDay-11ConfirmedCases']
chart_data = pd.DataFrame({
'x': my_x,
'y': my_y})
 sns.lineplot(x='x', y='y', data=chart_data, color=cnd[1] )<remove_duplicates>
results.to_csv('predict.csv' )