Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.neighbors import KNeighborsClassifier
df_plot = train_df.loc[: , ['Date', 'Country', 'ConfirmedCases', 'Fatalities']].groupby(['Date', 'Country'] ).max().reset_index()
df_plot.loc[:, 'Size'] = np.power(df_plot["ConfirmedCases"]+1,0.3)-1
fig = px.scatter_geo(df_plot,
locations="Country",
locationmode = "country names",
hover_name="Country",
color="ConfirmedCases",
animation_frame="Date",
size='Size',
projection="natural earth",
title="Global Spread of Covid-19",
width=1500, height=800)
 fig.layout.updatemenus[0].buttons[0].args[1]["frame"]["duration"] = 5
fig.show() 
<merge>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
data_leak = pd.merge(train_df,test_df, how='inner', on='Date')['Date'].unique().tolist()
data_leak.append('2020-04-01')
 data_leak.sort()
print("Both data sets contain the following dates: {}".format(data_leak))<feature_engineering>
train_len = len(train)
 test_copy = test.copy()
def create_features(df):
df['Day_num'] = le.fit_transform(df['Date'])
 df['Date'] = pd.to_datetime(df['Date'])
 df['Day'] = df['Date'].dt.day
df['Week'] = df['Date'].dt.week
df['Month'] = df['Date'].dt.month
df['DayOfWeek'] = df['Date'].dt.dayofweek
df['Country'] = le.fit_transform(df['Country'])
 country_dict = dict(zip(le.inverse_transform(df['Country']), df['Country']))
df['State'] = le.fit_transform(df['State'])
 state_dict = dict(zip(le.inverse_transform(df['State']), df['State']))
return df, country_dict, state_dict<feature_engineering>
total = train.append(test)
 total.isnull().sum()
df_all, country_dict, state_dict = create_features(df_all )<prepare_x_and_y>
total[total.Fare.isnull() ]
def train_test_split_extend(df,d,day,filter_col_confirmed,filter_col_fatalities):
df=df.loc[df['Day_num'] >= day]
df_train = df.loc[df['Day_num'] < d]
X_train = df_train
Y_train_1 = df_train['ConfirmedCases']
Y_train_2 = df_train['Fatalities']
X_train_1 = X_train.drop(columns=filter_col_fatalities ).drop(columns='ConfirmedCases')
 X_train_2 = X_train.drop(columns=filter_col_confirmed ).drop(columns='Fatalities')
 df_test = df.loc[df['Day_num'] == d]
x_test = df_test
x_test_1 = x_test.drop(columns=filter_col_fatalities ).drop(columns='ConfirmedCases')
 x_test_2 = x_test.drop(columns=filter_col_confirmed ).drop(columns='Fatalities')
 x_test.drop(['ConfirmedCases', 'Fatalities'], axis=1, inplace=True)
 X_train_1.drop('Id', inplace=True, errors='ignore', axis=1)
 X_train_1.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 X_train_2.drop('Id', inplace=True, errors='ignore', axis=1)
 X_train_2.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 x_test_1.drop('Id', inplace=True, errors='ignore', axis=1)
 x_test_1.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 x_test_2.drop('Id', inplace=True, errors='ignore', axis=1)
 x_test_2.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 return X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2<train_model>
total['Fare'].fillna(value = total[total.Pclass==3]['Fare'].median() , inplace = True )
def lin_reg(X_train, Y_train, x_test):
regr = linear_model.LinearRegression()
regr.fit(X_train, Y_train)
 pred = regr.predict(x_test)
 return regr, pred<drop_column>
titles = list(total.Title.unique())
 for title in titles:
age = total.groupby('Title')['Age'].median().loc[title]
total.loc[(total.Age.isnull())&(total.Title == title),'Age'] = age
def country_calculation(df_all,country,date,day):
df_country = df_all.copy()
df_country = df_country.loc[df_country['Date'] >= date]
df_country = df_country.loc[df_country['Country'] == country_dict[country]]
features = ['Id', 'State', 'Country','ConfirmedCases', 'Fatalities', 'Day_num']
df_country = df_country[features]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
filter_col_confirmed = [col for col in df_country if col.startswith('Confirmed')]
filter_col_fatalities= [col for col in df_country if col.startswith('Fataliti')]
filter_col = np.append(filter_col_confirmed, filter_col_fatalities)
 df_country[filter_col] = df_country[filter_col].apply(lambda x: np.log1p(x))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 start = df_country[df_country['Id']==-999].Day_num.min()
end = df_country[df_country['Id']==-999].Day_num.max()
for d in range(start,end+1):
X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2 = train_test_split_extend(df_country,d,day,filter_col_confirmed,filter_col_fatalities)
 regr_1, pred_1 = lin_reg(X_train_1, Y_train_1, x_test_1)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'ConfirmedCases'] = pred_1[0]
regr_2, pred_2 = lin_reg(X_train_2, Y_train_2, x_test_2)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'Fatalities'] = pred_2[0]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 print("Calculation done.")
 return df_country<drop_column>
total['Family_Size'] = total['Parch'] + total['SibSp']
def country_state_calculation(df_all,country, state, date,day):
df_country = df_all.copy()
df_country = df_country.loc[df_country['Date'] >= date]
df_country = df_country.loc[df_country['Country'] == country_dict[country] &(df_country['State']==state_dict[state])]
features = ['Id', 'State', 'Country','ConfirmedCases', 'Fatalities', 'Day_num']
df_country = df_country[features]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
filter_col_confirmed = [col for col in df_country if col.startswith('Confirmed')]
filter_col_fatalities= [col for col in df_country if col.startswith('Fataliti')]
filter_col = np.append(filter_col_confirmed, filter_col_fatalities)
 df_country[filter_col] = df_country[filter_col].apply(lambda x: np.log1p(x))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 start = df_country[df_country['Id']==-999].Day_num.min()
end = df_country[df_country['Id']==-999].Day_num.max()
for d in range(start,end+1):
X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2 = train_test_split_extend(df_country,d,day,filter_col_confirmed,filter_col_fatalities)
 regr_1, pred_1 = lin_reg(X_train_1, Y_train_1, x_test_1)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country])&(df_country['State'] == state_dict[state]), 'ConfirmedCases'] = pred_1[0]
regr_2, pred_2 = lin_reg(X_train_2, Y_train_2, x_test_2)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'Fatalities'] = pred_2[0]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 10))
df_country = lag_feature(df_country, 'Fatalities', range(1,8))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 print("Calculation done.")
 return df_country<data_type_conversions>
total['Last_Name'] = total['Name'].apply(lambda x: str.split(x, ",")[0])
 total['Fare'].fillna(total['Fare'].mean() , inplace=True)
 default_survival_rate = 0.5
total['Family_Survival'] = default_survival_rate
for grp, grp_df in total[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
total.loc[total['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
total.loc[total['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
total.loc[total['Family_Survival']!=0.5].shape[0] )
<install_modules>
for _, grp_df in total.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
total.loc[total['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
total.loc[total['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(total[total['Family_Survival']!=0.5].shape[0]))
!pip install pyramid.arima 
<load_from_csv>
total.Sex.replace({'male':0, 'female':1}, inplace = True)
 features = ['Survived','Pclass','Sex','Family_Size','Family_Survival','Fare_Bin','Age_Bin']
total = total[features]
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
train = total[:train_len]
x_train = train.drop(columns = ['Survived'])
 y_train = train['Survived'].astype(int)
 x_test = total[train_len:].drop(columns = ['Survived'] )
feature_day = [1,5,10,15,20,30,40,50,75,100,150,200,300,400,500,750,1000,1250,1500,1750,2000,2250,2500,2750,3000,3250,3500,3750,4000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]<prepare_x_and_y>
scaler = StandardScaler()
scaler.fit(x_train)
 x_train = scaler.transform(x_train)
 x_test = scaler.transform(x_test )
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
print(country + ' and ' + province)
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
if len(adjusted_y_train_confirmed)< 1:
adjusted_y_train_confirmed = np.zeros(3)
 else:
if len(adjusted_y_train_confirmed)< 2:
adjusted_y_train_confirmed = np.append(adjusted_y_train_confirmed,adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1],adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1])
 else:
if len(adjusted_y_train_confirmed)< 3:
adjusted_y_train_confirmed = np.append(adjusted_y_train_confirmed,adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1])
 else:
pass
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 if len(adjusted_y_train_fatalities)< 1:
adjusted_y_train_fatalities = np.zeros(3)
 else:
if len(adjusted_y_train_fatalities)< 2:
adjusted_y_train_fatalities = np.append(adjusted_y_train_fatalities,adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1],adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1])
 else:
if len(adjusted_y_train_fatalities)< 3:
adjusted_y_train_fatalities = np.append(adjusted_y_train_fatalities,adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1])
 else:
pass
model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data )<save_to_csv>
clf = KNeighborsClassifier()
params = {'n_neighbors':[6,8,10,12,14,16,18,20],
'leaf_size':list(range(1,50,5)) }
gs = GridSearchCV(clf, param_grid= params, cv = 5,scoring = "roc_auc",verbose=1)
 gs.fit(x_train, y_train)
 print(gs.best_score_)
 print(gs.best_estimator_)
 print(gs.best_params_ )
<prepare_x_and_y><EOS> 
preds = gs.predict(x_test)
 pd.DataFrame({'PassengerId': test_copy['PassengerId'], 'Survived': preds} ).to_csv('submission.csv', index = False)