Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_train_metric>
print(os.listdir("./"))
train = pd.read_csv('.. /input/train.csv',index_col = "PassengerId")
 print(train.shape)
 train.head()
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
for i in range(X_c.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
gr_base.append(np.clip(np.diff(temp[temp > threshold])[-num_days:].mean() , 0, params['growth rate max']))
else:
gr_base.append(params['growth rate default'])
 gr_base = np.array(gr_base)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor'])**i
preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 30,
"last N days": 5,
"growth rate default": 0.20,
"growth rate max": 0.3,
"growth rate factor": -0.03,
}
preds_c = run_c(params, np.log(1+train_p_c.values)[:,:TRAIN_N])
<train_model>
test = pd.read_csv('.. /input/test.csv',index_col = "PassengerId")
 print(test.shape)
 test.head()
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.06, 0.4)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
delta = X_c[:, train_size+i-params['length']+1] - X_c[:, train_size+i-params['length']]
delta = params['absolute growth'] + delta*X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 8,
"absolute growth": 0.02,
"fatality_rate_lower": 0.035,
"fatality_rate_upper": 0.40,
}
preds_f = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f = np.log(1+preds_f)
<compute_test_metric>
%matplotlib inline
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<categorify>
pd.pivot_table(train, index = "Embarked", values = "Survived" )
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()<save_to_csv>
train.loc[train["Sex"] == "male", "enc_sex"] = 0
train.loc[train["Sex"] == "female", "enc_sex"] = 1
print(train.shape)
 train[["Sex","enc_sex"]].head()
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )<sort_values>
test.loc[test["Sex"] == "male", "enc_sex"] = 0
test.loc[test["Sex"] == "female", "enc_sex"] = 1
print(test.shape)
 test[["Sex","enc_sex"]].head()
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country/Region']}, {rec['Province/State']}")
<import_modules>
train["Emb_C"] = train["Embarked"] == "C"
train["Emb_S"] = train["Embarked"] == "S"
train["Emb_Q"] = train["Embarked"] == "Q"
print(train.shape)
 train[["Embarked","Emb_C","Emb_S","Emb_Q"]].head()
import numpy as np
import pandas as pd
import xgboost as xgb
from xgboost import plot_importance, plot_tree
from sklearn.metrics import mean_squared_error, mean_absolute_error
from google.cloud import bigquery<load_from_csv>
test["Emb_C"] = test["Embarked"] == "C"
test["Emb_S"] = test["Embarked"] == "S"
test["Emb_Q"] = test["Embarked"] == "Q"
print(test.shape)
 test[["Embarked","Emb_C","Emb_S","Emb_Q"]].head()
train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv" )<create_dataframe>
train[train["Fare"].isnull() ]
%%time
client = bigquery.Client()
dataset_ref = client.dataset("noaa_gsod", project="bigquery-public-data")
 dataset = client.get_dataset(dataset_ref)
 tables = list(client.list_tables(dataset))
table_ref = dataset_ref.table("stations")
 table = client.get_table(table_ref)
 stations_df = client.list_rows(table ).to_dataframe()
table_ref = dataset_ref.table("gsod2020")
 table = client.get_table(table_ref)
 twenty_twenty_df = client.list_rows(table ).to_dataframe()
stations_df['STN'] = stations_df['usaf'] + '-' + stations_df['wban']
twenty_twenty_df['STN'] = twenty_twenty_df['stn'] + '-' + twenty_twenty_df['wban']
cols_1 = ['STN', 'mo', 'da', 'temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']
cols_2 = ['STN', 'country', 'state', 'call', 'lat', 'lon', 'elev']
weather_df = twenty_twenty_df[cols_1].join(stations_df[cols_2].set_index('STN'), on='STN')
 weather_df.tail(10 )<feature_engineering>
test[test["Fare"].isnull() ]
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in train.index:
df = train.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.Id, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 train['closest_station'] = C
train = train.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 train.sort_values(by=['Id'], inplace=True)
 train.head()<feature_engineering>
test.loc[test["Fare"].isnull() , "fillinFare"] = 0
test.loc[test["Fare"].isnull() , ["Fare", "fillinFare"]]
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in test.index:
df = test.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.ForecastId, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 test['closest_station'] = C
test = test.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 test.sort_values(by=['ForecastId'], inplace=True)
 test.head()<data_type_conversions>
train.loc[train["Name"].str.contains("Mr"), "title"] = "Mr"
train.loc[train["Name"].str.contains("Miss"), "title"] = "Miss"
train.loc[train["Name"].str.contains("Mrs"), "title"] = "Mrs"
train.loc[train["Name"].str.contains("Master"), "title"] = "Master"
print(train.shape)
 train[["Name", "title"]].head(10 )
train["wdsp"] = pd.to_numeric(train["wdsp"])
 test["wdsp"] = pd.to_numeric(test["wdsp"] )<data_type_conversions>
train["Master"] = train["Name"].str.contains("Master")
 print(train.shape)
 train[["Name", "Master"]].head(20 )
train["fog"] = pd.to_numeric(train["fog"])
 test["fog"] = pd.to_numeric(test["fog"] )<drop_column>
test["Master"] = test["Name"].str.contains("Master")
 print(test.shape)
 test[["Name", "Master"]].head(20 )
X_train = train.drop(["Fatalities", "ConfirmedCases"], axis=1 )<define_variables>
test["Child"] = test["Age"] < 14
print(test.shape)
 test[["Age", "Child"]].head(10 )
countries = X_train["Country/Region"]<drop_column>
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
print(train.shape)
 train[["SibSp", "Parch", "FamilySize"]].head()
X_train = X_train.drop(["Id"], axis=1)
 X_test = test.drop(["ForecastId"], axis=1 )<data_type_conversions>
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1
print(test.shape)
 test[["SibSp", "Parch", "FamilySize"]].head()
X_train['Date']= pd.to_datetime(X_train['Date'])
 X_test['Date']= pd.to_datetime(X_test['Date'] )<rename_columns>
train["Single"] = train["FamilySize"] == 1
train["Middle"] =(train["FamilySize"] > 1)&(train["FamilySize"] < 5)
 train["Big"] = train["FamilySize"] >= 5
print(train.shape)
 train[["FamilySize", "Single", "Middle", "Big"]].head(10 )
X_train = X_train.set_index(['Date'])
 X_test = X_test.set_index(['Date'] )<feature_engineering>
test["Single"] = test["FamilySize"] == 1
test["Middle"] =(test["FamilySize"] > 1)&(test["FamilySize"] < 5)
 test["Big"] = test["FamilySize"] >= 5
print(test.shape)
 test[["FamilySize", "Single", "Middle", "Big"]].head(10 )
def create_time_features(df):

 df['date'] = df.index
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X<feature_engineering>
feature = ["Pclass", "enc_sex", "Emb_C", "Emb_S", "Emb_Q","fillinFare",
"Master","Child", "Single", "Middle", "Big"]
feature
create_time_features(X_train)
 create_time_features(X_test )<drop_column>
label = "Survived"
label
X_train.drop("date", axis=1, inplace=True)
 X_test.drop("date", axis=1, inplace=True )<load_from_csv>
model = DecisionTreeClassifier(max_depth=9, random_state=0)
 model
world_happiness_index = pd.read_csv(".. /input/world-bank-datasets/World_Happiness_Index.csv" )<groupby>
model.fit(X_train, y_train )
world_happiness_grouped = world_happiness_index.groupby('Country name' ).nth(-1 )<drop_column>
model.fit(X_train, y_train )
world_happiness_grouped.drop("Year", axis=1, inplace=True )<merge>
tree = export_graphviz(model,
feature_names=feature,
class_names=["Perish", "Survived"],
out_file=None)
 graphviz.Source(tree )
X_train = pd.merge(left=X_train, right=world_happiness_grouped, how='left', left_on='Country/Region', right_on='Country name')
 X_test = pd.merge(left=X_test, right=world_happiness_grouped, how='left', left_on='Country/Region', right_on='Country name' )<load_from_csv>
prediction = model.predict(X_test)
 print(prediction.shape)
 prediction[0:9]
malaria_world_health = pd.read_csv(".. /input/world-bank-datasets/Malaria_World_Health_Organization.csv" )<merge>
submission = pd.read_csv('.. /input/gender_submission.csv',index_col = "PassengerId")
 print(submission.shape)
 submission.tail(10 )
X_train = pd.merge(left=X_train, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country' )<drop_column>
submission["Survived"] = prediction
print(submission.shape)
 submission.tail(10 )
<load_from_csv><EOS> 
submission.to_csv("tree.csv" )