Kaggle competition: Titanic - Machine Learning from Disaster
for col in country_meta_columns:
country_data[col] =(
 country_data[col] 
.fillna(-1000)
.apply(lambda v: v.replace(",", ".")if isinstance(v, str)else v)
.astype(np.float32)
 )<load_from_csv>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
lockdown_meta_df = pd.read_csv(".. /input/covid19-lockdown-dates-by-country/countryLockdowndates.csv")
 lockdown_meta_df.rename(
 columns={
"Country/Region": "Country_Region",
"Province": "Province_State",
"Date": "LockdownDate",
"Type": "LockdownType"
},
inplace=True 
)
 lockdown_meta_df.drop("Reference", axis=1, inplace=True)
 def convert_lockdown_date(d):
if isinstance(d, str):
return "-".join(reversed(d.split("/")))
 else:
return "2020-12-31"
lockdown_meta_df.LockdownDate = lockdown_meta_df.LockdownDate.apply(convert_lockdown_date )<merge>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )
train_df = train_df.merge(lockdown_meta_df, on=["Province_State", "Country_Region"], how="left")
 train_df = train_df.merge(country_data, on=["Country_Region"], how="left" )<categorify>
df_train['Survived'].value_counts()
train_df["days_since_lockdown"] = np.clip(
(pd.to_datetime(train_df.Date)- pd.to_datetime(train_df.LockdownDate)).dt.days,
a_min=-1,
a_max=None 
)
 train_df["lockdown_type"] = [
t if d >= 0 else "None" for t, d in zip(train_df.LockdownType, train_df.days_since_lockdown)
 ]
lockdown_type_encoder = LabelEncoder().fit(train_df.lockdown_type)
 train_df["lockdown_type"] = lockdown_type_encoder.transform(train_df.lockdown_type )<feature_engineering>
df_train[['Pclass','Survived']].groupby(['Pclass'] ).mean()
train_df["ConfirmedCases"] = np.log1p(train_df.ConfirmedCases)
 train_df["Fatalities"] = np.log1p(train_df.Fatalities )<categorify>
df_test['Survived'] = 0
train_test = df_train.append(df_test )
def extract_region(df):
return(
 df.Country_Region +
df.Province_State.fillna("" ).apply(lambda s: " + " + s if s else s)
)
 region_encoder = LabelEncoder().fit(extract_region(train_df))
train_df["region_id"] = region_encoder.transform(extract_region(train_df))
test_df["region_id"] = region_encoder.transform(extract_region(test_df))
region_metadata["region_id"] = region_encoder.transform(extract_region(region_metadata))
region_metadata.drop(["Province_State", "Country_Region"], axis=1, inplace=True )<merge>
train_test = pd.get_dummies(train_test,columns=['Pclass'])
 train_test.head()
stats_df = train_df[["Fatalities", "ConfirmedCases", "region_id"]].groupby("region_id" ).sum()
stats_df["fatalities_to_cases"] = stats_df.Fatalities - stats_df.ConfirmedCases
stats_df.drop(["Fatalities", "ConfirmedCases"], axis=1, inplace=True)
 train_df = train_df.merge(stats_df, on="region_id", how="left")
 stats_columns = ["fatalities_to_cases"]<categorify>
train_test['Sex'] = pd.factorize(train_test['Sex'])[0]
train_test.head()
def extract_country_from_region(region):
if "+" in region:
country, state = region.split(" + ")
 else:
country = region
return country
region_to_country = dict(
 zip(
 range(len(region_encoder.classes_)) ,
map(extract_country_from_region, region_encoder.classes_)
)
 )<merge>
train_test['SibSp_Parch'] = train_test['SibSp'] + train_test['Parch']
train_test = train_test.drop(['SibSp', 'Parch'], axis=1)
 train_test.head()
train_df = train_df.merge(region_metadata, on="region_id", how="left" )<define_variables>
train_test = pd.get_dummies(train_test,columns=["Embarked"])
 train_test.head()
meta_columns = [
"density",
"population",
"area"
]<define_variables>
train_test.loc[train_test["Age"].isnull() ,"age_nan"] = 1
train_test.loc[train_test["Age"].notnull() ,"age_nan"] = 0
train_test = pd.get_dummies(train_test,columns=['age_nan'])
 train_test.head()
categorical_columns = [
"region_id",
"lockdown_type"
]<feature_engineering>
train_test = train_test.drop(['Cabin', 'Ticket', 'Name', 'PassengerId'], axis=1)
 train_test.head()
def compute_historical_features(df, target_column, past_horizon, create_target=True):
features = dict()
for col in meta_columns + country_meta_columns:
features[col] = df[col].unique().item()
for col in geo_location_columns:
features[col] = df[col].unique().item()
for col in stats_columns:
features[col] = df[col].unique().item()
features["region_id"] = df["region_id"].unique().item()
last = -1 if create_target else len(df)
 target_values = df[target_column].values
for lag in range(past_horizon):
features["value_{}_{}".format(target_column, lag)] = target_values[last-lag-1]
features["days_since_lockdown"] = df.days_since_lockdown.max()
features["lockdown_type"] = df.lockdown_type.values[last-1]
if create_target:
features[target_column] = df[target_column].values[-1]
return features<sort_values>
train_test = train_test.fillna(0 )
def prepare_datasets(
 train_df,
last_train_date,
target_column,
train_start_offset=50,
step=1, 
):
dates = train_df.Date.unique()
train_dates = dates[dates <= last_train_date]
val_dates = dates[dates > last_train_date]
train_features = []
train_subdf = train_df[train_df.Date <= last_train_date]
val_subdf = train_df[train_df.Date > last_train_date]
for region_id in tqdm(sorted(train_df.region_id.unique()), desc="Making train features"):
subdf = train_subdf[train_subdf.region_id == region_id]
for date in train_dates[train_start_offset::step]:
features = compute_historical_features(
 subdf[subdf.Date <= date],
target_column=target_column,
past_horizon=train_start_offset,
create_target=True 
)
 train_features.append(features)
 return pd.DataFrame(train_features), train_subdf, val_subdf<predict_on_test>
train_data = train_test[:891]
test_data = train_test[891:]
train_data_X = train_data.drop(['Survived'],axis=1)
 train_data_Y = train_data['Survived']
test_data_X = test_data.drop(['Survived'],axis=1 )
def autoregressive_predict(model, dates, df, target_column, past_horizon):
n_regions = len(region_encoder.classes_)
 original_size = len(df)
 future_df = pd.DataFrame({
"Date": np.repeat(dates, repeats=n_regions),
"region_id": np.tile(np.arange(n_regions), len(dates))
})
 future_df["Country_Region"] = future_df.region_id.map(region_to_country)
 combined_df = pd.concat([
df.drop(meta_columns + country_meta_columns + geo_location_columns + stats_columns, axis=1),
future_df
], sort=True)
 combined_df = combined_df.merge(region_metadata, on="region_id", how="left")
 combined_df = combined_df.merge(country_data, on=["Country_Region"], how="left")
 combined_df = combined_df.merge(stats_df, on="region_id", how="left")
 for date in tqdm(dates, "Prediction"):
date_features = []
for region_id in range(len(region_encoder.classes_)) :
cond =(combined_df.Date < date)&(combined_df.region_id == region_id)
 features = compute_historical_features(
 combined_df[cond],
target_column=target_column,
past_horizon=past_horizon,
create_target=False 
)
 date_features.append(features)
 date_predictions = model.predict(pd.DataFrame(date_features))
combined_df.loc[combined_df.Date == date, target_column] = date_predictions
return combined_df[["Date", "region_id", target_column]].loc[original_size:].reset_index(drop=True )<compute_test_metric>
clf = RandomForestClassifier(n_estimators=150,min_samples_leaf=2,max_depth=6,oob_score=True)
 clf.fit(train_data_X,train_data_Y)
 clf.oob_score_
df_test["Survived"] = clf.predict(test_data_X)
 result = df_test[['PassengerId','Survived']].set_index('PassengerId')
 result.to_csv('result1.csv' )