Kaggle competition: Titanic - Machine Learning from Disaster
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<data_type_conversions>
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]<feature_engineering>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )<feature_engineering>
y_train = train.iloc[:, 1].values
year_map = building_meta_df.year_built.value_counts()
building_meta_df['year_cnt'] = building_meta_df.year_built.map(year_map)
 bid_map = train_df.building_id.value_counts()
train_df['bid_cnt'] = train_df.building_id.map(bid_map )<categorify>
extra_eda_cols = ['SibSp', 'Parch', 'Family_Size', 'Fare_Range', 'Alone']
train = train.drop(extra_eda_cols, axis = 1, inplace = False)
 train.head()
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()<feature_engineering>
extra_cols = ['PassengerId', 'Name', 'Ticket', 'Fare', 'Cabin']
train = train.drop(extra_cols, axis = 1, inplace = False)
 train.head()
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )<feature_engineering>
x_train = train.drop('Survived', axis = 1, inplace = False)
 print(x_train )
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique() :
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
 df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
 df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
 df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
 df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
 df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2 )<feature_engineering>
train.isnull().sum()
add_sg(weather_train_df )<define_variables>
train.isnull().sum()
category_cols = ['building_id', 'primary_use', 'groupNum_train']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr',
'sea_level_pressure',
'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72',
'wind_speed_mean_lag72',
'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3',
'sea_level_pressure_mean_lag3',
]<merge>
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
 imputer.fit(x_train[['Age']])
 x_train[['Age']]= imputer.transform(x_train[['Age']])
 imputers = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
 imputers.fit(x_train[['Embarked']])
 x_train[['Embarked']]= imputers.transform(x_train[['Embarked']] )
train_df = train_df.merge(building_meta_df, on=['building_id','meter'], how='left')
 train_df = train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df = reduce_mem_usage(train_df, use_float16=True)
 gc.collect()<prepare_x_and_y>
x_train.isnull().sum().any()
def create_X_y(train_df, groupNum_train):
target_train_df = train_df[train_df['groupNum_train'] == groupNum_train].copy()
X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train<train_model>
label_encoder = preprocessing.LabelEncoder()
x_train['Sex']= label_encoder.fit_transform(x_train['Sex'])
 x_train['Embarked']= label_encoder.fit_transform(x_train['Embarked'] )
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 50
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<choose_model_class>
sc = StandardScaler()
x_train = sc.fit_transform(x_train )
seed = 666
shuffle = False
kf = StratifiedKFold(n_splits=folds )<split>
test.isnull().sum().any()
for groupNum_train in building_meta_df['groupNum_train'].unique() :
X_train, y_train = create_X_y(train_df, groupNum_train=groupNum_train)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('groupNum_train', groupNum_train, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 exec('models' +str(groupNum_train)+ '=[]')
 train_df_site = train_df[train_df['groupNum_train']==groupNum_train].copy()
for train_idx, valid_idx in kf.split(train_df_site, train_df_site['building_id']):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df_site.iloc[valid_idx,:].month.unique()
print(mindex)
 print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
exec('models' +str(groupNum_train)+ '.append([mindex, model])')
 gc.collect()
if debug:
break
exec('oof' +str(groupNum_train)+ '= mean_squared_error(y_train, y_valid_pred_total)')
 try:
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 plt.show()
except:
pass
del X_train, y_train
gc.collect()
print('-------------------------------------------------------------' )<feature_engineering>
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
 imputer.fit(test[['Age']])
 test[['Age']]= imputer.transform(test[['Age']])
 imputers = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
 imputers.fit(test[['Embarked']])
 test[['Embarked']]= imputers.transform(test[['Embarked']] )
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather_test_df = weather_test_df.drop_duplicates(['timestamp', 'site_id'])
 set_local(weather_test_df)
 add_holiyday(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_sg(weather_test_df)
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 test_df['bid_cnt'] = test_df.building_id.map(bid_map)
 test_df = test_df.merge(building_meta_df[['building_id','meter','groupNum_train']], on=['building_id','meter'], how='left')
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
print(test_df.shape )<load_from_disk>
extra_cols_test = ['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Fare', 'Cabin']
test = test.drop(extra_cols_test, axis = 1, inplace = False)
 test.head()
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission)
 print(sample_submission.shape )<merge>
label_encoder = preprocessing.LabelEncoder()
test['Sex']= label_encoder.fit_transform(test['Sex'])
 test['Embarked']= label_encoder.fit_transform(test['Embarked'] )
def create_X(test_df, groupNum_train):
target_test_df = test_df[test_df['groupNum_train'] == groupNum_train].copy()
target_test_df = target_test_df.merge(building_meta_df, on=['building_id','meter','groupNum_train'], how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test<predict_on_test>
sc_x = StandardScaler()
test = sc_x.fit_transform(test )
def pred_all(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i,(mindex, model)in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def pred(X_test, models, batch_size=1000000):
if predmode == 'valid':
print('valid pred')
 return pred_valid(X_test, models, batch_size=1000000)
 elif predmode == 'train':
print('train pred')
 return pred_train(X_test, models, batch_size=1000000)
 else:
print('all pred')
 return pred_all(X_test, models, batch_size=1000000 )<feature_engineering>
classifier = LogisticRegression(random_state = 0)
 classifier.fit(x_train, y_train )
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118<feature_engineering>
y_pred = classifier.predict(test)
 acc_Tree = cross_val_score(classifier, x_train, y_train, cv=10, scoring='accuracy' ).mean()
acc_Tree
if clip0:
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0<save_to_csv>
classifier = KNeighborsClassifier(n_neighbors=5)
 classifier.fit(x_train, y_train)
 y_pred = classifier.predict(test )
if not debug:
sample_submission.to_csv('submission_no_leak.csv', index=False, float_format='%.4f' )<feature_engineering>
y_pred = classifier.predict(test)
 acc_Tree = cross_val_score(classifier, x_train, y_train, cv=10, scoring='accuracy' ).mean()
acc_Tree
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 print(leak_df.duplicated().sum())
 print(leak_df.meter.value_counts())
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']].drop_duplicates() , on='building_id', how='left' )<compute_test_metric>
classifier = DecisionTreeClassifier()
classifier.fit(x_train, y_train)
 y_pred = classifier.predict(test )
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))<feature_engineering>
y_pred = classifier.predict(test)
 acc_Tree = cross_val_score(classifier, x_train, y_train, cv=10, scoring='accuracy' ).mean()
acc_Tree
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<save_to_csv>
classifier = RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 0)
 classifier.fit(x_train, y_train )
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<compute_test_metric>
y_pred = classifier.predict(test)
 acc_Tree = cross_val_score(classifier, x_train, y_train, cv=10, scoring='accuracy' ).mean()
acc_Tree
print('LV score=', leak_score )<set_options>
classifier = SVC()
classifier.fit(x_train, y_train)
 y_pred = classifier.predict(test )
%matplotlib inline 
<load_from_disk>
y_pred = classifier.predict(test)
 acc_Tree = cross_val_score(classifier, x_train, y_train, cv=10, scoring='accuracy' ).mean()
acc_Tree
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/"building_metadata.feather" )<feature_engineering>
accuracy = {'Model' : ['Logistic Regression', 'K- Nearest Neighbor', 'SVC', 'Decision Tree', 'Random Forest'],
'Accuracy' : [0.7890, 0.8047, 0.8226, 0.7935, 0.8037]
}
all_cross_val_scores = pd.DataFrame(accuracy, columns = ['Model', 'Accuracy'])
 all_cross_val_scores.head()
<count_duplicates><EOS> 
test_df = pd.read_csv('.. /input/titanic/test.csv')
 submission = pd.DataFrame({
'PassengerId': test_df['PassengerId'],
'Survived': y_pred
})
 submission.to_csv('titanic_prediction.csv', index=False)
 print('File Saved' )