Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
%%time
!cp.. /input/rapids/rapids.0.13.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz
sys.path = ["/opt/conda/envs/rapids/lib"] + ["/opt/conda/envs/rapids/lib/python3.6"] + ["/opt/conda/envs/rapids/lib/python3.6/site-packages"] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_mean'] = test_df['building_id'].map(building_mean)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 test_df['building_min'] = test_df['building_id'].map(building_min)
 test_df['building_max'] = test_df['building_id'].map(building_max)
 test_df['building_std'] = test_df['building_id'].map(building_std)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<load_pretrained>
import cupy, cudf, cuml
from cuml.linear_model import LogisticRegression
from cuml.ensemble import RandomForestClassifier
from cuml.svm import SVC
import os
with open('.. /input/make-test-tuples/test_tuples.pickle', 'rb')as handle:
test_tuples = pickle.load(handle )<load_from_disk>
train = cudf.read_csv('/kaggle/input/titanic/train.csv')
 test = cudf.read_csv('/kaggle/input/titanic/test.csv' )
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )<merge>
train = train.drop(columns= ['Name','Ticket','Cabin'])
 test = test.drop(columns= ['Name','Ticket','Cabin'] )
def create_X(test_df, target_meter, site_id):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 target_test_df = target_test_df[target_test_df['site_id']==site_id]
X_test = target_test_df[feature_cols + category_cols]
return X_test, target_test_df['row_id']<predict_on_test>
train['Embarked_S'] =(train['Embarked'] == 'S' ).astype(int)
 train['Embarked_C'] =(train['Embarked'] == 'C' ).astype(int)
 train['Embarked_Q'] =(train['Embarked'] == 'Q' ).astype(int)
 train['Gender'] =(train['Sex'] == 'male' ).astype(int )
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total 
<prepare_x_and_y>
test['Embarked_S'] =(test['Embarked'] == 'S' ).astype(int)
 test['Embarked_C'] =(test['Embarked'] == 'C' ).astype(int)
 test['Embarked_Q'] =(test['Embarked'] == 'Q' ).astype(int)
 test['Gender'] =(test['Sex'] == 'male' ).astype(int )
y_tests = np.empty(( 4,16), dtype=list)
 y_tests_locs = np.empty(( 4,16), dtype=list )<feature_engineering>
train = train.drop(columns= ['Embarked','Sex',])
 test = test.drop(columns= ['Embarked','Sex',])
for tpl in test_tuples:
site_id = tpl[0]
target_meter = tpl[1]
sample_submission.loc[y_tests_locs[target_meter, site_id], 'meter_reading'] = \
np.expm1(y_tests[target_meter, site_id])
<save_to_csv>
train.fillna(0,inplace=True)
 test.fillna(0,inplace=True )
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<define_variables>
X = train.drop(columns = ['Survived'])
 y = train['Survived'].astype('int32' )
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 200
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'<import_modules>
model = RandomForestClassifier(n_estimators = 100, max_depth = 6)
 model.fit(X, y )
import gc
import os
from pathlib import Path
import random
import sys
from tqdm import tqdm_notebook as tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.core.display import display, HTML
from sklearn import preprocessing
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
from sklearn.metrics import mean_squared_error<set_options>
yhat_train = model.predict(X, predict_model = 'CPU')
 yhat_test = model.predict(test, predict_model = 'CPU' )
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<feature_engineering>
print(sum(y == yhat_train)/ len(y))
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_local(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )<load_from_disk>
submission = cudf.DataFrame({'PassengerId': test.PassengerId, 'Survived': yhat_test})
 submission.to_csv('submission.csv', index = False )