Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<predict_on_test>
%matplotlib inline
sns.set(style='white', context='notebook', palette='deep' )
y_pred = model.predict(test, num_iteration=model.best_iteration)
 y_pred = np.round(np.clip(y_pred, 0, 10)).astype(int )<save_to_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 IDtest = test["PassengerId"]
sub = pd.read_csv(BASE_PATH + 'sample_submission.csv')
 sub['open_channels'] = np.array(np.round(y_pred,0), np.int)
 sub.to_csv('submission.csv', index=False, float_format='%.4f')
 sub.head(10 )<import_modules>
train = train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
import os
import matplotlib.pyplot as plt
import glob
import numpy as np
import pandas as pd
import tensorflow as tf
from keras.layers import Dense, Dropout, Reshape, Conv1D, BatchNormalization, Activation, AveragePooling1D, GlobalAveragePooling1D, Lambda, Input, Concatenate, Add, UpSampling1D, Multiply, MaxPooling1D
from keras.models import Model
from keras.objectives import mean_squared_error
from keras import backend as K
from keras.losses import binary_crossentropy, categorical_crossentropy
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau,LearningRateScheduler
from keras.initializers import random_normal
from keras.optimizers import Adam, RMSprop, SGD
from keras.callbacks import Callback
from sklearn.metrics import cohen_kappa_score, f1_score
from sklearn.model_selection import KFold, train_test_split
from keras.constraints import unit_norm
from sklearn.manifold import TSNE<load_from_csv>
train_len = len(train)
 dataset = pd.concat(objs=[train, test], axis=0 ).reset_index(drop=True )
df_train = pd.read_csv(".. /input/liverpool-ion-switching/train.csv")
 df_test = pd.read_csv(".. /input/liverpool-ion-switching/test.csv")
 train_input = df_train["signal"].values.reshape(-1,5000,1)
 train_input_mean = train_input.mean()
train_input_sigma = train_input.std()
train_input =(train_input-train_input_mean)/train_input_sigma
test_input = df_test["signal"].values.reshape(-1,5000,1)
 test_input =(test_input-train_input_mean)/train_input_sigma
train_target = np.array([[i]*int(train_input.shape[0]/10)for i in range(10)] ).flatten()
idx = np.arange(train_input.shape[0])
 train_idx, val_idx = train_test_split(idx, random_state = 111,test_size = 0.2, stratify = train_target)
 val_input = train_input[val_idx]
train_input = train_input[train_idx]
val_target = train_target[val_idx]
train_target = train_target[train_idx]
print("train_input:{}, val_input:{}, train_target:{}, val_target:{}".format(train_input.shape, val_input.shape, train_target.shape, val_target.shape))<define_search_model>
dataset = dataset.fillna(np.nan)
 dataset.isnull().sum()
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def se_block(x_in, layer_n):
x = GlobalAveragePooling1D()(x_in)
 x = Dense(layer_n//8, activation="relu" )(x)
 x = Dense(layer_n, activation="sigmoid" )(x)
 x_out=Multiply()([x_in, x])
 return x_out
def resblock(x_in, layer_n, kernel, dilation, use_se=True):
x = cbr(x_in, layer_n, kernel, 1, dilation)
 x = cbr(x, layer_n, kernel, 1, dilation)
 if use_se:
x = se_block(x, layer_n)
 x = Add()([x_in, x])
 return x
def Classifier(input_shape=(None,1)) :
layer_n = 96
kernel_size = 7
depth = 3
input_layer = Input(input_shape)
 input_layer_1 = AveragePooling1D(5 )(input_layer)
 input_layer_2 = AveragePooling1D(25 )(input_layer)
 x = cbr(input_layer, layer_n, kernel_size, 1, 1)
 for i in range(depth):
x = resblock(x, layer_n, kernel_size, 1)
 out_0 = x
x = cbr(x, layer_n*2, kernel_size, 5, 1)
 for i in range(depth):
x = resblock(x, layer_n*2, kernel_size, 1)
 out_1 = x
x = Concatenate()([x, input_layer_1])
 x = cbr(x, layer_n*3, kernel_size, 5, 1)
 for i in range(depth):
x = resblock(x, layer_n*3, kernel_size, 1)
 out_2 = x
x = Concatenate()([x, input_layer_2])
 x = cbr(x, layer_n*4, kernel_size, 5, 1)
 for i in range(depth):
x = resblock(x, layer_n*4, kernel_size, 1)
 x = GlobalAveragePooling1D()(x)
 x = Dense(128, activation="linear" )(x)
 x = Lambda(lambda x: 3*x/tf.sqrt(tf.reduce_sum(x**2, axis=-1, keepdims=True)+1e-7))(x)
 out = Dense(10, activation="softmax", kernel_constraint = unit_norm() , name="out" )(x)
 model=Model(input_layer, out)
 return model
def augmentations(input_data, target_data):
if np.random.rand() <0.5:
input_data = input_data[::-1]
return input_data, target_data
def Datagen(input_dataset, target_dataset, batch_size, is_train=False):
x=[]
y=[]
count=0
idx = np.arange(len(input_dataset))
np.random.shuffle(idx)
 while True:
for i in range(len(input_dataset)) :
input_data = input_dataset[idx[i]]
target_data = target_dataset[idx[i]]
if is_train:
input_data, target_data = augmentations(input_data, target_data)
 x.append(input_data)
 y.append(target_data)
 count+=1
if count==batch_size:
x = np.array(x, dtype=np.float32)
 y = np.identity(10)[y].astype(np.float32)
 inputs = x
targets = y
x = []
y = []
count=0
yield inputs, targets
def model_fit(model, train_inputs, train_targets, val_inputs, val_targets, n_epoch, batch_size=32):
hist = model.fit_generator(
 Datagen(train_inputs, train_targets, batch_size, is_train=True),
steps_per_epoch = len(train_inputs)// batch_size,
epochs = n_epoch,
validation_data=Datagen(val_inputs, val_targets, batch_size),
validation_steps = len(val_inputs)// batch_size,
callbacks = [lr_schedule],
shuffle = False,
verbose = 1 
)
 return hist
def lrs(epoch):
if epoch<40:
lr = learning_rate
elif epoch<60:
lr = learning_rate/10
else:
lr = learning_rate/100
return lr<train_model>
dataset["Fare"] = dataset["Fare"].fillna(dataset["Fare"].median() )
K.clear_session()
model = Classifier()
learning_rate=0.0075
n_epoch=80
batch_size=64
lr_schedule = LearningRateScheduler(lrs)
 model.compile(loss=categorical_crossentropy,
optimizer=Adam(lr=learning_rate),
metrics=["accuracy"])
 hist = model_fit(model, train_input, train_target, val_input, val_target, n_epoch, batch_size )<predict_on_test>
dataset["Fare"] = dataset["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )
def tSNE_visualization(model, train_inputs, train_batch, test_inputs, test_batch):
inputs = np.concatenate(( train_inputs,test_inputs), axis=0)
 latent_model = Model(inputs=model.input, outputs=model.layers[-2].output)
 latent_features = latent_model.predict(inputs)
 tsne_model = TSNE(n_components=2, perplexity=30, n_iter=1000)
 pred = tsne_model.fit_transform(latent_features)
 train_pred = pred[:len(train_inputs)]
test_pred = pred[len(train_inputs):]
fig, ax = plt.subplots(2, 1, figsize=(7,14))
cmap = plt.get_cmap("tab10")
 for i in range(10):
ax[0].scatter(train_pred[train_batch==i,0], train_pred[train_batch==i,1], alpha=0.5, s=int(40), color=cmap(i), edgecolors=None, label="train_batch_No.{}".format(i))
ax[0].legend(bbox_to_anchor=(1.4,1), loc="upper right", borderaxespad=0, fontsize=12)
 ax[0].set_title('t-SNE Latent space -TRAIN-', fontsize=20)
 for i in range(4):
ax[1].scatter(test_pred[test_batch==i,0], test_pred[test_batch==i,1], alpha=0.5, s=int(40), color=cmap(i), edgecolors=None, label="test_batch_No.{}".format(i))
ax[1].legend(bbox_to_anchor=(1.4,1), loc="upper right", borderaxespad=0, fontsize=12)
 ax[1].set_title('t-SNE Latent space -TEST-', fontsize=20)
 plt.show()
test_batch = np.array([[i]*int(500000/5000)for i in range(4)] ).flatten()
tSNE_visualization(model, val_input, val_target, test_input, test_batch)
<feature_engineering>
dataset["Embarked"] = dataset["Embarked"].fillna("S" )
def add_feature(x):
x_ = np.roll(x[:,:,0],1)
 x_[:,0] = 0
x_ =(x_[:,:,np.newaxis] - x)
 x__ = np.roll(x[:,:,0],-1)
 x__[:,-1] = 0
x__ =(x__[:,:,np.newaxis] - x)
 x = np.concatenate(( x,x_,x__), axis=-1)
 return x
train_input = df_train["signal"].values.reshape(-1,5000,1)
 train_input = add_feature(train_input)
 train_input_mean = np.mean(train_input.reshape(len(df_train),-1), axis=0 ).reshape(1,1,-1)
 train_input_sigma =np.std(train_input.reshape(len(df_train),-1), axis=0 ).reshape(1,1,-1)
 train_input =(train_input-train_input_mean)/train_input_sigma
train_target = pd.get_dummies(df_train["open_channels"] ).values.reshape(-1,5000,11)
 test_input = df_test["signal"].values.reshape(-1,10000,1)
 test_input = add_feature(test_input)
 test_input =(test_input-train_input_mean)/train_input_sigma
batch_number = np.array([[i]*int(500000/5000)for i in range(10)] ).flatten()
idx = np.arange(train_input.shape[0])
 idx = [idx[i] for i in range(len(batch_number)) if batch_number[i] in [0,1,2,3,4,5,6,7]]
train_idx, val_idx = train_test_split(idx, random_state = 111,test_size = 0.2)
 val_input = train_input[val_idx]
train_input = train_input[train_idx]
val_target = train_target[val_idx]
train_target = train_target[train_idx]
print("train_input:{}, val_input:{}, train_target:{}, val_target:{}".format(train_input.shape, val_input.shape, train_target.shape, val_target.shape))<concatenate>
dataset["Sex"] = dataset["Sex"].map({"male": 0, "female":1} )
def aggregation_block(x_shallow, x_deep, deep_ch, out_ch):
x_deep = UpSampling1D(5 )(x_deep)
 x_deep = Conv1D(deep_ch, kernel_size=7, strides=1, padding="same" )(x_deep)
 x_deep = BatchNormalization()(x_deep)
 x_deep = Activation("relu" )(x_deep)
 x = Concatenate()([x_shallow, x_deep])
 x = Conv1D(out_ch, kernel_size=1, strides=1, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def aggregation(skip_connections,output_layer_n,name=""):
skip_connections_1=[]
n=len(skip_connections)
 m=0
for i in range(n):
x= cbr(skip_connections[i], output_layer_n, 1, 1, [1])
 skip_connections_1.append(x)
 x_0= cbr(skip_connections[0], output_layer_n, 1, 1, [1])
 x_0 = aggregation_block(x_0, skip_connections[1], output_layer_n, output_layer_n)
 x_1= cbr(skip_connections[1], output_layer_n, 1, 1, [1])
 x_1 = aggregation_block(x_1, skip_connections[2], output_layer_n, output_layer_n)
 x_0 = aggregation_block(x_0, x_1, output_layer_n, output_layer_n)
 x_2= cbr(skip_connections[2], output_layer_n, 1, 1, [1])
 skip_connections_out=[x_0,x_1,x_2]
return skip_connections_out
def MinPooling1D(x, stride):
x = Lambda(lambda x: -x )(x)
 x = MaxPooling1D(stride )(x)
 x = Lambda(lambda x: -x )(x)
 return x
def min_max_average_pooling(x_in, stride):
x_av = AveragePooling1D(stride )(x_in)
 x_max = MaxPooling1D(stride )(x_in)
 x_min = MinPooling1D(x_in, stride)
 x_out = Concatenate()([x_av,x_max,x_min])
 return x_out
def Unet(input_shape=(None,3)) :
layer_n = int(64*1.5)
 kernel_size = 7
depth = int(2*1.5)
 dilations = [1]
skip_connections = []
input_layer = Input(input_shape)
 input_layer_1 = min_max_average_pooling(input_layer, 5)
 input_layer_2 = min_max_average_pooling(input_layer, 25)
 input_layer_3 = min_max_average_pooling(input_layer, 125)
 x = cbr(input_layer, layer_n, kernel_size, 1, 1)
 for i in range(depth):
x = resblock(x, layer_n, kernel_size, dilations)
 out_0 = x
skip_connections.append(x)
 x = cbr(x, layer_n*2, kernel_size, 5, 1)
 for i in range(depth):
x = resblock(x, layer_n*2, kernel_size, dilations)
 out_1 = x
skip_connections.append(x)
 x = Concatenate()([x, input_layer_1])
 x = cbr(x, layer_n*3, kernel_size, 5, 1)
 for i in range(depth):
x = resblock(x, layer_n*3, kernel_size, dilations)
 out_2 = x
skip_connections.append(x)
 x = Concatenate()([x, input_layer_2])
 x = cbr(x, layer_n*4, kernel_size, 5, 1)
 for i in range(depth):
x = resblock(x, layer_n*4, kernel_size, dilations)
 x = Concatenate()([x, input_layer_3])
 x = cbr(x, layer_n*4, kernel_size, 1, 1)
 x = resblock(x, layer_n*4, kernel_size, dilations)
 skip_connections=aggregation(skip_connections,layer_n)
 x = UpSampling1D(5 )(x)
 x = Concatenate()([x, skip_connections[2]])
 x = cbr(x, layer_n*3, kernel_size, 1, 1)
 x = UpSampling1D(5 )(x)
 x = Concatenate()([x, skip_connections[1]])
 x = cbr(x, layer_n*2, kernel_size, 1, 1)
 x = UpSampling1D(5 )(x)
 x = Concatenate()([x, skip_connections[0]])
 x = cbr(x, layer_n, kernel_size, 1, 1)
 x = Conv1D(11, kernel_size=kernel_size, strides=1, padding="same" )(x)
 out = Activation("softmax", name="out" )(x)
 model=Model(input_layer, out)
 return model
def augmentations(input_data, target_data):
if np.random.rand() <0.5:
input_data = input_data[::-1]
target_data = target_data[::-1]
return input_data, target_data
def Datagen(input_dataset, target_dataset, batch_size, is_train=False):
x = []
y = []
count = 0
idx = np.arange(len(input_dataset))
np.random.shuffle(idx)
 while True:
for i in range(len(input_dataset)) :
input_data = input_dataset[idx[i]]
target_data = target_dataset[idx[i]]
if is_train:
input_data, target_data = augmentations(input_data, target_data)
 x.append(input_data)
 y.append(target_data)
 count += 1
if count==batch_size:
x = np.array(x, dtype=np.float32)
 y = np.array(y, dtype=np.float32)
 inputs = x
targets = y
x = []
y = []
count=0
yield inputs, targets
class macroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis=2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis=2 ).reshape(-1)
 f1_val = f1_score(self.targets, pred, average="macro")
 print("val_f1_macro_score: ", f1_val)
 def model_fit(model, train_inputs, train_targets, val_inputs, val_targets, n_epoch, batch_size=32):
hist = model.fit_generator(
 Datagen(train_inputs, train_targets, batch_size, is_train=True),
steps_per_epoch = len(train_inputs)// batch_size,
epochs = n_epoch,
validation_data=Datagen(val_inputs, val_targets, batch_size),
validation_steps = len(val_inputs)// batch_size,
callbacks = [lr_schedule, macroF1(model, val_inputs, val_targets)],
shuffle = False,
verbose = 1 
)
 return hist
def cos_lrs(epoch):
if epoch<200:
lr = learning_rate - learning_rate*(1-np.cos(np.pi*epoch/200)) /2 + learning_rate/200
else:
lr = learning_rate/200
return lr 
<train_model>
index_NaN_age = list(dataset["Age"][dataset["Age"].isnull() ].index)
 for i in index_NaN_age :
age_med = dataset["Age"].median()
age_pred = dataset["Age"][(( dataset['SibSp'] == dataset.iloc[i]["SibSp"])&(dataset['Parch'] == dataset.iloc[i]["Parch"])&(dataset['Pclass'] == dataset.iloc[i]["Pclass"])) ].median()
if not np.isnan(age_pred):
dataset['Age'].iloc[i] = age_pred
else :
dataset['Age'].iloc[i] = age_med
K.clear_session()
model = Unet()
n_epoch=230
batch_size=64
learning_rate=0.00075
lr_schedule = LearningRateScheduler(cos_lrs)
 model.compile(loss=categorical_crossentropy,
optimizer=Adam(lr=learning_rate),
metrics=["accuracy"])
 hist = model_fit(model, train_input, train_target, val_input, val_target, n_epoch, batch_size )<save_to_csv>
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in dataset["Name"]]
dataset["Title"] = pd.Series(dataset_title)
 dataset["Title"].head()
pred = np.argmax(( model.predict(test_input)+model.predict(test_input[:,::-1,:])[:,::-1,:])/2, axis=2 ).reshape(-1)
 df_sub = pd.read_csv(".. /input/liverpool-ion-switching/sample_submission.csv", dtype={'time':str})
 df_sub.open_channels = np.array(np.round(pred,0), np.int)
 df_sub.to_csv("submission.csv",index=False)
 print(df_sub.head() )<define_variables>
dataset["Title"] = dataset["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset["Title"] = dataset["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 dataset["Title"] = dataset["Title"].astype(int )
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 200
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'<import_modules>
dataset.drop(labels = ["Name"], axis = 1, inplace = True )
import gc
import os
from pathlib import Path
import random
import sys
from tqdm import tqdm_notebook as tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.core.display import display, HTML
from sklearn import preprocessing
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
from sklearn.metrics import mean_squared_error<set_options>
dataset["Fsize"] = dataset["SibSp"] + dataset["Parch"] + 1
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<feature_engineering>
dataset['Single'] = dataset['Fsize'].map(lambda s: 1 if s == 1 else 0)
 dataset['SmallF'] = dataset['Fsize'].map(lambda s: 1 if s == 2 else 0)
 dataset['MedF'] = dataset['Fsize'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 dataset['LargeF'] = dataset['Fsize'].map(lambda s: 1 if s >= 5 else 0 )
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_local(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )<load_from_disk>
dataset = pd.get_dummies(dataset, columns = ["Title"])
 dataset = pd.get_dummies(dataset, columns = ["Embarked"], prefix="Em" )
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 root_median = Path('.. /input/site-meter-median-bland-by-leak')
 df_groupNum_median = pd.read_pickle(root_median/'df_groupNum_median.pickle')
<merge>
dataset["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in dataset['Cabin'] ] )
building_meta_df = building_meta_df.merge(train_df[['building_id','meter']].drop_duplicates() , on='building_id' )<data_type_conversions>
dataset = pd.get_dummies(dataset, columns = ["Cabin"],prefix="Cabin" )
building_meta_df['groupNum_train'] = building_meta_df['site_id'].astype('int')*10 + building_meta_df['meter'].astype('int' )<filter>
Ticket = []
for i in list(dataset.Ticket):
if not i.isdigit() :
Ticket.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket.append("X")
 dataset["Ticket"] = Ticket
dataset["Ticket"].head()
train_df = train_df [ train_df['building_id'] != 1099 ]<feature_engineering>
dataset = pd.get_dummies(dataset, columns = ["Ticket"], prefix="T" )
building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count<feature_engineering>
dataset["Pclass"] = dataset["Pclass"].astype("category")
 dataset = pd.get_dummies(dataset, columns = ["Pclass"],prefix="Pc" )
en_holidays = holidays.England()
ir_holidays = holidays.Ireland()
ca_holidays = holidays.Canada()
us_holidays = holidays.UnitedStates()
def add_holiyday(df_weather):
en_idx = df_weather.query('site_id == 1 or site_id == 5' ).index
ir_idx = df_weather.query('site_id == 12' ).index
ca_idx = df_weather.query('site_id == 7 or site_id == 11' ).index
us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15' ).index
df_weather['IsHoliday'] = 0
df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))
df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))
df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))
df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))
holiday_idx = df_weather['IsHoliday'] != 0
df_weather.loc[holiday_idx, 'IsHoliday'] = 1
df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8 )<feature_engineering>
dataset.drop(labels = ["PassengerId"], axis = 1, inplace = True )
set_local(weather_train_df)
 add_holiyday(weather_train_df )<merge>
train = dataset[:train_len]
test = dataset[train_len:]
test.drop(labels=["Survived"],axis = 1,inplace=True )
train_df_black = train_df.copy()
train_df_black = train_df_black.merge(building_meta_df, on=['building_id', 'meter'], how='left')
 train_df_black = train_df_black.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df_black['black_count']=0
for bid in train_df_black.building_id.unique() :
df = train_df_black[train_df_black.building_id==bid]
for meter in df.meter.unique() :
dfm = df[df.meter == meter]
b =(dfm.meter_reading == 0 ).astype(int)
 train_df_black.loc[(train_df_black.building_id==bid)&(train_df_black.meter == meter), 'black_count'] = b.groupby(( ~b.astype(bool)).cumsum() ).cumsum()<merge>
train["Survived"] = train["Survived"].astype(int)
 Y_train = train["Survived"]
X_train = train.drop(labels = ["Survived"],axis = 1 )
train_df = train_df.merge(train_df_black[['timestamp','building_id','meter','black_count']], on=['timestamp','building_id','meter'])
 train_df = train_df[train_df['black_count'] < 24*black_day].drop('black_count', axis=1 )<set_options>
lr = LogisticRegression()
del train_df_black
gc.collect()<filter>
from sklearn.naive_bayes import GaussianNB
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<data_type_conversions>
gnb = GaussianNB()
( train_df[train_df.building_id == 954].meter_reading==0 ).astype(int ).plot()<filter>
from keras.models import Sequential
train_df = train_df.query('not(building_id == 954 & meter_reading == 0)')
 train_df = train_df.query('not(building_id == 1221 & meter_reading == 0)' )<feature_engineering>
from keras.layers import Dense,BatchNormalization,Dropout
funny_bids = [993, 1168, 904, 954, 778, 1021]
print('before', len(train_df))
if outlier:
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 0)&(train_df.meter_reading > 30000), 'meter_reading'] = 31921
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 1)&(train_df.meter_reading > 90000), 'meter_reading'] = 96545.5
train_df = train_df[(( train_df.building_id == 1168)&(train_df.meter == 0)&(train_df.meter_reading >10000)) == False]
train_df.loc[(train_df.building_id == 904)&(train_df.meter == 0)&(train_df.meter_reading >10000), 'meter_reading'] = 11306
train_df = train_df[(( train_df.building_id == 954)&(train_df.meter_reading >10000)) ==False]
if rescale:
train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading'] = train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading']/ 100
train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading'] = train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading']/ 1000
train_df = train_df.reset_index()
print('after', len(train_df))
gc.collect()<feature_engineering>
from keras import callbacks
train_df.loc[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0)]['meter_reading'] * 0.2931<feature_engineering>
model = Sequential()
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )<feature_engineering>
model.add(Dense(32,activation = "relu",input_shape =(66,)))
 model.add(Dense(64,activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(1,activation = 'sigmoid'))
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["day"] = df["timestamp"].dt.day
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek 
<categorify>
model.compile(optimizer = "adam",loss ="binary_crossentropy",metrics = ['accuracy'])
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='acc', factor=0.2,patience=3, min_lr=0.0001)
 model.fit(X_train,Y_train,epochs = 30,callbacks = [reduce_lr] )
preprocess(train_df )<sort_values>
y = model.predict(test )
if use_ucf:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()<categorify>
def sig(y):
pred= []
for i in y:
if i > 0.5:
a = 1
else :
a = 0
pred.append(a)
 return pred
<count_missing_values>
y = sig(y)
 y
weather_train_df.isna().sum()<groupby>
ans2 = pd.DataFrame(IDtest )
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<groupby>
ans2["Survived"] = y
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<groupby>
ans2.to_csv("sub.csv",index = False )