Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import random
from keras.models import Sequential
from keras.layers.core import Dense
from keras.optimizers import adam
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import MinMaxScaler
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.metrics import precision_score,confusion_matrix, recall_score, f1_score, roc_auc_score, accuracy_score, classification_report
class IonDataset(Dataset):
def __init__(self, data, labels=None, type='train', transform=None):
self.data = data
self.labels = labels
self.type = type
self.transform = transform
def __getitem__(self, i):
signal = self.data[i].astype(np.float32)
 if self.type == 'train':
label = self.labels[i].astype(np.int64)
 if self.transform is not None:
signal, label = self.transform(signal, label)
 return signal, label
else:
return signal
def __len__(self):
return len(self.data )<define_search_model>
train_titanic = pd.read_csv(".. /input/titanic/train.csv")
 real_test_titanic = pd.read_csv(".. /input/titanic/test.csv")
class SEModule(nn.Module):
def __init__(self, in_channels, reduction=4):
super().__init__()
self.conv1 = nn.Conv1d(in_channels, in_channels//reduction, kernel_size=1, padding=0)
 self.conv2 = nn.Conv1d(in_channels//reduction, in_channels, kernel_size=1, padding=0)
 def forward(self, x):
s = F.adaptive_avg_pool1d(x, 1)
 s = self.conv1(s)
 s = F.relu(s, inplace=True)
 s = self.conv2(s)
 x = x + torch.sigmoid(s)
 return x
class ConvBR1d(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=1, padding=0, dilation=1, stride=1, groups=1, is_activation=True):
super().__init__()
self.conv = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=padding, dilation=dilation, stride=stride, groups=groups, bias=False)
 self.bn = nn.BatchNorm1d(out_channels)
 self.is_activation = is_activation
if is_activation:
self.relu = nn.ReLU(inplace=True)
 def forward(self, x):
x = self.bn(self.conv(x))
if self.is_activation:
x = self.relu(x)
 return x
class SENextBottleneck(nn.Module):
def __init__(self, in_channels, out_channels, stride=1, groups=32, reduction=16, pool=None, is_shortcut=False):
super().__init__()
mid_channels = out_channels // 2
self.conv1 = ConvBR1d(in_channels, mid_channels, 1, 0, 1,)
 self.conv2 = ConvBR1d(mid_channels, mid_channels, 3, 1, 1, groups=groups)
 self.conv3 = ConvBR1d(mid_channels, out_channels, 1, 0, 1, is_activation=False)
 self.se = SEModule(out_channels, reduction)
 self.stride = stride
self.is_shortcut = is_shortcut
if is_shortcut:
self.shortcut = ConvBR1d(in_channels, out_channels, 1, 0, 1, is_activation=False)
 if stride > 1:
if pool == 'max':
self.pool = nn.MaxPool1d(stride, stride)
 elif pool == 'avg':
self.pool = nn.AvgPool1d(stride, stride)
 def forward(self, x):
s = self.conv1(x)
 s = self.conv2(s)
 if self.stride > 1:
s = self.pool(s)
 s = self.conv3(s)
 s = self.se(s)
 if self.is_shortcut:
if self.stride > 1:
x = F.avg_pool1d(x, self.stride, self.stride)
 x = self.shortcut(x)
 x = x + s
x = F.relu(x, inplace=True)
 return x
class Encoder(nn.Module):
def __init__(self, num_features=1):
super().__init__()
self.block0 = nn.Sequential(
 ConvBR1d(num_features, encoder_channels[0], kernel_size=5, stride=1, padding=2),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1), 
)
 self.block1 = nn.Sequential(
 SENextBottleneck(encoder_channels[0], encoder_channels[1], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[1], encoder_channels[1], stride=1, is_shortcut=False)for i in range(backbone[0])] 
)
 self.block2 = nn.Sequential(
 SENextBottleneck(encoder_channels[1], encoder_channels[2], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[2], encoder_channels[2], stride=1, is_shortcut=False)for i in range(backbone[1])] 
)
 self.block3 = nn.Sequential(
 SENextBottleneck(encoder_channels[2], encoder_channels[3], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[3], encoder_channels[3], stride=1, is_shortcut=False)for i in range(backbone[2])] 
)
 self.block4 = nn.Sequential(
 SENextBottleneck(encoder_channels[3], encoder_channels[4], stride=stride, is_shortcut=True, pool='avg'),
*[SENextBottleneck(encoder_channels[4], encoder_channels[4], stride=1, is_shortcut=False)for i in range(backbone[3])] 
)
 def forward(self, x):
x0 = self.block0(x)
 x1 = self.block1(x0)
 x2 = self.block2(x1)
 x3 = self.block3(x2)
 x4 = self.block4(x3)
 return [x0, x1, x2, x3, x4]
class SCSEModule(nn.Module):
def __init__(self, in_channels, reduction=16):
super().__init__()
self.cSE = nn.Sequential(
 nn.AdaptiveAvgPool1d(1),
nn.Conv1d(in_channels, in_channels // reduction, 1),
nn.ReLU(inplace=True),
nn.Conv1d(in_channels // reduction, in_channels, 1),
nn.Sigmoid() , 
)
 self.sSE = nn.Sequential(nn.Conv1d(in_channels, 1, 1), nn.Sigmoid())
 def forward(self, x):
return x * self.cSE(x)+ x * self.sSE(x)
 class DecoderBlock(nn.Module):
def __init__(self, in_channels, skip_channels, out_channels):
super().__init__()
self.conv1 = ConvBR1d(in_channels + skip_channels, out_channels, kernel_size=3, padding=1)
 self.conv2 = ConvBR1d(out_channels, out_channels, kernel_size=3, padding=1)
 def forward(self, x, skip=None):
x = F.interpolate(x, scale_factor=stride, mode="linear", align_corners=True)
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.conv1(x)
 x = self.conv2(x)
 return x
class Decoder(nn.Module):
def __init__(self):
super().__init__()
self.block4 = DecoderBlock(encoder_channels[-1], encoder_channels[-2], decoder_channels[0])
 self.block3 = DecoderBlock(decoder_channels[0], encoder_channels[-3], decoder_channels[1])
 self.block2 = DecoderBlock(decoder_channels[1], encoder_channels[-4], decoder_channels[2])
 self.block1 = DecoderBlock(decoder_channels[2], encoder_channels[-5], decoder_channels[3])
 def forward(self, xs):
x = self.block4(xs[4], xs[3])
 x = self.block3(x, xs[2])
 x = self.block2(x, xs[1])
 x = self.block1(x, xs[0])
 return x
class Unet(nn.Module):
def __init__(self, num_features=1, num_classes=11):
super().__init__()
self.num_features = num_features
self.num_classes = num_classes
self.encoder = Encoder(num_features=num_features)
 self.decoder = Decoder()
self.segmentation_head = nn.Conv1d(decoder_channels[-1], num_classes, kernel_size=1, padding=0, stride=1)
 def forward(self, x):
features = self.encoder(x)
 x = self.decoder(features)
 x = self.segmentation_head(x)
 return x<init_hyperparams>
train_titanic.isnull().sum()
class ClassificationMeter:

 def __init__(self, nCls, eps=1e-5, names=None):
self.nCls = nCls
self.names = names
self.eps = eps
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 self._measure = None
def clear(self):
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 def prepare_inputs(self, outputs, truths):

 if torch.is_tensor(outputs):
outputs = outputs.detach().cpu().numpy()
if torch.is_tensor(truths):
truths = truths.detach().cpu().numpy()
return outputs, truths
def update(self, preds, truths):
preds, truths = self.prepare_inputs(preds, truths)
 self.N += np.prod(truths.shape)
 for Cls in range(self.nCls):
true_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths == Cls))
true_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths != Cls))
false_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths != Cls))
false_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths == Cls))
self.table[Cls] += [true_positive, true_negative, false_positive, false_negative]
def measure(self):
return self._measure
def better(self, A, B):
return A > B
def write(self, writer, global_step, prefix=""):
writer.add_scalar(os.path.join(prefix, "Accuracy"), self.measure() , global_step)
 def report(self, each_class=False):
precisions = []
recalls = []
f1s = []
for Cls in range(self.nCls):
recall = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,3] + self.eps)
 precision = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,2] + self.eps)
 f1 = 2 * precision * recall /(precision + recall + self.eps)
 precisions.append(precision)
 recalls.append(recall)
 f1s.append(f1)
 total_TP = np.sum(self.table[:, 0])
 accuracy = total_TP/self.N
accuracy_mean_class = np.mean(precisions)
 macro_f1 = np.mean(f1s)
 self._measure = macro_f1
text = f"Macro F1 = {macro_f1:.4f}
"
text += f"\tOverall Accuracy = {accuracy:.4f}({total_TP}/{self.N})
 "
text += f"\tMean-class Accuracy = {accuracy_mean_class:.4f}
"
if each_class:
for Cls in range(self.nCls):
text += f"\tClass {str(Cls)+'('+self.names[Cls]+')' if self.names is not None else Cls}: precision = {precisions[Cls]:.3f} recall = {recalls[Cls]:.3f}
"
return text<prepare_x_and_y>
randomvalue = [i for i in range(age_sd,age_mean)]
for _ in range(train_titanic.isnull().sum().Age):
	number_to_insert = choice(randomvalue)
 	train_titanic['Age'].fillna(number_to_insert, inplace = True )
if TRAIN:
batch = 1; a = 500000*(batch-1); b = 500000*batch
batch = 2; c = 500000*(batch-1); d = 500000*batch
X_train_1s = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1s = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 3; a = 500000*(batch-1); b = 500000*batch
batch = 7; c = 500000*(batch-1); d = 500000*batch
X_train_1f = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1f = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 4; a = 500000*(batch-1); b = 500000*batch
batch = 8; c = 500000*(batch-1); d = 500000*batch
X_train_3 = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_3 = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 6; a = 500000*(batch-1); b = 500000*batch
batch = 9; c = 500000*(batch-1); d = 500000*batch
X_train_5 = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_5 = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 5; a = 500000*(batch-1); b = 500000*batch
batch = 10; c = 500000*(batch-1); d = 500000*batch
X_train_10 = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_10 = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
for X, y, num_classes, model_name, max_epoch in zip(
 [X_train_1s, X_train_1f, X_train_3, X_train_5, X_train_10],
[y_train_1s, y_train_1f, y_train_3, y_train_5, y_train_10],
np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'],
[200, 100, 100, 100, 100], 
):
X = X.reshape(-1, 1, time_step)
 y = y.reshape(-1, time_step)
 idx = np.arange(X.shape[0])
 kf = model_selection.KFold(5, shuffle=True, random_state=seed)
 train_idx, val_idx = list(kf.split(idx)) [fold]
print("model_name:", model_name)
 print("train dataset shape:", X[train_idx].shape, y[train_idx].shape)
 train_dataset = IonDataset(X[train_idx], y[train_idx], 'train', train_augment)
 valid_dataset = IonDataset(X[val_idx], y[val_idx], 'train', valid_augment)
 loaders = {
'train': DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True),
'valid': DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True),
}
model = Unet(num_classes=num_classes)
 loss_function = nn.CrossEntropyLoss()
optimizer = optim.AdamW(model.parameters() , lr=0.001, weight_decay=1e-5)
 scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=10, factor=0.1, min_lr=1e-8)
 metrics = [ClassificationMeter(num_classes),]
trainer = Trainer(device, workspace, model, optimizer, scheduler, loss_function, loaders, metrics,
model_name=model_name,
report_step_interval=-1, 
)
 trainer.train(max_epoch )<load_from_csv>
train_titanic.isnull().sum()
if PREDICT:
sub = pd.read_csv(".. /input/liverpool-ion-switching/sample_submission.csv", dtype={'time':str})
 predictors = {}
for num_classes, model_name in zip(
 np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'] 
):
if not ENSEMBLE:
models = [Unet(num_classes=num_classes),]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join(workspace, f"{model_name}_best.pth.tar"), 0)
 else:
models = [
Unet(num_classes=num_classes),
]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join("workspace/unet-wide-4k-f0", f"{model_name}_best.pth.tar"), 0)
 predictor.load_checkpoint(os.path.join("workspace/unet-wide-4k-f1", f"{model_name}_best.pth.tar"), 1)
 predictors[model_name] = predictor
for start, end, model_type in zip(
 np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])*100000,
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20])*100000,
['1s', '3', '5', '1s', '1f', '10', '5', '10', '1s', '3', '1s'], 
):
X = test.signal.values[start:end].reshape(-1, 1, time_step_test)
 test_dataset = IonDataset(X, type='test')
 test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 res = predictors[model_type].predict(test_loader)
 sub.iloc[start:end,1] = res
sub.to_csv("pred.csv", index=False)
 plt.figure(figsize=(20,5))
res = 1000
let = ['A','B','C','D','E','F','G','H','I','J']
plt.plot(range(0,test.shape[0],res),sub.open_channels[0::res])
 for i in range(5): plt.plot([i*500000,i*500000],[-5,12.5],'r')
 for i in range(21): plt.plot([i*100000,i*100000],[-5,12.5],'r:')
 for k in range(4): plt.text(k*500000+250000,10,str(k+1),size=20)
 for k in range(10): plt.text(k*100000+40000,7.5,let[k],size=16)
 plt.title('Test Data Predictions',size=16)
 plt.show()<import_modules>
print("The most frequent value in 'Embarked' column is :", train_titanic['Embarked'].value_counts().idxmax() )
import pandas as pd
import numpy as np
from cuml.ensemble import RandomForestRegressor
from sklearn.pipeline import make_pipeline
from sklearn.base import BaseEstimator, TransformerMixin
import cudf<prepare_x_and_y>
train_titanic['Embarked'] = train_titanic['Embarked'].fillna(train_titanic['Embarked'].value_counts().idxmax())
 train_titanic.isnull().sum()
class ShiftedFeatureMaker(BaseEstimator, TransformerMixin):
def __init__(self, periods=[1], column="signal", add_minus=False, fill_value=None, copy=True):
self.periods = periods
self.column = column
self.add_minus = add_minus
self.fill_value = fill_value
self.copy = copy
def fit(self, X, y):

 return self
def transform(self, X: pd.DataFrame, y=None):
periods = np.asarray(self.periods, dtype=np.int32)
 if self.add_minus:
periods = np.append(periods, -periods)
 X_transformed = X.copy() if self.copy else X
for p in periods:
X_transformed[f"{self.column}_shifted_{p}"] = X_transformed[self.column].shift(
 periods=p, fill_value=self.fill_value 
)
 X_cudf = cudf.from_pandas(X_transformed)
 return X_cudf
class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, columns=None):
self.columns = columns
def fit(self, X, y):

 return self
def transform(self, X: pd.DataFrame, y=None):
return X[[c for c in X.columns if c not in self.columns]]
def add_category(train, test):
train["category"] = 0
test["category"] = 0
train.loc[2_000_000:2_500_000-1, 'category'] = 1
train.loc[4_500_000:5_000_000-1, 'category'] = 1
test.loc[500_000:600_000-1, "category"] = 1
test.loc[700_000:800_000-1, "category"] = 1
train["category"] = train["category"].astype(np.float32)
 test ["category"] = test ["category"].astype(np.float32)
 return train, test
def read_input() :
train = pd.read_csv(".. /input/data-without-drift/train_clean.csv" ).sort_values("time" ).reset_index(drop=True)
 test = pd.read_csv(".. /input/data-without-drift/test_clean.csv" ).sort_values("time" ).reset_index(drop=True)
 train.signal = train.signal.astype('float32')
 train.open_channels = train.open_channels.astype('float32')
 test.signal = test.signal.astype('float32')
 return train, test
def save_submission(y_test):
submission = pd.read_csv(".. /input/liverpool-ion-switching/sample_submission.csv")
 submission["open_channels"] = np.asarray(y_test, dtype=np.int32)
 submission.to_csv("submission.csv", index=False, float_format="%.4f" )<train_model>
real_test_titanic.isnull().sum()
%%time
shifted_rfc = make_pipeline(
 ShiftedFeatureMaker(
 periods=range(1, 20),
add_minus=True,
fill_value=0 
),
ColumnDropper(
 columns=["open_channels", "time" ] 
),
RandomForestRegressor(
 n_estimators=150,
max_depth=19,
max_features=10,
split_algo=0,
bootstrap=False 
)
)
 train, test = read_input()
train, test = add_category(train, test)
 shifted_rfc.fit(train, cudf.from_pandas(train[['open_channels']] ).open_channels)
 open_channels = np.round(shifted_rfc.predict(test ).to_array() ).astype(np.int32)
 save_submission(open_channels )<install_modules>
age_mean_test = int(( round(real_test_titanic['Age'].mean() ,2)))
 age_sd_test = int(round(real_test_titanic['Age'].std() ,2))
randomvalue = [i for i in range(age_sd_test,age_mean_test)]
for _ in range(real_test_titanic.isnull().sum().Age):
number_to_insert = choice(randomvalue)
 real_test_titanic['Age'].fillna(number_to_insert, inplace = True )
!pip install tensorflow_addons==0.9.1<import_modules>
real_test_titanic.isnull().sum()
Add, AveragePooling1D, Multiply, GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN,
RepeatVector, Conv1D, MaxPooling1D, Concatenate, GlobalAveragePooling1D, UpSampling1D)
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline<categorify>
real_test_titanic['Fare'] = real_test_titanic['Fare'].fillna(int(real_test_titanic['Fare'].mean()))
def plot_cm(y_true, y_pred, title):
figsize=(14,14)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax)
 def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def lr_schedule(epoch):
if epoch < 40:
lr = LR
elif epoch < 20:
lr = LR / 10
else:
lr = LR / 100
return lr
def augment(X, y):
X = np.vstack(( X, np.flip(X, axis=1)))
 y = np.vstack(( y, np.flip(y, axis=1)))
 return X, y<categorify>
real_test_titanic.isnull().sum()
def Classifier(shape_):
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = wave_block(inp, 16, 3, 8)
 x = wave_block(x, 32, 3, 12)
 x = wave_block(x, 64, 3, 4)
 x = wave_block(x, 128, 3, 1)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
class Metric(Callback):
def __init__(self, model, callbacks, data):
super().__init__()
self.model = model
self.callbacks = callbacks
self.data = data
def on_train_begin(self, logs=None):
for callback in self.callbacks:
callback.on_train_begin(logs)
 def on_train_end(self, logs=None):
for callback in self.callbacks:
callback.on_train_end(logs)
 def on_epoch_end(self, batch, logs=None):
X_valid, y_valid = self.data[1][0], self.data[1][1]
y_valid = np.argmax(y_valid, axis=2 ).reshape(-1)
 valid_pred = np.argmax(self.model.predict(X_valid), axis=2 ).reshape(-1)
 valid_recall = recall_score(y_valid, valid_pred, average="macro")
 valid_precision = precision_score(y_valid, valid_pred, average="macro")
 valid_score = f1_score(y_valid, valid_pred, average="macro")
 logs['Valid_F1Macro'] = valid_score
logs['Valid_Recall'] = valid_recall
logs['Valid_Precision'] = valid_precision
print(f"Validation F1 Macro {valid_score:1.6f} Validation Recall {valid_recall:1.6f} Validation Precision {valid_precision:1.6f}")
 for callback in self.callbacks:
callback.on_epoch_end(batch, logs)
 gc.collect()<init_hyperparams>
total_passengers = train_titanic['Sex'].count()
total_males = train_titanic['Sex'].value_counts() ['male']
total_females = train_titanic['Sex'].value_counts() ['female']
survived_males = train_titanic.query('Survived==1')['Sex'].value_counts() ['male']
survived_females = train_titanic.query('Survived==1')['Sex'].value_counts() ['female']
survived_total = survived_males + survived_females
print("=" * 70)
 print("TRAINING SET ANALYSIS BASED ON GENDER")
 print("=" * 70)
 print("Total number of travellers : ",total_passengers)
 print("Percentage of males : ",(100 * total_males /total_passengers))
print("Percentage of females : ",(100 * total_females /total_passengers))
print("=" * 70)
 print("Total number of people survived : ", survived_total)
 print("Total number of survived males : ", survived_males)
 print("Total number of survived females : ", survived_females)
 print("=" * 70)
 print("Percentage of males survived : ", 100 *(survived_males/total_passengers))
print("Percentage of females survived : ", 100 *(survived_females/total_passengers))
print("=" * 70)
 print("Percentage of total people survived : ", 100 *(survived_total/total_passengers))
print("=" * 70)
 print("*" * 70)
 print("=" * 70)
 total_1stclass_passengers = train_titanic['Pclass'].value_counts() [1]
total_2ndclass_passengers = train_titanic['Pclass'].value_counts() [2]
total_3rdclass_passengers = train_titanic['Pclass'].value_counts() [3]
survived_1stclass_passengers = train_titanic.query('Survived==1')['Pclass'].value_counts() [1]
survived_2ndclass_passengers = train_titanic.query('Survived==1')['Pclass'].value_counts() [2]
survived_3rdclass_passengers = train_titanic.query('Survived==1')['Pclass'].value_counts() [3]
print("TRAINING SET ANALYSIS BASED ON PASSENGER CLASS")
 print("=" * 70)
 print("Total number of first class passengers : ", total_1stclass_passengers)
 print("Total number of second class passengers : ", total_2ndclass_passengers)
 print("Total number of third class passengers : ", total_3rdclass_passengers)
 print("=" * 70)
 print("Total number of survived first class passengers : ", survived_1stclass_passengers)
 print("Total number of survived second class passengers : ", survived_2ndclass_passengers)
 print("Total number of survived third class passengers : ", survived_3rdclass_passengers)
 print("Percentage of survived first class passengers : ", 100 * survived_1stclass_passengers/ total_passengers)
 print("Percentage of survived second class passengers : ", 100 * survived_2ndclass_passengers/ total_passengers)
 print("Percentage of survived first class passengers : ", 100 * survived_3rdclass_passengers/ total_passengers)
 print("=" * 70)
EPOCHS = 70
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.001
SPLITS = 5
SLIDE = 800
seed_everything(SEED )<load_from_csv>
train_titanic = train_titanic.drop(['Ticket', 'Cabin','Name'], axis=1)
 real_test_titanic = real_test_titanic.drop(['Ticket', 'Cabin','Name'], axis=1 )
train = pd.read_csv("/kaggle/input/remove-trends-giba/train_clean_giba.csv", usecols=["signal","open_channels"], dtype={'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv("/kaggle/input/remove-trends-giba/test_clean_giba.csv", usecols=["signal"], dtype={'signal': np.float32})
 train['group'] = np.arange(train.shape[0])//500_000
aug_df = train[train["group"] == 5].copy()
aug_df["group"] = 10
for col in ["signal", "open_channels"]:
aug_df[col] += train[train["group"] == 8][col].values
train = train.append(aug_df, sort=False ).reset_index(drop=True)
 del aug_df
gc.collect()<prepare_x_and_y>
train_survived = train_titanic['Survived']
train["mlp"] = np.load("/kaggle/input/into-the-wild-mlp-regression/mlp_reg.npz")['valid']
test["mlp"] = np.load("/kaggle/input/into-the-wild-mlp-regression/mlp_reg.npz")['test']
train["lgb"] = np.load("/kaggle/input/into-the-wild-lgb-regression/lgb_reg.npz")['valid']
test["lgb"] = np.load("/kaggle/input/into-the-wild-lgb-regression/lgb_reg.npz")['test']
Y_train_proba = np.load("/kaggle/input/into-the-wild-rfc-classification/rfc_clf.npz")["valid"]
Y_test_proba = np.load("/kaggle/input/into-the-wild-rfc-classification/rfc_clf.npz")["test"]
for i in range(11):
train[f"rfc_{i}"] = Y_train_proba[:, i]
test[f"rfc_{i}"] = Y_test_proba[:, i]
del Y_train_proba, Y_test_proba
gc.collect()<feature_engineering>
train_titanic['Sex'] = train_titanic['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 train_titanic['Embarked'] = train_titanic['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
for item in ['signal','mlp','lgb']:
if item in train.columns:
print(item)
 train_input_mean = train[item].mean()
train_input_sigma = train[item].std()
train[item]=(train[item] - train_input_mean)/ train_input_sigma
test[item] =(test[item] - train_input_mean)/ train_input_sigma
train['batch'] = train.groupby(train.index//GROUP_BATCH_SIZE, sort=False)['signal'].agg(['ngroup'] ).values
train['batch'] = train['batch'].astype(np.uint16)
 test['batch'] = test.groupby(test.index//100_000, sort=False)['signal'].agg(['ngroup'] ).values
test['batch'] = test['batch'].astype(np.uint16)
 feats = [col for col in train.columns if col not in ['index','batch', 'open_channels', 'time', 'group']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for col in feats:
col_mean = pd.concat([train[col], test[col]], axis=0 ).mean()
train[col] = train[col].fillna(col_mean)
 test[col] = test[col].fillna(col_mean)
 print(feats )<train_on_grid>
real_test_titanic['Sex'] = real_test_titanic['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 real_test_titanic['Embarked'] = real_test_titanic['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )
def run_cv_model_by_batch(train, test, n_splits, feats, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
oof_tta = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['batch']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['batch']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['batch']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('batch' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 del tr, group
gc.collect()
train = np.array(list(train.groupby('batch' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('batch' ).apply(lambda x: x[feats].values)))
 test_flip = np.flip(test, axis=1)
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
print("="*11,f'Training Started for Fold {n_fold+1}',"="*11,"
")
 train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
train_x, train_y = augment(train_x, train_y)
 gc.collect()
valid_x_flip = np.flip(valid_x, axis=1)
 shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cp = ModelCheckpoint(f"model_fold{n_fold+1}.h5", monitor='Valid_F1Macro', mode='max',save_best_only=True, verbose=1, period=1)
 cp.set_model(model)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 metric = Metric(model, [cp], [(train_x, train_y),(valid_x, valid_y)])
 model.fit(train_x, train_y,epochs = nn_epochs,
callbacks = [cb_lr_schedule, metric],
batch_size = nn_batch_size, verbose = 0,
validation_data =(valid_x, valid_y))
model.load_weights(f"model_fold{n_fold+1}.h5")
 preds_f = model.predict(valid_x)
 preds_f_tta = np.flip(model.predict(valid_x_flip), axis=1)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.F1 Macro : {f1_score_ :1.5f}')
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f + preds_f_tta, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.F1 Macro with TTA : {f1_score_ :1.5f}','
')
 oof_[val_orig_idx,:] += preds_f.reshape(-1, preds_f.shape[-1])
 oof_tta[val_orig_idx,:] += preds_f_tta.reshape(-1, preds_f.shape[-1])
 def sliding_predict(model, x):
pred = np.zeros(( x.shape[0], 100_000, 11))
for begin in range(0, 100_000, SLIDE):
end = begin + GROUP_BATCH_SIZE
pred[:, begin:end, :] += model.predict(x[:, begin:end, :])
 return pred
te_preds = sliding_predict(model, test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds
te_preds = np.flip(sliding_predict(model, test_flip), axis=1)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.OOF F1 Macro : {f1_score_:1.5f}')
 f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_ + oof_tta, axis = 1), average = 'macro')
 print(f'Training completed.OOF F1 Macro with TTA : {f1_score_:1.5f}')
 return preds_, oof_, oof_tta<compute_train_metric>
train_titanic = train_titanic.set_index('PassengerId')
 real_test_titanic = real_test_titanic.set_index('PassengerId' )
preds, oof, oof_tta = run_cv_model_by_batch(train, test, n_splits=5, feats=feats, nn_epochs=EPOCHS, nn_batch_size=NNBATCHSIZE )<compute_test_metric>
X = train_titanic.drop(['Survived'], axis = 1)
 y = train_titanic["Survived"]
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = 0.10)
 print("Dimension of Train data :", x_train.shape)
 print("Dimension of Test data :", x_test.shape )
oof_df = train[['signal','open_channels']].copy()
oof_df["oof"] = np.argmax(oof+oof_tta, axis=1)
 oof_df = oof_df.head(5000_000)
 gc.collect()
oof_f1 = f1_score(oof_df['open_channels'],oof_df['oof'],average = 'macro')
 oof_recall = recall_score(oof_df['open_channels'],oof_df['oof'],average = 'macro')
 oof_precision = precision_score(oof_df['open_channels'],oof_df['oof'],average = 'macro')
 oof_loss = log_loss(oof_df["open_channels"],(oof/2+oof_tta/2)[:5000_000])
 print(f"OOF F1 Macro Score: {oof_f1:.6f} - OOF Recall Score: {oof_recall:.6f} - OOF Precision Score: {oof_precision:.6f}, OOF Log Loss Score: {oof_loss:.6f} " )<concatenate>
scaler = MinMaxScaler()
x_train_scaled = scaler.fit_transform(x_train)
 x_test_scaled = scaler.transform(x_test )
np.savez_compressed(f'wavenet.npz',valid=oof, test=preds,tta=oof_tta )<save_to_csv>
mlp_model = MLPClassifier(hidden_layer_sizes=(150,150,150),activation ='relu', max_iter=500, alpha=0.0001,
solver='sgd', verbose=10, learning_rate = 'adaptive', momentum=0.9)
 mlp_model.fit(x_train_scaled, y_train)
 y_pred = mlp_model.predict(x_test_scaled)
sample_submission = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 sample_submission['open_channels'] = np.argmax(preds, axis=1 ).astype(int)
 sample_submission.to_csv(f'submission.csv', index=False, float_format='%.4f')
 print(sample_submission.open_channels.mean())
 display(sample_submission.head() )<load_from_csv>
confusion = confusion_matrix(y_test, y_pred)
 print('Confusion Matrix
', confusion)
 print('Accuracy: {:.2f}'.format(accuracy_score(y_test, y_pred)) )
df = pd.read_csv("/kaggle/input/remove-trends-giba/train_clean_giba.csv", usecols=["signal","open_channels"], dtype={'signal': np.float32, 'open_channels':np.int32})
 test_df = pd.read_csv("/kaggle/input/remove-trends-giba/test_clean_giba.csv", usecols=["signal"], dtype={'signal': np.float32})
 df.shape, test_df.shape<feature_engineering>
classifier = MLPClassifier()
parameter_space = {
'hidden_layer_sizes': [(50,50,50),(100,100,100),(150,150,150),(200,200,200)],
'activation': ['tanh', 'relu', 'logistic'],
'solver': ['sgd', 'adam'],
'alpha': [0.0001, 0.05],
'learning_rate': ['constant','adaptive'],
}
df['group'] = np.arange(df.shape[0])//500_000
aug_df = df[df["group"] == 5].copy()
aug_df["group"] = 10
for col in ["signal", "open_channels"]:
aug_df[col] += df[df["group"] == 8][col].values
df = df.append(aug_df, sort=False ).reset_index(drop=True)
 df.shape
del aug_df
gc.collect()<feature_engineering>
model = GridSearchCV(classifier, parameter_space, n_jobs=-1, cv=3)
 model.fit(x_train_scaled, y_train )
wavenet_oof =(np.load("/kaggle/input/into-the-wild-wavenet/wavenet.npz")["valid"] + np.load("/kaggle/input/into-the-wild-wavenet/wavenet.npz")["tta"])/2
wavenet_test = np.load("/kaggle/input/into-the-wild-wavenet/wavenet.npz")["test"]
for i in range(wavenet_oof.shape[1]):
df["prob_{}".format(i)] = wavenet_oof[:, i]
test_df["prob_{}".format(i)] = wavenet_test[:, i]
df["wave_pred"] = np.argmax(wavenet_oof, axis=1)
 test_df["wave_pred"] = np.argmax(wavenet_test, axis=1)
 df['noise'] = df['signal'].values - df['wave_pred'].values
test_df['noise'] = test_df['signal'].values - test_df['wave_pred'].values<normalization>
print('Best parameters calculated :', model.best_params_ )
def get_margin(x):
return np.log(x)
 M = get_margin(wavenet_oof)
 M_test = get_margin(wavenet_test )<import_modules>
predicted_y = model.predict(x_test_scaled)
f1_score(df["open_channels"], df["wave_pred"], average="macro" )<compute_test_metric>
confusion = confusion_matrix(y_test, predicted_y)
 print('Confusion Matrix after hyperparameter optimization
', confusion)
 print('Accuracy after hyperparameter optimization: {:.2f}'.format(accuracy_score(y_test, predicted_y)) )
log_loss(df["open_channels"], wavenet_oof )<feature_engineering>
layer1 = Dense(units = 10,activation = 'relu', input_dim = 7)
 layer2 = Dense(units = 15, activation = 'relu')
 layer3 = Dense(units = 2, activation = 'sigmoid')
 model = Sequential([layer1, layer2, layer3])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
 clf = model.fit(x_train_scaled, y_train, batch_size = 25, epochs = 300 )
NUM_FOLDS = 5
df["mg"] = df.index//100_000
test_df["mg"] = test_df.index//100_000
df["fold"] = df["mg"] % NUM_FOLDS<categorify>
predicted_on_test = model.predict_classes(x_test_scaled, batch_size = 25)
 confusion = confusion_matrix(y_test, predicted_on_test)
 print('Confusion Matrix
', confusion)
 print('Accuracy: {:.2f}'.format(accuracy_score(y_test, predicted_on_test)) )
for data in [df, test_df]:
y_time_since = np.empty(( data.shape[0], 11))
y_time_till = np.empty(( data.shape[0], 11))
y_pred = data["wave_pred"].values
for sec in range(data.shape[0]//100_000):
begin, end = sec*100_000,(sec+1)*100_000
last_seen = np.array([np.nan]*11)
 for index in range(begin, end):
y_time_since[index] = index - last_seen
last_seen[y_pred[index]] = index
last_seen = np.array([np.nan]*11)
 for index in reversed(range(begin, end)) :
y_time_till[index] = last_seen - index
last_seen[y_pred[index]] = index
for i in range(11):
f = "time_since_{}".format(i)
 data[f] = y_time_since[:, i]
data[f] = np.clip(data[f].fillna(np.inf), 0, 1_000)
 f = "time_till_{}".format(i)
 data[f] = y_time_till[:, i]
data[f] = np.clip(data[f].fillna(np.inf), 0, 1_000)
 test_df.sample(2 ).T<define_variables>
real_test_titanic_scaled = scaler.transform(real_test_titanic )
features = ["signal", "noise",
"prob_0", "prob_1", "prob_2", "prob_3", "prob_4", "prob_5", "prob_6", "prob_7", "prob_8", "prob_9", "prob_10"]
for i in range(11):
f = "time_since_{}".format(i)
 features.append(f)
 f = "time_till_{}".format(i)
 features.append(f )<prepare_x_and_y>
predicted_on_actual = model.predict_classes(real_test_titanic_scaled, batch_size = 25 )
<compute_test_metric><EOS> 
actual_test = pd.read_csv(".. /input/titanic/test.csv")
 for_submission = pd.DataFrame({"PassengerId": actual_test['PassengerId'],
"Survived":predicted_on_actual.astype(int)})
 for_submission.to_csv("prediction_file_by_arunjith.csv",index=False )