Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<load_from_csv>
seed = 42
test_size = 0.2
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 workspace = "./unet"
backbone = [1, 1, 1, 1]
encoder_channels = np.array([64, 128, 256, 512, 1024])*2
decoder_channels = np.array([512, 256, 128, 64])*2
fold = 0
time_step = 4000
time_step_test = 10000
stride = 2
batch_size = 8
TRAIN = True
PREDICT = True
ENSEMBLE = False
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 print(device )
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )<prepare_output>
df_train = pd.read_csv(".. /input/liverpool-ion-switching/train.csv")
 df_test = pd.read_csv(".. /input/liverpool-ion-switching/test.csv" )
submission['label'] = predictions<save_to_csv>
def do_identity(image, label):
return image, label
def do_flip(image, label):
image = np.ascontiguousarray(np.flip(image, axis=1))
label = np.ascontiguousarray(np.flip(label))
return image, label
def train_augment(image, label):
for op in np.random.choice([
lambda image, label: do_identity(image, label),
lambda image, label: do_flip(image, label),
], 1):
image, label = op(image, label)
 return image, label
def valid_augment(image, label):
return image, label
submission.to_csv("submission.csv",index=False )<define_variables>
class IonDataset(Dataset):
def __init__(self, data, labels=None, type='train', transform=None):
self.data = data
self.labels = labels
self.type = type
self.transform = transform
def __getitem__(self, i):
signal = self.data[i].astype(np.float32)
 if self.type == 'train':
label = self.labels[i].astype(np.int64)
 if self.transform is not None:
signal, label = self.transform(signal, label)
 return signal, label
else:
return signal
def __len__(self):
return len(self.data )
IMG_SIZE = 28
N_CHANNELS = 1<load_from_csv>
class SEModule(nn.Module):
def __init__(self, in_channels, reduction=4):
super().__init__()
self.conv1 = nn.Conv1d(in_channels, in_channels//reduction, kernel_size=1, padding=0)
 self.conv2 = nn.Conv1d(in_channels//reduction, in_channels, kernel_size=1, padding=0)
 def forward(self, x):
s = F.adaptive_avg_pool1d(x, 1)
 s = self.conv1(s)
 s = F.relu(s, inplace=True)
 s = self.conv2(s)
 x = x + torch.sigmoid(s)
 return x
class ConvBR1d(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=1, padding=0, dilation=1, stride=1, groups=1, is_activation=True):
super().__init__()
self.conv = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=padding, dilation=dilation, stride=stride, groups=groups, bias=False)
 self.bn = nn.BatchNorm1d(out_channels)
 self.is_activation = is_activation
if is_activation:
self.relu = nn.ReLU(inplace=True)
 def forward(self, x):
x = self.bn(self.conv(x))
if self.is_activation:
x = self.relu(x)
 return x
class SENextBottleneck(nn.Module):
def __init__(self, in_channels, out_channels, stride=1, groups=32, reduction=16, pool=None, is_shortcut=False):
super().__init__()
mid_channels = out_channels // 2
self.conv1 = ConvBR1d(in_channels, mid_channels, 1, 0, 1,)
 self.conv2 = ConvBR1d(mid_channels, mid_channels, 3, 1, 1, groups=groups)
 self.conv3 = ConvBR1d(mid_channels, out_channels, 1, 0, 1, is_activation=False)
 self.se = SEModule(out_channels, reduction)
 self.stride = stride
self.is_shortcut = is_shortcut
if is_shortcut:
self.shortcut = ConvBR1d(in_channels, out_channels, 1, 0, 1, is_activation=False)
 if stride > 1:
if pool == 'max':
self.pool = nn.MaxPool1d(stride, stride)
 elif pool == 'avg':
self.pool = nn.AvgPool1d(stride, stride)
 def forward(self, x):
s = self.conv1(x)
 s = self.conv2(s)
 if self.stride > 1:
s = self.pool(s)
 s = self.conv3(s)
 s = self.se(s)
 if self.is_shortcut:
if self.stride > 1:
x = F.avg_pool1d(x, self.stride, self.stride)
 x = self.shortcut(x)
 x = x + s
x = F.relu(x, inplace=True)
 return x
class Encoder(nn.Module):
def __init__(self, num_features=1):
super().__init__()
self.block0 = nn.Sequential(
 ConvBR1d(num_features, encoder_channels[0], kernel_size=5, stride=1, padding=2),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1), 
)
 self.block1 = nn.Sequential(
 SENextBottleneck(encoder_channels[0], encoder_channels[1], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[1], encoder_channels[1], stride=1, is_shortcut=False)for i in range(backbone[0])] 
)
 self.block2 = nn.Sequential(
 SENextBottleneck(encoder_channels[1], encoder_channels[2], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[2], encoder_channels[2], stride=1, is_shortcut=False)for i in range(backbone[1])] 
)
 self.block3 = nn.Sequential(
 SENextBottleneck(encoder_channels[2], encoder_channels[3], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[3], encoder_channels[3], stride=1, is_shortcut=False)for i in range(backbone[2])] 
)
 self.block4 = nn.Sequential(
 SENextBottleneck(encoder_channels[3], encoder_channels[4], stride=stride, is_shortcut=True, pool='avg'),
*[SENextBottleneck(encoder_channels[4], encoder_channels[4], stride=1, is_shortcut=False)for i in range(backbone[3])] 
)
 def forward(self, x):
x0 = self.block0(x)
 x1 = self.block1(x0)
 x2 = self.block2(x1)
 x3 = self.block3(x2)
 x4 = self.block4(x3)
 return [x0, x1, x2, x3, x4]
class SCSEModule(nn.Module):
def __init__(self, in_channels, reduction=16):
super().__init__()
self.cSE = nn.Sequential(
 nn.AdaptiveAvgPool1d(1),
nn.Conv1d(in_channels, in_channels // reduction, 1),
nn.ReLU(inplace=True),
nn.Conv1d(in_channels // reduction, in_channels, 1),
nn.Sigmoid() , 
)
 self.sSE = nn.Sequential(nn.Conv1d(in_channels, 1, 1), nn.Sigmoid())
 def forward(self, x):
return x * self.cSE(x)+ x * self.sSE(x)
 class DecoderBlock(nn.Module):
def __init__(self, in_channels, skip_channels, out_channels):
super().__init__()
self.conv1 = ConvBR1d(in_channels + skip_channels, out_channels, kernel_size=3, padding=1)
 self.conv2 = ConvBR1d(out_channels, out_channels, kernel_size=3, padding=1)
 def forward(self, x, skip=None):
x = F.interpolate(x, scale_factor=stride, mode="linear", align_corners=True)
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.conv1(x)
 x = self.conv2(x)
 return x
class Decoder(nn.Module):
def __init__(self):
super().__init__()
self.block4 = DecoderBlock(encoder_channels[-1], encoder_channels[-2], decoder_channels[0])
 self.block3 = DecoderBlock(decoder_channels[0], encoder_channels[-3], decoder_channels[1])
 self.block2 = DecoderBlock(decoder_channels[1], encoder_channels[-4], decoder_channels[2])
 self.block1 = DecoderBlock(decoder_channels[2], encoder_channels[-5], decoder_channels[3])
 def forward(self, xs):
x = self.block4(xs[4], xs[3])
 x = self.block3(x, xs[2])
 x = self.block2(x, xs[1])
 x = self.block1(x, xs[0])
 return x
class Unet(nn.Module):
def __init__(self, num_features=1, num_classes=11):
super().__init__()
self.num_features = num_features
self.num_classes = num_classes
self.encoder = Encoder(num_features=num_features)
 self.decoder = Decoder()
self.segmentation_head = nn.Conv1d(decoder_channels[-1], num_classes, kernel_size=1, padding=0, stride=1)
 def forward(self, x):
features = self.encoder(x)
 x = self.decoder(features)
 x = self.segmentation_head(x)
 return x
train_df = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test_df = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')
 pred_df = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv' )<concatenate>
class ClassificationMeter:

 def __init__(self, nCls, eps=1e-5, names=None):
self.nCls = nCls
self.names = names
self.eps = eps
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 self._measure = None
def clear(self):
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 def prepare_inputs(self, outputs, truths):

 if torch.is_tensor(outputs):
outputs = outputs.detach().cpu().numpy()
if torch.is_tensor(truths):
truths = truths.detach().cpu().numpy()
return outputs, truths
def update(self, preds, truths):
preds, truths = self.prepare_inputs(preds, truths)
 self.N += np.prod(truths.shape)
 for Cls in range(self.nCls):
true_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths == Cls))
true_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths != Cls))
false_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths != Cls))
false_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths == Cls))
self.table[Cls] += [true_positive, true_negative, false_positive, false_negative]
def measure(self):
return self._measure
def better(self, A, B):
return A > B
def write(self, writer, global_step, prefix=""):
writer.add_scalar(os.path.join(prefix, "Accuracy"), self.measure() , global_step)
 def report(self, each_class=False):
precisions = []
recalls = []
f1s = []
for Cls in range(self.nCls):
recall = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,3] + self.eps)
 precision = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,2] + self.eps)
 f1 = 2 * precision * recall /(precision + recall + self.eps)
 precisions.append(precision)
 recalls.append(recall)
 f1s.append(f1)
 total_TP = np.sum(self.table[:, 0])
 accuracy = total_TP/self.N
accuracy_mean_class = np.mean(precisions)
 macro_f1 = np.mean(f1s)
 self._measure = macro_f1
text = f"Macro F1 = {macro_f1:.4f}
"
text += f"\tOverall Accuracy = {accuracy:.4f}({total_TP}/{self.N})
 "
text += f"\tMean-class Accuracy = {accuracy_mean_class:.4f}
"
if each_class:
for Cls in range(self.nCls):
text += f"\tClass {str(Cls)+'('+self.names[Cls]+')' if self.names is not None else Cls}: precision = {precisions[Cls]:.3f} recall = {recalls[Cls]:.3f}
"
return text
train_df = train_df.append(test_df )<train_model>
def five_type_remove_drift(train):
a=500000; b=600000
train.loc[train.index[a:b],'signal'] = train.signal[a:b].values - 3*(train.time.values[a:b] - 50)/10.
 def f(x,low,high,mid):
return -(( -low+high)/625)*(x-mid)**2+high -low
batch = 7; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,-1.817,3.186,325)
 batch = 8; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,-0.094,4.936,375)
 batch = 9; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,1.715,6.689,425)
 batch = 10; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,3.361,8.45,475)
 return train
MNIST like datatset for Kannada handwritten digits
print(f'Training set: {train_df.shape}')
 print(f'To be Predicted: {pred_df.shape}' )<prepare_x_and_y>
if TRAIN:
df_train = five_type_remove_drift(df_train)
 batch = 1; a = 500000*(batch-1); b = 500000*batch
batch = 2; c = 500000*(batch-1); d = 500000*batch
X_train_1s = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1s = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 3; a = 500000*(batch-1); b = 500000*batch
batch = 7; c = 500000*(batch-1); d = 500000*batch
X_train_1f = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1f = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 4; a = 500000*(batch-1); b = 500000*batch
batch = 8; c = 500000*(batch-1); d = 500000*batch
X_train_3 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_3 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 6; a = 500000*(batch-1); b = 500000*batch
batch = 9; c = 500000*(batch-1); d = 500000*batch
X_train_5 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_5 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 5; a = 500000*(batch-1); b = 500000*batch
batch = 10; c = 500000*(batch-1); d = 500000*batch
X_train_10 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_10 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
for X, y, num_classes, model_name, max_epoch in zip(
 [X_train_1s, X_train_1f, X_train_3, X_train_5, X_train_10],
[y_train_1s, y_train_1f, y_train_3, y_train_5, y_train_10],
np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'],
[30, 30, 60, 60, 120], 
):
X = X.reshape(-1, 1, time_step)
 y = y.reshape(-1, time_step)
 idx = np.arange(X.shape[0])
 kf = model_selection.KFold(5, shuffle=True, random_state=seed)
 train_idx, val_idx = list(kf.split(idx)) [fold]
print("model_name:", model_name)
 print("train dataset shape:", X[train_idx].shape, y[train_idx].shape)
 train_dataset = IonDataset(X[train_idx], y[train_idx], 'train', train_augment)
 valid_dataset = IonDataset(X[val_idx], y[val_idx], 'train', valid_augment)
 loaders = {
'train': DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True),
'valid': DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True),
}
model = Unet(num_classes=num_classes)
 loss_function = nn.CrossEntropyLoss()
optimizer = optim.AdamW(model.parameters() , lr=0.001, weight_decay=1e-5)
 scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=10, factor=0.1, min_lr=1e-8)
 metrics = [ClassificationMeter(num_classes),]
trainer = Trainer(device, workspace, model, optimizer, scheduler, loss_function, loaders, metrics,
model_name=model_name,
report_step_interval=-1, 
)
 trainer.train(max_epoch)
<split><EOS> 
if PREDICT:
start=500
a = 0; b = 100000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=510
a = 100000; b = 200000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=540
a = 400000; b = 500000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=560
a = 600000; b = 700000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=570
a = 700000; b = 800000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=580
a = 800000; b = 900000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 def f(x):
return -(0.00788)*(x-625)**2+2.345 +2.58
a = 1000000; b = 1500000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - f(df_test.time[a:b].values)
 sub = pd.read_csv(".. /input/liverpool-ion-switching/sample_submission.csv", dtype={'time':str})
 predictors = {}
for num_classes, model_name in zip(
 np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'] 
):
if not ENSEMBLE:
models = [Unet(num_classes=num_classes),]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join(workspace, f"{model_name}_best.pth.tar"), 0)
 else:
models = [
Unet(num_classes=num_classes),
]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join("workspace/unet-wide-4k-f0", f"{model_name}_best.pth.tar"), 0)
 predictors[model_name] = predictor
for start, end, model_type in zip(
 np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])*100000,
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20])*100000,
['1s', '3', '5', '1s', '1f', '10', '5', '10', '1s', '3', '1s'], 
):
X = df_test.signal.values[start:end].reshape(-1, 1, time_step_test)
 test_dataset = IonDataset(X, type='test')
 test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 res = predictors[model_type].predict(test_loader)
 sub.iloc[start:end,1] = res
sub.to_csv("pred.csv", index=False)
 plt.figure(figsize=(20,5))
res = 1000
let = ['A','B','C','D','E','F','G','H','I','J']
plt.plot(range(0,df_test.shape[0],res),sub.open_channels[0::res])
 for i in range(5): plt.plot([i*500000,i*500000],[-5,12.5],'r')
 for i in range(21): plt.plot([i*100000,i*100000],[-5,12.5],'r:')
 for k in range(4): plt.text(k*500000+250000,10,str(k+1),size=20)
 for k in range(10): plt.text(k*100000+40000,7.5,let[k],size=16)
 plt.title('Test Data Predictions',size=16)
 plt.show()