Kaggle competition: Digit Recognizer
df = pd.DataFrame({'id':names,'predicted':preds})
 df.to_csv('submission.csv',index=False )<load_from_zip>
tf.__version__
!mkdir -p /tmp/pip/cache/
!cp.. /input/segmentation/segmentation_models/efficientnet_pytorch-0.6.3.xyz /tmp/pip/cache/efficientnet_pytorch-0.6.3.tar.gz
!cp.. /input/segmentation/segmentation_models/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz
!cp.. /input/segmentation/segmentation_models/segmentation-models-pytorch-0.1.2.xyz /tmp/pip/cache/segmentation_models_pytorch-0.1.2.tar.gz
!cp.. /input/segmentation/segmentation_models/timm-0.1.20-py3-none-any.whl /tmp/pip/cache/
!cp.. /input/segmentation/segmentation_models/timm-0.2.1-py3-none-any.whl /tmp/pip/cache/
!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch
!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch<set_options>
from tensorflow.keras import layers
import tensorflow.keras.backend as K
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from tensorflow.keras.layers import Conv2D, Dense, Flatten, MaxPool2D, Dropout, Reshape, Input
from tensorflow.keras.losses import CategoricalCrossentropy
from tensorflow.keras.optimizers import Adam, Adadelta, Adagrad, Adamax, Nadam, RMSprop, Ftrl
warnings.filterwarnings("ignore")
<load_from_csv>
device = torch.device("cuda" )
sz = 512
reduce = 2
TH = 0.4
DATA = '.. /input/hubmap-kidney-segmentation/test/'
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 overlap = 230
DEVICE<categorify>
df_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 df_test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 df_sub = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 df_Id = df_sub["ImageId"]
mean = np.array([0.65459856,0.48386562,0.69428385])
 std = np.array([0.15167958,0.23584107,0.13146145])
 identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 def img2tensor(img,dtype:np.dtype=np.float32):
if img.ndim==2 : img = np.expand_dims(img,2)
 img = np.transpose(img,(2,0,1))
return torch.from_numpy(img.astype(dtype, copy=False))<define_search_space>
X = df_train.drop("label", axis=1 ).values
y = df_train["label"].values
X_test = df_test.values
X = X / 255
X_test = X_test / 255
X = X.reshape(( -1, 1, 28, 28))
X_test = X_test.reshape(( -1, 1, 28, 28))
model = [1,2,3]
[model[0],model[1]]<train_on_grid>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=123 )
WINDOW = 1024
DATA_PATH = '.. /input/hubmap-kidney-segmentation/'
p = pathlib.Path(DATA_PATH)
 numk = 0
for i, filename in enumerate(p.glob('test/*.tiff')) :
file = rasterio.open(filename.as_posix())
 slices = make_grid(file.shape, window=WINDOW, min_overlap=overlap)
 preds = np.zeros(file.shape, dtype=np.float16)
 print('staring predict test data:',filename)
 print(file.shape)
 print(slices)
 models = []
model = torch.load(f'.. /input/b7-models/best_0_7.pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b7-models/best_4.pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b7-models/best_3.pth')
 models.append(model)
 del model; gc.collect()
if file.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in file.subdatasets]
for(x1,x2,y1,y2)in slices:
if file.count == 3:
image = file.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(image.shape[1]//reduce,image.shape[0]//reduce),
interpolation = cv2.INTER_AREA)
 image = img2tensor(( image/255 - mean)/std)
 with torch.no_grad() :
image = image.to(DEVICE)
 score = model_pred([models[0]], image[None,...])
 score2 = model_pred([models[1]], torch.flip(image[None,...], [0, 3]))
score2 = torch.flip(score2[None,None,...], [3, 0])[0][0]
score3 = model_pred([models[2]], torch.flip(image[None,...], [1, 2]))
score3 = torch.flip(score3[None,None,...], [2, 1])[0][0]
score =(score + score2 + score3)/3
score = score.cpu()
score = cv2.resize(np.float32(score),(1024, 1024))
preds[x1:x2,y1:y2] = score*score/(preds[x1:x2,y1:y2] + score)+ preds[x1:x2,y1:y2]*preds[x1:x2,y1:y2]/(preds[x1:x2,y1:y2] + score)
 del score,score2,score3,image
del slices,models; gc.collect()
slices = make_grid(file.shape, window=WINDOW, min_overlap=32)
 models = []
model = torch.load(f'.. /input/0924model/best_0_924.pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_0(5 ).pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_4(1 ).pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_3(1 ).pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_1(1 ).pth')
 models.append(model)
 del model; gc.collect()
if file.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in file.subdatasets]
for(x1,x2,y1,y2)in slices:
if file.count == 3:
image = file.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(image.shape[1]//reduce,image.shape[0]//reduce),
interpolation = cv2.INTER_AREA)
 image = img2tensor(( image/255 - mean)/std)
 with torch.no_grad() :
image = image.to(DEVICE)
 score = model_pred([models[0]], image[None,...])
 score2 = model_pred([models[1], models[2]], torch.flip(image[None,...], [0, 3]))
score2 = torch.flip(score2[None,None,...], [3, 0])[0][0]
score3 = model_pred([models[3], models[4]], torch.flip(image[None,...], [1, 2]))
score3 = torch.flip(score3[None,None,...], [2, 1])[0][0]
score =(score+score2+score3)/3
score = score.cpu()
score = cv2.resize(np.float32(score),(1024, 1024))
preds[x1:x2,y1:y2] =(score + preds[x1:x2,y1:y2])/2
preds =(preds > TH ).astype(np.uint8)
 subm[i] = {'id':filename.stem, 'predicted': rle_numba_encode(preds)}
del preds; gc.collect()<save_to_csv>
t_X_train = torch.from_numpy(X_train ).float()
t_y_train = torch.from_numpy(y_train ).long()
t_X_val = torch.from_numpy(X_val ).float()
t_y_val = torch.from_numpy(y_val ).long()
dataset_train = TensorDataset(t_X_train, t_y_train)
 dataset_val = TensorDataset(t_X_val, t_y_val)
 loader_train = DataLoader(dataset_train, batch_size=20, shuffle=True)
 loader_val = DataLoader(dataset_val, batch_size=20 )
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv(f'submission.csv', index=False)
 submission
%reset -f out<install_modules>
activation_relu = torch.nn.ReLU()
class cnn(nn.Module):
def __init__(self):
super(cnn, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3, padding=1)
 self.conv2 = nn.Conv2d(32, 32, 3, padding=1)
 self.conv3 = nn.Conv2d(32, 32, 3, padding=1)
 self.conv4 = nn.Conv2d(32, 32, 3, padding=1)
 self.maxpool1 = nn.MaxPool2d(kernel_size=2, stride=2)
 self.conv5 = nn.Conv2d(32, 64, 3, padding=1)
 self.conv6 = nn.Conv2d(64, 64, 3, padding=1)
 self.conv7 = nn.Conv2d(64, 64, 3, padding=1)
 self.conv8 = nn.Conv2d(64, 64, 3, padding=1)
 self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2)
 self.flatten = nn.Flatten()
self.drop1 = nn.Dropout(p=0.5)
 self.fc1 = nn.Linear(7*7*64, 1024)
 self.drop2 = nn.Dropout(p=0.5)
 self.fc2 = nn.Linear(1024, 10)
 def forward(self, x):
x = activation_relu(self.conv1(x))
x = activation_relu(self.conv2(x))
x = activation_relu(self.conv3(x))
x = activation_relu(self.conv4(x))
x = activation_relu(self.maxpool1(x))
x = activation_relu(self.conv5(x))
x = activation_relu(self.conv6(x))
x = activation_relu(self.conv7(x))
x = activation_relu(self.conv8(x))
x = activation_relu(self.maxpool2(x))
x = self.flatten(x)
 x = self.drop1(x)
 x = activation_relu(self.fc1(x))
x = self.drop2(x)
 x = self.fc2(x)
 return x
cnn = cnn()
!pip install --no-deps.. /input/hubmap-packages/pretrained-models.pytorch-master/pretrained-models.pytorch-master > /dev/null<install_modules>
device = torch.device("cuda")
 cnn.to(device )
!pip install --no-deps.. /input/hubmap-packages/efficientnet_pytorch-0.6.3/efficientnet_pytorch-0.6.3 > /dev/null<install_modules>
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(cnn.parameters() , lr=0.0002 )
!pip install --no-deps.. /input/hubmap-packages/timm-0.4.5-py3-none-any.whl > /dev/null<install_modules>
train_history = []
val_history = []
for epoch in range(10):
total_train_loss = 0
total_val_loss = 0
total_train_acc = 0
total_val_acc = 0
total_train = 0
total_val = 0
for inputs, labels in loader_train:
inputs, labels = inputs.to(device), labels.to(device)
 cnn.train()
optimizer.zero_grad()
outputs = cnn(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
total_train_loss += loss
with torch.no_grad() :
y_pred = torch.argmax(cnn(inputs), dim=1)
 acc_train = int(( y_pred==labels ).sum())
 total_train_acc += acc_train
total_train += len(labels)
 for inputs, labels in loader_val:
inputs, labels = inputs.to(device), labels.to(device)
 cnn.eval()
outputs = cnn(inputs)
 loss = criterion(outputs, labels)
 total_val_loss += loss
with torch.no_grad() :
y_pred = torch.argmax(cnn(inputs), dim=1)
 total_acc_val = int(( y_pred==labels ).sum())
 total_val += len(labels)
 avg_train_acc = total_train_acc / total_train
avg_val_acc = total_acc_val / total_val
train_history.append([total_train_loss, total_train_acc])
 val_history.append([total_val_loss, total_val_acc])
 print(f"Epoch {epoch+1} ---> train_loss: {total_train_loss:.5f} | val_loss: {total_val_loss:.5f} || "\
f"train_acc {avg_train_acc:.5f} | val_acc {avg_val_acc:.5f}" )
!pip install --no-deps.. /input/hubmap-packages/segmentation_models.pytorch-master/segmentation_models.pytorch-master > /dev/null<set_options>
x = torch.tensor([[1, 2, 3, 4, 5],
[2, 2, 2, 2, 2]])
 y = torch.tensor([1, 3, 4, 4, 5])
 type(len(x))
warnings.filterwarnings("ignore" )<load_from_csv>
cnn1 = tf.keras.models.Sequential([Conv2D(input_shape=(28, 28, 1), filters=16, kernel_size=(3, 3), activation="relu", padding="same"),
Conv2D(filters=16, kernel_size=(3, 3), activation="relu", padding="same"),
Conv2D(filters=16, kernel_size=(3, 3), activation="relu", padding="same"),
Conv2D(filters=16, kernel_size=(3, 3), activation="relu", padding="same"),
MaxPool2D(pool_size=(2, 2), strides=(2, 2)) ,
Conv2D(filters=32, kernel_size=(3, 3), activation="relu", padding="same"),
Conv2D(filters=32, kernel_size=(3, 3), activation="relu", padding="same"),
Conv2D(filters=32, kernel_size=(3, 3), activation="relu", padding="same"),
Conv2D(filters=32, kernel_size=(3, 3), activation="relu", padding="same"),
MaxPool2D(pool_size=(2, 2), strides=(2, 2)) ,
Flatten() ,
Dense(units=1024, activation="relu"),
Dropout(rate=0.5),
Dense(units=10, activation="softmax")] )
sz = 512
step = 176
reduce = 2
TH = 0.4
S_TH = 0.55
DATA = '.. /input/hubmap-kidney-segmentation/test/'
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 bs = 8
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<categorify>
def seq_train_val(X, y, model, optimizer=Adam, validation_data=None, learning_rate=0.0001, loss="categorical_crossentropy", batch_size=10, epochs=40):
model.summary()
learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor="val_loss", mode="min", patiencs=5, verbose=1, factor=0.5, min_lr=0.000001)
 model.compile(optimizer=optimizer(learning_rate=learning_rate), loss=loss, metrics=["acc"])
 model.fit(X_train, y_train, validation_data=validation_data, batch_size=batch_size, verbose=2, epochs=epochs, callbacks=[learning_rate_reduction])
 return None
def predict(X, model):
y_pred = model.predict(X)
 y_pred = np.argmax(y_pred, axis=1)
 return y_pred
def enc2mask(encs, shape):
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for m,enc in enumerate(encs):
if isinstance(enc,np.float)and np.isnan(enc): continue
s = enc.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1 + m
return img.reshape(shape ).T
def mask2enc(mask, n=1):
pixels = mask.T.flatten()
encs = []
for i in range(1,n+1):
p =(pixels == i ).astype(np.int8)
 if p.sum() == 0: encs.append(np.nan)
 else:
p = np.concatenate([[0], p, [0]])
 runs = np.where(p[1:] != p[:-1])[0] + 1
runs[1::2] -= runs[::2]
encs.append(' '.join(str(x)for x in runs))
return encs
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<categorify>
seq_train_val(X=X_train_std, y=y_train, validation_data=(X_val_std, y_val), model=cnn1 )
mean = np.array([0.63701425, 0.47097038, 0.68173952])
 std = np.array([0.15979014, 0.22442915, 0.14194921])
 s_th = 40
p_th = 200*sz//256
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 def img2tensor(img,dtype:np.dtype=np.float32):
if img.ndim==2 : img = np.expand_dims(img,2)
 img = np.transpose(img,(2,0,1))
return torch.from_numpy(img.astype(dtype, copy=False))
class HuBMAPDataset(Dataset):
def __init__(self, idx, sz=sz, step=step, reduce=reduce):
self.data = rasterio.open(os.path.join(DATA,idx+'.tiff'), transform = identity,
num_threads='all_cpus')
 if self.data.count != 3:
print('Image file with subdatasets as channels')
 self.layers = [rasterio.open(subd)for subd in self.data.subdatasets]
self.shape = self.data.shape
self.reduce = reduce
self.sz = reduce*sz
self.step = reduce*step
self.pad0 =(self.sz - self.shape[0]%self.step)%self.step
self.pad1 =(self.sz - self.shape[1]%self.step)%self.step
self.n0max =(self.shape[0] + self.pad0 - self.sz)//self.step + 1
self.n1max =(self.shape[1] + self.pad1 - self.sz)//self.step + 1
def __len__(self):
return self.n0max*self.n1max
def __getitem__(self, idx):
n0,n1 = idx//self.n1max, idx%self.n1max
x0,y0 = -self.pad0//2 + n0*self.step, -self.pad1//2 + n1*self.step
p00,p01 = max(0,x0), min(x0+self.sz,self.shape[0])
 p10,p11 = max(0,y0), min(y0+self.sz,self.shape[1])
 img = np.zeros(( self.sz,self.sz,3),np.uint8)
 if self.data.count == 3:
img[(p00-x0):(p01-x0),(p10-y0):(p11-y0)] = np.moveaxis(self.data.read([1,2,3],
window=Window.from_slices(( p00,p01),(p10,p11))), 0, -1)
 else:
tile = np.zeros(( p01-p00, p11-p10, 3), dtype=np.uint8)
 for fl in range(3):
tile[:,:,fl] = self.layers[fl].read(window=Window.from_slices(( p00,p01),(p10,p11)))
 img[(p00-x0):(p01-x0),(p10-y0):(p11-y0)] = tile
if self.reduce != 1:
img = cv2.resize(img,(self.sz//self.reduce,self.sz//self.reduce),
interpolation = cv2.INTER_AREA)
 hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
 h,s,v = cv2.split(hsv)
 if(s>s_th ).sum() <= p_th or img.sum() <= p_th:
return img2tensor(( img/255.0 - mean)/std), -1, x0, y0, p00, p01, p10, p11
else: return img2tensor(( img/255.0 - mean)/std), idx, x0, y0, p00, p01, p10, p11<import_modules>
validation(X_val_std, y_val, model=cnn )
class Model_pred:
def __init__(self, models, dl, reduce, tta:bool=True):
self.models = models
self.dl = dl
self.reduce = reduce
self.tta = tta
def __iter__(self):
count=0
with torch.no_grad() :
for x,y,x0,y0,p00,p01,p10,p11 in iter(self.dl):
if(( y>=0 ).sum() > 0):
x = x[y>=0].to(device)
 x0 = x0[y>=0]
y0 = y0[y>=0]
p00 = p00[y>=0]
p01 = p01[y>=0]
p10 = p10[y>=0]
p11 = p11[y>=0]
y = y[y>=0]
with autocast(enabled=True):
py = None
for model in self.models:
p = model(x)
 p = torch.sigmoid(p ).detach()
if py is None: py = p
else: py += p
if self.tta:
flips = [[-1],[-2],[-2,-1]]
for f in flips:
xf = torch.flip(x,f)
 for model in self.models:
p = model(xf)
 p = torch.flip(p,f)
 py += torch.sigmoid(p ).detach()
py /=(1+len(flips))
py /= len(self.models)
 py = py.permute(0,2,3,1 ).half().cpu()
batch_size = len(py)
 for i in range(batch_size):
yield py[i],y[i],x0[i],y0[i],p00[i],p01[i],p10[i],p11[i]
count += 1
def __len__(self):
return len(self.dl.dataset )<choose_model_class>
y_pred = predict(X_test_std, model=cnn)
 y_pred = pd.DataFrame(y_pred, columns=["Label"])
 sub = pd.concat([df_Id, y_pred], axis=1)
 sub.to_csv("submission.csv", index=False )
class FReLU(nn.Module):
def __init__(self, c1, k=3):
super().__init__()
self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1, bias=False)
 self.bn = nn.BatchNorm2d(c1)
 def forward(self, x):
return torch.max(x, self.bn(self.conv(x)))
 class Conv2dAct(nn.Sequential):
def __init__(
 self,
in_channels,
out_channels,
kernel_size,
padding=0,
stride=1,
use_batchnorm=True,
act_layer='relu' 
):
conv = nn.Conv2d(
 in_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding,
bias=not(use_batchnorm), 
)
 if act_layer == 'frelu':
act = FReLU(out_channels)
 else:
act = create_act_layer(act_layer)
 if use_batchnorm:
bn = nn.BatchNorm2d(out_channels)
 else:
bn = nn.Identity()
super(Conv2dAct, self ).__init__(conv, bn, act)
 class DecoderBlockV2(nn.Module):
def __init__(
 self,
in_channels,
skip_channels,
out_channels,
use_batchnorm=True,
act_layer='relu',
attention_type=None, 
):
super().__init__()
self.conv1 = Conv2dAct(
 in_channels + skip_channels,
out_channels,
kernel_size=3,
padding=1,
use_batchnorm=use_batchnorm,
act_layer=act_layer 
)
 self.attention1 = md.Attention(attention_type, in_channels=in_channels + skip_channels)
 self.conv2 = Conv2dAct(
 out_channels,
out_channels,
kernel_size=3,
padding=1,
use_batchnorm=use_batchnorm,
act_layer=act_layer 
)
 self.attention2 = md.Attention(attention_type, in_channels=out_channels)
 def forward(self, x, skip=None):
x = F.interpolate(x, scale_factor=2, mode="nearest")
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.attention1(x)
 x = self.conv1(x)
 x = self.conv2(x)
 x = self.attention2(x)
 return x
class UnetDecoderV2(nn.Module):
def __init__(
 self,
encoder_channels,
decoder_channels,
n_blocks=5,
use_batchnorm=True,
act_layer='relu',
attention_type=None, 
):
super().__init__()
if n_blocks != len(decoder_channels):
raise ValueError(
 "Model depth is {}, but you provide `decoder_channels` for {} blocks.".format(
 n_blocks, len(decoder_channels)
)
)
 encoder_channels = encoder_channels[1:]
encoder_channels = encoder_channels[::-1]
head_channels = encoder_channels[0]
in_channels = [head_channels] + list(decoder_channels[:-1])
 skip_channels = list(encoder_channels[1:])+ [0]
out_channels = decoder_channels
self.center = nn.Identity()
kwargs = dict(use_batchnorm=use_batchnorm, act_layer=act_layer, attention_type=attention_type)
 blocks = [
DecoderBlockV2(in_ch, skip_ch, out_ch, **kwargs)
 for in_ch, skip_ch, out_ch in zip(in_channels, skip_channels, out_channels)
 ]
self.blocks = nn.ModuleList(blocks)
 def forward(self, *features):
features = features[1:]
features = features[::-1]
head = features[0]
skips = features[1:]
x = self.center(head)
 for i, decoder_block in enumerate(self.blocks):
skip = skips[i] if i < len(skips)else None
x = decoder_block(x, skip)
 return x<categorify>
ac_relu = layers.Activation("relu")
 ac_softmax = layers.Activation("softmax")
 class cnn2(tf.keras.Model):
def __init__(self, *args, **kwargs):
super(cnn2, self ).__init__(*args, **kwargs)
 self.conv1 = Conv2D(filters=8, kernel_size=(3, 3), padding="same")
 self.conv2 = Conv2D(filters=8, kernel_size=(3, 3), padding="same")
 self.conv3 = Conv2D(filters=8, kernel_size=(3, 3), padding="same")
 self.maxpool1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding="same")
 self.conv4 = Conv2D(filters=16, kernel_size=(3, 3), padding="same")
 self.conv5 = Conv2D(filters=16, kernel_size=(3, 3), padding="same")
 self.conv6 = Conv2D(filters=16, kernel_size=(3, 3), padding="same")
 self.maxpool2 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding="same")
 self.conv7 = Conv2D(filters=32, kernel_size=(3, 3), padding="same")
 self.conv8 = Conv2D(filters=32, kernel_size=(3, 3), padding="same")
 self.conv9 = Conv2D(filters=32, kernel_size=(3, 3), padding="same")
 self.conv10 = Conv2D(filters=32, kernel_size=(3, 3), padding="same")
 self.maxpool3 = MaxPool2D(pool_size=(2, 2), strides=(2, 2))
self.conv11 = Conv2D(filters=64, kernel_size=(3, 3), padding="same")
 self.conv12 = Conv2D(filters=64, kernel_size=(3, 3), padding="same")
 self.conv13 = Conv2D(filters=64, kernel_size=(3, 3), padding="same")
 self.conv14 = Conv2D(filters=64, kernel_size=(3, 3), padding="same")
 self.maxpool4 = MaxPool2D(pool_size=(2, 2), strides=(2, 2))
self.flatten = Flatten()
self.dense1 = Dense(units=1024)
 self.drop = Dropout(rate=0.5)
 self.dense2 = Dense(units=10)
 def call(self, inputs, training=None):
x2 = ac_relu(self.conv1(inputs))
x3 = ac_relu(self.conv2(x2))
x4 = ac_relu(self.conv3(x3))
x5 = ac_relu(self.maxpool1(x4))
x6 = ac_relu(self.conv4(x5))
x7 = ac_relu(self.conv5(x6))
x8 = ac_relu(self.conv6(x7))
x9 = ac_relu(self.maxpool2(x8))
x10 = ac_relu(self.conv7(x9))
x11 = ac_relu(self.conv8(x10))
x12 = ac_relu(self.conv9(x11))
x13 = ac_relu(self.conv10(x12))
x14 = ac_relu(self.maxpool3(x13))
x15 = ac_relu(self.conv11(x14))
x16 = ac_relu(self.conv12(x15))
x17 = ac_relu(self.conv13(x16))
x18 = ac_relu(self.conv14(x17))
x19 = ac_relu(self.maxpool4(x18))
x20 = self.flatten(x19)
 x21 = ac_relu(self.dense1(x20))
x22 = self.drop(x21)
 outputs = ac_softmax(self.dense2(x22))
return outputs
cnn2 = cnn2()
class CSPDarkNet53Encoder(nn.Module):
def __init__(self, pretrain=True, act_layer='leaky_relu'):
super(CSPDarkNet53Encoder, self ).__init__()
base = timm.create_model('cspdarknet53', pretrained=pretrain, act_layer=get_act_layer(act_layer))
self.stem = base.stem
self.layer0 = base.stages[0]
self.layer1 = base.stages[1]
self.layer2 = base.stages[2]
self.layer3 = base.stages[3]
self.layer4 = base.stages[4]
del base
self.depth = 5
self.out_channels =(32, 64, 128, 256, 512, 1024)
 self.in_channels = 3
def get_stages(self):
return [
self.stem,
self.layer0,
self.layer1,
self.layer2,
self.layer3,
self.layer4,
]
def forward(self, x):
stages = self.get_stages()
features = []
for i in range(self.depth + 1):
x = stages[i](x)
 features.append(x)
 return features
class CSPDarkNet53FPN(smp.base.SegmentationModel):
def __init__(
 self,
pretrain=True,
decoder_pyramid_channels: int = 256,
decoder_segmentation_channels: int = 128,
decoder_merge_policy: str = "add",
decoder_dropout: float = 0.2,
in_channels: int = 3,
classes: int = 1,
activation: Optional[str] = None,
upsampling: int = 4,
aux_params: Optional[dict] = None, 
):
super().__init__()
self.encoder = CSPDarkNet53Encoder(pretrain)
 self.decoder = smp.fpn.decoder.FPNDecoder(
 encoder_channels=self.encoder.out_channels,
encoder_depth=self.encoder.depth,
pyramid_channels=decoder_pyramid_channels,
segmentation_channels=decoder_segmentation_channels,
dropout=decoder_dropout,
merge_policy=decoder_merge_policy, 
)
 self.segmentation_head = smp.base.SegmentationHead(
 in_channels=self.decoder.out_channels,
out_channels=classes,
activation=activation,
kernel_size=1,
upsampling=upsampling, 
)
 if aux_params is not None:
self.classification_head = smp.base.ClassificationHead(
 in_channels=self.encoder.out_channels[-1], **aux_params 
)
 else:
self.classification_head = None
self.name = "fpn-cspdarknet53"
self.initialize()
class CSPDarkNet53Unet(smp.base.SegmentationModel):
def __init__(
 self,
pretrain=True,
act_layer='leaky_relu',
decoder_use_batchnorm: bool = True,
decoder_channels: List[int] =(256, 128, 64, 32, 16),
decoder_attention_type: Optional[str] = None,
in_channels: int = 3,
classes: int = 1,
activation: Optional[str] = None,
aux_params: Optional[dict] = None, 
):
super().__init__()
self.encoder = CSPDarkNet53Encoder(pretrain, act_layer)
 self.decoder = smp.unet.decoder.UnetDecoder(
 encoder_channels=self.encoder.out_channels,
decoder_channels=decoder_channels,
n_blocks=self.encoder.depth,
use_batchnorm=decoder_use_batchnorm,
center=False,
attention_type=decoder_attention_type 
)
 self.segmentation_head = smp.base.SegmentationHead(
 in_channels=decoder_channels[-1],
out_channels=classes,
activation=activation,
kernel_size=3, 
)
 if aux_params is not None:
self.classification_head = smp.base.ClassificationHead(
 in_channels=self.encoder.out_channels[-1], **aux_params 
)
 else:
self.classification_head = None
self.name = "unet-cspdarknet53"
self.initialize()
class CSPDarkNet53UnetV2(smp.base.SegmentationModel):
def __init__(
 self,
pretrain=True,
act_layer='leaky_relu',
decoder_use_batchnorm: bool = True,
decoder_channels: List[int] =(256, 128, 64, 32, 16),
decoder_act_layer: str = 'relu',
decoder_attention_type: Optional[str] = None,
in_channels: int = 3,
classes: int = 1,
activation: Optional[str] = None,
aux_params: Optional[dict] = None, 
):
super().__init__()
self.encoder = CSPDarkNet53Encoder(pretrain, act_layer)
 self.decoder = UnetDecoderV2(
 encoder_channels=self.encoder.out_channels,
decoder_channels=decoder_channels,
n_blocks=self.encoder.depth,
use_batchnorm=decoder_use_batchnorm,
act_layer=decoder_act_layer,
attention_type=decoder_attention_type 
)
 self.segmentation_head = smp.base.SegmentationHead(
 in_channels=decoder_channels[-1],
out_channels=classes,
activation=activation,
kernel_size=3, 
)
 if aux_params is not None:
self.classification_head = smp.base.ClassificationHead(
 in_channels=self.encoder.out_channels[-1], **aux_params 
)
 else:
self.classification_head = None
self.name = "unetv2-cspdarknet53"
self.initialize()<choose_model_class>
def sub_train_val(X_train, y_train, X_val, y_val, model, optimizer=Adam, learning_rate=0.0001, loss_object=CategoricalCrossentropy() ,
batch_size=20, y_label=10, epochs=5):
X_trainn = X_train.astype("float32")
 if y_label:
y_trainn = tf.one_hot(y_train, 10, dtype="float32")
 else:
y_trainn = y_train.astype("float32")
 train_sliced = tf.data.Dataset.from_tensor_slices(( X_trainn, y_trainn))
train_dataset = train_sliced.shuffle(100000 ).batch(batch_size)
 X_vall = X_val.astype("float32")
 if y_label:
y_vall = tf.one_hot(y_val, 10, dtype="float32")
 else:
y_vall = y_vall.astype("float32")
 val_sliced = tf.data.Dataset.from_tensor_slices(( X_vall, y_vall))
val_dataset = val_sliced.batch(batch_size)
 optimizer = optimizer(learning_rate=learning_rate)
 loss_object = loss_object
train_loss = tf.keras.metrics.Mean(name="train_loss")
 val_loss = tf.keras.metrics.Mean(name="val_loss")
 train_loss_list = []
val_loss_list = []
@tf.function
def train_step(X_train, y_train, model, optimizer, loss_object, train_loss):
training = True
K.set_learning_phase(training)
 with tf.GradientTape() as tape:
y_pred_tra = model(X_train, training=training)
 loss = loss_object(y_train, y_pred_tra)
 gradient = tape.gradient(loss, model.trainable_weights)
 optimizer.apply_gradients(zip(gradient, model.trainable_weights))
train_loss(loss)
 @tf.function
def val_step(X_val, y_val, model, optimizer, loss_object, val_loss):
training = False
y_pred_val = model(X_val, training=training)
 loss = loss_object(y_val, y_pred_val)
 val_loss(loss)
 for epoch in range(epochs):
print(f"Epoch is {str(epoch+1 ).zfill(3)} ----->", end=" ")
 for X, y in train_dataset:
train_step(X, y, model=model, optimizer=optimizer, loss_object=loss_object, train_loss=train_loss)
 print("train_loss: {:.6f}".format(float(train_loss.result())) , end=" | ")
 train_loss_list.append(float(train_loss.result()))
train_loss.reset_states()
for X, y in val_dataset:
val_step(X, y, model=model, optimizer=optimizer, loss_object=loss_object, val_loss=val_loss)
 print("val_loss: {:.6f}".format(float(val_loss.result())))
 val_loss_list.append(float(val_loss.result()))
val_loss.reset_states()
print()
print("-"*100)
 print()
print("Best val_loss is {:.6f}, and when the epoch is {}".format(min(val_loss_list), epoch+1))
fig = plt.figure(figsize=(10, 8))
plt.plot(range(1, epochs+1), train_loss_list, marker="o", c="b", label="train_loss")
 plt.plot(range(1, epochs+1), val_loss_list, marker="v", c="r", label="validation_loss")
 plt.legend()
plt.xlabel("epochs")
 plt.ylabel("loss")
 plt.title("train_and_validation_loss" )
class PlModel(pl.LightningModule):
def __init__(self, learning_rate = 1e-3):
super(PlModel, self ).__init__()
self.learning_rate = learning_rate
self.model = smp.Unet('timm-efficientnet-b3', decoder_attention_type = None, encoder_weights = None)
 self.criterion = None
def forward(self, x):
return self.model(x )<find_best_params>
train_val(X_train=X_train_std, y_train=y_train, X_val=X_val_std, y_val=y_val, model=cnn2, learning_rate=0.0001, epochs=40, optimizer=Adam )
models = []
B3_MODELS = [
'.. /input/nfl-submissions/Mar291351_0_epoch20_val_metric0.940.ckpt',
'.. /input/nfl-submissions/Mar291351_1_epoch20_val_metric0.940.ckpt',
'.. /input/nfl-submissions/Mar291351_2_epoch18_val_metric0.947.ckpt',
'.. /input/nfl-submissions/Mar291351_3_epoch20_val_metric0.939.ckpt',
]
for path in B3_MODELS:
model = PlModel()
model = model.load_from_checkpoint(path)
 model.float()
model.eval()
model.to(device)
 models.append(model )<load_pretrained>
def validate(X, y, model):
X_tensor = tf.convert_to_tensor(X)
 y_pred = model(X_tensor)
 y_pred = y_pred.numpy()
y_pred = np.argmax(y_pred, axis=1)
 acc = accuracy_score(y, y_pred)
 return acc
def predict(X, model):
X_tensor = tf.convert_to_tensor(X)
 y_pred = model(X_tensor)
 y_pred = y_pred.numpy()
y_pred = np.argmax(y_pred, axis=1)
 return y_pred
D53_MODELS = [ 
('.. /input/hubmap-trained-models/unetv2-d53_sf-672-pseudo5-external_s2.pth', 'frelu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sf_valid_8242609fa_best.pth', 'frelu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sf_valid_b2dc8411c_best.pth', 'frelu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sr_valid_4ef6695ce_best.pth', 'relu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sf_valid_2f6ecfcdf_best.pth', 'frelu')
 ]
for path, act in D53_MODELS:
state_dict = torch.load(path,map_location=torch.device('cpu'))
model = CSPDarkNet53UnetV2(
 pretrain=False,
act_layer='swish',
decoder_act_layer=act,
in_channels=3,
classes=1 ).cuda()
model.load_state_dict(state_dict['model'])
 model.float()
model.eval()
model.to(device)
 models.append(model)
 del state_dict<train_model>
validate(X_train_std, y_train, cnn2 )
print(f'Total number of trained models {len(models)}' )<feature_engineering>
x = torch.empty(2, 3)
 print(x)
 x = torch.rand(2, 3)
 print(x)
 x = torch.zeros(2, 3, dtype=torch.float)
 print(x)
 x = torch.ones(2, 3, dtype=torch.int)
 print(x)
 x = torch.tensor([[0.0, 0.1, 0.2],
[1.0, 1.1, 1.2]])
 print(x)
 y = x.new_ones(1, 2)
 print(y)
 y = torch.ones_like(x, dtype=torch.int)
 print(y )
def get_gauss_filter(half):
y,x = np.mgrid[-half:half,-half:half]
y = half-abs(y)
 x = half-abs(x)
 w = np.minimum(x,y)
 w = w/w.max()
w = np.minimum(w,1)
 return w<categorify>
b = x.numpy()
print('PyTorch計算→NumPy反映：')
 print(b); x.add_(y); print(b)
 print('NumPy計算→PyTorch反映：')
 print(x); np.add(b, b, out=b); print(x)
 c = np.ones(( 2, 3), dtype=np.float64)
 d = torch.from_numpy(c)
 print('NumPy計算→PyTorch反映：')
 print(d); np.add(c, c, out=c); print(d)
 print('PyTorch計算→NumPy反映：')
 print(c); d.add_(y); print(c )
def reduce_fp(predict, shape, s, t=0.4):
mask =(predict > t ).astype(np.uint8)
 mask = ms.label(mask)
 l_probs = []
for i in range(1, mask.max() +1):
l_probs.append(predict[mask == i].max())
 l_probs = np.array(l_probs)
 mask[np.isin(mask,(np.where(l_probs < s)[0] + 1)) ] = 0
return cv2.resize(( mask > 0 ).astype(np.uint8), shape, interpolation=cv2.INTER_NEAREST )<categorify>
if torch.cuda.is_available() :
print('CUDA（GPU）が利用できる環境')
 print(f'CUDAデバイス数： {torch.cuda.device_count() }')
 print(f'現在のCUDAデバイス番号： {torch.cuda.current_device() }')
 print(f'1番目のCUDAデバイス名： {torch.cuda.get_device_name(0)}')
 device = torch.device("cuda")
 device0 = torch.device("cuda:0")
 g = torch.ones(2, 3, device=device)
 g = x.to(device)
 g = x.cuda(device)
 f = x.cpu()
g = x.to("cuda")
 f = x.to("cpu")
 g = x.cuda()
model.cuda()
model.cpu()
else:
print('CUDA（GPU）が利用できない環境' )
<save_to_csv><EOS> 
!pip install playground-data
PROBLEM_DATA_TYPE = pg.DatasetType.ClassifyCircleData
TRAINING_DATA_RATIO = 0.5
DATA_NOISE = 0.0
data_list = pg.generate_data(PROBLEM_DATA_TYPE, DATA_NOISE)
 X_train, y_train, X_valid, y_valid = pg.split_data(data_list, training_size=TRAINING_DATA_RATIO)
 print('X_train:'); print(X_train[:5])
 print('y_train:'); print(y_train[:5])
 print('X_valid:'); print(X_valid[:5])
 print('y_valid:'); print(y_valid[:5] )