Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<categorify>
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
imgaug.seed(1)
 t_start = time.time()
SPS_CODE = dict(zip(train.sci_name, train.ebird_code))
def get_sps_from_bg(s):
if type(s)!= str: s = str(s)
 return re.findall('\((.*?)\)', s)
 def get_sps_from_sec(l):
return [re.split('_', s[1:-1])[0] for s in re.split(',\s*', l[1:-1])]
def map_sps_to_code(l):
res = []
for s in l:
if s == '':
continue
elif s in SPS_CODE:
res.append(SPS_CODE[s])
 else:
None
return res
train['labels_bg'] = train['background'].apply(get_sps_from_bg ).apply(map_sps_to_code)
 train['labels_sec'] = train['secondary_labels'].apply(get_sps_from_sec ).apply(map_sps_to_code)
 train['secondary_codes'] = [list(set(a+b)) for a,b in zip(train['labels_sec'].values, train['labels_bg'].values)]<feature_engineering>
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )
filter_limit = 30
train['secondary_codes'] = train['secondary_codes'].where(train['duration'] < filter_limit, '[]')
 train.secondary_codes = train.secondary_codes.apply(lambda x: list(x for x in map(BIRD_CODE.get, x)if x is not None))<groupby>
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )
def downsampling(df):
main_data = pd.DataFrame()
for ebird_code in list(df.groupby('ebird_code' ).groups.keys()):
main_data = pd.concat([main_data, df[df['ebird_code'] == ebird_code].sort_values('duration')[:80]])
 return main_data
def upsampling(df):
def duplicate_long_data(df, ebird_code, dif):
long_data = df[df['ebird_code'] == ebird_code].query('duration > 40' ).sort_values('duration', ascending=False)[:dif]
df = pd.concat([df, long_data])
 return df
for i in range(5):
ebird_code_count = df.groupby('ebird_code' ).count().filename
for ebird_code in ebird_code_count.keys() :
dif = 60 - ebird_code_count[ebird_code]
if dif > 0:
df = duplicate_long_data(df, ebird_code, dif)
 else:
continue
return df<define_variables>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )
GCS_PATH0 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-00')
 GCS_PATH1 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-01')
 GCS_PATH2 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-02')
 GCS_PATH3 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-03')
 GCS_PATH4 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-04')
 train_all = train.copy()
train_all['file_path'] = ''
def f(x):
fl = x[0]
def get_gcp_path(fl):
if re.compile('[a-b]' ).fullmatch(fl):
return GCS_PATH0
elif re.compile('[c-f]' ).fullmatch(fl):
return GCS_PATH1
elif re.compile('[g-m]' ).fullmatch(fl):
return GCS_PATH2
elif re.compile('[n-r]' ).fullmatch(fl):
return GCS_PATH3
elif re.compile('[s-y]' ).fullmatch(fl):
return GCS_PATH4
else:
raise Exception('no matching code')
 return get_gcp_path(fl)+ '/' + x + '/'
train_all['file_path'] = train_all['ebird_code'].map(f)+ train_all['resampled_filename']
train_all = downsampling(train_all)
 skf = StratifiedKFold(n_splits=N_SPLIT, shuffle=True, random_state=1)
 train_all["fold"] = -1
for fold_id,(train_index, val_index)in enumerate(skf.split(train_all, train_all["ebird_code"])) :
train_all.iloc[val_index, -1] = fold_id
train_all = upsampling(train_all)
 use_fold = USE_FOLD
train_file_list = train_all.query("fold != @use_fold")[["file_path", "ebird_code", "secondary_codes"]]
val_file_list = train_all.query("fold == @use_fold")[["file_path", "ebird_code", "secondary_codes"]]
print("[fold {}] train: {}, val: {}".format(use_fold, len(train_file_list), len(val_file_list)) )<categorify>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
train_file_list.ebird_code = train_file_list.ebird_code.map(BIRD_CODE)
 val_file_list.ebird_code = val_file_list.ebird_code.map(BIRD_CODE)
 train_file_list.secondary_codes = [list(set(s + [p])) for s , p in zip(train_file_list.secondary_codes, train_file_list.ebird_code)]
val_file_list.secondary_codes = [list(set(s + [p])) for s , p in zip(val_file_list.secondary_codes, val_file_list.ebird_code)]<categorify>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
def mixup(batch_size, alpha, images, labels):

 if tf.random.uniform([])< 0.5:
return images,(tf.zeros([batch_size, 1, NUM_CLASSES]),labels)
 mix_weight = tfp.distributions.Beta(alpha, alpha ).sample([batch_size, 1])
 mix_weight = tf.maximum(mix_weight, 1.- mix_weight)
 images_mix_weight = tf.reshape(mix_weight, [batch_size, 1, 1, 1])
 images_mix =(
 images * images_mix_weight + images[::-1] *(1.- images_mix_weight))
labels_mix = labels * mix_weight + labels[::-1] *(1.- mix_weight)
 return images_mix,(tf.zeros([batch_size, 1, NUM_CLASSES]),labels_mix)
 def augment_freq_time_mask(spectrogram,
frequency_masking_para=15,
time_masking_para=15,
frequency_mask_num=1,
time_mask_num=1):
time_max = tf.shape(spectrogram)[1]
freq_max = tf.shape(spectrogram)[2]
for _ in range(frequency_mask_num):
f = tf.random.uniform(shape=() , minval=0, maxval=frequency_masking_para, dtype=tf.dtypes.int32)
 f0 = tf.random.uniform(shape=() , minval=0, maxval=freq_max - f, dtype=tf.dtypes.int32)
 value_ones_freq_prev = tf.ones(shape=[1, time_max, f0])
 value_zeros_freq = tf.zeros(shape=[1, time_max, f])
 value_ones_freq_next = tf.ones(shape=[1, time_max, freq_max-(f0+f)])
 freq_mask = tf.concat([value_ones_freq_prev, value_zeros_freq, value_ones_freq_next], axis=2)
 spectrogram = spectrogram*freq_mask
for _ in range(time_mask_num):
t = tf.random.uniform(shape=() , minval=0, maxval=time_masking_para, dtype=tf.dtypes.int32)
 t0 = tf.random.uniform(shape=() , minval=0, maxval=time_max - t, dtype=tf.dtypes.int32)
 value_zeros_time_prev = tf.ones(shape=[1, t0, freq_max])
 value_zeros_time = tf.zeros(shape=[1, t, freq_max])
 value_zeros_time_next = tf.ones(shape=[1, time_max-(t0+t), freq_max])
 time_mask = tf.concat([value_zeros_time_prev, value_zeros_time, value_zeros_time_next], axis=1)
 spectrogram = spectrogram*time_mask
return spectrogram
def augment_pitch_and_tempo(spectrogram,
max_tempo=1.2,
max_pitch=1.1,
min_pitch=0.95):
original_shape = tf.shape(spectrogram)
 choosen_pitch = tf.random.uniform(shape=() , minval=min_pitch, maxval=max_pitch)
 choosen_tempo = tf.random.uniform(shape=() , minval=1, maxval=max_tempo)
 new_freq_size = tf.cast(tf.cast(original_shape[2], tf.float32)*choosen_pitch, tf.int32)
 new_time_size = tf.cast(tf.cast(original_shape[1], tf.float32)/(choosen_tempo), tf.int32)
 spectrogram_aug = tf.image.resize(tf.expand_dims(spectrogram, -1), [new_time_size, new_freq_size])
 spectrogram_aug = tf.image.crop_to_bounding_box(spectrogram_aug, offset_height=0, offset_width=0, target_height=tf.shape(spectrogram_aug)[1], target_width=tf.minimum(original_shape[2], new_freq_size))
spectrogram_aug = tf.cond(choosen_pitch < 1,
lambda: tf.image.pad_to_bounding_box(spectrogram_aug, offset_height=0, offset_width=0,
target_height=tf.shape(spectrogram_aug)[1], target_width=original_shape[2]),
lambda: spectrogram_aug)
 return spectrogram_aug[:, :, :, 0]
def augment_speed_up(spectrogram,
speed_std=0.1):
original_shape = tf.shape(spectrogram)
 choosen_speed = tf.math.abs(tf.random.normal(shape=() , stddev=speed_std))
choosen_speed = 1 + choosen_speed
new_freq_size = tf.cast(tf.cast(original_shape[2], tf.float32), tf.int32)
 new_time_size = tf.cast(tf.cast(original_shape[1], tf.float32)/(choosen_speed), tf.int32)
 spectrogram_aug = tf.image.resize(tf.expand_dims(spectrogram, -1), [new_time_size, new_freq_size])
 return spectrogram_aug[:, :, :, 0]
def augment_dropout(spectrogram,
keep_prob=0.9):
return tf.nn.dropout(spectrogram, rate=1-keep_prob)
 def transform_spectrogram(spectrogram,y):
spectrogram = tf.expand_dims(spectrogram, axis=0)
 if tf.random.uniform([])> 0.7:
spectrogram = augment_freq_time_mask(spectrogram)
 rand = tf.random.uniform([])
 if rand < 0.3:
spectrogram = augment_dropout(spectrogram, keep_prob=1-rand)
 if tf.random.uniform([])< 0.2:
spectrogram = augment_pitch_and_tempo(spectrogram)
 spectrogram = tf.squeeze(spectrogram, axis=0)
 return spectrogram,y<create_dataframe>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
def paths_and_labels_to_dataset(audio_paths, labels, sec_labels):

 path_ds = tf.data.Dataset.from_tensor_slices(audio_paths)
 audio_ds = path_ds.map(lambda x: path_to_audio(x), num_parallel_calls=AUTO)
 label_ds = tf.data.Dataset.from_tensor_slices(to_categorical(labels))
sec_labels_ds = tf.data.Dataset.from_tensor_slices(sec_labels.apply(lambda x: sum(list(map(lambda x: to_categorical(x, NUM_CLASSES), x)))).tolist())
 label_ds = tf.data.Dataset.from_tensor_slices(sec_labels.apply(lambda x: sum(list(map(lambda x: to_categorical(x, NUM_CLASSES), x)))).tolist())
 return tf.data.Dataset.zip(( audio_ds, label_ds))
def path_to_audio(path):

 audio = tf.io.read_file(path)
 audio, sample_rate = tf.audio.decode_wav(audio, 1)
 return audio
def split_audio(audio, y):
len_audio = len(audio)
 effective_length = SAMPLE_RATE * PERIOD
if len_audio <= effective_length:
audio = tf.concat([audio, tf.expand_dims(tf.zeros([effective_length - len_audio]), 1)], axis=0)
 else:
start = tf.random.uniform([1], maxval = len_audio - effective_length, dtype = tf.int32)[0]
audio = audio[start:start + effective_length]
return audio, y
def audio_to_fft(audio, y):
audio = tf.squeeze(audio, axis=-1)
 fft = tf.signal.fft(
 tf.cast(tf.complex(real=audio, imag=tf.zeros_like(audio)) , tf.complex64)
)
 fft = tf.expand_dims(fft, axis=-1)
 return tf.math.abs(fft[:(audio.shape[0] // 2), :]), y
def audio_to_mel_spectrogram(audio, y):
audio = tf.squeeze(audio, axis=-1)
 spectrogram = tfio.experimental.audio.spectrogram(
 audio, nfft=2048, window=2048, stride=320)
 mel_spectrogram = tfio.experimental.audio.melscale(
 spectrogram, rate=SAMPLE_RATE, mels=500, fmin=50, fmax=14000)
 dbscale_mel_spectrogram = tfio.experimental.audio.dbscale(
 mel_spectrogram, top_db=80)
 return dbscale_mel_spectrogram, y
def mono_to_color(audio, y):
eps=1e-6
X = audio
X = tf.stack([X, X, X], axis=-1)
 mean = tf.math.reduce_mean(X)
 X = X - mean
std = tf.math.reduce_std(X)
 Xstd = X /(std + eps)
 _min, _max = tf.math.reduce_min(Xstd), tf.math.reduce_max(Xstd)
 norm_max = _max
norm_min = _min
if(_max - _min)> eps:
V = Xstd
V = 255 *(V - norm_min)/(norm_max - norm_min)
 else:
V = tf.zeros_like(Xstd)
 img_size = 224
image = tf.image.resize(V,(img_size,img_size))
return preprocess_input(image), y 
<categorify>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )
train_ds =(
 paths_and_labels_to_dataset(train_file_list['file_path'].values.tolist() , train_file_list['ebird_code'].values.tolist() , train_file_list['secondary_codes'])
.cache() 
.map(split_audio, num_parallel_calls=AUTO)
.map(audio_to_mel_spectrogram, num_parallel_calls=AUTO)
.map(transform_spectrogram, num_parallel_calls=AUTO)
.map(mono_to_color, num_parallel_calls=AUTO)
.shuffle(1024)
.repeat() 
.batch(BATCH_SIZE)
.map(functools.partial(mixup, BATCH_SIZE, 0.2), num_parallel_calls=AUTO)
.prefetch(AUTO)
)
 valid_ds =(
 paths_and_labels_to_dataset(val_file_list['file_path'].values.tolist() , val_file_list['ebird_code'].values.tolist() , val_file_list['secondary_codes'])
.map(split_audio, num_parallel_calls=AUTO)
.map(audio_to_mel_spectrogram, num_parallel_calls=AUTO)
.map(mono_to_color, num_parallel_calls=AUTO)
.cache() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<init_hyperparams>
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x
class F1Callback(Callback):
def __init__(self):
self.f1s = []
def on_epoch_end(self, epoch, logs):
eps = np.finfo(np.float32 ).eps
recall = logs["val_clipwise_output_true_positives"] /(logs["val_clipwise_output_possible_positives"] + eps)
 precision = logs["val_clipwise_output_true_positives"] /(logs["val_clipwise_output_predicted_positives"] + eps)
 f1 = 2*precision*recall /(precision+recall+eps)
 print("f1_val(from log)=", f1)
 self.f1s.append(f1)
 def true_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 def possible_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true, 0, 1)))
 def predicted_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_pred, 0, 1)))
 def F1(y_true, y_pred):
TPFN = possible_positives(y_true, y_pred)
 TPFP = predicted_positives(y_true, y_pred)
 TP = true_positives(y_true, y_pred)
 return(TP * 2)/(TPFN + TPFP + K.epsilon())
 f1cb = F1Callback() 
<choose_model_class>
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer
with strategy.scope() :
base_model = ResNet50(include_top=False, weights='imagenet')
 x = base_model.output
x = tf.reduce_mean(x, axis=2)
 x1 = L.MaxPooling1D(pool_size=3, strides=1, padding='same' )(x)
 x2 = L.AveragePooling1D(pool_size=3, strides=1, padding='same' )(x)
 x = x1 + x2
x = L.Dropout(0.5 )(x)
 x = L.Dense(1024, activation='relu' )(x)
 x = L.Dropout(0.5 )(x)
 norm_att = L.Conv1D(filters=NUM_CLASSES, kernel_size=1, padding='same' )(x)
 norm_att = tf.keras.activations.tanh(norm_att/10)*10
norm_att = tf.keras.activations.softmax(norm_att, axis=-2)
 segmentwise_output = L.Conv1D(filters=NUM_CLASSES, kernel_size=1, padding='same', activation='sigmoid', name='segmentwise_output' )(x)
 clipwise_output = tf.math.reduce_sum(norm_att * segmentwise_output, axis=1)
 clipwise_output = L.Lambda(lambda x: x, name="clipwise_output" )(clipwise_output)
 output = [segmentwise_output, clipwise_output]
model = tf.keras.models.Model(inputs=base_model.input, outputs=output)
 optimizer= tfa.optimizers.RectifiedAdam(
 lr=1e-3,
total_steps=10000,
warmup_proportion=0.1,
min_lr=1e-8, 
)
 model.compile(optimizer, loss=[None, "binary_crossentropy"],loss_weights=[0,1], metrics=[[],["accuracy", F1,true_positives,possible_positives,predicted_positives]])
 model.summary()<train_on_grid>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )
%%time
es = tf.keras.callbacks.EarlyStopping(monitor='loss', verbose=1, patience=10)
 sv = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', verbose=1, factor=0.5, patience=2, min_delta=0.0001, cooldown=1, min_lr=1e-7)
 STEPS_PER_EPOCH = len(train_file_list)// BATCH_SIZE
VALIDATION_STEP = len(val_file_list)// BATCH_SIZE
history = model.fit(
 train_ds,
epochs=100,
steps_per_epoch=STEPS_PER_EPOCH,
callbacks=[reduce_lr, es, sv, f1cb],
validation_data=valid_ds,
validation_steps = VALIDATION_STEP
)<import_modules>
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.elu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss
<import_modules>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )
import math
import cv2
import audioread
import collections
import logging
import os
import random
import re
import time
import warnings
import librosa
import numpy as np
import pandas as pd
import soundfile as sf
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as data
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Optional
from fastprogress import progress_bar
from sklearn.metrics import f1_score
from torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU
from torch.nn.modules.utils import _pair<set_options>
learning_rate = 0.01
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 c = optimizers.adam(lr = learning_rate)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_logger(out_file=None):
logger = logging.getLogger()
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 logger.handlers = []
logger.setLevel(logging.INFO)
 handler = logging.StreamHandler()
handler.setFormatter(formatter)
 handler.setLevel(logging.INFO)
 logger.addHandler(handler)
 if out_file is not None:
fh = logging.FileHandler(out_file)
 fh.setFormatter(formatter)
 fh.setLevel(logging.INFO)
 logger.addHandler(fh)
 logger.info("logger set up")
 return logger
@contextmanager
def timer(name: str, logger: Optional[logging.Logger] = None):
t0 = time.time()
msg = f"[{name}] start"
if logger is None:
print(msg)
 else:
logger.info(msg)
 yield
msg = f"[{name}] done in {time.time() - t0:.2f} s"
if logger is None:
print(msg)
 else:
logger.info(msg )<set_options>
early_stopping = EarlyStopping(monitor='my_iou_metric', mode = 'max',patience=10, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 100
batch_size = 32
history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint,reduce_lr],
verbose=2 )
logger = get_logger("main.log")
 set_seed(1213 )<define_variables>
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = learning_rate)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
TARGET_SR = 32000
TEST = Path(".. /input/birdsong-recognition/test_audio" ).exists()<load_from_csv>
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 200
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2 )
if TEST:
DATA_DIR = Path(".. /input/birdsong-recognition/")
 else:
DATA_DIR = Path(".. /input/birdcall-check/")
 test = pd.read_csv(DATA_DIR / "test.csv")
 test_audio = DATA_DIR / "test_audio"
test.head()<save_to_csv>
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss})
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv("submission.csv", index=False )<train_model>
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2
def init_layer(layer):
nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, "bias"):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):
bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.0)
 def interpolate(x: torch.Tensor, ratio: int):

(batch_size, time_steps, classes_num)= x.shape
upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
 upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)
 return upsampled
def pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):

 pad = framewise_output[:, -1:, :].repeat(
 1, frames_num - framewise_output.shape[1], 1)

output = torch.cat(( framewise_output, pad), dim=1)

return output
class ConvBlock(nn.Module):
def __init__(self, in_channels: int, out_channels: int):
super().__init__()
self.conv1 = nn.Conv2d(
 in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.conv2 = nn.Conv2d(
 in_channels=out_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size=(2, 2), pool_type='avg'):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 if pool_type == 'max':
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg':
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg+max':
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception('Incorrect argument!')
 return x
class AttBlock(nn.Module):
def __init__(self,
in_features: int,
out_features: int,
activation="linear",
temperature=1.0):
super().__init__()
self.activation = activation
self.temperature = temperature
self.att = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.cla = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.init_weights()
def init_weights(self):
init_layer(self.att)
 def forward(self, x):
norm_att = torch.softmax(torch.tanh(self.att(x)) , dim=-1)
 cla = self.nonlinear_transform(self.cla(x))
x = torch.sum(norm_att * cla, dim=2)
 return x, norm_att, cla
def nonlinear_transform(self, x):
if self.activation == 'linear':
return x
elif self.activation == 'sigmoid':
return torch.sigmoid(x )<define_search_model>
preds_valid = predict_result(model,x_valid,img_size_target )
class SplAtConv2d(Module):

 def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0),
dilation=(1, 1), groups=1, bias=True,
radix=2, reduction_factor=4,
rectify=False, rectify_avg=False, norm_layer=None,
dropblock_prob=0.0, **kwargs):
super(SplAtConv2d, self ).__init__()
padding = _pair(padding)
 self.rectify = rectify and(padding[0] > 0 or padding[1] > 0)
 self.rectify_avg = rectify_avg
inter_channels = max(in_channels*radix//reduction_factor, 32)
 self.radix = radix
self.cardinality = groups
self.channels = channels
self.dropblock_prob = dropblock_prob
if self.rectify:
self.conv = RFConv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, average_mode=rectify_avg, **kwargs)
 else:
self.conv = Conv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, **kwargs)
 self.use_bn = norm_layer is not None
if self.use_bn:
self.bn0 = norm_layer(channels*radix)
 self.relu = ReLU(inplace=True)
 self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality)
 if self.use_bn:
self.bn1 = norm_layer(inter_channels)
 self.fc2 = Conv2d(inter_channels, channels*radix, 1, groups=self.cardinality)
 if dropblock_prob > 0.0:
self.dropblock = DropBlock2D(dropblock_prob, 3)
 self.rsoftmax = rSoftMax(radix, groups)
 def forward(self, x):
x = self.conv(x)
 if self.use_bn:
x = self.bn0(x)
 if self.dropblock_prob > 0.0:
x = self.dropblock(x)
 x = self.relu(x)
 batch, rchannel = x.shape[:2]
if self.radix > 1:
if torch.__version__ < '1.5':
splited = torch.split(x, int(rchannel//self.radix), dim=1)
 else:
splited = torch.split(x, rchannel//self.radix, dim=1)
 gap = sum(splited)
 else:
gap = x
gap = F.adaptive_avg_pool2d(gap, 1)
 gap = self.fc1(gap)
 if self.use_bn:
gap = self.bn1(gap)
 gap = self.relu(gap)
 atten = self.fc2(gap)
 atten = self.rsoftmax(atten ).view(batch, -1, 1, 1)
 if self.radix > 1:
if torch.__version__ < '1.5':
attens = torch.split(atten, int(rchannel//self.radix), dim=1)
 else:
attens = torch.split(atten, rchannel//self.radix, dim=1)
 out = sum([att*split for(att, split)in zip(attens, splited)])
 else:
out = atten * x
return out.contiguous()
class rSoftMax(nn.Module):
def __init__(self, radix, cardinality):
super().__init__()
self.radix = radix
self.cardinality = cardinality
def forward(self, x):
batch = x.size(0)
 if self.radix > 1:
x = x.view(batch, self.cardinality, self.radix, -1 ).transpose(1, 2)
 x = F.softmax(x, dim=1)
 x = x.reshape(batch, -1)
 else:
x = torch.sigmoid(x)
 return x
class DropBlock2D(object):
def __init__(self, *args, **kwargs):
raise NotImplementedError
class GlobalAvgPool2d(nn.Module):
def __init__(self):

 super(GlobalAvgPool2d, self ).__init__()
def forward(self, inputs):
return nn.functional.adaptive_avg_pool2d(inputs, 1 ).view(inputs.size(0), -1)
 class Bottleneck(nn.Module):

 expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None,
radix=1, cardinality=1, bottleneck_width=64,
avd=False, avd_first=False, dilation=1, is_first=False,
rectified_conv=False, rectify_avg=False,
norm_layer=None, dropblock_prob=0.0, last_gamma=False):
super(Bottleneck, self ).__init__()
group_width = int(planes *(bottleneck_width / 64.))* cardinality
self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)
 self.bn1 = norm_layer(group_width)
 self.dropblock_prob = dropblock_prob
self.radix = radix
self.avd = avd and(stride > 1 or is_first)
 self.avd_first = avd_first
if self.avd:
self.avd_layer = nn.AvgPool2d(3, stride, padding=1)
 stride = 1
if dropblock_prob > 0.0:
self.dropblock1 = DropBlock2D(dropblock_prob, 3)
 if radix == 1:
self.dropblock2 = DropBlock2D(dropblock_prob, 3)
 self.dropblock3 = DropBlock2D(dropblock_prob, 3)
 if radix >= 1:
self.conv2 = SplAtConv2d(
 group_width, group_width, kernel_size=3,
stride=stride, padding=dilation,
dilation=dilation, groups=cardinality, bias=False,
radix=radix, rectify=rectified_conv,
rectify_avg=rectify_avg,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif rectified_conv:
self.conv2 = RFConv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False,
average_mode=rectify_avg)
 self.bn2 = norm_layer(group_width)
 else:
self.conv2 = nn.Conv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False)
 self.bn2 = norm_layer(group_width)
 self.conv3 = nn.Conv2d(
 group_width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = norm_layer(planes*4)
 if last_gamma:
zeros_(self.bn3.weight)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.dilation = dilation
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock1(out)
 out = self.relu(out)
 if self.avd and self.avd_first:
out = self.avd_layer(out)
 out = self.conv2(out)
 if self.radix == 0:
out = self.bn2(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock2(out)
 out = self.relu(out)
 if self.avd and not self.avd_first:
out = self.avd_layer(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out<choose_model_class>
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )
class ResNest(nn.Module):

 def __init__(self, block, layers, radix=1, groups=1, bottleneck_width=64,
num_classes=1000, dilated=False, dilation=1,
deep_stem=False, stem_width=64, avg_down=False,
rectified_conv=False, rectify_avg=False,
avd=False, avd_first=False,
final_drop=0.0, dropblock_prob=0,
last_gamma=False, norm_layer=nn.BatchNorm2d):
self.cardinality = groups
self.bottleneck_width = bottleneck_width
self.inplanes = stem_width*2 if deep_stem else 64
self.avg_down = avg_down
self.last_gamma = last_gamma
self.radix = radix
self.avd = avd
self.avd_first = avd_first
super(ResNest, self ).__init__()
self.rectified_conv = rectified_conv
self.rectify_avg = rectify_avg
if rectified_conv:
conv_layer = RFConv2d
else:
conv_layer = nn.Conv2d
conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {}
if deep_stem:
self.conv1 = nn.Sequential(
 conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width*2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs), 
)
 else:
self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3,
bias=False, **conv_kwargs)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer)
 if dilated or dilation == 4:
self.layer3 = self._make_layer(block, 256, layers[2], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=4, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif dilation==2:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilation=1, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 else:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.avgpool = GlobalAvgPool2d()
self.drop = nn.Dropout(final_drop)if final_drop > 0.0 else None
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, norm_layer):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1, dilation=1, norm_layer=None,
dropblock_prob=0.0, is_first=True):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
down_layers = []
if self.avg_down:
if dilation == 1:
down_layers.append(nn.AvgPool2d(kernel_size=stride, stride=stride,
ceil_mode=True, count_include_pad=False))
else:
down_layers.append(nn.AvgPool2d(kernel_size=1, stride=1,
ceil_mode=True, count_include_pad=False))
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=1, bias=False))
else:
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False))
down_layers.append(norm_layer(planes * block.expansion))
downsample = nn.Sequential(*down_layers)
 layers = []
if dilation == 1 or dilation == 2:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=1, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
elif dilation == 4:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=2, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
else:
raise RuntimeError("=> unknown dilation size: {}".format(dilation))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=dilation, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 if self.drop:
x = self.drop(x)
 x = self.fc(x)
 multilabel_proba = torch.sigmoid(x)
 multiclass_proba = torch.softmax(x, dim=1)
 return {
"logits": x,
"multilabel_proba": multilabel_proba,
"multiclass_proba": multiclass_proba
}<normalization>

 def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_tensors[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 assert p >= 0 and p <= 1, 'p must be in range of [0,1]'
if not training:
return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_width_and_height_from_size(x):

 if isinstance(x, int):
return x, x
if isinstance(x, list)or isinstance(x, tuple):
return x
else:
raise TypeError()
def calculate_output_image_size(input_image_size, stride):

 if input_image_size is None:
return None
image_height, image_width = get_width_and_height_from_size(input_image_size)
 stride = stride if isinstance(stride, int)else stride[0]
image_height = int(math.ceil(image_height / stride))
image_width = int(math.ceil(image_width / stride))
return [image_height, image_width]
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, stride, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw =(image_size, image_size)if isinstance(image_size, int)else image_size
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
def get_same_padding_maxPool2d(image_size=None):

 if image_size is None:
return MaxPool2dDynamicSamePadding
else:
return partial(MaxPool2dStaticSamePadding, image_size=image_size)
 class MaxPool2dDynamicSamePadding(nn.MaxPool2d):

 def __init__(self, kernel_size, stride, padding=0, dilation=1, return_indices=False, ceil_mode=False):
super().__init__(kernel_size, stride, padding, dilation, return_indices, ceil_mode)
 self.stride = [self.stride] * 2 if isinstance(self.stride, int)else self.stride
self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int)else self.kernel_size
self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int)else self.dilation
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.kernel_size
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.max_pool2d(x, self.kernel_size, self.stride, self.padding,
self.dilation, self.ceil_mode, self.return_indices)
 class MaxPool2dStaticSamePadding(nn.MaxPool2d):

 def __init__(self, kernel_size, stride, image_size=None, **kwargs):
super().__init__(kernel_size, stride, **kwargs)
 self.stride = [self.stride] * 2 if isinstance(self.stride, int)else self.stride
self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int)else self.kernel_size
self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int)else self.dilation
assert image_size is not None
ih, iw =(image_size, image_size)if isinstance(image_size, int)else image_size
kh, kw = self.kernel_size
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.max_pool2d(x, self.kernel_size, self.stride, self.padding,
self.dilation, self.ceil_mode, self.return_indices)
 return x
class Identity(nn.Module):

 def __init__(self):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
'efficientnet-b8':(2.2, 3.6, 672, 0.5),
'efficientnet-l2':(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 num_repeat=int(options['r']),
kernel_size=int(options['k']),
stride=[int(options['s'][0])],
expand_ratio=int(options['e']),
input_filters=int(options['i']),
output_filters=int(options['o']),
se_ratio=float(options['se'])if 'se' in options else None,
id_skip=('noskip' not in block_string))
@staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, image_size=None,
dropout_rate=0.2, drop_connect_rate=0.2, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25',
'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25',
'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25',
'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
image_size=image_size,
dropout_rate=dropout_rate,
num_classes=num_classes,
batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
drop_connect_rate=drop_connect_rate,
depth_divisor=8,
min_depth=None, 
)
 return blocks_args, global_params
GlobalParams = collections.namedtuple('GlobalParams', [
'width_coefficient', 'depth_coefficient', 'image_size', 'dropout_rate',
'num_classes', 'batch_norm_momentum', 'batch_norm_epsilon',
'drop_connect_rate', 'depth_divisor', 'min_depth'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'num_repeat', 'kernel_size', 'stride', 'expand_ratio',
'input_filters', 'output_filters', 'se_ratio', 'id_skip'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields )<set_options>
preds_test = predict_result(model,x_test,img_size_target )
VALID_MODELS =(
 'efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3',
'efficientnet-b4', 'efficientnet-b5', 'efficientnet-b6', 'efficientnet-b7',
'efficientnet-b8',
'efficientnet-l2' 
)
 class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params, image_size=None):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 image_size = calculate_output_image_size(image_size, s)
 if self.has_se:
Conv2d = get_same_padding_conv2d(image_size=(1,1))
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 self._swish = MemoryEfficientSwish()
def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._expand_conv(inputs)
 x = self._bn0(x)
 x = self._swish(x)
 x = self._depthwise_conv(x)
 x = self._bn1(x)
 x = self._swish(x)
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_reduce(x_squeezed)
 x_squeezed = self._swish(x_squeezed)
 x_squeezed = self._se_expand(x_squeezed)
 x = torch.sigmoid(x_squeezed)* x
x = self._project_conv(x)
 x = self._bn2(x)
 input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
image_size = global_params.image_size
Conv2d = get_same_padding_conv2d(image_size=image_size)
 in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 image_size = calculate_output_image_size(image_size, 2)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
image_size = calculate_output_image_size(image_size, block_args.stride)
 if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._avg_pooling = nn.AdaptiveAvgPool2d(1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def extract_endpoints(self, inputs):

 endpoints = dict()
x = self._swish(self._bn0(self._conv_stem(inputs)))
 prev_x = x
for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 if prev_x.size(2)> x.size(2):
endpoints[f'reduction_{len(endpoints)+1}'] = prev_x
prev_x = x
x = self._swish(self._bn1(self._conv_head(x)))
 endpoints[f'reduction_{len(endpoints)+1}'] = x
return endpoints
def extract_features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 x = self.extract_features(inputs)
 x = self._avg_pooling(x)
 x = x.flatten(start_dim=1)
 x = self._dropout(x)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, in_channels=3, **override_params):

 cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 model = cls(blocks_args, global_params)
 model._change_in_channels(in_channels)
 return model
@classmethod
def from_pretrained(cls, model_name, weights_path=None, advprop=False,
in_channels=3, num_classes=1000, **override_params):

 model = cls.from_name(model_name, num_classes = num_classes, **override_params)
 load_pretrained_weights(model, model_name, weights_path=weights_path, load_fc=(num_classes == 1000), advprop=advprop)
 model._change_in_channels(in_channels)
 return model
@classmethod
def get_image_size(cls, model_name):

 cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 if model_name not in VALID_MODELS:
raise ValueError('model_name should be one of: ' + ', '.join(VALID_MODELS))
def _change_in_channels(self, in_channels):

 if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size = self._global_params.image_size)
 out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False )<choose_model_class>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )
<choose_model_class><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )