Kaggle competition: Recursion Cellular Image Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: recursion-cellular-image-classification<choose_model_class>
%matplotlib inline
warnings.filterwarnings("ignore")
%%time
fold = Fold(input_df, input_X, seed=555); torch_seed(fold.seed)
 learn2 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )<choose_model_class>
fold = 1; head_num = 4; head_run = 3
target = 'sirna_ct'
resolution = 256
dropout_rate = 0.4
weight_decay = 2e-4
channel_size = 200
margin = 0.45
learning_rate = 0.035
warmup_learning_rate = learning_rate/100
focal_gamma = 3.5
remove_head = False
train_to = -18
freeze_at_1st_n_last = False
device = 'cuda' if torch.cuda.is_available() else 'cpu'
use_amp = True if device == 'cuda' else False
training = True if device == 'cuda' else False
train_batch_size = 96 if device == 'cuda' else 16
eval_batch_size = 16 if device == 'cuda' else 8
debug = False
create_sub =(device == 'cuda' or debug)
 print(f'Interactive:{is_interactive() } Debug:{debug} Device:{device} Training:{training}' )
%%time
fold = Fold(input_df, input_X, seed=1111); torch_seed(fold.seed)
 learn3 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )<choose_model_class>
path_data = '/kaggle/input/recursion-cellular-image-classification'
model_path = glob.glob(f'/kaggle/input/*/fold{fold}*.pth')[0]
best_sub = glob.glob(f'/kaggle/input/*/submission.csv')[0]
print(f'Training from model {model_path}')
 print(f'with pseudo labels from {best_sub}' )
%%time
fold = Fold(input_df, input_X, seed=1111); torch_seed(fold.seed)
 learn3 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )<choose_model_class>
seed_everything(42000 + head_num*100 + head_run*10 + fold )
%%time
fold = Fold(input_df, input_X, seed=3333); torch_seed(fold.seed)
 learn4 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )<find_best_params>
print(torch.__version__, ignite.__version__, os.cpu_count())
 if device == 'cuda': print(torch.cuda.get_device_name() )
%%time
test_idx = fold.test_idx
hold_idx = fold.holdout_idx
hold_y = input_df.loc[fold.holdout_idx, 'target']
ensemble_hold = []
ensemble_test = []
for li, learn in enumerate([learn1, learn2, learn3, learn4]):
for ep in [5]:
if ep >= len(learn.epoch_params): continue
learn.restore_epoch(ep)
 ensemble_hold.append(learn.predict(idx=hold_idx))
f1 = f1_curve(hold_y, ensemble_hold[-1] ).max()
print('learn%d ep%d %.6f' %(li+1, ep, f1))
ensemble_test.append(learn.predict(idx=test_idx))
ensemble_test_s = pd.Series(np.mean(ensemble_test, axis=0), index=fold.data.loc[test_idx, 'qid'])
 score = f1_curve(hold_y, np.mean(ensemble_hold, axis=0))
thresh = score.idxmax()
print('F1score=',score.max())
 submission_df = pd.read_csv('.. /input/sample_submission.csv')
 submission_df['prediction'] =(ensemble_test_s.loc[submission_df.qid] >= thresh ).astype(int ).values
submission_df.to_csv('submission.csv', index=False )<import_modules>
if use_amp and training:
try:
except ImportError:
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/*/*/NVIDIA-apex*
import time
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Lambda
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, BatchNormalization
from keras.optimizers import Adam, Nadam
from keras.models import Model
from keras import backend as K
from keras.callbacks import Callback
from keras import initializers, regularizers, constraints, optimizers, layers
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.engine.topology import Layer
import gc, re
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score, confusion_matrix, auc, precision_recall_curve<define_variables>
class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 class Flatten(nn.Module):
def forward(self, x):
return x.reshape(x.shape[0], -1 )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"ain't" : "will not",
"didn't": "did not"}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<load_from_csv>
class SqueezeExcitation(nn.Module):
def __init__(self, inplanes, se_planes):
super(SqueezeExcitation, self ).__init__()
self.reduce_expand = nn.Sequential(
 nn.Conv2d(inplanes, se_planes,
kernel_size=1, stride=1, padding=0, bias=True),
Swish() ,
nn.Conv2d(se_planes, inplanes,
kernel_size=1, stride=1, padding=0, bias=True),
nn.Sigmoid() 
)
 def forward(self, x):
x_se = torch.mean(x, dim=(-2, -1), keepdim=True)
 x_se = self.reduce_expand(x_se)
 return x_se * x
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<feature_engineering>
class MBConv(nn.Module):
def __init__(self, inplanes, planes, kernel_size, stride,
expand_rate=1.0, se_rate=0.25,
drop_connect_rate=0.2):
super(MBConv, self ).__init__()
expand_planes = int(inplanes * expand_rate)
 se_planes = max(1, int(inplanes * se_rate))
self.expansion_conv = None
if expand_rate > 1.0:
self.expansion_conv = nn.Sequential(
 nn.Conv2d(inplanes, expand_planes,
kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),
Swish() 
)
 inplanes = expand_planes
self.depthwise_conv = nn.Sequential(
 nn.Conv2d(inplanes, expand_planes,
kernel_size=kernel_size, stride=stride,
padding=kernel_size // 2, groups=expand_planes,
bias=False),
nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),
Swish() 
)
 self.squeeze_excitation = SqueezeExcitation(expand_planes, se_planes)
 self.project_conv = nn.Sequential(
 nn.Conv2d(expand_planes, planes,
kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(planes, momentum=0.01, eps=1e-3), 
)
 self.with_skip = stride == 1
self.drop_connect_rate = torch.tensor(drop_connect_rate, requires_grad=False)
 def _drop_connect(self, x):
keep_prob = 1.0 - self.drop_connect_rate
drop_mask = torch.rand(x.shape[0], 1, 1, 1)+ keep_prob
drop_mask = drop_mask.type_as(x)
 drop_mask.floor_()
return drop_mask * x / keep_prob
def forward(self, x):
z = x
if self.expansion_conv is not None:
x = self.expansion_conv(x)
 x = self.depthwise_conv(x)
 x = self.squeeze_excitation(x)
 x = self.project_conv(x)
 if x.shape == z.shape and self.with_skip:
if self.training and self.drop_connect_rate is not None:
self._drop_connect(x)
 x += z
return x
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
splits = list(StratifiedKFold(n_splits=10,random_state=2018 ).split(train_X,train_df['target'].values))<split>
def init_weights(module):
if isinstance(module, nn.Conv2d):
nn.init.kaiming_normal_(module.weight, a=0, mode='fan_out')
 elif isinstance(module, nn.Linear):
init_range = 1.0 / math.sqrt(module.weight.shape[1])
 nn.init.uniform_(module.weight, a=-init_range, b=init_range)
 class EfficientNet(nn.Module):
def _setup_repeats(self, num_repeats):
return int(math.ceil(self.depth_coefficient * num_repeats))
def _setup_channels(self, num_channels):
num_channels *= self.width_coefficient
new_num_channels = math.floor(num_channels / self.divisor + 0.5)* self.divisor
new_num_channels = max(self.divisor, new_num_channels)
 if new_num_channels < 0.9 * num_channels:
new_num_channels += self.divisor
return new_num_channels
def __init__(self, num_classes,
width_coefficient=1.0,
depth_coefficient=1.0,
se_rate=0.25,
dropout_rate=0.2,
drop_connect_rate=0.2):
super(EfficientNet, self ).__init__()
self.width_coefficient = width_coefficient
self.depth_coefficient = depth_coefficient
self.divisor = 8
list_channels = [32, 16, 24, 40, 80, 112, 192, 320, 1280]
list_channels = [self._setup_channels(c)for c in list_channels]
list_num_repeats = [1, 2, 2, 3, 3, 4, 1]
list_num_repeats = [self._setup_repeats(r)for r in list_num_repeats]
expand_rates = [1, 6, 6, 6, 6, 6, 6]
strides = [1, 2, 2, 2, 1, 2, 1]
kernel_sizes = [3, 3, 5, 3, 5, 5, 3]
self.stem = nn.Sequential(
 nn.Conv2d(3, list_channels[0], kernel_size=3, stride=2, padding=1, bias=False),
nn.BatchNorm2d(list_channels[0], momentum=0.01, eps=1e-3),
Swish() 
)
 blocks = []
counter = 0
num_blocks = sum(list_num_repeats)
 for idx in range(7):
num_channels = list_channels[idx]
next_num_channels = list_channels[idx + 1]
num_repeats = list_num_repeats[idx]
expand_rate = expand_rates[idx]
kernel_size = kernel_sizes[idx]
stride = strides[idx]
drop_rate = drop_connect_rate * counter / num_blocks
name = "MBConv{}_{}".format(expand_rate, counter)
 blocks.append((
name,
MBConv(num_channels, next_num_channels,
kernel_size=kernel_size, stride=stride, expand_rate=expand_rate,
se_rate=se_rate, drop_connect_rate=drop_rate)
))
counter += 1
for i in range(1, num_repeats):
name = "MBConv{}_{}".format(expand_rate, counter)
 drop_rate = drop_connect_rate * counter / num_blocks
blocks.append((
name,
MBConv(next_num_channels, next_num_channels,
kernel_size=kernel_size, stride=1, expand_rate=expand_rate,
se_rate=se_rate, drop_connect_rate=drop_rate)
))
counter += 1
self.blocks = nn.Sequential(OrderedDict(blocks))
self.head = nn.Sequential(
 nn.Conv2d(list_channels[-2], list_channels[-1],
kernel_size=1, bias=False),
nn.BatchNorm2d(list_channels[-1], momentum=0.01, eps=1e-3),
Swish() ,
nn.AdaptiveAvgPool2d(1),
Flatten() ,
nn.Dropout(p=dropout_rate),
nn.Linear(list_channels[-1], num_classes)
)
 self.apply(init_weights)
 def forward(self, x):
f = self.stem(x)
 f = self.blocks(f)
 y = self.head(f)
 return y
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<choose_model_class>
import torchvision.utils as vutils
import pandas as pd
from sklearn import preprocessing
from sklearn.utils import shuffle
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
le = preprocessing.LabelEncoder()
le.fit_transform(['HEPG2', 'HUVEC', 'RPE', 'U2OS'] )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
df = pd.read_csv(path_data+'/train.csv')
 df['set'] = 'train'
df_test = pd.read_csv(path_data+'/test.csv')
 df_test['set'] = 'test'
best = pd.read_csv(best_sub, index_col=0)
 df_test_pseudo = df_test.set_index('id_code' ).join(best, how='inner' ).reset_index()
df_ctrl = pd.read_csv(path_data+'/train_controls.csv')
 df_ctrl['set'] = 'train'
df_ctrl_test = pd.read_csv(path_data+'/test_controls.csv')
 df_ctrl_test['set'] = 'test'
df_ctrl = pd.concat([df_ctrl, df_ctrl_test], ignore_index=False, sort=False)
 del df_ctrl_test
n_targets = df.sirna.nunique() + df_ctrl.sirna.nunique()
for d in [df, df_test, df_test_pseudo, df_ctrl]:
d['cell_type'] = d.experiment.apply(lambda s: s[:-3])
 d['cell_type_n'] = le.transform(d.cell_type)
 if d is not df_test:
d['sirna_ct'] = d.sirna +(d.cell_type_n * n_targets)
 n_classes = df.sirna_ct.nunique() + df_ctrl.sirna_ct.nunique()
print(n_targets, n_classes )
pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_noemb_val_y>thresh ).astype(int))))<predict_on_test>
if training or not debug:
for label,n in target_dist.items() :
copies = min(int(round(max_n/n)- 1), 4)
 if copies != 0:
addon = [df_train[df_train[target] == label]] * copies
df_train = pd.concat([df_train] + addon, ignore_index=False, sort=False )
pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
df_train = pd.concat([df_train, df_ctrl[(df_ctrl['cell_type_n'] == 4)]],
ignore_index=False, sort=False)
 df_train = pd.concat([df_train, df_ctrl[(
 df_ctrl['cell_type_n'].isin([0,2])) &((
df_ctrl['plate'] != fold)|(
 df_ctrl.experiment.isin(df_test.experiment.unique())| df_ctrl.experiment.isin(valid_experiments)))]],
ignore_index=False, sort=False)
 df_train = pd.concat([df_train, df_ctrl[(
 df_ctrl['cell_type_n'] == 1)&(
 df_ctrl['plate'] == fold)&(
 df_ctrl.experiment.isin(df_test.experiment.unique())| df_ctrl.experiment.isin(valid_experiments)) ]],
ignore_index=False, sort=False )
del model, inp, x
time.sleep(10 )<statistical_test>
if debug:
df_train = df_train[df_train[target].isin(df_train[target].sample(50)) ]
df_valid = df_valid[df_valid[target].isin(df_train[target].unique())]
print(df_train.shape, df_valid.shape, df_test.shape )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
class_weights = class_weights.values.ravel()
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
if not is_interactive() :
del df; del df_ctrl; del target_dist
gc.collect()
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
class ExpNormTwinDataset(Dataset):

 def __init__(self, df, img_dir, target, mode='train', sites=[1,2], channels=[1,2,3,4,5,6],
img_stats=None, transform=None):
self.df = df
self.channels = channels
self.sites = sites
self.target = target
self.mode = mode
self.img_dir = img_dir
self.stats = img_stats
self.transform = transform
@staticmethod
def _load_channel(file_name):
img = cv2.imread(file_name, cv2.IMREAD_UNCHANGED)
 return np.float32(img)
 def _get_img(self, rec, site):
dset, experiment, well, plate = rec.set, rec.experiment, rec.well, rec.plate
paths = [os.path.join(self.img_dir, dset, experiment, f'Plate{plate}', f'{well}_s{site}_w{channel}.png')
 for channel in self.channels]
img = [self._load_channel(img_path)for img_path in paths]
if self.stats is not None:
stats = self.stats.loc[experiment, ['mean', 'std']]
img = [i-m for i,m in zip(img, stats['mean'].values)]
img = [i/s for i,s in zip(img, stats['std'].values)]
img = np.stack(img, axis=-1)
 if self.transform:
img = self.transform(image=img)['image']
return img
def _get_twin(self, rec, site):
experiment, target = rec.experiment, rec[self.target]
twin = self.df[(self.df[self.target] == target)&(self.df.experiment != experiment)].sample(1 ).iloc[0]
return self._get_img(twin, site)
 def __getitem__(self, index):
rec = self.df.iloc[index]
if self.mode == 'train':
img = self._get_img(rec, random.choice(self.sites))
twin = self._get_twin(rec, random.choice(self.sites))
return img, twin, rec[self.target]
elif self.mode == 'eval':
img = self._get_img(rec, random.choice(self.sites))
return img, rec[self.target]
elif self.mode == 'test':
return [self._get_img(rec, site)for site in self.sites]
def __len__(self):

 return len(self.df)
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
from albumentations import *
from albumentations.pytorch import ToTensor
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
num_workers = os.cpu_count()
print('num_workers:', num_workers)
 train_loader = DataLoader(train_dataset, batch_size=train_batch_size, num_workers=num_workers,
shuffle=True, drop_last=True, pin_memory=True)
 eval_train_loader = DataLoader(train_eval_dataset, batch_size=eval_batch_size, num_workers=num_workers,
shuffle=False, drop_last=False, pin_memory=True)
 valid_loader = DataLoader(valid_dataset, batch_size=eval_batch_size, num_workers=num_workers,
shuffle=False, drop_last=False, pin_memory=True)
 test_loader = DataLoader(test_dataset, batch_size=eval_batch_size, num_workers=num_workers,
shuffle=False, drop_last=False, pin_memory=True)
 print(len(train_loader), len(eval_train_loader), len(valid_loader), len(test_loader))
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
del batch
torch.cuda.empty_cache()
gc.collect()
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
n_channels = len(test_dataset.channels )
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
def load_n_remap(model, model_path, device):
model_state = torch.load(model_path, map_location=device)
 mapping = { k:v for k,v in zip(model_state.keys() , model.state_dict().keys())}
mapped_model_state = OrderedDict([ 
(mapping[k], v)for k,v in model_state.items() if k in mapping.keys()
])
 model.load_state_dict(mapped_model_state, strict=False)
 return model
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super().__init__()
self.p = nn.Parameter(torch.ones(1)*p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(self.eps)+ ')'
def get_model(model_path=None, n_classes=n_classes):
model = EfficientNet(num_classes=n_classes,
width_coefficient=1.4, depth_coefficient=1.8,
dropout_rate=dropout_rate)
 orig_stem = model.stem[0]
model.stem[0] = nn.Conv2d(
 in_channels=n_channels,
out_channels=model.stem[0].out_channels,
kernel_size=model.stem[0].kernel_size,
stride=model.stem[0].stride,
padding=model.stem[0].padding,
bias=model.stem[0].bias)
 del model.head[6]
if model_path is not None:
model = load_n_remap(model, model_path, device)
 return model
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
class ArcModule(nn.Linear):
def __init__(self, in_features, out_features, s, m):
super().__init__(in_features=in_features, out_features=out_features, bias=False)
 self.s = s
self.m = m
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.th = math.cos(math.pi - m)
 self.mm = math.sin(math.pi - m)* m
def forward(self, inputs, labels=None):
cos_th = F.linear(inputs, F.normalize(self.weight))
if labels is None:
assert not self.training
return cos_th * self.s
cos_th = cos_th.clamp(-1, 1)
 sin_th = torch.sqrt(1.0 - torch.pow(cos_th, 2))
cos_th_m = cos_th * self.cos_m - sin_th * self.sin_m
cos_th_m = torch.where(cos_th > self.th, cos_th_m, cos_th - self.mm)
 cond_v = cos_th - self.th
cond = cond_v <= 0
cos_th_m[cond] =(cos_th - self.mm)[cond]
if labels.dim() == 1:
labels = labels.unsqueeze(-1)
 onehot = torch.zeros_like(cos_th)
 onehot.scatter_(1, labels, 1)
 outputs = onehot * cos_th_m +(1.0 - onehot)* cos_th
outputs = outputs * self.s
return outputs
class ArcNet(nn.Module):
def __init__(self, criterion, backbone, in_features, num_classes, channel_size, s=65, m=margin):
super().__init__()
assert num_classes % 2 == 0
self.num_classes = num_classes
self.criterion = criterion
self.backbone = backbone
self.feat = nn.Sequential(
 nn.Linear(in_features, channel_size, bias=False),
nn.BatchNorm1d(channel_size, momentum=0.01), 
)
 self.arc = ArcModule(channel_size, num_classes, s=s, m=m)
 def features(self, images, labels=None):
features = self.backbone(images)
 features = self.feat(features)
 features = F.normalize(features, dim=-1)
 return features
def forward(self, images, twins=None, labels=None):
if labels is None:
features = self.features(images)
 return self.arc(features)
 else:
return self.loss(images, twins, labels)
 def loss(self, images, twins, labels, cos_w=3):
f0 = self.features(images)
 f1 = self.features(twins)
 targets =(labels % n_targets)
 same_target = targets.expand(size=(len(targets), len(targets)))
 same_target = 2*(same_target == same_target.t() ).type_as(f0)- 1
cos_dist = 1 -(torch.mm(f0, f1.t())* same_target)
 cos_dist = cos_dist**2
loss = cos_w *(torch.mean(cos_dist))
pred0 = self.arc(f0, labels)
 pred1 = self.arc(f1, labels)
 return loss + 0.5 *(self.criterion(pred0, labels)+ self.criterion(pred1, labels))
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
backbone = get_model(model_path if remove_head else None, n_classes)
 model = ArcNet(criterion=FocalLoss(gamma=focal_gamma, alpha=class_weights),
backbone=backbone,
in_features=backbone.head[0].out_channels,
num_classes=n_classes,
channel_size=channel_size)
 if not remove_head:
model = load_n_remap(model, model_path, device )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
lr = warmup_learning_rate
if train_to:
training_blocks = model.backbone.blocks[train_to:].parameters()
else:
training_blocks = chain(model.backbone.stem.parameters() , model.backbone.blocks.parameters())
 params2freeze = None
if freeze_at_1st_n_last:
params2freeze = chain(model.backbone.stem.named_parameters() , model.backbone.blocks.named_parameters())
 elif train_to:
params2freeze = chain(model.backbone.stem.named_parameters() , model.backbone.blocks[:train_to].named_parameters())
 if params2freeze is not None:
for name, param in params2freeze:
print(f'"{name}" is frozen')
 param.requires_grad = False
optimizer = optim.SGD([
{ "params": training_blocks,
"lr": lr * 0.01,
},
{ "params": chain(model.feat[0].parameters() , model.backbone.head.parameters()),
"lr": lr * 0.2,
},
{ "params": chain(model.feat[1].parameters() , model.arc.parameters()),
"lr": lr
}],
momentum=0.9, weight_decay=1e-6, nesterov=True)
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
if use_amp and training:
model, optimizer = amp.initialize(model, optimizer, opt_level="O2", num_losses=1 )
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
def update_fn(engine, batch):
x0 = convert_tensor(batch[0], device=device, non_blocking=True)
 x1 = convert_tensor(batch[1], device=device, non_blocking=True)
 y = convert_tensor(batch[2], device=device, non_blocking=True)
 model.train()
loss = model(x0, x1, y)
 optimizer.zero_grad()
if use_amp:
with amp.scale_loss(loss, optimizer, loss_id=0)as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
return {
"batchloss": loss.item() ,
}
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<compute_test_metric>
from ignite.engine import Engine, Events, create_supervised_evaluator
from ignite.metrics import RunningAverage, Accuracy, Precision, Recall, Loss, TopKCategoricalAccuracy
from ignite.contrib.handlers import TensorboardLogger
from ignite.contrib.handlers.tensorboard_logger import OutputHandler, OptimizerParamsHandler
pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))<save_to_csv>
trainer = Engine(update_fn)
 def siRNAaccuracyMax1108(pred, y):
" Check predictions only on classes present in the validation and test sets "
pred_ = pred.reshape([pred.shape[0], n_classes // n_targets, n_targets] ).max(dim=1 ).values
pred_ = torch.argmax(pred_[...,:1108], dim=-1)
 y_ =(y % n_targets)
 assert pred_.shape == y_.shape
return(pred_ == y_ ).float().mean().cpu().numpy()
metrics = {
'Score': Loss(siRNAaccuracyMax1108),
'Accuracy': Accuracy() ,
'Top5Accuracy': TopKCategoricalAccuracy(k=5),
}
evaluator = create_supervised_evaluator(model, metrics=metrics, device=device, non_blocking=True)
 history = Metrics(evaluator, eval_train_loader, output_transform=lambda out: out['batchloss'], interactive=is_interactive())
 history.attach(trainer, "batchloss" )
pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<save_to_csv>
log_path = f"./log"
tb_logger = TensorboardLogger(log_dir=log_path)
 tb_logger.attach(trainer,
log_handler=OutputHandler('training', ['batchloss', ]),
event_name=Events.ITERATION_COMPLETED )
coefs = [0.35,0.25,0.2,0.2]
pred_test_y = np.sum([outputs[i][1]*coefs[i] for i in range(len(coefs)) ], axis = 0)
 pred_test_y =(pred_test_y > results['threshold'] ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<set_options>
sim_epochs = 2
epoch_size = len(train_loader)
 cycle_mult = 1
lr_sched_params = {'param_name':'lr', 'cycle_size':epoch_size*2, 'cycle_mult':cycle_mult,
'start_value_mult':0.7, 'end_value_mult':0.5}
mom_sched_params = {'param_name':'momentum', 'cycle_size':epoch_size*2, 'cycle_mult':cycle_mult,
'start_value':0.82, 'end_value':0.92}
wd_sched_params = {'param_name':'weight_decay', 'cycle_size':epoch_size*sim_epochs, 'cycle_mult':cycle_mult,
'start_value':weight_decay/10, 'end_value':weight_decay}
lr = learning_rate
body_sched_params = {**lr_sched_params, 'start_value':lr/10, 'end_value':lr/(100*2**head_run)}
body_sched = CosineAnnealingScheduler(optimizer.param_groups[0], **body_sched_params)
 feat_sched_params = {**lr_sched_params, 'start_value':lr/5, 'end_value':lr/(50*2**head_run)}
feat_sched = CosineAnnealingScheduler(optimizer.param_groups[1], **feat_sched_params)
 head_sched_params = {**lr_sched_params, 'start_value':lr, 'end_value':lr/(20*2**head_run)}
head_sched = CosineAnnealingScheduler(optimizer.param_groups[2], **feat_sched_params)
 mom_sched = CosineAnnealingScheduler(optimizer, **mom_sched_params)
 wd_sched = CosineAnnealingScheduler(optimizer, **wd_sched_params)
 schedulers = [body_sched, feat_sched, head_sched, mom_sched, wd_sched]
names = ["lr(body)", "lr(feat)", "lr(head)", "momentum", "wd"]
lr_values0 = np.array(body_sched.simulate_values(num_events=epoch_size*sim_epochs, **body_sched_params))
lr_values1 = np.array(feat_sched.simulate_values(num_events=epoch_size*sim_epochs, **feat_sched_params))
lr_values2 = np.array(head_sched.simulate_values(num_events=epoch_size*sim_epochs, **head_sched_params))
wd_values = np.array(wd_sched.simulate_values(num_events=epoch_size*sim_epochs, **wd_sched_params))
mom_values = np.array(mom_sched.simulate_values(num_events=epoch_size*sim_epochs, **mom_sched_params))
fig = plt.figure(figsize=(16, 4))
ax = plt.subplot()
plt.title(f"Cosine annealing with start_value_mult={lr_sched_params['start_value_mult']}")
 ax.plot(lr_values0[:, 0], lr_values0[:, 1], label=names[0])
 ax.plot(lr_values1[:, 0], lr_values1[:, 1], label=names[1])
 ax.plot(lr_values2[:, 0], lr_values2[:, 1], label=names[2])
 ax.plot(wd_values[:, 0], wd_values[:, 1], label=names[4])
 ax.set_yscale('log')
 ax.set_xlabel('Batches processed')
 ax.set_ylabel("learning rate")
 ax.legend(frameon=False, loc='upper right')
 ax2 = ax.twinx()
ax2.plot(mom_values[:, 0], mom_values[:, 1], label=names[3])
 ax2.set_ylabel("momentum")
 ax2.legend(frameon=False, loc='lower right')
 _ = ax.plot()
%matplotlib inline
pd.set_option('max_colwidth',400)
 warnings.filterwarnings("ignore", message="F-score is ill-defined and being set to 0.0 due to no predicted samples.")
<set_options>
scheduler = ParamGroupScheduler(schedulers=schedulers, names=names)
 trainer.add_event_handler(Events.ITERATION_STARTED, scheduler )
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<load_from_csv>
tb_logger.attach(trainer,
log_handler=OptimizerParamsHandler(optimizer, "lr"),
event_name=Events.EPOCH_STARTED )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 sub = pd.read_csv('.. /input/sample_submission.csv' )<count_values>
def setup_logger(logger):
handler = logging.StreamHandler()
formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
 handler.setFormatter(formatter)
 logger.addHandler(handler)
 logger.setLevel(logging.INFO )
train["target"].value_counts()<define_variables>
trainer.add_event_handler(Events.ITERATION_COMPLETED, TerminateOnNan())
 def default_score_fn(engine):
score = engine.state.metrics['Score']
return score
best_model_handler = ModelCheckpoint(dirname=log_path,
filename_prefix="best",
n_saved=10,
score_name="Score",
score_function=default_score_fn,
require_empty=False)
 evaluator.add_event_handler(Events.COMPLETED, best_model_handler, {'model': model, })
 @trainer.on(Events.EPOCH_COMPLETED)
 @evaluator.on(Events.COMPLETED)
 def empty_cuda_cache(engine):
if debug and device == 'cuda': print('max_memory_allocated:', torch.cuda.max_memory_allocated())
 torch.cuda.empty_cache()
gc.collect()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 train["question_text"] = train["question_text"].apply(lambda x: clean_text(x.lower()))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x.lower()))
train["question_text"] = train["question_text"].apply(lambda x: clean_numbers(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_numbers(x))
train["question_text"] = train["question_text"].apply(lambda x: replace_typical_misspell(x))
test["question_text"] = test["question_text"].apply(lambda x: replace_typical_misspell(x))<train_model>
NUM_EPOCHS =(4 if debug else sim_epochs)
 if freeze_at_1st_n_last:
@trainer.on(Events.EPOCH_STARTED)
 def turn_on_layers(engine):
epoch = engine.state.epoch
if(epoch == 1)or(epoch == NUM_EPOCHS):
for name, param in model.named_parameters() :
if('backbone' not in name)or('head' in name):
pbar.log_message(f'training "{name}"')
 param.requires_grad = True
else:
param.requires_grad = False
elif(epoch == 2):
count = 0
for param in training_blocks:
if param.requires_grad == False:
param.requires_grad = True
count += 1
pbar.log_message(f"Epoch {epoch}: training all layers({count})" )
max_features = 120000
tk = Tokenizer(lower = True, filters='', num_words=max_features)
 full_text = list(train['question_text'].values)+ list(test['question_text'].values)
 tk.fit_on_texts(full_text )<string_transform>
if training:
state = trainer.run(train_loader, max_epochs=NUM_EPOCHS )
train_tokenized = tk.texts_to_sequences(train['question_text'].fillna('missing'))
test_tokenized = tk.texts_to_sequences(test['question_text'].fillna('missing'))<categorify>
best_model = model
max_len = 72
maxlen = 72
X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_test = pad_sequences(test_tokenized, maxlen = max_len )<prepare_x_and_y>
!ls {log_path}
checkpoints = next(os.walk(log_path)) [2]
checkpoints = sorted(filter(lambda f: f.endswith(".pth"), checkpoints))
scores = [c.split('=')[-1][:-4] for c in checkpoints]
best_epoch = 0
if len(scores)> 0:
best_epoch = np.argmax(scores)
 print(best_epoch, scores)
 if not checkpoints:
print('No weight files in {}'.format(log_path))
else:
model_path = f'fold{fold}h{head_num}r{head_run}_{best_epoch}_{scores[best_epoch]}.pth'
!cp {os.path.join(log_path, checkpoints[best_epoch])} {model_path}
best_model.load_state_dict(torch.load(model_path))
print(model_path)
 if not is_interactive() :
!rm {log_path}/*.pth
y_train = train['target'].values<compute_test_metric>
best_model = best_model.to(device ).eval()
def sigmoid(x):
return 1 /(1 + np.exp(-x))<split>
with torch.no_grad() :
y_pred = best_model(batch[0][:1].to(device))
for idx in torch.topk(y_pred, k=10)[1].squeeze(0 ).tolist() :
prob = torch.softmax(y_pred, dim=-1)[0, idx].item()
print('{label:<75}({p:.8f}%)'.format(label=str(idx), p=prob*100))
del batch
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10 ).split(X_train, y_train))<statistical_test>
TTA = 9
def tta(sites):
pred = []
for site in sites:
site = site.to(device)
 for i in range(TTA):
x = tta9crop(site, i, resolution)
 pred.append(best_model(x ).unsqueeze(0))
pred = torch.cat(pred)
 pred = pred.reshape([pred.shape[0], pred.shape[1], n_classes // n_targets, n_targets] ).max(dim=-2 ).values
return pred[...,:1108].softmax(dim=-1 ).mean(dim=0 ).cpu()
embed_size = 300
embedding_path = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))
emb_mean,emb_std = -0.005838499, 0.48782197
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<compute_train_metric>
def inference_update_with_tta(engine, batch):
global preds, allpreds
with torch.no_grad() :
sites = batch
pred = tta(sites)
 allpreds = np.concatenate([allpreds,(-np.log(pred.numpy())).astype(np.float16)])
 preds +=(torch.argmax(pred, dim=-1)% n_targets ).tolist()
embedding_path = ".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_path, encoding='utf-8', errors='ignore')if len(o)>100)
 emb_mean,emb_std = -0.0053247833, 0.49346462
embedding_matrix1 = np.random.normal(emb_mean, emb_std,(nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix1[i] = embedding_vector<compute_test_metric>
if create_sub:
pd.DataFrame(allpreds, index=df_valid.id_code ).to_pickle(f'df_valid_fold{fold}_log.pkl.gz')
 print(glob.glob('*.gz'))
embedding_matrix = np.mean([embedding_matrix, embedding_matrix1], axis=0)
 del embedding_matrix1<normalization>
if create_sub:
test_y = np.exp(-pd.read_pickle('df_test_log.pkl.gz' ).astype(np.float64)).fillna(0)
 valid_y = np.exp(-pd.read_pickle(f'df_valid_fold{fold}_log.pkl.gz' ).astype(np.float64)).fillna(0)
 else:
test_y = np.exp(-pd.read_pickle(glob.glob(f'.. /input/*/df_test_log.pkl.gz')[0] ).astype(np.float64)).fillna(0)
 valid_y = np.exp(-pd.read_pickle(glob.glob(f'.. /input/*/df_valid_fold{fold}_log.pkl.gz')[0] ).astype(np.float64)).fillna(0)
 test_y = test_y / test_y.sum(axis=0)
 valid_y = valid_y / valid_y.sum(axis=0)
 df_valid['pred'] = valid_y.values.argmax(axis=1)
 df_valid['acc'] =(df_valid.pred == df_valid.sirna ).astype(int)
 print('validation accuracy(norm):', df_valid.acc.mean())
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 128
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(1024, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<choose_model_class>
preds = remove_leaked_sirna(df_valid.set_index('id_code'), valid_y.copy())
 best_score = -1
bestpreds = None
scores = []
for k in tqdm(range(10)) :
preds, newscore, sirna_preds = normalize_both_ways(preds, df_valid.sirna.values)
 print(f'Newscore:{newscore} Iter:{k}')
 scores.append(newscore)
 if newscore > best_score:
best_score = newscore
best_iter = k
bestpreds = sirna_preds
m = NeuralNet()<train_model>
test_preds = remove_leaked_sirna(df_test.set_index('id_code'), test_y.copy())
 for i in tqdm(range(best_iter+1)) :
test_preds, _, _ = normalize_both_ways(test_preds)
 preds = assign_sirna(test_preds )
<create_dataframe><EOS> 
if create_sub:
submission = pd.DataFrame({'id_code': pd.read_csv(path_data+'/sample_submission.csv' ).id_code.values,
'sirna': np.squeeze(preds ).astype(int)})
 submission.to_csv('submission.csv', index=False)
 submission.hist(bins=111)
 submission.head()