Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<define_variables>
tqdm.pandas(desc='Progress')
PATH = './'
TRAIN = '.. /input/human-protein-atlas-image-classification/train/'
TEST = '.. /input/human-protein-atlas-image-classification/test/'
LABELS = '.. /input/human-protein-atlas-image-classification/train.csv'
SAMPLE = '.. /input/human-protein-atlas-image-classification/sample_submission.csv'<define_variables>
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED = 10
debug =0
name_label_dict = {
0: 'Nucleoplasm',
1: 'Nuclear membrane',
2: 'Nucleoli',
3: 'Nucleoli fibrillar center',
4: 'Nuclear speckles',
5: 'Nuclear bodies',
6: 'Endoplasmic reticulum',
7: 'Golgi apparatus',
8: 'Peroxisomes',
9: 'Endosomes',
10: 'Lysosomes',
11: 'Intermediate filaments',
12: 'Actin filaments',
13: 'Focal adhesion sites',
14: 'Microtubules',
15: 'Microtubule ends',
16: 'Cytokinetic bridge',
17: 'Mitotic spindle',
18: 'Microtubule organizing center',
19: 'Centrosome',
20: 'Lipid droplets',
21: 'Plasma membrane',
22: 'Cell junctions',
23: 'Mitochondria',
24: 'Aggresome',
25: 'Cytosol',
26: 'Cytoplasmic bodies',
27: 'Rods & rings' }<choose_model_class>
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum' )
nw = 2
arch = resnet34<split>
def seed_everything(seed=10):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
train_names = list({f[:36] for f in os.listdir(TRAIN)})
 test_names = list({f[:36] for f in os.listdir(TEST)})
 tr_n, val_n = train_test_split(train_names, test_size=0.1, random_state=42 )<data_type_conversions>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
else:
embedding_vector = embeddings_index.get(word.capitalize())
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return np.stack(img, axis=-1 )<prepare_x_and_y>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.labels = pd.read_csv(LABELS ).set_index('Id')
 self.labels['Target'] = [[int(i)for i in s.split() ] for s in self.labels['Target']]
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_rgby(self.path,self.fnames[i])
 if self.sz == 512: return img
else: return cv2.resize(img,(self.sz, self.sz),cv2.INTER_AREA)
 def get_y(self, i):
if(self.path == TEST): return np.zeros(len(name_label_dict),dtype=np.int)
 else:
labels = self.labels.loc[self.fnames[i]]['Target']
return np.eye(len(name_label_dict),dtype=np.float)[labels].sum(axis=0)
 @property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return len(name_label_dict )<create_dataframe>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
if bool(re.search(r'\d', x)) :
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
def get_data(sz,bs):
aug_tfms = [RandomRotate(30, tfm_y=TfmType.NO),
RandomDihedral(tfm_y=TfmType.NO)]
stats = A([0.00505, 0.00331, 0.00344, 0.00519], [0.10038, 0.08131, 0.08284, 0.10179])
 tfms = tfms_from_stats(stats, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(pdFilesDataset,(tr_n[:-(len(tr_n)%bs)],TRAIN), 
(val_n,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md<train_model>
def parallelize_apply(df,func,colname,num_process,newcolnames):
pool =Pool(processes=num_process)
 arraydata = pool.map(func,tqdm(df[colname].values))
pool.close()
newdf = pd.DataFrame(arraydata,columns = newcolnames)
 df = pd.concat([df,newdf],axis=1)
 return df
def parallelize_dataframe(df, func):
df_split = np.array_split(df, 4)
 pool = Pool(4)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df["lower_question_text"] = df["question_text"].apply(lambda x: x.lower())
 df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
if debug:
train_df = pd.read_csv(".. /input/train.csv")[:80000]
test_df = pd.read_csv(".. /input/test.csv")[:20000]
else:
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)

train = parallelize_dataframe(train_df, add_features)
 test = parallelize_dataframe(test_df, add_features)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
features = train[['num_unique_words','words_vs_unique']].fillna(0)
 test_features = test[['num_unique_words','words_vs_unique']].fillna(0)
 ss = StandardScaler()
pc = PCA(n_components=5)
 ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
features = features[trn_idx]
return train_X, test_X, train_y, features, test_features, tokenizer.word_index
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()<compute_test_metric>
start = time.time()
x_train, x_test, y_train, features, test_features, word_index = load_and_prec()
print(time.time() -start )
def acc(preds,targs,thresh=0.0):
preds =(preds > thresh ).int()
targs = targs.int()
return(preds==targs ).float().mean()<choose_model_class>
seed_everything()
if debug:
paragram_embeddings = np.random.randn(120000,300)
 glove_embeddings = np.random.randn(120000,300)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 else:
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0 )
class ConvnetBuilder_custom() :
def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0,
custom_head=None, pretrained=True):
self.f,self.c,self.is_multi,self.is_reg,self.xtra_cut = f,c,is_multi,is_reg,xtra_cut
if xtra_fc is None: xtra_fc = [512]
if ps is None: ps = [0.25]*len(xtra_fc)+ [0.5]
self.ps,self.xtra_fc = ps,xtra_fc
if f in model_meta: cut,self.lr_cut = model_meta[f]
else: cut,self.lr_cut = 0,0
cut-=xtra_cut
layers = cut_model(f(pretrained), cut)
 w = layers[0].weight
layers[0] = nn.Conv2d(4,64,kernel_size=(7,7),stride=(2,2),padding=(3, 3), bias=False)
 layers[0].weight = torch.nn.Parameter(torch.cat(( w,torch.zeros(64,1,7,7)) ,dim=1))
self.nf = model_features[f] if f in model_features else(num_features(layers)*2)
 if not custom_head: layers += [AdaptiveConcatPool2d() , Flatten() ]
self.top_model = nn.Sequential(*layers)
 n_fc = len(self.xtra_fc)+1
if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc
if custom_head: fc_layers = [custom_head]
else: fc_layers = self.get_fc_layers()
self.n_fc = len(fc_layers)
 self.fc_model = to_gpu(nn.Sequential(*fc_layers))
if not custom_head: apply_init(self.fc_model, kaiming_normal)
 self.model = to_gpu(nn.Sequential(*(layers+fc_layers)))
 @property
def name(self): return f'{self.f.__name__}_{self.xtra_cut}'
def create_fc_layer(self, ni, nf, p, actn=None):
res=[nn.BatchNorm1d(num_features=ni)]
if p: res.append(nn.Dropout(p=p))
res.append(nn.Linear(in_features=ni, out_features=nf))
if actn: res.append(actn)
 return res
def get_fc_layers(self):
res=[]
ni=self.nf
for i,nf in enumerate(self.xtra_fc):
res += self.create_fc_layer(ni, nf, p=self.ps[i], actn=nn.ReLU())
 ni=nf
final_actn = nn.Sigmoid() if self.is_multi else nn.LogSoftmax()
if self.is_reg: final_actn = None
res += self.create_fc_layer(ni, self.c, p=self.ps[-1], actn=final_actn)
 return res
def get_layer_groups(self, do_fc=False):
if do_fc:
return [self.fc_model]
idxs = [self.lr_cut]
c = children(self.top_model)
 if len(c)==3: c = children(c[0])+c[1:]
lgs = list(split_by_idxs(c,idxs))
return lgs+[self.fc_model]
class ConvLearner(Learner):
def __init__(self, data, models, precompute=False, **kwargs):
self.precompute = False
super().__init__(data, models, **kwargs)
 if hasattr(data, 'is_multi')and not data.is_reg and self.metrics is None:
self.metrics = [accuracy_thresh(0.5)] if self.data.is_multi else [accuracy]
if precompute: self.save_fc1()
self.freeze()
self.precompute = precompute
def _get_crit(self, data):
if not hasattr(data, 'is_multi'): return super()._get_crit(data)
 return F.l1_loss if data.is_reg else F.binary_cross_entropy if data.is_multi else F.nll_loss
@classmethod
def pretrained(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
pretrained=True, **kwargs):
models = ConvnetBuilder_custom(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=pretrained)
 return cls(data, models, precompute, **kwargs)
 @classmethod
def lsuv_learner(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=False, **kwargs):
models = ConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=False)
 convlearn=cls(data, models, precompute, **kwargs)
 convlearn.lsuv_init()
return convlearn
@property
def model(self): return self.models.fc_model if self.precompute else self.models.model
def half(self):
if self.fp16: return
self.fp16 = True
if type(self.model)!= FP16: self.models.model = FP16(self.model)
 if not isinstance(self.models.fc_model, FP16): self.models.fc_model = FP16(self.models.fc_model)
 def float(self):
if not self.fp16: return
self.fp16 = False
if type(self.models.model)== FP16: self.models.model = self.model.module.float()
if type(self.models.fc_model)== FP16: self.models.fc_model = self.models.fc_model.module.float()
@property
def data(self): return self.fc_data if self.precompute else self.data_
def create_empty_bcolz(self, n, name):
return bcolz.carray(np.zeros(( 0,n), np.float32), chunklen=1, mode='w', rootdir=name)
 def set_data(self, data, precompute=False):
super().set_data(data)
 if precompute:
self.unfreeze()
self.save_fc1()
self.freeze()
self.precompute = True
else:
self.freeze()
def get_layer_groups(self):
return self.models.get_layer_groups(self.precompute)
 def summary(self):
precompute = self.precompute
self.precompute = False
res = super().summary()
self.precompute = precompute
return res
def get_activations(self, force=False):
tmpl = f'_{self.models.name}_{self.data.sz}.bc'
names = [os.path.join(self.tmp_path, p+tmpl)for p in('x_act', 'x_act_val', 'x_act_test')]
if os.path.exists(names[0])and not force:
self.activations = [bcolz.open(p)for p in names]
else:
self.activations = [self.create_empty_bcolz(self.models.nf,n)for n in names]
def save_fc1(self):
self.get_activations()
act, val_act, test_act = self.activations
m=self.models.top_model
if len(self.activations[0])!=len(self.data.trn_ds):
predict_to_bcolz(m, self.data.fix_dl, act)
 if len(self.activations[1])!=len(self.data.val_ds):
predict_to_bcolz(m, self.data.val_dl, val_act)
 if self.data.test_dl and(len(self.activations[2])!=len(self.data.test_ds)) :
if self.data.test_dl: predict_to_bcolz(m, self.data.test_dl, test_act)
 self.fc_data = ImageClassifierData.from_arrays(self.data.path, 
(act, self.data.trn_y),(val_act, self.data.val_y), self.data.bs, classes=self.data.classes,
test = test_act if self.data.test_dl else None, num_workers=8)
 def freeze(self):
self.freeze_to(-1)
 def unfreeze(self):
self.freeze_to(0)
 self.precompute = False
def predict_array(self, arr):
precompute = self.precompute
self.precompute = False
pred = super().predict_array(arr)
 self.precompute = precompute
return pred<choose_model_class>
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
sz = 256
bs = 64
md = get_data(sz,bs)
 learner = ConvLearner.pretrained(arch, md, ps=0.5)
 learner.opt_fn = optim.Adam
learner.crit = FocalLoss()
learner.metrics = [acc]<load_pretrained>
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self,index):
data,target = self.dataset[index]
return data,target,index
def __len__(self):
return len(self.dataset )
learner.load('ResNet34_256_1' )<train_model>
def pytorch_model_run_cv(x_train,y_train,features,x_test, model_obj, feats = False,clip = True):
seed_everything()
avg_losses_f = []
avg_val_losses_f = []
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(x_test)))
 splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
for i,(train_idx, valid_idx)in enumerate(splits):
seed_everything(i*1000+i)
 x_train = np.array(x_train)
 y_train = np.array(y_train)
 if feats:
features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
if feats:
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = copy.deepcopy(model_obj)
 model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))
valid = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))
train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
if feats:
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 else:
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
if clip:
nn.utils.clip_grad_norm_(model.parameters() ,1)
 optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(x_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch,index)in enumerate(valid_loader):
if feats:
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
else:
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
if feats:
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
else:
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
 return train_preds, test_preds
lr = 2e-2
learner.fit(lr,1 )<define_search_space>
class Alex_NeuralNet_Meta(nn.Module):
def __init__(self,hidden_size,lin_size, embedding_matrix=embedding_matrix):
super(Alex_NeuralNet_Meta, self ).__init__()
self.hidden_size = hidden_size
drp = 0.1
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.lstm.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 for name, param in self.gru.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.linear = nn.Linear(hidden_size*6 + features.shape[1], lin_size)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(drp)
 self.out = nn.Linear(lin_size, 1)
 def forward(self, x):

 h_embedding = self.embedding(x[0])
 embeddings = h_embedding.unsqueeze(2)
 embeddings = embeddings.permute(0, 3, 2, 1)
 embeddings = self.embedding_dropout(embeddings)
 embeddings = embeddings.permute(0, 3, 2, 1)
 h_embedding = embeddings.squeeze(2)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, hh_gru = self.gru(h_lstm)
 hh_gru = hh_gru.view(-1, 2*self.hidden_size)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( hh_gru, avg_pool, max_pool,f), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
learner.unfreeze()
lrs=np.array([lr/10,lr/3,lr] )<train_model>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )
learner.fit(lrs/4,4,cycle_len=2,use_clr=(10,20))<train_model>
train_preds , test_preds = pytorch_model_run_cv(x_train,y_train,features,x_test,Alex_NeuralNet_Meta(70,16, embedding_matrix=embedding_matrix), feats = True )
learner.fit(lrs/4,2,cycle_len=4,use_clr=(10,20))<train_model>
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta , tmp[2]
delta, _ = bestThresshold(y_train,train_preds )
learner.fit(lrs/8,1,cycle_len=8,use_clr=(5,20))<save_model>
if debug:
df_test = pd.read_csv(".. /input/test.csv")[:20000]
else:
df_test = pd.read_csv(".. /input/test.csv")
 submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )