Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<save_to_csv>
%load_ext Cython
submission = pd.DataFrame({
"ImageId": ImageId,
"Label": class_score
})
 submission.to_csv("submission.csv", index=False)
 display(submission.head(3))
display(submission.tail(3))<set_options>
modules =
os.environ['DATADIR'] = '/kaggle/input'
%reload_ext autoreload
%autoreload 2
%matplotlib inline<set_options>
def load_qiqc(n_rows=None):
train_df = pd.read_csv(f'{os.environ["DATADIR"]}/train.csv', nrows=n_rows)
 submit_df = pd.read_csv(f'{os.environ["DATADIR"]}/test.csv', nrows=n_rows)
 n_labels = {
0:(train_df.target == 0 ).sum() ,
1:(train_df.target == 1 ).sum() ,
}
train_df['target'] = train_df.target.astype('f')
 train_df['weights'] = train_df.target.apply(lambda t: 1 / n_labels[t])
 return train_df, submit_df
def build_datasets(train_df, submit_df, holdout=False, seed=0):
submit_dataset = QIQCDataset(submit_df)
 if holdout:
splitter = sklearn.model_selection.StratifiedShuffleSplit(
 n_splits=1, test_size=0.1, random_state=seed)
 train_indices, test_indices = list(splitter.split(
 train_df, train_df.target)) [0]
train_indices.sort() , test_indices.sort()
train_dataset = QIQCDataset(
 train_df.iloc[train_indices].reset_index(drop=True))
test_dataset = QIQCDataset(
 train_df.iloc[test_indices].reset_index(drop=True))
else:
train_dataset = QIQCDataset(train_df)
 test_dataset = QIQCDataset(train_df.head(0))
return train_dataset, test_dataset, submit_dataset
class QIQCDataset(object):
def __init__(self, df):
self.df = df
@property
def tokens(self):
return self.df.tokens.values
@tokens.setter
def tokens(self, tokens):
self.df['tokens'] = tokens
@property
def positives(self):
return self.df[self.df.target == 1]
@property
def negatives(self):
return self.df[self.df.target == 0]
def build(self, device):
self._X = self.tids
self.X = torch.Tensor(self._X ).type(torch.long ).to(device)
 if 'target' in self.df:
self._t = self.df.target[:, None]
self._W = self.df.weights
self.t = torch.Tensor(self._t ).type(torch.float ).to(device)
 self.W = torch.Tensor(self._W ).type(torch.float ).to(device)
 if hasattr(self, '_X2'):
self.X2 = torch.Tensor(self._X2 ).type(torch.float ).to(device)
 else:
self._X2 = np.zeros(( self._X.shape[0], 1), 'f')
 self.X2 = torch.Tensor(self._X2 ).type(torch.float ).to(device)
 def build_labeled_dataset(self, indices):
return torch.utils.data.TensorDataset(
 self.X[indices], self.X2[indices],
self.t[indices], self.W[indices] )
warnings.simplefilter('ignore' )<import_modules>
NORMALIZER_REGISTRY = {}
TOKENIZER_REGISTRY = {}
WORD_EMBEDDING_FEATURIZER_REGISTRY = {}
WORD_EXTRA_FEATURIZER_REGISTRY = {}
SENTENCE_EXTRA_FEATURIZER_REGISTRY = {}
ENCODER_REGISTRY = {}
AGGREGATOR_REGISTRY = {}
ATTENTION_REGISTRY = {}
def register_preprocessor(name):
def register_cls(cls):
NORMALIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_tokenizer(name):
def register_cls(cls):
TOKENIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_word_embedding_features(name):
def register_cls(cls):
WORD_EMBEDDING_FEATURIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_word_extra_features(name):
def register_cls(cls):
WORD_EXTRA_FEATURIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_sentence_extra_features(name):
def register_cls(cls):
SENTENCE_EXTRA_FEATURIZER_REGISTRY[name] = cls
return cls
return register_cls
def register_encoder(name):
def register_cls(cls):
ENCODER_REGISTRY[name] = cls
return cls
return register_cls
def register_aggregator(name):
def register_cls(cls):
AGGREGATOR_REGISTRY[name] = cls
return cls
return register_cls
def register_attention(name):
def register_cls(cls):
ATTENTION_REGISTRY[name] = cls
return cls
return register_cls
from fastai import *
from fastai.vision import *
import torchvision.models<import_modules>
class WordVocab(object):
def __init__(self, mincount=1):
self.counter = Counter()
self.n_documents = 0
self._counters = {}
self._n_documents = defaultdict(int)
 self.mincount = mincount
def __len__(self):
return len(self.token2id)
 def add_documents(self, documents, name):
self._counters[name] = Counter()
for document in documents:
bow = dict.fromkeys(document, 1)
 self._counters[name].update(bow)
 self.counter.update(bow)
 self.n_documents += 1
self._n_documents[name] += 1
def build(self):
counter = dict(self.counter.most_common())
 self.word_freq = {
**{'<PAD>': 0},
**counter,
}
self.token2id = {
**{'<PAD>': 0},
**{word: i + 1 for i, word in enumerate(counter)}
}
self.lfq = np.array(list(self.word_freq.values())) < self.mincount
self.hfq = ~self.lfq
import os
from pathlib import Path
import pandas as pd<load_from_csv>
%%cython
c
cdef class StringReplacer:
cpdef public dict rule
cpdef list keys
cpdef list values
cpdef int n_rules
def __init__(self, dict rule):
self.rule = rule
self.keys = list(rule.keys())
 self.values = list(rule.values())
 self.n_rules = len(rule)
 def __call__(self, str x):
cdef int i
for i in range(self.n_rules):
if self.keys[i] in x:
x = x.replace(self.keys[i], self.values[i])
 return x
def __getstate__(self):
return(self.rule, self.keys, self.values, self.n_rules)
 def __setstate__(self, state):
self.rule, self.keys, self.values, self.n_rules = state
cdef class RegExpReplacer:
cdef dict rule
cdef list keys
cdef list values
cdef regexp
cdef int n_rules
def __init__(self, dict rule):
self.rule = rule
self.keys = list(rule.keys())
 self.values = list(rule.values())
 self.regexp = re.compile('(%s)' % '|'.join(self.keys))
self.n_rules = len(rule)
 @property
def rule(self):
return self.rule
def __call__(self, str x):
def replace(match):
x = match.group(0)
 if x in self.rule:
return self.rule[x]
else:
for i in range(self.n_rules):
x = re.sub(self.keys[i], self.values[i], x)
 return x
return self.regexp.sub(replace, x)
 cpdef str cylower(str x):
return x.lower()
Cache = {}
is_alphabet = re.compile(r'[a-zA-Z]')
 cpdef str unidecode_weak(str string):

 cdef list retval = []
cdef int i = 0
cdef int n = len(string)
 cdef str char
for i in range(n):
char = string[i]
codepoint = ord(char)
 if codepoint < 0x80:
retval.append(char)
 continue
if codepoint > 0xeffff:
continue
section = codepoint >> 8
position = codepoint % 256
try:
table = Cache[section]
except KeyError:
try:
mod = __import__('unidecode.x%03x'%(section), [], [], ['data'])
 except ImportError:
Cache[section] = None
continue
Cache[section] = table = mod.data
if table and len(table)> position:
if table[position] == '[?]' or is_alphabet.match(table[position]):
retval.append(' ' + char + ' ')
 else:
retval.append(table[position])
 return ''.join(retval)
train_df = pd.read_csv(INPUT/"train.csv")
 train_df.head(3 )<load_from_csv>
class PunctSpacer(StringReplacer):
def __init__(self, edge_only=False):
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
if edge_only:
rule = {
**dict([(f' {p}', f' {p} ')for p in puncts]),
**dict([(f'{p} ', f' {p} ')for p in puncts]),
}
else:
rule = dict([(p, f' {p} ')for p in puncts])
 super().__init__(rule)
 class NumberReplacer(RegExpReplacer):
def __init__(self, with_underscore=False):
prefix, suffix = '', ''
if with_underscore:
prefix += ' __'
suffix = '__ '
rule = {
'[0-9]{5,}': f'{prefix}
'[0-9]{4}': f'{prefix}
'[0-9]{3}': f'{prefix}
'[0-9]{2}': f'{prefix}
}
super().__init__(rule)
 class KerasFilterReplacer(StringReplacer):
def __init__(self):
filters = '!"
'
rule = dict([(f, ' ')for f in filters])
 super().__init__(rule)
 class MisspellReplacer(StringReplacer):
def __init__(self):
rule = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd'y": "how do you",
"how'd": "how did",
"how'll": "how will",
"how's": "how is",
"i'd've": "i would have",
"i'd": "i would",
"i'll've": "i will have",
"i'll": "i will",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd've": "it would have",
"it'd": "it would",
"it'll've": "it will have",
"it'll": "it will",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't've": "might not have",
"mightn't": "might not",
"must've": "must have",
"mustn't've": "must not have",
"mustn't": "must not",
"needn't've": "need not have",
"needn't": "need not",
"o'clock": "of the clock",
"oughtn't've": "ought not have",
"oughtn't": "ought not",
"shan't've": "shall not have",
"shan't": "shall not",
"sha'n't": "shall not",
"she'd've": "she would have",
"she'd": "she would",
"she'll've": "she will have",
"she'll": "she will",
"she's": "she is",
"should've": "should have",
"shouldn't've": "should not have",
"shouldn't": "should not",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd've": "that would have",
"that'd": "that would",
"that's": "that is",
"there'd've": "there would have",
"there'd": "there would",
"there's": "there is",
"here's": "here is",
"they'd've": "they would have",
"they'd": "they would",
"they'll've": "they will have",
"they'll": "they will",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd've": "we would have",
"we'd": "we would",
"we'll've": "we will have",
"we'll": "we will",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll've": "what will have",
"what'll": "what will",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll've": "who will have",
"who'll": "who will",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't've": "will not have",
"won't": "will not",
"would've": "would have",
"wouldn't've": "would not have",
"wouldn't": "would not",
"y'all'd've": "you all would have",
"y'all'd": "you all would",
"y'all're": "you all are",
"y'all've": "you all have",
"y'all": "you all",
"you'd've": "you would have",
"you'd": "you would",
"you'll've": "you will have",
"you'll": "you will",
"you're": "you are",
"you've": "you have",
"colour": "color",
"centre": "center",
"favourite": "favorite",
"travelling": "traveling",
"counselling": "counseling",
"theatre": "theater",
"cancelled": "canceled",
"labour": "labor",
"organisation": "organization",
"wwii": "world war 2",
"citicise": "criticize",
"youtu ": "youtube ",
"qoura": "quora",
"sallary": "salary",
"whta": "what",
"narcisist": "narcissist",
"howdo": "how do",
"whatare": "what are",
"howcan": "how can",
"howmuch": "how much",
"howmany": "how many",
"whydo": "why do",
"doi": "do i",
"thebest": "the best",
"howdoes": "how does",
"mastrubation": "masturbation",
"mastrubate": "masturbate",
"mastrubating": "masturbating",
"pennis": "penis",
"etherium": "ethereum",
"narcissit": "narcissist",
"bigdata": "big data",
"2k17": "2017",
"2k18": "2018",
"qouta": "quota",
"exboyfriend": "ex boyfriend",
"airhostess": "air hostess",
"whst": "what",
"watsapp": "whatsapp",
"demonitisation": "demonetization",
"demonitization": "demonetization",
"demonetisation": "demonetization",
}
super().__init__(rule)
 register_preprocessor('lower' )(cylower)
 register_preprocessor('punct' )(PunctSpacer())
 register_preprocessor('unidecode' )(unidecode)
 register_preprocessor('unidecode_weak' )(unidecode_weak)
 register_preprocessor('number' )(NumberReplacer())
 register_preprocessor('number+underscore' )(
 NumberReplacer(with_underscore=True))
register_preprocessor('misspell' )(MisspellReplacer())
 register_preprocessor('keras' )(KerasFilterReplacer() )
test_df = pd.read_csv(INPUT/"test.csv")
 test_df.head(3 )<define_variables>
def load_pretrained_vectors(names, token2id, test=False):
assert isinstance(names, list)
 with Pool(processes=len(names)) as pool:
f = partial(load_pretrained_vector, token2id=token2id, test=test)
 vectors = pool.map(f, names)
 return dict([(n, v)for n, v in zip(names, vectors)])
 def load_pretrained_vector(name, token2id, test=False):
loader = dict(
 gnews=GNewsPretrainedVector,
wnews=WNewsPretrainedVector,
paragram=ParagramPretrainedVector,
glove=GlovePretrainedVector, 
)
 return loader[name].load(token2id, test)
 class BasePretrainedVector(object):
@classmethod
def load(cls, token2id, test=False, limit=None):
embed_shape =(len(token2id), 300)
 freqs = np.zeros(( len(token2id)) , dtype='f')
 if test:
np.random.seed(0)
 vectors = np.random.normal(0, 1, embed_shape)
 vectors[0] = 0
vectors[len(token2id)// 2:] = 0
else:
vectors = np.zeros(embed_shape, dtype='f')
 path = f'{os.environ["DATADIR"]}/{cls.path}'
for i, o in enumerate(
 open(path, encoding="utf8", errors='ignore')) :
token, *vector = o.split(' ')
 token = str.lower(token)
 if token not in token2id or len(o)<= 100:
continue
if limit is not None and i > limit:
break
freqs[token2id[token]] += 1
vectors[token2id[token]] += np.array(vector, 'f')
 vectors[freqs != 0] /= freqs[freqs != 0][:, None]
vec = KeyedVectors(300)
 vec.add(list(token2id.keys()), vectors, replace=True)
 return vec
class GNewsPretrainedVector(object):
name = 'GoogleNews-vectors-negative300'
path = f'embeddings/{name}/{name}.bin'
@classmethod
def load(cls, tokens, limit=None):
raise NotImplementedError
path = f'{os.environ["DATADIR"]}/{cls.path}'
return KeyedVectors.load_word2vec_format(
 path, binary=True, limit=limit)
 class WNewsPretrainedVector(BasePretrainedVector):
name = 'wiki-news-300d-1M'
path = f'embeddings/{name}/{name}.vec'
class ParagramPretrainedVector(BasePretrainedVector):
name = 'paragram_300_sl999'
path = f'embeddings/{name}/{name}.txt'
class GlovePretrainedVector(BasePretrainedVector):
name = 'glove.840B.300d'
path = f'embeddings/{name}/{name}.txt'
TRAIN = Path(".. /train")
 TEST = Path(".. /test" )<data_type_conversions>
@register_word_embedding_features('pretrained')
 class PretrainedVectorFeaturizer(object):
def __init__(self, config, vocab):
self.config = config
self.vocab = vocab
@classmethod
def add_args(self, parser):
pass
def __call__(self, features, datasets):
return features
class Any2VecFeaturizer(object):
def __init__(self, config, vocab):
self.config = config
self.vocab = vocab
def build_fillvalue(self, mode, initialW):
n_embed = initialW.shape[1]
n_fill = initialW[self.vocab.unk].shape
assert mode in {'zeros', 'mean', 'noise'}
if mode == 'zeros':
return np.zeros(n_embed, 'f')
 elif mode == 'mean':
return initialW.mean(axis=0)
 elif mode == 'noise':
mean, std = initialW.mean() , initialW.std()
return np.random.normal(mean, std,(n_fill, n_embed))
def __call__(self, features, datasets):
tokens = np.concatenate([d.tokens for d in datasets])
 model = self.build_model()
model.build_vocab_from_freq(self.vocab.word_freq)
 initialW = features.copy()
initialW[self.vocab.unk] = self.build_fillvalue(
 self.config.finetune_word2vec_init_unk, initialW)
 idxmap = np.array(
 [self.vocab.token2id[w] for w in model.wv.index2entity])
 model = self.initialize(model, initialW, idxmap)
 model.train(tokens, total_examples=len(tokens), epochs=model.epochs)
 finetunedW = np.zeros(( initialW.shape), 'f')
 for i, word in enumerate(self.vocab.token2id):
if word in model.wv:
finetunedW[i] = model.wv.get_vector(word)
 return finetunedW
@register_word_embedding_features('word2vec')
 class Word2VecFeaturizer(Any2VecFeaturizer):
@classmethod
def add_args(self, parser):
parser.add_argument('--finetune-word2vec-init-unk', type=str,
choices=['zeros', 'mean', 'noise'])
 parser.add_argument('--finetune-word2vec-mincount', type=int)
 parser.add_argument('--finetune-word2vec-workers', type=int)
 parser.add_argument('--finetune-word2vec-iter', type=int)
 parser.add_argument('--finetune-word2vec-size', type=int)
 parser.add_argument('--finetune-word2vec-window', type=int, default=5)
 parser.add_argument('--finetune-word2vec-sorted-vocab', type=int,
default=0)
 parser.add_argument('--finetune-word2vec-sg', type=int, choices=[0, 1])
 def build_model(self):
model = Word2Vec(
 min_count=self.config.finetune_word2vec_mincount,
workers=self.config.finetune_word2vec_workers,
iter=self.config.finetune_word2vec_iter,
size=self.config.finetune_word2vec_size,
window=self.config.finetune_word2vec_window,
sg=self.config.finetune_word2vec_sg, 
)
 return model
def initialize(self, model, initialW, idxmap):
model.wv.vectors[:] = initialW[idxmap]
model.trainables.syn1neg[:] = initialW[idxmap]
return model
@register_word_embedding_features('fasttext')
 class FastTextFeaturizer(Any2VecFeaturizer):
@classmethod
def add_args(self, parser):
parser.add_argument('--finetune-fasttext-init-unk', type=str,
choices=['zeros', 'mean', 'noise'])
 parser.add_argument('--finetune-fasttext-mincount', type=int)
 parser.add_argument('--finetune-fasttext-workers', type=int)
 parser.add_argument('--finetune-fasttext-iter', type=int)
 parser.add_argument('--finetune-fasttext-size', type=int)
 parser.add_argument('--finetune-fasttext-sg', type=int, choices=[0, 1])
 parser.add_argument('--finetune-fasttext-min_n', type=int)
 parser.add_argument('--finetune-fasttext-max_n', type=int)
 def build_model(self):
model = FastText(
 min_count=self.config.finetune_fasttext_mincount,
workers=self.config.finetune_fasttext_workers,
iter=self.config.finetune_fasttext_iter,
size=self.config.finetune_fasttext_size,
sg=self.config.finetune_fasttext_sg,
min_n=self.config.finetune_fasttext_min_n,
max_n=self.config.finetune_fasttext_max_n, 
)
 return model
def initialize(self, model, initialW, idxmap):
model.wv.vectors[:] = initialW[idxmap]
model.wv.vectors_vocab[:] = initialW[idxmap]
model.trainables.syn1neg[:] = initialW[idxmap]
return model
def saveDigit(digit, filepath):
digit = digit.reshape(28,28)
 digit = digit.astype(np.uint8)
 img = Image.fromarray(digit)
 img.save(filepath )<define_variables>
@register_word_extra_features('idf')
 class IDFWordFeaturizer(object):
def __call__(self, vocab):
dfs = np.array(list(vocab.word_freq.values()))
dfs[0] = vocab.n_documents
features = np.log(vocab.n_documents / dfs)
 features = features[:, None]
return features
@register_word_extra_features('unk')
 class UnkWordFeaturizer(object):
def __call__(self, vocab):
features = vocab.unk.astype('f')
 features[0] = 0
features = features[:, None]
return features
for index, row in train_df.iterrows() :
label,digit = row[0], row[1:]
folder = TRAIN/str(label)
 filename = f"{index}.jpg"
filepath = folder/filename
digit = digit.values
saveDigit(digit, filepath )<save_model>
@register_sentence_extra_features('char')
 class CharacterStatisticsFeaturizer(object):
n_dims = 3
def __call__(self, sentence):
feature = {}
feature['n_chars'] = len(sentence)
 feature['n_caps'] = sum(1 for char in sentence if char.isupper())
 feature['caps_rate'] = feature['n_caps'] / feature['n_chars']
features = np.array(list(feature.values()))
return features
@register_sentence_extra_features('word')
 class WordStatisticsFeaturizer(object):
n_dims = 3
def __call__(self, sentence):
feature = {}
tokens = sentence.split()
feature['n_words'] = len(tokens)
 feature['unique_words'] = len(set(tokens))
feature['unique_rate'] = feature['unique_words'] / feature['n_words']
features = np.array(list(feature.values()))
return features
for index, digit in test_df.iterrows() :
folder = TEST
filename = f"{index}.jpg"
filepath = folder/filename
digit = digit.values
saveDigit(digit, filepath )<feature_engineering>
register_tokenizer('space' )(cysplit)
 register_tokenizer('word_tokenize' )(nltk.word_tokenize )
tfms = get_transforms(do_flip=False )<load_pretrained>
class TextNormalizerWrapper(object):
registry = NORMALIZER_REGISTRY
default_config = None
def __init__(self, config):
self.normalizers = [self.registry[n] for n in config.normalizers]
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument(
 '--normalizers', nargs='+', choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 def __call__(self, x):
for normalizer in self.normalizers:
x = normalizer(x)
 return x
class TextTokenizerWrapper(object):
registry = TOKENIZER_REGISTRY
default_config = None
def __init__(self, config):
self.tokenizer = self.registry[config.tokenizer]
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument('--tokenizer', choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 def __call__(self, x):
return self.tokenizer(x)
 class WordEmbeddingFeaturizerWrapper(object):
registry = WORD_EMBEDDING_FEATURIZER_REGISTRY
default_config = None
default_extra_config = None
def __init__(self, config, vocab):
self.config = config
self.vocab = vocab
self.featurizers = {
k: self.registry[k](config, vocab)
 for k in config.word_embedding_features}
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument(
 '--use-pretrained-vectors', nargs='+',
choices=['glove', 'paragram', 'wnews', 'gnews'])
 parser.add_argument(
 '--word-embedding-features', nargs='+', choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
assert isinstance(cls.default_extra_config, dict)
 for featurizer in config.word_embedding_features:
cls.registry[featurizer].add_args(parser)
 parser.set_defaults(**cls.default_extra_config)
 def __call__(self, features, datasets):
return {k: feat(features, datasets)
 for k, feat in self.featurizers.items() }
class WordExtraFeaturizerWrapper(object):
registry = WORD_EXTRA_FEATURIZER_REGISTRY
default_config = None
def __init__(self, config, vocab):
self.config = config
self.vocab = vocab
self.featurizers = {
k: self.registry[k]() for k in config.word_extra_features}
@classmethod
def add_args(cls, parser):
parser.add_argument(
 '--word-extra-features', nargs='+', choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 def __call__(self, vocab):
empty = np.empty([len(vocab), 0])
 return np.concatenate([empty, *[
f(vocab)for f in self.featurizers.values() ]], axis=1)
 class SentenceExtraFeaturizerWrapper(object):
registry = SENTENCE_EXTRA_FEATURIZER_REGISTRY
default_config = None
def __init__(self, config):
self.config = config
self.featurizers = {
k: self.registry[k]() for k in config.sentence_extra_features}
self.n_dims = sum(list(f.n_dims for f in self.featurizers.values()))
@classmethod
def add_args(cls, parser):
parser.add_argument(
 '--sentence-extra-features', nargs='+', choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 def __call__(self, sentence):
empty = np.empty(( 0,))
return np.concatenate([empty, *[
f(sentence)for f in self.featurizers.values() ]], axis=0)
 def fit_standardize(self, features):
assert features.ndim == 2
self.mean = features.mean(axis=0)
 self.std = features.std(axis=0)
 self.std = np.where(self.std != 0, self.std, 1)
 return(features - self.mean)/ self.std
def standardize(self, features):
assert hasattr(self, 'mean'), hasattr(self, 'std')
 return(features - self.mean)/ self.std
data = ImageDataBunch.from_folder(
 path = TRAIN,
test = TEST,
valid_pct = 0.2,
bs = 256,
size = 28,
num_workers = 5,
ds_tfms = tfms
).normalize(mnist_stats )<choose_model_class>
class WordbasedPreprocessor() :
def tokenize(self, datasets, normalizer, tokenizer):
tokenize = Pipeline(normalizer, tokenizer)
 apply_tokenize = ApplyNdArray(tokenize, processes=2, dtype=object)
 tokens = [apply_tokenize(d.df.question_text.values)for d in datasets]
return tokens
def build_vocab(self, datasets, config):
train_dataset, test_dataset, submit_dataset = datasets
vocab = WordVocab(mincount=config.vocab_mincount)
 vocab.add_documents(train_dataset.positives.tokens, 'train-pos')
 vocab.add_documents(train_dataset.negatives.tokens, 'train-neg')
 vocab.add_documents(test_dataset.positives.tokens, 'test-pos')
 vocab.add_documents(test_dataset.negatives.tokens, 'test-neg')
 vocab.add_documents(submit_dataset.df.tokens, 'submit')
 vocab.build()
return vocab
def build_tokenids(self, datasets, vocab, config):
token2id = lambda xs: pad_sequence(
 [vocab.token2id[x] for x in xs], config.maxlen)
 apply_token2id = ApplyNdArray(
 token2id, processes=1, dtype='i', dims=(config.maxlen,))
tokenids = [apply_token2id(d.df.tokens.values)for d in datasets]
return tokenids
def build_sentence_features(self, datasets, sentence_extra_featurizer):
train_dataset, test_dataset, submit_dataset = datasets
apply_featurize = ApplyNdArray(
 sentence_extra_featurizer, processes=1, dtype='f',
dims=(sentence_extra_featurizer.n_dims,))
_X2 = [apply_featurize(d.df.question_text.values)for d in datasets]
_train_X2, _test_X2, _submit_X2 = _X2
train_X2 = sentence_extra_featurizer.fit_standardize(_train_X2)
 test_X2 = sentence_extra_featurizer.standardize(_test_X2)
 submit_X2 = sentence_extra_featurizer.standardize(_submit_X2)
 return train_X2, test_X2, submit_X2
def build_embedding_matrices(self, datasets, word_embedding_featurizer,
vocab, pretrained_vectors):
pretrained_vectors_merged = np.stack(
 [wv.vectors for wv in pretrained_vectors.values() ] ).mean(axis=0)
 vocab.unk =(pretrained_vectors_merged == 0 ).all(axis=1)
 vocab.known = ~vocab.unk
embedding_matrices = word_embedding_featurizer(
 pretrained_vectors_merged, datasets)
 return embedding_matrices
def build_word_features(self, word_embedding_featurizer,
embedding_matrices, word_extra_features):
embedding = np.stack(list(embedding_matrices.values()))
embedding = embedding.mean(axis=0)
 word_features = np.concatenate(
 [embedding, word_extra_features], axis=1)
 return word_features
resnet_learn = Learner(data, torchvision.models.resnet50(pretrained=True), metrics=[error_rate, accuracy, top_k_accuracy], model_dir="/tmp/models", callback_fns=ShowGraph)
 googlenet_learn = Learner(data, torchvision.models.googlenet(pretrained=True), metrics=[error_rate, accuracy, top_k_accuracy], model_dir="/tmp/models", callback_fns=ShowGraph)
 resnext_learn = Learner(data, torchvision.models.resnext50_32x4d(pretrained=True), metrics=[error_rate, accuracy, top_k_accuracy], model_dir="/tmp/models", callback_fns=ShowGraph)
 wideres_learn = Learner(data, torchvision.models.wide_resnet50_2(pretrained=True), metrics=[error_rate, accuracy, top_k_accuracy], model_dir="/tmp/models", callback_fns=ShowGraph)
 mobilenet_learn = Learner(data, torchvision.models.mobilenet_v2(pretrained=True), metrics=[error_rate, accuracy, top_k_accuracy], model_dir="/tmp/models", callback_fns=ShowGraph )<define_variables>
class RNNEncoderBase(nn.Module):
def __init__(self, config, modules, in_size):
super().__init__()
rnns = []
input_size = in_size
for module in modules:
rnn = module(
 input_size=input_size,
hidden_size=config.encoder_n_hidden,
bidirectional=config.encoder_bidirectional,
batch_first=True, 
)
 n_direction = int(config.encoder_bidirectional)+ 1
input_size = n_direction * config.encoder_n_hidden
rnns.append(rnn)
 self.rnns = nn.ModuleList(rnns)
 self.out_size = n_direction * config.encoder_n_hidden
@classmethod
def add_args(self, parser):
parser.add_argument('--encoder-bidirectional', type=bool, default=True)
 parser.add_argument('--encoder-dropout', type=float, default=0.)
parser.add_argument('--encoder-n-hidden', type=int)
 parser.add_argument('--encoder-n-layers', type=int)
 parser.add_argument('--encoder-aggregator', type=str,
choices=AGGREGATOR_REGISTRY)
 def forward(self, input, mask):
h = input
for rnn in self.rnns:
h, _ = rnn(h)
 return h
@register_encoder('lstm')
 class LSTMEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
modules = [nn.LSTM] * config.encoder_n_layers
super().__init__(config, modules, in_size)
 @register_encoder('gru')
 class GRUEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
assert config.encoder_n_layers > 1
modules = [nn.GRU] * config.encoder_n_layers
super().__init__(config, modules, in_size)
 @register_encoder('lstmgru')
 class LSTMGRUEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
assert config.encoder_n_layers > 1
modules = [nn.LSTM] *(config.encoder_n_layers - 1)+ [nn.GRU]
super().__init__(config, modules, in_size)
 @register_encoder('grulstm')
 class GRULSTMEncoder(RNNEncoderBase):
def __init__(self, config, in_size):
assert config.encoder_n_layers > 1
modules = [nn.GRU] *(config.encoder_n_layers - 1)+ [nn.LSTM]
super().__init__(config, modules, in_size )
model = [resnet_learn, googlenet_learn, resnext_learn, wideres_learn, mobilenet_learn]<find_best_params>
class BaseEnsembler(metaclass=ABCMeta):
def __init__(self, config, models, results):
super().__init__()
self.config = config
self.models = models
self.results = results
@abstractmethod
def fit(self, X, t, test_size=0.1):
pass
@abstractmethod
def predict_proba(self, X, X2):
pass
def predict(self, X, X2):
y = self.predict_proba(X, X2)
 return(y > self.threshold ).astype('i')
 class AverageEnsembler(BaseEnsembler):
def __init__(self, config, models, results):
self.config = config
self.models = models
self.results = results
self.device = config.device
self.batchsize_train = config.batchsize
self.batchsize_valid = config.batchsize_valid
self.threshold_cv = np.array(
 [m.threshold for m in models] ).mean()
self.threshold = self.threshold_cv
def fit(self, X, X2, t, test_size=0.1):
pass
def predict_proba(self, X, X2):
pred_X = X.to(self.device)
 pred_X2 = X2.to(self.device)
 dataset = torch.utils.data.TensorDataset(pred_X, pred_X2)
 iterator = DataLoader(
 dataset, batch_size=self.batchsize_valid, shuffle=False)
 ys = defaultdict(list)
 for batch in tqdm(iterator, desc='submit', leave=False):
for i, model in enumerate(self.models):
model.eval()
ys[i].append(model.predict_proba(*batch))
ys = np.concatenate(
 [np.concatenate(_ys)for _ys in ys.values() ], axis=1)
 y = ys.mean(axis=1, keepdims=True)
 return y
for learn in model:
learn.lr_find()
learn.recorder.plot(suggestion=True )<train_model>
class BinaryClassifier(nn.Module):
default_config = None
def __init__(self, embedding, encoder, aggregator, mlp, out, lossfunc):
super().__init__()
self.embedding = embedding
self.encoder = encoder
self.aggregator = aggregator
self.mlp = mlp
self.out = out
self.lossfunc = lossfunc
def calc_loss(self, X, X2, t, W=None):
y = self.forward(X, X2)
 loss = self.lossfunc(y, t)
 output = dict(
 y=torch.sigmoid(y ).cpu().detach().numpy() ,
t=t.cpu().detach().numpy() ,
loss=loss.cpu().detach().numpy() , 
)
 return loss, output
def to_device(self, device):
self.device = device
self.to(device)
 return self
def forward(self, X, X2):
h = self.predict_features(X, X2)
 out = self.out(h)
 return out
def predict_proba(self, X, X2):
y = self.forward(X, X2)
 proba = torch.sigmoid(y ).cpu().detach().numpy()
return proba
def predict_features(self, X, X2):
mask = X != 0
maxlen =(mask == 1 ).any(dim=0 ).sum()
X = X[:, :maxlen]
mask = mask[:, :maxlen]
h = self.embedding(X)
 h = self.encoder(h, mask)
 h = self.aggregator(h, mask)
 h = self.mlp(h, X2)
 return h
resnet_learn.fit_one_cycle(10 )<train_model>
class NNModuleWrapperBase(nn.Module, metaclass=ABCMeta):
@abstractmethod
def add_args(cls, parser):
raise NotImplementedError()
@abstractmethod
def add_extra_args(cls, parser):
raise NotImplementedError()
class EmbeddingWrapper(NNModuleWrapperBase):
default_config = None
def __init__(self, config, embedding_matrix):
super().__init__()
self.config = config
self.module = nn.Embedding.from_pretrained(
 torch.Tensor(embedding_matrix), freeze=True)
 if self.config.embedding_dropout1d > 0:
self.dropout1d = nn.Dropout(config.embedding_dropout1d)
 if self.config.embedding_dropout2d > 0:
self.dropout2d = nn.Dropout2d(config.embedding_dropout2d)
 if self.config.embedding_spatial_dropout > 0:
self.spatial_dropout = nn.Dropout2d(
 config.embedding_spatial_dropout)
 self.out_size = embedding_matrix.shape[1]
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument('--embedding-dropout1d', type=float, default=0.)
parser.add_argument('--embedding-dropout2d', type=float, default=0.)
parser.add_argument('--embedding-spatial-dropout',
type=float, default=0.)
parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
pass
def forward(self, X):
h = self.module(X)
 if self.config.embedding_dropout1d > 0:
h = self.dropout1d(h)
 if self.config.embedding_dropout2d > 0:
h = self.dropout2d(h)
 if self.config.embedding_spatial_dropout > 0:
h = h.permute(0, 2, 1)
 h = self.spatial_dropout(h)
 h = h.permute(0, 2, 1)
 return h
class EncoderWrapper(nn.Module):
registry = ENCODER_REGISTRY
def __init__(self, config, in_size):
super().__init__()
self.config = config
self.module = self.registry[config.encoder](config, in_size)
 self.out_size = self.module.out_size
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument(
 '--encoder', choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
assert isinstance(cls.default_extra_config, dict)
 cls.registry[config.encoder].add_args(parser)
 parser.set_defaults(**cls.default_extra_config)
 def forward(self, X, mask):
h = self.module(X, mask)
 return h
class AggregatorWrapper(NNModuleWrapperBase):
default_config = None
registry = AGGREGATOR_REGISTRY
def __init__(self, config):
super().__init__()
self.config = config
self.module = self.registry[config.aggregator]()
@classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument('--aggregator',
choices=cls.registry)
 parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
pass
def forward(self, X, mask):
h = self.module(X, mask)
 return h
class MLPWrapper(NNModuleWrapperBase):
default_config = None
def __init__(self, config, in_size):
super().__init__()
self.in_size = in_size
self.config = config
assert isinstance(config.mlp_n_hiddens, list)
 layers = []
if config.mlp_bn0:
layers.append(nn.BatchNorm1d(in_size))
if config.mlp_dropout0 > 0:
layers.append(nn.Dropout(config.mlp_dropout0))
for n_hidden in config.mlp_n_hiddens:
layers.append(nn.Linear(in_size, n_hidden))
if config.mlp_actfun is not None:
layers.append(config.mlp_actfun)
 if config.mlp_bn:
layers.append(nn.BatchNorm1d(n_hidden))
if config.mlp_dropout > 0:
layers.append(nn.Dropout(config.mlp_dropout))
in_size = n_hidden
self.layers = nn.Sequential(*layers)
 @classmethod
def add_args(cls, parser):
assert isinstance(cls.default_config, dict)
 parser.add_argument('--mlp-n-hiddens', type=list)
 parser.add_argument('--mlp-bn', type=bool)
 parser.add_argument('--mlp-bn0', type=bool)
 parser.add_argument('--mlp-dropout', type=float, default=0.)
parser.add_argument('--mlp-dropout0', type=float, default=0.)
parser.add_argument('--mlp-actfun', default=0.)
parser.set_defaults(**cls.default_config)
 @classmethod
def add_extra_args(cls, parser, config):
pass
def forward(self, X, X2):
h = X
if X.shape[1] + X2.shape[1] == self.in_size:
h = torch.cat([h, X2], dim=1)
 h = self.layers(h)
 return h
googlenet_learn.fit_one_cycle(10 )<train_model>
class ExperimentConfigBuilderPresets(ExperimentConfigBuilderBase):
default_config = dict(
 maxlen=72,
vocab_mincount=5,
scale_batchsize=[],
validate_from=2, 
)
 class TextNormalizerPresets(TextNormalizerWrapper):
default_config = dict(
 normalizers=[
'lower',
'misspell',
'punct',
'number+underscore'
] 
)
 class TextTokenizerPresets(TextTokenizerWrapper):
default_config = dict(
 tokenizer='space' 
)
 class WordEmbeddingFeaturizerPresets(WordEmbeddingFeaturizerWrapper):
default_config = dict(
 use_pretrained_vectors=['glove', 'paragram'],
word_embedding_features=['pretrained', 'word2vec'], 
)
 default_extra_config = dict(
 finetune_word2vec_init_unk='zeros',
finetune_word2vec_mincount=1,
finetune_word2vec_workers=1,
finetune_word2vec_iter=5,
finetune_word2vec_size=300,
finetune_word2vec_sg=0,
finetune_word2vec_sorted_vocab=0, 
)
 class WordExtraFeaturizerPresets(WordExtraFeaturizerWrapper):
default_config = dict(
 word_extra_features=[], 
)
 class SentenceExtraFeaturizerPresets(SentenceExtraFeaturizerWrapper):
default_config = dict(
 sentence_extra_features=[], 
)
 class PreprocessorPresets(WordbasedPreprocessor):
def build_word_features(self, word_embedding_featurizer,
embedding_matrices, word_extra_features):
embedding = np.stack(list(embedding_matrices.values()))
unk =(embedding[0] == 0 ).all(axis=1)
 mean, std = embedding[0, ~unk].mean() , embedding[0, ~unk].std()
unk_and_hfq = unk & word_embedding_featurizer.vocab.hfq
noise = np.random.normal(
 mean, std,(unk_and_hfq.sum() , embedding[0].shape[1]))
embedding[0, unk_and_hfq] = noise
embedding[0, 0] = 0
embedding = embedding.mean(axis=0)
 word_features = np.concatenate(
 [embedding, word_extra_features], axis=1)
 return word_features
class EmbeddingPresets(EmbeddingWrapper):
default_config = dict(
 embedding_dropout1d=0.2, 
)
 class EncoderPresets(EncoderWrapper):
default_config = dict(
 encoder='lstm', 
)
 default_extra_config = dict(
 encoder_bidirectional=True,
encoder_dropout=0.,
encoder_n_layers=2,
encoder_n_hidden=128, 
)
 class AggregatorPresets(AggregatorWrapper):
default_config = dict(
 aggregator='max', 
)
 class MLPPresets(MLPWrapper):
default_config = dict(
 mlp_n_hiddens=[128, 128],
mlp_bn0=False,
mlp_dropout0=0.,
mlp_bn=True,
mlp_actfun=nn.ReLU(True), 
)
 class EnsemblerPresets(AverageEnsembler):
pass
resnext_learn.fit_one_cycle(10 )<train_model>
def classification_metrics(ys, ts):
scores = {}
if len(np.unique(ts)) > 1:
precs, recs, thresholds = metrics.precision_recall_curve(ts, ys)
 thresholds = np.append(thresholds, 1.001)
 idx =(precs != 0)*(recs != 0)
 precs, recs, thresholds = precs[idx], recs[idx], thresholds[idx]
fbetas = 2 /(1 / precs + 1 / recs)
 best_idx = np.argmax(fbetas)
 threshold = thresholds[best_idx]
prec = precs[best_idx]
rec = recs[best_idx]
fbeta = fbetas[best_idx]
scores['ap'] = metrics.average_precision_score(ts, ys)
 scores['rocauc'] = metrics.roc_auc_score(ts, ys)
 scores['threshold'] = threshold
scores['prec'] = prec
scores['rec'] = rec
scores['fbeta'] = fbeta
return scores
class ClassificationResult(object):
def __init__(self, name, outdir=None, postfix=None, main_metrics='fbeta'):
self.initialize()
self.name = name
self.postfix = postfix
self.outdir = outdir
self.summary = None
self.main_metrics = main_metrics
self.n_trained = 0
def initialize(self):
self.losses = []
self.ys = []
self.ts = []
def add_record(self, loss, y, t):
self.losses.append(loss)
 self.ys.append(y)
 self.ts.append(t)
 self.n_trained += len(y)
 def calc_score(self, epoch):
loss = np.array(self.losses ).mean()
self.ys, self.ts = np.concatenate(self.ys), np.concatenate(self.ts)
 score = classification_metrics(self.ys, self.ts)
 summary = dict(name=self.name, loss=loss, **score)
 if len(score)> 0:
if self.summary is None:
self.summary = pd.DataFrame([summary], index=[epoch])
 self.summary.index.name = 'epoch'
else:
self.summary.loc[epoch] = summary
if self.best_epoch == epoch:
self.best_ys = self.ys
self.best_ts = self.ts
self.initialize()
def get_dict(self):
loss, fbeta, epoch = 0, 0, 0
if self.summary is not None:
row = self.summary.iloc[-1]
epoch = row.name
loss = row.loss
fbeta = row.fbeta
return {
'epoch': epoch,
'loss': loss,
'fbeta': fbeta,
}
@property
def fbeta(self):
if self.summary is None:
return 0
else:
return self.summary.fbeta[-1]
@property
def best_fbeta(self):
return self.summary[self.main_metrics].max()
@property
def best_epoch(self):
return self.summary[self.main_metrics].idxmax()
@property
def best_threshold(self):
idx = self.summary[self.main_metrics].idxmax()
return self.summary['threshold'][idx]
wideres_learn.fit_one_cycle(10 )<train_model>
%%cython
c
cdef class ApplyNdArray:
cdef func
cdef dtype
cdef dims
cdef int processes
def __init__(self, func, processes=1, dtype=object, dims=None):
self.func = func
self.processes = processes
self.dtype = dtype
self.dims = dims
def __call__(self, arr):
if self.processes == 1:
return self.apply(arr)
 else:
return self.apply_parallel(arr)
 cpdef apply(self, arr):
cdef int i
cdef int n = len(arr)
 if self.dims is not None:
shape =(n, *self.dims)
 else:
shape = n
cdef res = np.empty(shape, dtype=self.dtype)
 for i in range(n):
res[i] = self.func(arr[i])
 return res
cpdef apply_parallel(self, arr):
cdef list arrs = np.array_split(arr, self.processes)
 with Pool(processes=self.processes)as pool:
outputs = pool.map(self.apply, arrs)
 return np.concatenate(outputs, axis=0)
mobilenet_learn.fit_one_cycle(10 )<define_variables>
def load_module(filename):
assert isinstance(filename, Path)
 name = filename.stem
spec = importlib.util.spec_from_file_location(name, filename)
 mod = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(mod)
 sys.modules[mod.__name__] = mod
return mod
def rmtree_after_confirmation(path, force=False):
if Path(path ).exists() :
if not force and not prompter.yesno('Overwrite %s?' % path):
sys.exit(0)
 else:
shutil.rmtree(path)
 def pad_sequence(xs, length, padding_value=0):
assert isinstance(xs, list)
 n_padding = length - len(xs)
 return np.array(xs + [padding_value] * n_padding, 'i')[:length]
def set_seed(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
class Pipeline(object):
def __init__(self, *modules):
self.modules = modules
def __call__(self, x):
for module in self.modules:
x = module(x)
 return x
ImageId = [int(os.path.splitext(path)[0])+1 for path in os.listdir(TEST)]<save_to_csv>
%%time
exec(modules)
 config = ExperimentConfigBuilder().build(args=[])
 print(config)
 start = time.time()
set_seed(config.seed)
 train_df, submit_df = load_qiqc(n_rows=config.n_rows)
 datasets = build_datasets(train_df, submit_df, config.holdout, config.seed)
 train_dataset, test_dataset, submit_dataset = datasets
model_name = ['resnet', 'googlenet', 'resnext', 'wideres', 'mobilenet']
i = 0
for learn in model:
class_score, y = learn.get_preds(DatasetType.Test)
 class_score = np.argmax(class_score, axis=1)
 submission = pd.DataFrame({"ImageId": ImageId,"Label": class_score})
 submission.to_csv("submission_"+str(model_name[i])+".csv", index=False)
 i += 1<install_modules>
%%time
print('Tokenize texts...')
 preprocessor = Preprocessor()
normalizer = TextNormalizer(config)
 tokenizer = TextTokenizer(config)
 train_dataset.tokens, test_dataset.tokens, submit_dataset.tokens = \
preprocessor.tokenize(datasets, normalizer, tokenizer )
!pip install git+https://github.com/keras-team/keras-tuner.git -q<load_from_csv>
%%time
print('Build vocabulary...')
 vocab = preprocessor.build_vocab(datasets, config )
def load_train(path):
data = pd.read_csv(path)
 y = data["label"]
x = data.drop(labels=["label"], axis=1 ).values.reshape(-1, 28, 28, 1)
 return x, y
def load_test(path):
data = pd.read_csv(path)
 x = data.values.reshape(-1, 28, 28, 1)
 return x
x_train, y_train = load_train(".. /input/digit-recognizer/train.csv")
 x_test = load_test(".. /input/digit-recognizer/test.csv" )<normalization>
%%time
print('Build token ids...')
 train_dataset.tids, test_dataset.tids, submit_dataset.tids = \
preprocessor.build_tokenids(datasets, vocab, config )
def augment_images(x, hp):
use_rotation = hp.Boolean('use_rotation')
 if use_rotation:
x = layers.experimental.preprocessing.RandomRotation(
 hp.Float('rotation_factor', min_value=0.05, max_value=0.2)
 )(x)
 use_zoom = hp.Boolean('use_zoom')
 if use_zoom:
x = layers.experimental.preprocessing.RandomZoom(
 hp.Float('use_zoom', min_value=0.05, max_value=0.2)
 )(x)
 return x
def make_model(hp):
inputs = keras.Input(shape=(28, 28, 1))
x = layers.experimental.preprocessing.Rescaling(1./ 255 )(inputs)
 x = layers.experimental.preprocessing.Resizing(64, 64 )(x)
 x = augment_images(x, hp)
 num_block = hp.Int('num_block', min_value=2, max_value=5, step=1)
 num_filters = hp.Int('num_filters', min_value=32, max_value=128, step=32)
 for i in range(num_block):
x = layers.Conv2D(
 num_filters,
kernel_size=3,
activation='relu',
padding='same'
)(x)
 x = layers.Conv2D(
 num_filters,
kernel_size=3,
activation='relu',
padding='same'
)(x)
 x = layers.MaxPooling2D(2 )(x)
 reduction_type = hp.Choice('reduction_type', ['flatten', 'avg'])
 if reduction_type == 'flatten':
x = layers.Flatten()(x)
 else:
x = layers.GlobalAveragePooling2D()(x)
 x = layers.Dense(
 units=hp.Int('num_dense_units', min_value=32, max_value=512, step=32),
activation='relu'
)(x)
 x = layers.Dropout(
 hp.Float('dense_dropout', min_value=0., max_value=0.7)
 )(x)
 outputs = layers.Dense(10 )(x)
 model = keras.Model(inputs, outputs)
 learning_rate = hp.Float('learning_rate', min_value=3e-4, max_value=3e-3)
 optimizer = keras.optimizers.Adam(learning_rate=1e-3)
 model.compile(loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
optimizer=optimizer,
metrics=[keras.metrics.SparseCategoricalAccuracy(name='acc')])
 model.summary()
return model<train_model>
%%time
print('Build sentence extra features...')
 sentence_extra_featurizer = SentenceExtraFeaturizer(config)
 train_dataset._X2, test_dataset._X2, submit_dataset._X2 = \
preprocessor.build_sentence_features(
 datasets, sentence_extra_featurizer)
 [d.build(config.device)for d in datasets]
tuner = kt.tuners.RandomSearch(
 make_model,
objective='val_acc',
max_trials=100,
overwrite=True)
 callbacks=[keras.callbacks.EarlyStopping(monitor='val_acc', mode='max', patience=3, baseline=0.9)]
tuner.search(x_train, y_train, validation_split=0.2, callbacks=callbacks, verbose=1, epochs=100 )<train_model>
%%time
print('Load pretrained vectors...')
 pretrained_vectors = load_pretrained_vectors(
 config.use_pretrained_vectors, vocab.token2id, test=config.test )
best_hp = tuner.get_best_hyperparameters() [0]
model = make_model(best_hp)
 history = model.fit(x_train, y_train, validation_split=0.2, epochs=50 )<train_model>
%%time
print('Build word embedding matrix...')
 word_embedding_featurizer = WordEmbeddingFeaturizer(config, vocab)
 embedding_matrices = preprocessor.build_embedding_matrices(
 datasets, word_embedding_featurizer, vocab, pretrained_vectors )
val_acc_per_epoch = history.history['val_acc']
best_epoch = val_acc_per_epoch.index(max(val_acc_per_epoch)) + 1
model = make_model(best_hp)
 model.fit(x_train, y_train, epochs=best_epoch )<save_to_csv>
%%time
print('Build word extra features...')
 word_extra_featurizer = WordExtraFeaturizer(config, vocab)
 word_extra_features = word_extra_featurizer(vocab )
predictions = model.predict(x_test)
 submission = pd.DataFrame({"ImageId": list(range(1, len(predictions)+ 1)) ,
"Label": np.argmax(predictions, axis=-1)})
 submission.to_csv("submission.csv", index=False )<set_options>
%%time
print('Build models...')
 word_features_cv = [
preprocessor.build_word_features(
 word_embedding_featurizer, embedding_matrices, word_extra_features)
 for i in range(config.cv)]
models = [
build_model(
 config, word_features, sentence_extra_featurizer.n_dims 
)for word_features in word_features_cv]
%matplotlib inline 
<load_from_csv>
print(models[0] )
base_set = pd.read_csv('.. /input/digit-recognizer/train.csv' )<prepare_x_and_y>
%%time
print('Start training...')
 splitter = sklearn.model_selection.StratifiedKFold(
 n_splits=config.cv, shuffle=True, random_state=config.seed)
 train_results, valid_results = [], []
best_models = []
for i_cv,(train_indices, valid_indices)in enumerate(
 splitter.split(train_dataset.df, train_dataset.df.target)) :
if config.cv_part is not None and i_cv >= config.cv_part:
break
train_tensor = train_dataset.build_labeled_dataset(train_indices)
 valid_tensor = train_dataset.build_labeled_dataset(valid_indices)
 valid_iter = DataLoader(
 valid_tensor, batch_size=config.batchsize_valid)
 model = models.pop(0)
 model = model.to_device(config.device)
 model_snapshots = []
optimizer = torch.optim.Adam(model.parameters() , config.lr)
 train_result = ClassificationResult('train', config.outdir, str(i_cv))
valid_result = ClassificationResult('valid', config.outdir, str(i_cv))
batchsize = config.batchsize
for epoch in range(config.epochs):
if epoch in config.scale_batchsize:
batchsize *= 2
print(f'Batchsize: {batchsize}')
 epoch_start = time.time()
sampler = None
train_iter = DataLoader(
 train_tensor, sampler=sampler, drop_last=True,
batch_size=batchsize, shuffle=sampler is None)
 _summary = []
for i, batch in enumerate(
 tqdm(train_iter, desc='train', leave=False)) :
model.train()
optimizer.zero_grad()
loss, output = model.calc_loss(*batch)
 loss.backward()
optimizer.step()
train_result.add_record(**output)
 train_result.calc_score(epoch)
 _summary.append(train_result.summary.iloc[-1])
 if epoch >= config.validate_from:
for i, batch in enumerate(
 tqdm(valid_iter, desc='valid', leave=False)) :
model.eval()
loss, output = model.calc_loss(*batch)
 valid_result.add_record(**output)
 valid_result.calc_score(epoch)
 _summary.append(valid_result.summary.iloc[-1])
 _model = deepcopy(model)
 _model.threshold = valid_result.summary.threshold[epoch]
model_snapshots.append(_model)
 summary = pd.DataFrame(_summary ).set_index('name')
 epoch_time = time.time() - epoch_start
pbar = '
tqdm.write(f'
{pbar} cv: {i_cv} / {config.cv}, epoch {epoch}, '
f'time: {epoch_time}')
 tqdm.write(str(summary))
train_results.append(train_result)
 valid_results.append(valid_result)
 best_indices = valid_result.summary.fbeta.argsort() [::-1]
best_models.extend([model_snapshots[i] for i in
best_indices[:config.ensembler_n_snapshots]] )
label_column = 'label'
X = base_set.drop(columns=[label_column] ).values
Y = base_set[label_column]<prepare_x_and_y>
train_X, train_X2, train_t = \
train_dataset.X, train_dataset.X2, train_dataset.t
ensembler = Ensembler(config, best_models, valid_results)
 ensembler.fit(train_X, train_X2, train_t)
 scores = dict(
 valid_fbeta=np.array([r.best_fbeta for r in valid_results] ).mean() ,
valid_epoch=np.array([r.best_epoch for r in valid_results] ).mean() ,
threshold_cv=ensembler.threshold_cv,
threshold=ensembler.threshold,
elapsed_time=time.time() - start,
)
X = X / 255
def reshape_data(np_array):
return np_array.reshape(len(np_array), 28, 28, 1)
 X = reshape_data(X )<categorify>
if config.holdout:
test_X, test_X2, test_t = \
test_dataset.X, test_dataset.X2, test_dataset._t
y, t = ensembler.predict_proba(test_X, test_X2), test_t
y_pred = y > ensembler.threshold
y_pred_cv = y > ensembler.threshold_cv
result = classification_metrics(y_pred, t)
 result_cv = classification_metrics(y_pred_cv, t)
 result_theoretical = classification_metrics(y, t)
 scores.update(dict(
 test_fbeta=result['fbeta'],
test_fbeta_cv=result_cv['fbeta'],
test_fbeta_theoretical=result_theoretical['fbeta'],
test_threshold_theoretical=result_theoretical['threshold'], 
))
Y = to_categorical(Y, 10 )<split>
submit_y = ensembler.predict(submit_dataset.X, submit_dataset.X2)
 submit_df['prediction'] = submit_y
submit_df = submit_df[['qid', 'prediction']]
submit_df.to_csv(config.outdir / 'submission.csv', index=False )
train_to_valtest_ratio =.32
validate_to_test_ratio =.5 
(X_train,
X_validation_and_test,
Y_train,
Y_validation_and_test)= train_test_split(X, Y, test_size=train_to_valtest_ratio)
(X_validation,
X_test,
Y_validation,
Y_test)= train_test_split(X_validation_and_test, Y_validation_and_test, test_size=validate_to_test_ratio )<choose_model_class>
for i,(train, valid)in enumerate(zip(train_results, valid_results)) :
df = pd.concat([train.summary, valid.summary])
 df = df.set_index(['name', df.index], drop=True)
 display(df )
<choose_model_class><EOS> 
pd.DataFrame([scores] )