Kaggle competition: Quora Insincere Questions Classification
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
np.random.seed(481945 )
import tokenization<categorify>
from sklearn.metrics import pairwise
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn import metrics
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
def summary(model, input_size, batch_size=-1, device="cuda", input_type=torch.float32):
def register_hook(module):
def hook(module, input, output):
class_name = str(module.__class__ ).split(".")[-1].split("'")[0]
module_idx = len(summary)
 m_key = "%s-%i" %(class_name, module_idx + 1)
 summary[m_key] = OrderedDict()
summary[m_key]["input_shape"] = list(input[0].size())
 summary[m_key]["input_shape"][0] = batch_size
if isinstance(output,(list, tuple)) :
summary[m_key]["output_shape"] = [
[-1] + list(o.size())[1:] for o in output
]
else:
summary[m_key]["output_shape"] = list(output.size())
 summary[m_key]["output_shape"][0] = batch_size
params = 0
if hasattr(module, "weight")and hasattr(module.weight, "size"):
params += torch.prod(torch.LongTensor(list(module.weight.size())))
 summary[m_key]["trainable"] = module.weight.requires_grad
if hasattr(module, "bias")and hasattr(module.bias, "size"):
params += torch.prod(torch.LongTensor(list(module.bias.size())))
 summary[m_key]["nb_params"] = params
if(
 not isinstance(module, nn.Sequential)
 and not isinstance(module, nn.ModuleList)
 and not(module == model)
):
hooks.append(module.register_forward_hook(hook))
device = device.lower()
assert device in [
"cuda",
"cpu",
], "Input device is not valid, please specify 'cuda' or 'cpu'"
if device == "cuda" and torch.cuda.is_available() :
dtype = torch.cuda.FloatTensor
else:
dtype = torch.FloatTensor
net_device = next(model.parameters() ).device
x = torch.zeros(( 1,)+ input_size ,dtype=input_type ).to(net_device)
 summary = OrderedDict()
hooks = []
model.apply(register_hook)
 model(x)
 for h in hooks:
h.remove()
print("----------------------------------------------------------------")
 line_new = "{:>20} {:>25} {:>15}".format("Layer(type)", "Output Shape", "Param
print(line_new)
 print("================================================================")
 total_params = 0
total_output = 0
trainable_params = 0
for layer in summary:
line_new = "{:>20} {:>25} {:>15}".format(
 layer,
str(summary[layer]["output_shape"]),
"{0:,}".format(summary[layer]["nb_params"]), 
)
 total_params += summary[layer]["nb_params"]
total_output += np.prod(summary[layer]["output_shape"])
 if "trainable" in summary[layer]:
if summary[layer]["trainable"] == True:
trainable_params += summary[layer]["nb_params"]
print(line_new)
 total_input_size = abs(np.prod(input_size)* batch_size * 4./(1024 ** 2.))
 total_output_size = abs(2.* total_output * 4./(1024 ** 2.))
 total_params_size = abs(total_params.numpy() * 4./(1024 ** 2.))
 total_size = total_params_size + total_output_size + total_input_size
print("================================================================")
 print("Total params: {0:,}".format(total_params))
print("Trainable params: {0:,}".format(trainable_params))
print("Non-trainable params: {0:,}".format(total_params - trainable_params))
print("----------------------------------------------------------------")
 print("Input size(MB): %0.2f" % total_input_size)
 print("Forward/backward pass size(MB): %0.2f" % total_output_size)
 print("Params size(MB): %0.2f" % total_params_size)
 print("Estimated Total Size(MB): %0.2f" % total_size)
 print("----------------------" )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
class CyclicLR(_LRScheduler):

 def __init__(self,
optimizer,
base_lr=1e-3,
max_lr=6e-3,
step_size_up=2000,
step_size_down=None,
mode='triangular',
gamma=1.,
scale_fn=None,
scale_mode='cycle',
last_batch_idx=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
base_lrs = self._format_lr('base_lr', optimizer, base_lr)
 if last_batch_idx == -1:
for base_lr, group in zip(base_lrs, optimizer.param_groups):
group['lr'] = base_lr
self.max_lrs = self._format_lr('max_lr', optimizer, max_lr)
 step_size_down = step_size_down or step_size_up
self.total_size = float(step_size_up + step_size_down)
 self.step_ratio = float(step_size_up)/ self.total_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
super(CyclicLR, self ).__init__(optimizer, last_batch_idx)
 def _format_lr(self, name, optimizer, lr):

 if isinstance(lr,(list, tuple)) :
if len(lr)!= len(optimizer.param_groups):
raise ValueError("expected {} values for {}, got {}".format(
 len(optimizer.param_groups), name, len(lr)))
 return torch.tensor(lr)
 else:
return lr * torch.ones(len(optimizer.param_groups))
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):

 cycle = math.floor(1 + self.last_epoch / self.total_size)
 x = 1 + self.last_epoch / self.total_size - cycle
if x <= self.step_ratio:
scale_factor = x / self.step_ratio
else:
scale_factor =(x - 1)/(self.step_ratio - 1)
 lrs = []
for base_lr, max_lr in zip(self.base_lrs, self.max_lrs):
base_height =(max_lr - base_lr)* scale_factor
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_epoch)
 lrs.append(lr)
 return lrs
train_input = bert_encode([" ".join(i)for i in train.text], tokenizer, max_len=160)
 test_input = bert_encode([" ".join(i)for i in test.text], tokenizer, max_len=160 )<choose_model_class>
quora_data = pd.read_csv('.. /input/train.csv' )
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model<train_model>
quora_test_data = pd.read_csv('.. /input/test.csv' )
early = EarlyStopping(monitor='val_loss',mode='auto', baseline=None, restore_best_weights=False)
 train_history = model.fit(
 train_input, ydata,
validation_split=0.2,
epochs=100,
batch_size=16,
callbacks=[early], verbose=1 
)
 model.save('model.h5' )<save_to_csv>
for s in sample[sample.target == 1].question_text:
print(s )
predictions = model.predict(test_input)
 for i in [0.5, 0.6, 0.4]:
arr = [1 if j>i else 0 for j in predictions]
sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':arr})
 sub.to_csv('submission_bert'+str(i)+'.csv',index=False )<install_modules>
from nltk.tokenize import TweetTokenizer
%%capture
!apt update && apt install -y openjdk-8-jdk
!update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
!python -m pip install --upgrade pip
!pip install --upgrade language-check pycontractions commonregex ekphrasis<set_options>
print(nltk.tokenize.word_tokenize("Don't spoil the movie or I'll kill you"))
%%capture
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
<load_pretrained>
print(TweetTokenizer().tokenize("Don't spoil the movie or I'll kill you"))
%%capture
EMBEDDINGS = "glove-twitter-200"
cont = Contractions(api_key=EMBEDDINGS)
 cont.load_models()
text_processor = TextPreProcessor(
 normalize=['url', 'email', 'percent', 'money', 'phone', 'user', 'time', 'url', 'date', 'number'],
annotate={},
fix_html=True,
segmenter="twitter",
corrector="twitter",
unpack_hashtags=True,
unpack_contractions=True,
spell_correct_elong=True,
tokenizer=SocialTokenizer(lowercase=False ).tokenize,
dicts=[emoticons],
fix_text=True,
)<categorify>
def tokenize(questions):
tokenized_questions = []
for iteration, text in enumerate(questions):
if iteration % 50000 == 0:
print(iteration, "texts tokenized")
 tokenized_questions.append([t.lower() for t in TweetTokenizer().tokenize(text)])
 return tokenized_questions
def decontraction(corpus: List[str])-> List[str]:

 return list(cont.expand_texts(corpus))
def clean(corpus: List[str])-> List[str]:

 return list(map(lambda text: " ".join(text_processor.pre_process_doc(text)) , corpus))<load_from_csv>
dev_tokens = tokenize(quora_data.question_text )
%%capture
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 train["clean_text"] = decontraction(train["text"].tolist())
 train["clean_text"] = clean(train["clean_text"].tolist())
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 test["clean_text"] = decontraction(test["text"].tolist())
 test["clean_text"] = clean(test["clean_text"].tolist() )<choose_model_class>
test_tokens = tokenize(quora_test_data.question_text )
%%capture
model = Pipeline(
 [ 
('embedder', FeatureUnion(
 [ 
('char', TfidfVectorizer(sublinear_tf=True, lowercase=True, ngram_range=(1, 10), analyzer='char_wb')) , 
('word', TfidfVectorizer(sublinear_tf=True, lowercase=True, ngram_range=(1, 4), stop_words='english'))
])
), 
('classifier', VotingClassifier(
 [ 
('clf1', CalibratedClassifierCV(LinearSVC() , cv=5)) , 
('clf2', ComplementNB()), 
('clf3', SGDClassifier(alpha=1e-4, max_iter=50, penalty="elasticnet"))
])
)
 ]
)<train_on_grid>
import torchtext
macro_f1s = []
for seed in [0, 1, 2, 42, 56]:
train_, eval_ = tts(train, test_size=0.05, stratify=train["target"], random_state=seed)
 model.fit(train_["clean_text"], train_["target"])
 macro_f1s.append(classification_report(eval_["target"], model.predict(eval_["clean_text"]), output_dict=True)["macro avg"]["f1-score"])
 print(f"Max: {np.max(macro_f1s)} Min: {np.min(macro_f1s)} Avg: {np.mean(macro_f1s)}" )<train_model>
from collections import Counter
import itertools
model.fit(train["clean_text"], train["target"] )<save_to_csv>
train_tokens, val_tokens, train_labels, val_labels = train_test_split(dev_tokens, quora_data.target, test_size=0.1 )
model_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 model_submission['target'] = model.predict(test["clean_text"])
 model_submission.to_csv('model_submission.csv', index=False )<save_to_csv>
word_counts = Counter(itertools.chain(*train_tokens))
train.to_csv("clean_train.csv")
 test.to_csv("clean_test.csv" )<feature_engineering>
print(len(word_counts))
print(word_counts.most_common(10))
os.environ["WANDB_API_KEY"] = "0"<set_options>
from gensim.models.keyedvectors import KeyedVectors
%matplotlib inline
%config InlineBackend.figure_format='retina' 
<define_variables>
gensim_vectors = KeyedVectors.load_word2vec_format('.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True )
RANDOM_SEED = 42
MAX_LEN = 84
TRAIN_BATCH_SIZE = 16
VALID_BATCH_SIZE = 16
EPOCHS=6
LR=5e-5
TEST_SIZE = 0.15
TRAINING_FILE = ".. /input/nlp-getting-started/train.csv"
SAMPLE_FILE = ".. //nlp-getting-started/sample_submission.csv"
TEST_FILE = ".. /input/nlp-getting-started/test.csv"
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(DEVICE)
 MODEL_TYPE = "bert-large-uncased"
MODEL_FILENAME = 'model.bin'
DO_LOWER = True
CLEANSE_DATA = True
random.seed(RANDOM_SEED)
 np.random.seed(RANDOM_SEED)
 torch.manual_seed(RANDOM_SEED)
 torch.cuda.manual_seed(RANDOM_SEED)
 torch.cuda.manual_seed_all(RANDOM_SEED)
 torch.backends.cudnn.deterministic = True<load_pretrained>
def filter_vectors(gensim_vectors, words):
result = {}
for w in words:
if w in gensim_vectors.vocab:
result[w] = gensim_vectors[w].copy()
return result
TOKENIZER = transformers.BertTokenizer.from_pretrained(
 MODEL_TYPE,
do_lower_case=DO_LOWER,
)<categorify>
filtered_vectors = filter_vectors(gensim_vectors, word_counts.keys() )
def cleanse_text(t):
_re_rep = re.compile(r'(\S )(\1{2,})')
 _re_wrep = re.compile(r'(?:\s|^ )(\w+)\s+(( ?:\1\s+)+)\1(\s|\W|$)')
 def replace_rep(t):
"Replace repetitions at the character level, however there are exceptions"
def _replace_rep(m):
c,cc = m.groups()
return f'{c}{c}'
w = _re_rep.sub(_replace_rep, t)
 w = w.replace('LOOL','LOL' ).replace('lool','lol' ).replace('LooL','LoL' ).replace('Lool','Lol')
 return w
def fix_html(x):
"messy characters"
x = x.replace('
'
', "" ).replace('quot;', "'" ).replace('<br />', "" ).replace('
', "" ).replace('</br>', "" ).replace('&gt;','' ).replace('&lt;','' ).replace('|','' ).replace(
 '\"', '"' ).replace('<unk>','' ).replace(' @.@ ','.' ).replace(' @-@ ','-' ).replace('...',' …' ).replace('=>','' ).replace('.. ','' ).replace('Û_','' ).replace(
 'gooaal','goal' ).replace('!','' ).replace(']','' ).replace('ÛÊ','' ).replace(' - ','' ).replace('åÊ',' ' ).replace('åÊÛÒ',' ' ).replace('Ûª',' ' ).replace(
 '[','' ).replace('_','' ).replace(';)', ':)' ).replace('?','' ).replace('+','' ).replace('ÛÒ',' ' ).replace('{','' ).replace('}','' ).replace('%20',' ' ).replace('*',' ')
 return html.unescape(x)
 def replace_wrep(t):
"Replace word repetitions: w1 w1 w1 w1"
def _replace_wrep(m):
c,cc,e = m.groups()
return f'{c} {e}'
return _re_wrep.sub(_replace_wrep, t)
 def remove_url(t):
return re.split('https?:\/\/.*', str(t)) [0]
return remove_url(fix_html(replace_wrep(replace_rep(t))))<define_variables>
del gensim_vectors
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired",
"kno" : "know",
"fab" : "fabulous",
"oli" : "oil",
"tren" : "trend",
"swea" : "swear",
"stil" : "still",
"diff" : "different",
"appx" : "approximately",
"srsly" : "seriously",
"epicente" : "epicenter",
"evng" : "evening",
"lookg" : "looking",
"sayin" : "saying",
"tryin" : "trying",
"comin" : "Coming",
"jumpin" : "jumping",
"nothin" : "nothing",
"burnin" : "burning",
"killin" : "killing",
"thinkin" : "thinking",
"throwin" : "throwing",
"newss" : "news",
"memez" : "memes",
"fforecast" : "Forecast",
}
def convert_abbrev(w):
return abbreviations[w.lower() ] if w.lower() in abbreviations.keys() else w 
<load_from_csv>
min_occurences = 14
filtered_counts = {w:c for w,c in word_counts.items() if c >= min_occurences}
print(len(filtered_counts))
df_train = pd.read_csv(TRAINING_FILE)
 df_test = pd.read_csv(TEST_FILE)
 df_train['classes'] = np.where(df_train['target'] == 0, 'Not Disaster', 'Disaster' )<feature_engineering>
class VocabLike:
def __init__(self, itos, stoi):
self.itos = itos
self.stoi = stoi
if CLEANSE_DATA:
df_train['text'] = df_train['text'].apply(lambda x: convert_abbrev(cleanse_text(x)))
 df_train['tlen'] = df_train['text'].apply(len)
 df_train.head()<feature_engineering>
from collections import defaultdict
if CLEANSE_DATA:
df_test['text'] = df_test['text'].apply(lambda x: convert_abbrev(cleanse_text(x)))
 df_test['tlen'] = df_test['text'].apply(len)
 df_test.head()<set_options>
specials = ['<unk>', '<pad>', '<eos>']
filtered_counts.update({w:0 for w in specials})
 stoi = defaultdict(lambda:0)
 itos = [0] * len(filtered_counts)
 trainable_words = {w for w in filtered_counts.keys() if w not in specials and w not in filtered_vectors}
pretrained_words = {w for w in filtered_counts.keys() if w not in specials and w in filtered_vectors}
for i,w in enumerate(itertools.chain(specials, trainable_words, pretrained_words)) :
stoi[w] = i
itos[i] = w
vocab = VocabLike(itos, stoi )
sns.set(font_scale=1.25)
 plt.rcParams["figure.figsize"]=(8,6)
 sns.countplot(df_train.classes);
sns.set(style="dark")
 class_names = ['Not Disaster', 'Disaster'];<concatenate>
def extract_vectors(vocab, vec_dict, offset, total, vec_size):
vectors = np.zeros(( total, vec_size), dtype=np.float32)
 for i in range(total):
word = vocab.itos[i + offset]
assert word in vec_dict
vectors[i] = vec_dict[word]
return vectors
labels = df_train['target'].values
idx = len(labels)
 df = pd.concat([df_train, df_test])
 df = df.text.values
train= df[:idx]
test = df[idx:]
train.shape, test.shape<define_variables>
np_vectors = extract_vectors(vocab, filtered_vectors,
len(specials)+ len(trainable_words),
len(pretrained_words),
300 )
max_len=0
for t in df:
input_ids = TOKENIZER.encode(t, add_special_tokens=True)
 max_len = max(max_len, len(input_ids))
print('Max input_ids length = MAX_LEN: ', max_len)
 MAX_LEN = max_len<categorify>
def nearest_neighbors(vocab, embeddings, word, topn, use_offset=False):
offset = len(specials)+ len(trainable_words)if use_offset else 0
assert word in vocab.stoi
word_index = vocab.stoi[word] - offset
sims = pairwise.cosine_similarity(embeddings[word_index].reshape(1,-1),embeddings ).ravel()
indices = np.argsort(sims)[::-1]
print(word)
 for i in range(topn):
sim_word = vocab.itos[indices[i] + offset]
print(sim_word, sims[indices[i]] )
class DisasterTweetsDataset(data.Dataset):
def __init__(self,text,target,tokenizer,max_len):
self.text = text
self.target = target
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.text)
 def __getitem__(self, item):
text = str(self.text[item])
 text = " ".join(text.split())
 encoding = TOKENIZER.encode_plus(
 text,
add_special_tokens = True,
max_length = self.max_len,
truncation='longest_first',
return_token_type_ids=True,
pad_to_max_length=True,
return_attention_mask = True,
return_tensors='pt', 
)
 return {
'text': text,
'input_ids': encoding['input_ids'].flatten() ,
'attention_mask': encoding['attention_mask'].flatten() ,
'token_type_ids': encoding['token_type_ids'].flatten() ,
'targets': torch.tensor(self.target[item],dtype=torch.long)
 }<choose_model_class>
nearest_neighbors(vocab, np_vectors, 'we'll', 10, True )
class DisasterTweetsModel(nn.Module):
def __init__(self, n_classes):
super(DisasterTweetsModel,self ).__init__()
self.bert = transformers.BertModel.from_pretrained(MODEL_TYPE)
 self.bert_drop_1 = nn.Dropout(0.15)
 self.bn = nn.BatchNorm1d(self.bert.config.hidden_size)
 self.relu = nn.ReLU()
self.out1 = nn.utils.weight_norm(nn.Linear(self.bert.config.hidden_size,2048))
self.out2 = nn.utils.weight_norm(nn.Linear(2048,self.bert.config.hidden_size))
self.out3 = nn.utils.weight_norm(nn.Linear(self.bert.config.hidden_size,n_classes))
self.softmax = nn.Softmax(dim=1)
 def forward(self, input_ids, attention_mask,token_type_ids):
_,pooled_output = self.bert(
 input_ids = input_ids,
attention_mask = attention_mask,
token_type_ids = token_type_ids, 
)
 output = self.bert_drop_1(pooled_output)
 output = self.relu(output)
 output = self.out1(output)
 output = self.out2(output)
 output = self.bn(output)
 output = self.relu(output)
 output = self.out3(output)
 return self.softmax(output )<split>
gc.collect()
df_train, df_val = train_test_split(df_train, test_size=TEST_SIZE,random_state=RANDOM_SEED)
 df_train = df_train.reset_index(drop=True)
 df_val = df_val.reset_index(drop=True)
 df_train.shape, df_val.shape<categorify>
def to_word_indices(tokens, vocab, start_index, end_index):
return [[start_index] + [vocab.stoi[w] for w in sent] + [end_index] for sent in tokens]
def create_data_loader(df, tokenizer, max_len, bsz, shuffle):
ds = DisasterTweetsDataset(
 text=df.text.to_numpy() ,
target=df.target.to_numpy() ,
tokenizer=TOKENIZER,
max_len = MAX_LEN, 
)
 return data.DataLoader(
 ds,
batch_size=bsz,
num_workers=4,
shuffle=shuffle 
)
 train_dl = create_data_loader(df_train, tokenizer=TOKENIZER, max_len=MAX_LEN, bsz=TRAIN_BATCH_SIZE,shuffle=True)
 val_dl = create_data_loader(df_val, tokenizer=TOKENIZER, max_len=MAX_LEN, bsz=VALID_BATCH_SIZE,shuffle=False )<choose_model_class>
class TokenToIdDataset(torch.utils.data.Dataset):
def __init__(self, tokens, labels, vocab, max_size=-1, min_size=10, precompute=False, precomputed=False):
self._start_index = vocab.stoi['<sos>']
self._end_index = vocab.stoi['<eos>']
self._pad_index = vocab.stoi['<pad>']
if precompute and not precomputed:
tokens = to_word_indices(tokens, vocab, self._start_index, self._end_index)
 self._precomputed = precompute or precomputed
self._tokens = tokens
self._labels = labels
print(len(self._labels))
self._vocab = vocab
self._len = len(tokens)
 self._max_size = max_size
self._min_size = min_size
self._cache = []
self._batch_size = -1
def __len__(self):
return self._len
def compute_cache(self, batch_size):
start = 0
batch_num = 0
cache = []
while start < len(self):
batch_end = min(start + batch_size, len(self))
cache.append(self.collate([self[j] for j in range(start, batch_end)]))
start = batch_end
self._cache = cache
self._batch_size = batch_size
def get_cache(self, batch_size):
if not self._cache or self._batch_size != batch_size:
self.compute_cache(batch_size)
 return self._cache
def collate(self, samples):
if self._max_size == -1:
size = max(self._min_size, max(len(s[0])for s in samples))
else:
size = self._max_size
labels_tensor = torch.tensor([s[1] for s in samples], dtype=torch.float32)
 texts_tensor = torch.zeros(len(samples), size, dtype=torch.long)
 texts_tensor += self._pad_index
for i, pair in enumerate(samples):
text, _ = pair
text = text if len(text)<= size else text[:size]
texts_tensor[i, size - len(text):] = torch.tensor(text)
 return texts_tensor, labels_tensor
def __getitem__(self, index):
text = self._tokens[index]
if self._precomputed:
indices = text
else:
indices = [self._start_index]
indices.extend(self._vocab.stoi[tok] for tok in text)
 indices.append(self._end_index)
 return indices, self._labels[index]
model = DisasterTweetsModel(len(class_names))
model = model.to(DEVICE)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],'weight_decay':0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],'weight_decay':0.0}]
optimizer = AdamW(optimizer_parameters, lr=LR)
 steps = len(train_dl)* EPOCHS
scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps = 0,
num_training_steps = steps 
)
 loss_fn = nn.CrossEntropyLoss().to(DEVICE)
<train_model>
min_length=10
train_dataset = TokenToIdDataset(train_tokens,train_labels.values, vocab,min_size=min_length, precompute=True)
 val_dataset = TokenToIdDataset(val_tokens, val_labels.values, vocab,min_size=min_length, precompute=True )
def train_fn(model, dl, loss_fn, optimizer, device, scheduler, n_examples):
model.train()
losses = []
predictions = 0
for d in tqdm(dl, total=len(dl), position=0, leave=True):
input_ids = d['input_ids'].to(DEVICE)
 attention_mask = d['attention_mask'].to(DEVICE)
 token_type_ids = d['token_type_ids'].to(DEVICE, dtype=torch.long)
 targets = d['targets'].to(DEVICE)
 outputs = model(
 input_ids = input_ids,
attention_mask = attention_mask,
token_type_ids = token_type_ids 
)
 _,preds = torch.max(outputs, dim=1)
 loss = loss_fn(outputs, targets)
 predictions += torch.sum(preds==targets)
 losses.append(loss.item())
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() ,max_norm=1.0)
 optimizer.step()
scheduler.step()
optimizer.zero_grad()
return predictions.double() / n_examples, np.mean(losses)
 def eval_fn(model, dl, loss_fn, device, n_examples):
model.eval()
losses = []
predictions = 0
with torch.no_grad() :
for d in tqdm(dl, total=len(dl), position=0, leave=True):
input_ids = d['input_ids'].to(DEVICE)
 attention_mask = d['attention_mask'].to(DEVICE)
 token_type_ids = d['token_type_ids'].to(DEVICE, dtype=torch.long)
 targets = d['targets'].to(DEVICE)
 outputs = model(
 input_ids = input_ids,
attention_mask = attention_mask,
token_type_ids = token_type_ids 
)
 _,preds = torch.max(outputs, dim=1)
 loss = loss_fn(outputs, targets)
 predictions += torch.sum(preds==targets)
 losses.append(loss.item())
 return predictions.double() / n_examples, np.mean(losses )<train_model>
gc.collect()
%%time
hist = defaultdict(list)
 best_acc = 0
for epoch in range(EPOCHS):
print(f'
Epoch {epoch + 1} / {EPOCHS}')
 train_acc, train_loss = train_fn(model,train_dl,loss_fn,optimizer,DEVICE,scheduler,len(df_train))
print(f'Train loss {train_loss} Accuracy {train_acc}')
 val_acc, val_loss = eval_fn(model,val_dl,loss_fn,DEVICE,len(df_val))
print(f'Validation loss {val_loss} Accuracy {val_acc}')
 print()
hist['train_acc'].append(train_acc)
 hist['train_loss'].append(train_loss)
 hist['val_acc'].append(val_acc)
 hist['val_loss'].append(val_loss)
 if val_acc > best_acc:
torch.save(model.state_dict() ,MODEL_FILENAME)
 best_acc = val_acc<predict_on_test>
import tqdm
from tqdm import tqdm_notebook
def get_preds(model, data_loader):
model.eval()
predictions = []
prediction_proba = []
with torch.no_grad() :
for d in tqdm(data_loader, total=len(data_loader)) :
input_ids = d['input_ids'].to(DEVICE)
 attention_mask = d['attention_mask'].to(DEVICE)
 token_type_ids = d['token_type_ids'].to(DEVICE, dtype=torch.long)
 outputs = model(
 input_ids = input_ids,
attention_mask = attention_mask,
token_type_ids = token_type_ids 
)
 _,preds = torch.max(outputs, dim=1)
 predictions.extend(preds)
 prediction_proba.extend(outputs)
 predictions = torch.stack(predictions ).cpu()
prediction_proba = torch.stack(prediction_proba ).cpu()
return predictions, prediction_proba<load_pretrained>
class BestModel:
def __init__(self, model_path, optimizer_path, best_loss=10000):
self.best_loss = best_loss
self.model_path = model_path
self.optimizer_path = optimizer_path
def update(self, loss, model, optimizer=None):
self.best_loss = loss
torch.save(model.state_dict() , self.model_path)
 if optimizer:
torch.save(optimizer.state_dict() , self.optimizer_path)
 def load(self, model, optimizer=None):
model.load_state_dict(torch.load(self.model_path))
if optimizer:
optimizer.load_state_dict(torch.load(self.optimizer_path))
model.eval()
def copy(self, new_model_path, new_optimizer_path):
copyfile(self.model_path, new_model_path)
 copyfile(self.optimizer_path, new_optimizer_path)
 return BestModel(new_model_path, new_optimizer_path, self.best_loss )
model = DisasterTweetsModel(len(class_names))
model.load_state_dict(torch.load(MODEL_FILENAME))
model = model.to(DEVICE)
 df_test['target']=-1
test_dl = create_data_loader(df_test, tokenizer=TOKENIZER, max_len=MAX_LEN, bsz=TRAIN_BATCH_SIZE,shuffle=False )<predict_on_test>
def train_network(network, optimizer,
criterion,
train_loader, val_loader,
n_epochs, patience,
best_model, after_gradient=None, lr_scheduler=None, schedule_per_batch=False):
attempts_left = patience
for epoch in range(n_epochs):
network.train()
if lr_scheduler and not schedule_per_batch:
lr_scheduler.step()
print_every = len(train_loader)// 10
running_train_loss = 0.0
batches_since_last_print = 0
for i, batch in tqdm_notebook(enumerate(train_loader), total=len(train_loader)) :
if lr_scheduler and schedule_per_batch:
lr_scheduler.step()
X, y = batch
X, y = X.cuda() , y.cuda()
optimizer.zero_grad()
output = network(X)
 loss = criterion(output, y.view(-1,1))
running_train_loss += loss.item()
batches_since_last_print += 1
if i % print_every == print_every - 1:
learning_rate = next(iter(optimizer.param_groups)) ['lr']
print("Epoch {}, batch {}, loss {}, lr {}".format(epoch + 1,
i + 1,
running_train_loss / batches_since_last_print,
learning_rate))
running_train_loss = 0.0
batches_since_last_print = 0
loss.backward()
if after_gradient:
after_gradient(epoch, network)
 optimizer.step()
network.eval()
with torch.no_grad() :
running_val_loss = 0.0
for i, batch in tqdm_notebook(enumerate(val_loader),total=len(val_loader)) :
X, y = batch
X, y = X.cuda() , y.cuda()
output = network(X)
 loss = criterion(output, y.view(-1,1))
running_val_loss += loss.item()
running_val_loss /= len(val_loader)
 print("Epoch {}, validation loss {}".format(epoch + 1, running_val_loss))
if running_val_loss < best_model.best_loss:
print('Improved from {} to {}'.format(best_model.best_loss, running_val_loss))
best_model.update(running_val_loss, network, optimizer)
 attempts_left = patience
elif attempts_left > 0:
print('No improvement, attempts left = {}'.format(attempts_left))
attempts_left -= 1
else:
print('Early stopping, best_weight are saved')
 break
preds, proba = get_preds(model,test_dl)
 len(preds )<load_from_csv>
class GenericAttention(nn.Module):
def __init__(self, scoring_function, query_transform, key_transform, value_transform):
super().__init__()
self.scoring_function = scoring_function
self.query_transform = query_transform
self.key_transform = key_transform
self.value_transform = value_transform
def forward(self, query, keys, values):
query, keys, values = self.query_transform(query), self.key_transform(keys), self.value_transform(values)
 batch_size, output_length, query_dim = query.size()
_, keys_length, keys_dim = keys.size()
attention_scores = self.scoring_function(query, keys)
 at_b, at_q, at_k = attention_scores.size()
assert at_b == batch_size
assert at_q == output_length
assert at_q == keys_length
attention_weights = F.softmax(attention_scores, dim=2)
 alignments = torch.bmm(attention_weights, values)
 return alignments, attention_weights
df_sample = pd.read_csv(SAMPLE_FILE )<prepare_output>
class DotProductAttentionScoring(nn.Module):
def __init__(self, scale):
super().__init__()
self.scale = np.sqrt(scale)
 def forward(self, query, keys):
b_q, n_q, d_q = query.size()
b_k, n_k, d_k = keys.size()
assert b_q == b_k
dot_products = torch.bmm(query, torch.transpose(keys, 1, 2)) / self.scale
return dot_products
df_sample['target']=preds;df_sample.head()<save_to_csv>
class TimeInvariant(nn.Module):
def __init__(self, inner):
super().__init__()
self.inner = inner
def forward(self, data):
batch,time,dim = data.size()
result = self.inner(data.view(batch * time, dim))
result = result.view(batch, time, -1)
 return result
df_sample.to_csv("submission_final.csv",index=False )<import_modules>
class PositionalEncoding(nn.Module):
"Implement the PE function."
def __init__(self, d_model, max_len=5000):
super(PositionalEncoding, self ).__init__()
pe = torch.zeros(max_len, d_model,dtype=torch.float32)
 position = torch.arange(0., max_len ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0., d_model, 2)*
-(math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + Variable(self.pe[:, :x.size(1)],
requires_grad=False)
 return x
import pandas as pd
import numpy as np
import re<load_from_csv>
class SelfAttentionNet(nn.Module):
def __init__(self, vocab, embeddings, num_trainable):
super().__init__()
self.pos_encoding = PositionalEncoding(50,120)
 self.num_trainable = num_trainable
self.embedding = nn.Embedding(num_embeddings=len(vocab.itos), embedding_dim=300,padding_idx=vocab.stoi['<pad>'])
 self.embedding.weight.data[num_trainable:].copy_(torch.from_numpy(embeddings))
self.drop_emb = nn.Dropout2d(p=0.5)
 self.pretrained_projection = nn.Sequential(
 nn.Linear(300,300))
key_transform = TimeInvariant(
 nn.Sequential(
 nn.Linear(350,300),
nn.ReLU() ,
nn.Linear(300,300)
))
query_transform = TimeInvariant(
 nn.Sequential(
 nn.Linear(350,300),
nn.ReLU() ,
nn.Linear(300,300)
))
value_transform = TimeInvariant(
 nn.Sequential(
 nn.Linear(350,300),
nn.ReLU() ,
nn.Linear(300,300)
))
attention_scoring = DotProductAttentionScoring(300.)
self.attention1 = GenericAttention(attention_scoring, query_transform, key_transform,value_transform)
 self.fc = nn.Sequential(
 nn.Dropout(p=0.2),
nn.Linear(300,500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Linear(500, 1)
)
 def zero_embedding_grad(self):
self.embedding.weight.grad[self.num_trainable:] = 0
def set_emb_dropout(self, p):
self.emb_dropout = p
def apply_spatial_dropout(self,dropout_layer, x):
x = torch.transpose(x,1,2)
 x = x.view(x.shape[0], x.shape[1], x.shape[2], 1)
 x = dropout_layer(x)
 x = torch.squeeze(x,dim=3)
 x = torch.transpose(x,1,2)
 return x
def get_attention_weights(self, x):
x = self.embed(x)
 x,weights = self.attention1(x, x, x)
 return weights
def mixed_embedding(self, x):
e = self.embedding(x)
 pretrained_mask = x < self.num_trainable
pretrained = e.view(-1,300)[pretrained_mask.view(-1)]
e.view(-1,300)[pretrained_mask.view(-1)] = self.pretrained_projection(pretrained)
 return e
def embed(self, x):
x = self.mixed_embedding(x)
 x = self.apply_spatial_dropout(self.drop_emb, x)
 pe = torch.zeros(x.size(0), x.size(1), 50,device=x.device)
 pe = self.pos_encoding(pe)
 x = torch.cat(( x,pe), dim=2)
 return x
def forward(self, x):
x = self.embed(x)
 x,_ = self.attention1(x, x, x)
 reduced,_ = torch.max(x, dim=1)
 return self.fc(reduced)
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()<feature_engineering>
class Net(nn.Module):
def __init__(self, vocab, embeddings, num_trainable, normalize=False):
super().__init__()
self.num_trainable = num_trainable
self.embedding = nn.Embedding(num_embeddings=len(vocab.itos), embedding_dim=300,padding_idx=vocab.stoi['<pad>'])
 if normalize:
embeddings = embeddings / np.linalg.norm(embeddings,axis=1 ).reshape(-1,1)
 self.embedding.weight.data[num_trainable:].copy_(torch.from_numpy(embeddings))
self.drop_emb = nn.Dropout2d(p=0.5)
 self.rnn1 = nn.GRU(input_size=300, hidden_size=150,num_layers=1,bidirectional=True)
 self.drop_rnn = nn.Dropout(p=0.2)
 self.rnn2 = nn.GRU(input_size=300, hidden_size=150,num_layers=1,bidirectional=True)
 self.fc = nn.Sequential(
 nn.Dropout(p=0.3),
nn.Linear(300, 1)
)
 def embed(self, x):
return self.embedding(x)
 def zero_embedding_grad(self):
self.embedding.weight.grad[self.num_trainable:] = 0
def get_hidden(self, x):
x = self.embedding(x)
 x = torch.transpose(x,1,2)
 x = x.view(x.shape[0], x.shape[1], x.shape[2], 1)
 x = self.drop_emb(x)
 x = torch.squeeze(x,dim=3)
 x = torch.transpose(x,1,2)
 x = torch.transpose(x,0,1)
 x, hidden = self.rnn1(x)
 return x
def forward(self, x):
x = self.embedding(x)
 x = torch.transpose(x,1,2)
 x = x.view(x.shape[0], x.shape[1], x.shape[2], 1)
 x = self.drop_emb(x)
 x = torch.squeeze(x,dim=3)
 x = torch.transpose(x,1,2)
 x = torch.transpose(x,0,1)
 output, hidden = self.rnn1(x)
 output = self.drop_rnn(output)
 output = output + self.rnn2(output)[0]
output = output.transpose(0,1)
 x,_ = torch.max(output, dim=1)
 x = self.fc(x)
 return x
def htmlremove(text):
return re.sub('<[^<]+?>', '', text)
 train['text'] = train['text'].apply(htmlremove)
 test['text'] = test['text'].apply(htmlremove )<feature_engineering>
num_scratch = len(specials)+ len(trainable_words )
def urlremove(text):
return re.sub(r"(?:\@|https?\://)\S+", "", text)
 train['text'] = train['text'].apply(urlremove)
 test['text'] = test['text'].apply(urlremove )<categorify>
net = Net(vocab, np_vectors, num_scratch ).cuda()
best_model = BestModel('best_model', 'best_optimizer' )
def remove_emoji(text):
emoji_pattern = re.compile(
 u"(\ud83d[\ude00-\ude4f])|"
u"(\ud83c[\udf00-\uffff])|"
u"(\ud83d[\u0000-\uddff])|"
u"(\ud83d[\ude80-\udeff])|"
u"(\ud83c[\udde0-\uddff])"
"+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train['text'] = train['text'].apply(remove_emoji)
 test['text'] = test['text'].apply(remove_emoji )<categorify>
criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([1.])).cuda()
optimizer = torch.optim.Adam(net.parameters() , lr=0.0003 )
def decontracted(text):
if '’' in text:
text = text.replace("’", "'")
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"can't", "can not", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'s", " is", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'t", " not", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'m", " am", text)
 return text
train['text'] = train['text'].apply(decontracted)
 test['text'] = test['text'].apply(decontracted )<feature_engineering>
def truncate_batch(batch):
x,y = batch
if x.shape[1] > 100:
x = x[:,:100]
return x,y
def characterremove(text):
return re.sub('\W+|_', ' ', text)
 train.text = train.text.apply(characterremove)
 test.text = test.text.apply(characterremove )<feature_engineering>
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=256,
collate_fn=lambda samples: truncate_batch(train_dataset.collate(samples)) , shuffle=True)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=256,collate_fn=lambda samples: truncate_batch(val_dataset.collate(samples)))
def remove_numbers(text):
pattern = r'[^a-zA-z.,!?/:;"'\s]'
return re.sub(pattern, '', text)
 train.text = train.text.apply(remove_numbers)
 train.text = train.text.str.lower()
train.text = train.text.str.strip()
test.text = test.text.apply(remove_numbers)
 test.text = test.text.str.lower()
test.text = test.text.str.strip()<feature_engineering>
print(128 * 112 * 112 * 300 * 4 / 1024 / 1024 )
stopwords = nltk.corpus.stopwords.words('english')
 def get_top_20(ngram_range, new_corpus, stopwords, min_df):
init_vec = CountVectorizer(ngram_range=ngram_range, stop_words = stopwords, min_df=min_df)
 X = init_vec.fit_transform(new_corpus)
 count_list = X.toarray().sum(axis=0)
 word_list = init_vec.get_feature_names()
counts = dict(zip(word_list,count_list))
top_20 = sorted(counts.items() , key=lambda x:x[1], reverse=True)[:20]
return top_20, counts
top_20, counts = get_top_20(( 1,1), train.text, stopwords, 4)
 top_20<string_transform>
train_network(net, optimizer,criterion,train_loader,val_loader,16, 3,best_model,
after_gradient=lambda epoch, network: network.zero_embedding_grad())
 best_model.load(net, optimizer)
lemmer = WordNetLemmatizer()
train.text = [' '.join([lemmer.lemmatize(word)for word in text.split(' ')])for text in train.text]
test.text = [' '.join([lemmer.lemmatize(word)for word in text.split(' ')])for text in test.text]<prepare_x_and_y>
train_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model,
after_gradient=lambda epoch, network: network.zero_embedding_grad())
 best_model.load(net, optimizer)
X_text = train.text
y_text = train.target
X_test_text = test.text<features_selection>
best_model.load(net, optimizer )
nlp = spacy.load("en_core_web_lg")
 nlp.vocab["amp"].is_stop = True
docs = [nlp(d ).vector for d in X_text]
X_text = np.vstack(docs)
 docs_test = [nlp(d ).vector for d in X_test_text]
X_test_text = np.vstack(docs_test )<split>
best_model_with_fixed_embeddings = best_model.copy('best_model_fixed', 'best_optimizer_fixed' )
X_train, X_test, y_train, y_test = train_test_split(X_text, y_text, random_state=0 )<train_on_grid>
best_model_with_fixed_embeddings.load(net, optimizer )
param_grid = {'kernel': ['rbf','linear'], 'C': [0.001, 0.01, 0.1, 1, 1.5, 2, 3, 10]}
grid = GridSearchCV(SVC() , param_grid, cv=10)
 grid.fit(X_train, y_train)
 print(grid.best_params_)
 grid.score(X_test,y_test )<save_to_csv>
best_model_with_fixed_embeddings.load(net, optimizer)
 for g in optimizer.param_groups:
g['lr'] = 0.00001
train_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model_with_fixed_embeddings,
after_gradient=lambda epoch, network: network.zero_embedding_grad())
 best_model_with_fixed_embeddings.load(net, optimizer )
y_test_text = grid.predict(X_test_text)
 test['target'] = y_test_text
test_export = test.loc[:, ['id', 'target']]
test_export.to_csv('disaster_test_word2vec.csv', index=False )<install_modules>
best_model2 = best_model_with_fixed_embeddings.copy('best_model_low', 'best_optimizer_low')
 best_model2.load(net, optimizer)
 for g in optimizer.param_groups:
g['lr'] = 0.00003
train_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model2)
 best_model2.load(net, optimizer )
%pip install --upgrade tensorflow-gpu<compute_test_metric>
best_model2.load(net, optimizer )
pd.set_option("display.max_colwidth", None)
 set_config(display='diagram')
 def md(text: str):
display(Markdown(text))
def benchmark(y_test, y_hat):
print('_' * 80)
 print(f"f1 score: {metrics.f1_score(y_test, y_hat, average='macro'):.3f}")
 print("classification report:")
 print(metrics.classification_report(y_test, y_hat))<load_from_csv>
train_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model)
 best_model.load(net, optimizer )
folder = ".. /input/nlp-getting-started"
test = pd.read_csv(os.path.join(folder, "test.csv"), index_col="id")
 train = pd.read_csv(os.path.join(folder, "train.csv"), index_col="id")
 X = train["text"]
y = train["target"]
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42 )<train_model>
best_model.load(net, optimizer )
print(f"We are using `{len(X_train)}` rows for model training, and `{len(X_valid)}` rows for validation" )<choose_model_class>
def predict_loader(network, loader):
network.eval()
all_predictions = []
with torch.no_grad() :
for X,_ in tqdm_notebook(loader, total=len(loader)) :
X = X.cuda()
out = network(X ).view(-1)
 out = torch.sigmoid(out)
 all_predictions.extend(out.tolist())
 return np.array(all_predictions )
vectorizers = {
"hashing": HashingVectorizer(stop_words="english", strip_accents="unicode", alternate_sign=False),
"tf-idf": TfidfVectorizer(stop_words="english", strip_accents="unicode", sublinear_tf=True),
}
pipe = Pipeline([ 
("vectorizer", None), 
("model", None)
 ] )<choose_model_class>
network_pred = predict_loader(net, val_loader )
feature_options = {
"vectorizer": [vectorizers["hashing"], vectorizers["tf-idf"]]
}
baseline_params = [
{
"model": [
ComplementNB() ,
],
**feature_options,
},
{
"model": [SGDClassifier() ],
"model__penalty": ["elasticnet", "l2"],
**feature_options,
},
{
"model": [LinearSVC() ],
"model__penalty": ["l1", "l2"],
**feature_options
}
]
grid = GridSearchCV(pipe, param_grid=baseline_params, cv=5, scoring="f1", n_jobs=-1, verbose=3)
 _ = grid.fit(X_train, y_train )<find_best_params>
print(net.embedding.weight.requires_grad )
print(grid.best_params_)
 baseline = grid.best_estimator_
baseline<predict_on_test>
prc = metrics.precision_recall_curve(val_labels.values, network_pred, pos_label=1 )
failed =(
 X_valid 
.to_frame("text")
.assign(
 y_hat=baseline.predict(X_valid),
y_true=y_valid 
)
)
 false_negatives = failed.pipe(lambda df: df[df["y_hat"].eq(0)& df["y_true"].eq(1)])
 false_positives = failed.pipe(lambda df: df[df["y_hat"].eq(1)& df["y_true"].eq(0)])
 false_negatives[:20]<drop_column>
import matplotlib.pyplot as plt
del false_positives
del failed<drop_column>
exact_predictions = np.round(network_pred)
 print(metrics.classification_report(val_labels.values, exact_predictions))
del false_negatives<import_modules>
def try_different_thresholds(y_true, network_pred):
thresholds = np.arange(0,1,0.01)
 scores = []
for t in thresholds:
exact_pred =(network_pred > t ).astype(np.int8)
 f1_score = metrics.f1_score(y_true, exact_pred)
 scores.append(f1_score)
 return thresholds, np.array(scores )
def custom_tokenizer(nlp):

 tokenizer = nlp.tokenizer
special_cases = {
":)": [{"ORTH": ":)"}],
"&amp": [{"ORTH": "&"}]
}
for text, change in special_cases.items() :
tokenizer.add_special_case(text, change)
 infixes = nlp.Defaults.infixes +(r [\^\.] [\^\.] https?://
Pipeline step to merge hashtags together.
 Code based on examples here: https://spacy.io/usage/rule-based-matching 
<feature_engineering>
thresholds, f1_scores = try_different_thresholds(val_labels.values, network_pred )
nlp = create_nlp()
doc = nlp("^oo^ hello-world.:)http://t.co/ABC-DEF measures
pd.DataFrame({
"text": [token.text for token in doc],
"like_url": [token.like_url for token in doc],
"entity_type": [token.ent_type_ for token in doc],
"is_hashtag": [token._.is_hashtag for token in doc],
"is_ascii": [token.is_ascii for token in doc],
"is_punct": [token.is_punct for token in doc]
} )<string_transform>
max_f1_index = np.argmax(f1_scores)
 best_threshold = thresholds[max_f1_index]
print(best_threshold)
 print(f1_scores[max_f1_index] )
def is_valid_token(token: spacy.tokens.Token)-> bool:

 return(
 not token.like_url
and not token._.is_hashtag
and token.ent_type_ != "USERNAME"
and not token.is_punct
and token.text.strip() != ""
and token.is_ascii 
)
 def extract_text(
 tweet: spacy.tokens.Doc,
include_text: bool = True,
include_hashtags: bool = True,
include_handles: bool = False 
)-> str:
cleaned_tokens = []
for token in tweet:
if(token._.is_hashtag and include_hashtags):
cleaned_tokens.append(token.text.replace("
elif(token.ent_type_ == "USERNAME" and include_handles):
cleaned_tokens.append(token.text.replace("@", ""))
elif(is_valid_token(token)and include_text):
cleaned_tokens.append(token.text.lower())
 return " ".join(cleaned_tokens)
 def clean_tweets(tweets: List[str], **kwargs)-> List[str]:
nlp = create_nlp()
cleaned_tweets = []
for doc in nlp.pipe(tweets, n_process=1):
string = extract_text(doc, **kwargs)
 cleaned_tweets.append(string)
 return cleaned_tweets<categorify>
def thresholded_predictions(probs, threshold):
return(probs > threshold ).astype(np.int8 )
cleaned_grid = clone(grid)
 new_pipe = Pipeline([ 
("clean", FunctionTransformer(clean_tweets)) , 
("vectorizer", None), 
("model", None)
 ])
 cleaning_param_grid = {
"clean__kw_args": [
{"include_text": text, "include_hashtags": hashtags, "include_handles": handles}
for text in [True]
for hashtags in [True]
for handles in [False]
]
}
old_param_grid = cleaned_grid.get_params() ["param_grid"]
cleaned_grid.set_params(
 estimator=new_pipe,
param_grid={**{key: [value] for key,value in grid.best_params_.items() }, **cleaning_param_grid} 
)
 cleaned_grid.get_params() ["param_grid"]<train_model>
exact_predictions = thresholded_predictions(network_pred, best_threshold)
 print(metrics.classification_report(val_labels.values, exact_predictions))
cleaned_grid.fit(X_train, y_train )<find_best_params>
print(metrics.confusion_matrix(val_labels.values, exact_predictions))
print(cleaned_grid.best_params_)
 cleaned_baseline = cleaned_grid.best_estimator_<predict_on_test>
def test_weights(network):
network.eval()
with torch.no_grad() :
xx, yy = next(iter(val_loader))
xx, yy = xx.cuda() , yy.cuda()
print(xx[10], [vocab.itos[w] for w in xx[10]])
 weights = network.get_attention_weights(xx)
 print(weights.size())
 return weights.cpu()
benchmark(y_valid, cleaned_baseline.predict(X_valid))<import_modules>
attn = test_weights(net)
 show_attention(15,attn,val_tokens[:256] )
print(device_lib.list_local_devices() )<choose_model_class>
def print_examples(questions, labels, y_pred):
assert len(questions)== len(labels)== len(y_pred)
 sample = np.random.choice(len(questions),size=1000, replace=False)
 questions, labels, y_pred = questions[sample], labels[sample], y_pred[sample]
positive_pred = y_pred == 1
negative_pred = ~positive_pred
positives = labels == 1
negatives = ~positives
true_pos = questions[positive_pred & positives][:10]
false_pos = questions[positive_pred & negatives][:10]
true_neg = questions[negative_pred & negatives][:10]
false_neg = questions[negative_pred & positives][:10]
for name, examples in zip(( 'TP', 'FP', 'TN', 'FN'),(true_pos, false_pos, true_neg, false_neg)) :
print(name,":")
 for q in examples:
print("\t", q)
 print()
MAX_LENGTH = 64
PRETRAINED = "roberta-base"
def construct_bert(bert_layer, max_length: int = MAX_LENGTH, lr=5e-5, dropout=0.2):
attention_mask = tf.keras.Input(shape=(max_length,), dtype="int32", name="attention_mask")
 input_ids = tf.keras.Input(shape=(max_length,), dtype="int32", name="input_ids")
 output = bert_layer([input_ids, attention_mask])
 output = output[1]
output = tf.keras.layers.Dense(32,activation='relu' )(output)
 output = tf.keras.layers.Dropout(dropout )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 model = tf.keras.models.Model(inputs=[input_ids, attention_mask], outputs=output)
 model.compile(Adam(lr=lr), loss="binary_crossentropy", metrics=["accuracy"])
 return model
def bert_encode(tokenizer, texts: list, max_length: int = MAX_LENGTH):
encoded = tokenizer.batch_encode_plus(
 batch_text_or_text_pairs=texts,
max_length=max_length,
pad_to_max_length=True,
return_token_type_ids=False,
return_attention_mask=True,
return_tensors="tf" 
)
 return [encoded["input_ids"], encoded["attention_mask"]]<init_hyperparams>
val_questions = quora_data.iloc[list(val_labels.index)].question_text.values
EPOCHS = 5
def objective(trial: optuna.Trial):
lr = trial.suggest_categorical("lr", [1e-5, 2e-5, 3e-5])
 batch_size = trial.suggest_categorical("batch_size", [16, 32])
 dropout_rate = trial.suggest_uniform('dropout_rate', 0.0, 1.0)
 clean = trial.suggest_categorical("clean_tweets", [True, False])
 monitor = "val_accuracy"
tokenizer = RobertaTokenizer.from_pretrained(PRETRAINED, do_lower_case=True)
 if clean:
include_hashtags = trial.suggest_categorical("include_hashtags", [True, False])
 include_handles = trial.suggest_categorical("include_handles", [True, False])
 tweets = clean_tweets(X_train, include_hashtags=include_hashtags, include_handles=include_handles)
 else:
tweets = X_train
encoded_tweets = bert_encode(tokenizer, tweets)
 model = construct_bert(TFRobertaModel.from_pretrained(PRETRAINED), lr=lr, dropout=dropout_rate)
 fit_params = {
"batch_size": batch_size,
"epochs": EPOCHS,
"callbacks": [
tf.keras.callbacks.EarlyStopping(
 monitor=monitor,
patience=1,
restore_best_weights=True 
),
TFKerasPruningCallback(trial, monitor)
 ],
"validation_split": 0.2
}
history = model.fit(encoded_tweets, y_train.to_numpy() , verbose=2, **fit_params)
 return max(history.history[monitor] )<load_pretrained>
print_examples(val_questions, val_labels.values, exact_predictions )
if params["clean_tweets"]:
tweets = clean_tweets(X_train, include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets = X_train
tokenizer = RobertaTokenizer.from_pretrained(PRETRAINED, do_lower_case=True)
 encoded_tweets = bert_encode(tokenizer, tweets)
 model = construct_bert(TFRobertaModel.from_pretrained(PRETRAINED), lr=params["lr"], dropout=params["dropout_rate"])
 fit_params = {
"batch_size": params["batch_size"],
"epochs": 3,
"callbacks": [
tf.keras.callbacks.EarlyStopping(
 monitor="val_accuracy",
patience=0,
restore_best_weights=True 
)
 ],
"validation_split": 0.1
}
history = model.fit(encoded_tweets, y_train.to_numpy() , **fit_params )<predict_on_test>
print(net.embedding.weight.data )
if params["clean_tweets"]:
tweets_valid = clean_tweets(X_valid, include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets_valid = X_valid
encoded_tweets_valid = bert_encode(tokenizer, tweets_valid)
 preds = model.predict(encoded_tweets_valid )<compute_test_metric>
tuned_embeddings = net.embedding.weight.data.cpu().numpy()
benchmark(y_valid, preds > 0.5 )<train_model>
from sklearn import decomposition
model = construct_bert(TFRobertaModel.from_pretrained(PRETRAINED), lr=params["lr"], dropout=params["dropout_rate"])
 if params["clean_tweets"]:
tweets_all = clean_tweets(X, include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets_all = X
encoded_tweets_all = bert_encode(tokenizer, tweets_all)
 history = model.fit(encoded_tweets_all, y.to_numpy() , **fit_params )<categorify>
def get_trajectory(batch):
X = batch.cuda()
with torch.no_grad() :
states = net.get_hidden(X)
 probs = F.sigmoid(net(X)).view(-1 ).cpu().numpy()
print(states.shape)
 pca = decomposition.PCA(2)
 timesteps = states.shape[0]
batch_size = states.shape[1]
states = torch.transpose(states,0,1)
 pca_points = pca.fit_transform(states.cpu().reshape(-1,192 ).numpy())
 print(pca.explained_variance_)
 return pca_points.reshape(batch_size, timesteps, 2), probs
if params["clean_tweets"]:
tweets_test = clean_tweets(test["text"], include_hashtags=params["include_hashtags"], include_handles=params["include_handles"])
 else:
tweets_test = test["text"]
encoded_tweets_test = bert_encode(tokenizer, tweets_test )<predict_on_test>
batch, _ = next(iter(val_loader))
pca, probs = get_trajectory(batch)
 toks = val_tokens[:len(batch)]
target = model.predict(encoded_tweets_test )<prepare_output>
import matplotlib.colors
submission = pd.DataFrame({"target": target.reshape(-1 ).round().astype(int)}, index=test.index)
 submission.head()<save_to_csv>
cdict = {'red':(( 0.0, 0.0, 0.0), 
(1/6., 0.0, 0.0), 
(1/2., 0.8, 1.0), 
(5/6., 1.0, 1.0), 
(1.0, 0.4, 1.0)) ,
'green':(( 0.0, 0.0, 0.4), 
(1/6., 1.0, 1.0), 
(1/2., 1.0, 0.8), 
(5/6., 0.0, 0.0), 
(1.0, 0.0, 0.0)) ,
'blue':(( 0.0, 0.0, 0.0), 
(1/6., 0.0, 0.0), 
(1/2., 0.9, 0.9), 
(5/6., 0.0, 0.0), 
(1.0, 0.0, 0.0))
}
cmap=matplotlib.colors.LinearSegmentedColormap('rg',cdict, N=256 )
submission.to_csv("submission.csv" )<load_from_csv>
sel_word = 'iq'
nearest_neighbors(vocab, np_vectors, sel_word, 10, True)
 print('
after_training
')
 nearest_neighbors(vocab, tuned_embeddings, sel_word, 10, False )
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()<save_to_csv>
def ngram_neighbors(conv_layer, vocab, vectors, idx):
ngram = conv_layer.weight.data[idx].cpu().numpy()
ngram = ngram.T
sims = pairwise.cosine_similarity(ngram, vectors)
 ranking = np.argsort(sims,axis=1)[:,::-1]
for i, row in enumerate(ranking):
print(i)
 for j in range(10):
word_index = ranking[i,j]
sim = sims[i, word_index]
print(j+1, vocab.itos[word_index], sim)
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()<load_from_csv>
test_dataset = TokenToIdDataset(test_tokens,np.broadcast_to(np.zeros(1),shape=(len(test_tokens,))),vocab,min_size=25, precompute=True )
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()<define_variables>
test_loader = torch.utils.data.DataLoader(test_dataset,batch_size=256,collate_fn=test_dataset.collate )
print('Total number of question pairs for testing: {}'.format(len(df_test)) )<feature_engineering>
network_test_pred = predict_loader(net, test_loader )
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))<string_transform>
exact_test_predictions = thresholded_predictions(network_test_pred, best_threshold )
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )<init_hyperparams>
submission = pd.DataFrame({'qid': quora_test_data.qid, 'prediction': exact_test_predictions} )
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<sort_values>
submission.to_csv('submission.csv', index=False )
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<feature_engineering>
submission.to_csv('submission.csv', index=False )
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<compute_test_metric>
with open('submission.csv')as f:
for line in itertools.islice(f,0,5):
print(line )
<prepare_x_and_y><EOS> 
exact_test_predictions = thresholded_predictions(network_test_pred, 0.5)
 submission = pd.DataFrame({'qid': quora_test_data.qid, 'prediction': exact_test_predictions})
 submission.to_csv('submission_05.csv', index=False )