Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<define_variables>
tqdm.pandas()
sns.set_style('whitegrid' )
stopwords = ["they've", 'k', 'whom', "he'll", 'could', 'itself', 'hence', "when's", 'where', 'through',
'was', 'its', 'into', 'however', "she'd", "we'd", 'they', 'below', 'again', "she'll", "he's",
'did', 'my', 'are', 'our', "where's", 'above', 'ever', 'yourself', "i'd", 'just', 'we', "i'll",
'it', 'from', "we'll", 'that', 'he', 'cannot', "you're", 'her', 'this', "why's", 'once',
'am', 'ourselves', 'out', 'get', 'would', 'up', "it's", 'same', 'these', "you've", 'such', 'between',
'himself', 'also', "that's", 'r', 'www', 'all', "what's", 'if', 'http', 'herself', 'after', 'had',
'has', 'your', 'while', 'other', 'their', 'shall', 'more', 'off', 'as', 'hers', 'with', 'over',
'by', 'there', "we're", "they'll", 'any', 'to', 'no', 'about', 'both', "he'd", 'only', 'here',
'than', 'what', 'been', 'does', "we've", 'theirs', 'being', 'ought', "they'd", 'few', 'you', 'under',
'since', 'can', 'them', 'at', 'else', 'each', 'ours', 'therefore', 'most', 'before', 'then', 'his', 'me',
'a', 'further', "how's", 'during', 'of', 'like', 'on', 'themselves', 'why', 'those', 'in', 'too', 'she',
'because', "i've", "let's", 'how', 'very', "you'd", 'own', 'but', "she's", 'i', 'yourselves', 'down', 'should',
'and', 'do', 'or', 'were', 'some', 'an', "who's", 'otherwise', 'be', 'him', 'myself', 'have', 'which', "there's",
"i'm", 'when', 'doing', "you'll", 'com', 'for', 'who', 'yours', 'until', 'the', 'is', 'so', "they're", "here's",
'nor', 'having', 'will', 'may', 'one', 'now']<categorify>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
def preprocessor2(text):
text = re.sub('[^a-zA-Z]', ' ', text)
 text = re.sub(r"\s+[a-zA-Z]\s+", ' ', text)
 text = re.sub(r'\s+', ' ', text)
 text = text.lower()
return text<feature_engineering>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
train_df['text'] = train_df['text'].apply(lambda x:remove_url(x))<data_type_conversions>
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
train_df['merged'] = train_df['text'].astype(str)+' '+train_df['keyword']+' '+train_df['location']
test_df['merged'] = test_df['text'].astype(str)+' '+test_df['keyword']+' '+test_df['location']<feature_engineering>
embed_glove = load_embed(glove)
train_df['merged_cleaned'] = train_df['merged'].copy().apply(lambda x : clean(str(x)) ).apply(lambda x : preprocessor2(x)).apply(lambda x: ' '.join([word for word in x.split() if word not in(stopwords)]))
test_df['merged_cleaned'] = test_df['merged'].copy().apply(lambda x : clean(str(x)) ).apply(lambda x : preprocessor2(x)).apply(lambda x: ' '.join([word for word in x.split() if word not in(stopwords)]))<feature_engineering>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
train_df['totalwords'] = train_df['merged_cleaned'].str.count(' ')+ 1<define_variables>
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Se encontraron embeddings para {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
max_len = 50<drop_column>
df = pd.concat([train_df ,test_df])
 vocab = build_vocab(df['question_text'] )
train_df = train_df[['merged_cleaned', 'target']]
test_df = test_df[['id','merged_cleaned']]<define_variables>
print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove )
label_L = list(train_df.target )<load_pretrained>
df['question_text'] = df['question_text'].apply(lambda x: x.lower() )
configuration = BertConfig.from_pretrained(f'/kaggle/input/bert-tensorflow/bert-base-uncased-config.json' )<load_pretrained>
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Anadidas {count} palabras al embedding" )
slow_tokenizer = BertTokenizer.from_pretrained(f'/kaggle/input/bertbaseuncased/vocab.txt')
 save_path = "bert-base-uncased/"
if not os.path.exists(save_path):
os.makedirs(save_path)
 slow_tokenizer.save_pretrained(save_path)
 tokenizer = BertWordPieceTokenizer(f'/kaggle/input/bertbaseuncased/vocab.txt', lowercase=True)
<categorify>
print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove)
 add_lower(embed_glove, vocab)
 oov_glove = check_coverage(vocab, embed_glove )
class SquadExample:
def __init__(self, text):
self.text = text
self.skip = False
def preprocess(self):
text = self.text
text = str(text)
 tokenized_context = tokenizer.encode(text)
 input_ids = tokenized_context.ids
token_type_ids = [0] * len(tokenized_context.ids)
 attention_mask = [1] * len(input_ids)
 padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 attention_mask = attention_mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 elif padding_length < 0:
self.skip = True
self.input_ids = input_ids
self.token_type_ids = token_type_ids
self.attention_mask = attention_mask
def create_examples(raw_data):
squad_examples = []
for i in range(raw_data.shape[0]):
text = raw_data["merged_cleaned"][i]
squad_eg = SquadExample(text)
 squad_eg.preprocess()
squad_examples.append(squad_eg)
 return squad_examples<prepare_x_and_y>
contraction_mapping = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have",
"who'd": "who would",
"who're": "who are",
"'re": " are",
"tryin'": "trying",
"doesn'": "does not",
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
}
def create_inputs_targets(squad_examples,label):
dataset_dict = {
"input_ids": [],
"token_type_ids": [],
"attention_mask": [],
}
for item in squad_examples:
if item.skip == False:
for key in dataset_dict:
dataset_dict[key].append(getattr(item, key))
for key in dataset_dict:
dataset_dict[key] = np.array(dataset_dict[key])
 x = [
dataset_dict["input_ids"],
dataset_dict["token_type_ids"],
dataset_dict["attention_mask"],
]
y = [np.array(label)]
return x, y<categorify>
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
train_examples = create_examples(train_df )<categorify>
print("- Contracciones Conocidas -")
 print(" Glove :")
 print(known_contractions(embed_glove))
test_examples = create_examples(test_df )<prepare_x_and_y>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
x_train, y_train = create_inputs_targets(train_examples,label_L )<prepare_x_and_y>
df['question_text'] = df['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
x_test, y_test = create_inputs_targets(test_examples,None )<train_model>
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
print(f"{len(train_examples)} training points created." )<categorify>
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
<categorify>
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
base_path = '/kaggle/input/bert-tensorflow/bert-base-uncased-tf_model.h5'
def create_model(path = base_path):
encoder = TFBertForSequenceClassification.from_pretrained(path, config=configuration)
 input_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 token_type_ids = layers.Input(shape=(max_len,), dtype=tf.int32)
 attention_mask = layers.Input(shape=(max_len,), dtype=tf.int32)
 embedding = encoder(
 input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask 
)[0]
reshape_L = layers.Reshape(( 2,1))(embedding)
 lstm = layers.LSTM(64 )(reshape_L)
 label_class_0 = layers.Dense(128, name="process-1", use_bias=False )(lstm)
 label_class_1 = layers.Dense(128, name="process-2", use_bias=False )(label_class_0)
 label_class = layers.Dense(2, name="label_class", use_bias=False )(label_class_1)
 label_class = layers.Flatten()(label_class)
 label_probs = layers.Activation(keras.activations.softmax )(label_class)
 model = keras.Model(
 inputs=[input_ids, token_type_ids, attention_mask],
outputs=[label_probs], 
)
 loss = keras.losses.SparseCategoricalCrossentropy(from_logits=False)
 optimizer = keras.optimizers.Adam(lr=2e-5)
 model.compile(optimizer=optimizer, loss=[loss, loss])
 return model<train_model>
df['question_text'] = df['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
model.fit(
 x_train,
y_train,
epochs=3,
verbose=2,
batch_size=16
)<predict_on_test>
vocab = build_vocab(df['question_text'])
 print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove )
Y_pred = model.predict(x_test )<prepare_output>
mispell_dict = {'advanatges': 'advantages',
'irrationaol': 'irrational' ,
'defferences': 'differences',
'lamboghini':'lamborghini',
'hypothical':'hypothetical',
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'pokémon': 'pokemon'}
Y_pred = np.argmax(Y_pred,axis=1 )<create_dataframe>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
pred_df = pd.DataFrame(Y_pred, columns=['target'])
 result = pd.concat([test_df,pred_df], axis=1, join='outer', ignore_index=False, keys=None, sort = False)
 result = result[['id','target']]
print(Y_pred )<save_to_csv>
df['question_text'] = df['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
result.to_csv('sample_submission2.csv',index=False )<install_modules>
vocab = build_vocab(df['question_text'])
 print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove )
! pip install nlplot<import_modules>
def clean_numbers(x):
x = re.sub('[0-9]{5,}', ' number ', x)
 x = re.sub('[0-9]{4}', ' number ', x)
 x = re.sub('[0-9]{3}', ' number ', x)
 x = re.sub('[0-9]{2}', ' number ', x)
 return x
tqdm.pandas()
stop=set(stopwords.words('english'))
%matplotlib inline
pd.set_option('display.max_columns', 300)
 pd.set_option('display.max_rows', 300)
 pd.set_option('display.max_colwidth', 300)
 pd.options.display.float_format = '{:.3f}'.format<load_from_csv>
df['question_text'] = df['question_text'].apply(lambda x: clean_numbers(x))
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )<string_transform>
vocab = build_vocab(df['question_text'])
 print("Glove : ")
 oov_glove = check_coverage(vocab, embed_glove )
stopwords = npt.get_stopword(top_n=30, min_freq=0)
 stopwords<concatenate>
train_df['treated_question'] = train_df['question_text'].apply(lambda x: x.lower())
 train_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
train_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
train_df['treated_question'] = train_df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))
train_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_numbers(x))
df = pd.concat([train,test])
 df.shape<categorify>
test_df['treated_question'] = test_df['question_text'].apply(lambda x: x.lower())
 test_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
test_df['treated_question'] = test_df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))
test_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_numbers(x))
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html = re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punct(text):
table = str.maketrans('','',string.punctuation)
 return text.translate(table)
 def remove_number(text):
num = re.compile(r'\w*\d\w*')
 return num.sub(r'',text)
 def acronyms(text):
text = re.sub(r"MH370", "Malaysia Airlines Flight 370", text)
 text = re.sub(r"mÌ¼sica", "music", text)
 text = re.sub(r"okwx", "Oklahoma City Weather", text)
 text = re.sub(r"arwx", "Arkansas Weather", text)
 text = re.sub(r"gawx", "Georgia Weather", text)
 text = re.sub(r"scwx", "South Carolina Weather", text)
 text = re.sub(r"cawx", "California Weather", text)
 text = re.sub(r"tnwx", "Tennessee Weather", text)
 text = re.sub(r"azwx", "Arizona Weather", text)
 text = re.sub(r"alwx", "Alabama Weather", text)
 text = re.sub(r"wordpressdotcom", "wordpress", text)
 text = re.sub(r"usNWSgov", "United States National Weather Service", text)
 text = re.sub(r"Suruc", "Sanliurfa", text)
 text = re.sub(r"Bestnaijamade", "bestnaijamade", text)
 text = re.sub(r"SOUDELOR", "Soudelor", text)
 return text
def clean_text(text):

 text = acronyms(text)
 text = text.lower()
text = remove_URL(text)
 text = remove_html(text)
 text = remove_emoji(text)
 text = remove_punct(text)
 text = remove_number(text)
 return text<feature_engineering>
df['text'] = df['text'].progress_apply(lambda x: clean_text(x))<string_transform>
train, val = train_test_split(train_df, test_size=0.2, random_state=2)
def create_corpus(df):
corpus=[]
for tweet in tqdm(df['text']):
words=[word.lower() for word in word_tokenize(tweet)if(( word.isalpha() ==1)&(word not in stop)) ]
corpus.append(words)
 return corpus<statistical_test>
xtrain = train['question_text'].fillna('_na_' ).values
xval = val['question_text'].fillna('_na_' ).values
xtest = test_df['question_text'].fillna('_na_' ).values
corpus = create_corpus(df )<load_from_csv>
EMBED_SIZE = 300
MAX_FEATURES = 100000
MAXLEN = 60
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(list(xtrain))
xtrain = tokenizer.texts_to_sequences(xtrain)
 xval = tokenizer.texts_to_sequences(xval)
 xtest = tokenizer.texts_to_sequences(xtest )
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word=values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()<string_transform>
xtrain = pad_sequences(xtrain, maxlen=MAXLEN)
 xval = pad_sequences(xval, maxlen=MAXLEN)
 xtest = pad_sequences(xtest, maxlen=MAXLEN )
MAX_LEN = 100
tokenizer_obj = Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences = tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad = pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )<count_unique_values>
ytrain = train['target'].values
yval = val['target'].values
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))<define_variables>
def load_glove_matrix(word_index, embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
EMBED_SIZE = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, EMBED_SIZE))
for word, i in word_index.items() :
if i >= MAX_FEATURES:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i > num_words:
continue
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec<split>
np.random.seed(2)
 trn_idx = np.random.permutation(len(xtrain))
val_idx = np.random.permutation(len(xval))
xtrain = xtrain[trn_idx]
ytrain = ytrain[trn_idx]
xval = xval[val_idx]
yval = yval[val_idx]
embedding_matrix_glove = load_glove_matrix(tokenizer.word_index, embed_glove )
train_tweet = tweet_pad[:train.shape[0]]
test_tweet = tweet_pad[train.shape[0]:]<split>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
X_train,X_test,y_train,y_test=train_test_split(train_tweet, train['target'].values,test_size=0.1)
 print('Shape of train',X_train.shape)
 print("Shape of Validation ",X_test.shape )<choose_model_class>
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def build_BLSTM() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights = [embedding_matrix],
input_length=MAX_LEN))
model.add(SpatialDropout1D(0.3))
model.add(Bidirectional(LSTM(MAX_LEN, return_sequences = True, recurrent_dropout=0.2)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(MAX_LEN, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(MAX_LEN, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
def model_lstm_att(embedding_matrix):
inp = Input(shape=(MAXLEN,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)
 att = Attention(MAXLEN )(x)
 y = Dense(32, activation='relu' )(att)
 y = Dropout(0.1 )(y)
 outp = Dense(1, activation='sigmoid' )(y)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy', 'accuracy'])
 return model
checkpoint = ModelCheckpoint(
 'model.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 reduce_lr = ReduceLROnPlateau(
 monitor = 'val_loss',
factor = 0.2,
verbose = 1,
patience = 5,
min_lr = 0.001
)<train_model>
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embedding_matrix_para = load_glove_matrix(tokenizer.word_index, load_embed(paragram))
history = model.fit(
 X_train,
y_train,
epochs = 15,
batch_size = 64,
validation_data = [X_test, y_test],
verbose = 1,
callbacks = [reduce_lr, checkpoint]
)<compute_test_metric>
embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_para], axis=0 )
loss, accuracy = model.evaluate(X_test, y_test)
 print('Loss:', loss)
 print('Accuracy:', accuracy )<predict_on_test>
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(xtrain, ytrain, batch_size=512, epochs=3, validation_data=(xval, yval))
pred_val_y = model.predict([xval], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val puntuacion F1: {:.4f}".format(best_score))
pred_test_y = model.predict([xtest], batch_size=1024, verbose=1)
 return pred_val_y, pred_test_y, best_score
preds = model.predict_classes(X_test)
 metrics(preds, y_test )<predict_on_test>
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_lstm, epochs=2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'model_lstm_att only Glove'] )
y_pred = model.predict(test_tweet)
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub = pd.DataFrame({'id':submission['id'].values.tolist() ,'target':y_pred} )<save_to_csv>
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("La puntuación F1 en los limites {0} y {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
sub.to_csv('submission.csv',index=False )<import_modules>
print("Mejor limite:", best_thresh, "y puntuacion F1 ", thresholds[0][1] )
stop_words = stopwords.words('english')
<load_from_csv>
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int )
test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv" )<count_unique_values>
sub = pd.read_csv('.. /input/sample_submission.csv')
 out_df = pd.DataFrame({"qid":sub["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )