Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<filter>
import numpy as np
import pandas as pd
import spacy
from spacy.matcher import Matcher
from spacy.tokens import Span
from spacy import displacy
last_amount = test.loc[(test['Country/Region']=='Italy')&(test['Date']=='2020-03-24'),'ConfirmedCases_x']<filter>
nlp=spacy.load("en_core_web_sm" )
last_fat = test.loc[(test['Country/Region']=='Italy')&(test['Date']=='2020-03-24'),'Fatalities_x']<define_variables>
train=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
i = 0
k = 35<feature_engineering>
stopwords = list(STOP_WORDS)
 punct=string.punctuation
def text_data_cleaning(sentence):
doc = nlp(sentence)
 tokens = []
for token in doc:
if token.lemma_ != "-PRON-":
temp = token.lemma_.lower().strip()
else:
temp = token.lower_
tokens.append(temp)
 cleaned_tokens = []
for token in tokens:
if token not in stopwords and token not in punct:
cleaned_tokens.append(token)
 return cleaned_tokens
for date in dates:
k = k-1
i = i + 1
test.loc[(test['Country/Region']=='Italy')&(test['Date']==date),'ConfirmedCases_x'] = last_amount.values[0]+i*(5000-(100*i))
test.loc[(test['Country/Region']=='Italy')&(test['Date']==date),'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))<filter>
from sklearn.svm import LinearSVC
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
test.loc[(test['Country/Region']=='Italy')]<prepare_output>
tfidf = TfidfVectorizer(tokenizer = text_data_cleaning)
 classifier = LinearSVC()
sub = test[['ForecastId', 'ConfirmedCases_x','Fatalities_x']]<rename_columns>
x = train['text']
y = train['target']
sub.columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 42 )
sub.loc[sub['ConfirmedCases']<0, 'ConfirmedCases'] = 0<feature_engineering>
clf = Pipeline([('tfidf', tfidf),('clf', classifier)] )
sub.loc[sub['Fatalities']<0, 'Fatalities'] = 0<save_to_csv>
clf.fit(X_train,y_train )
sub.to_csv('submission.csv',index=False )<set_options>
y_pred = clf.predict(X_test )
warnings.filterwarnings("ignore")
 sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 np.random.seed(42)
 tf.random.set_seed(42)
 print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE<set_options>
print(classification_report(y_test, y_pred))
MIXED_PRECISION = False
XLA_ACCELERATE = True
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled' )<load_from_csv>
y_pred=clf.predict(test['text'] )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 cols = [c for c in sub.columns.values if c != 'sig_id']<compute_test_metric>
sub_file=pd.DataFrame({'id':test['id'],'target':y_pred.round().astype(int)} )
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<define_variables>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
top_feats = [ 0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 14, 15,
16, 18, 19, 20, 21, 23, 24, 25, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 44, 45, 46,
48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76,
78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 92,
93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 105, 106, 107,
108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120,
121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 132, 133, 134,
135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
149, 150, 151, 152, 153, 154, 155, 157, 159, 160, 161, 163, 164,
165, 166, 167, 168, 169, 170, 172, 173, 175, 176, 177, 178, 180,
181, 182, 183, 184, 186, 187, 188, 189, 190, 191, 192, 193, 195,
197, 198, 199, 202, 203, 205, 206, 208, 209, 210, 211, 212, 213,
214, 215, 218, 219, 220, 221, 222, 224, 225, 227, 228, 229, 230,
231, 232, 233, 234, 236, 238, 239, 240, 241, 242, 243, 244, 245,
246, 248, 249, 250, 251, 253, 254, 255, 256, 257, 258, 259, 260,
261, 263, 265, 266, 268, 270, 271, 272, 273, 275, 276, 277, 279,
282, 283, 286, 287, 288, 289, 290, 294, 295, 296, 297, 299, 300,
301, 302, 303, 304, 305, 306, 308, 309, 310, 311, 312, 313, 315,
316, 317, 320, 321, 322, 324, 325, 326, 327, 328, 329, 330, 331,
332, 333, 334, 335, 338, 339, 340, 341, 343, 344, 345, 346, 347,
349, 350, 351, 352, 353, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 368, 369, 370, 371, 372, 374, 375, 376, 377,
378, 379, 380, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391,
392, 393, 394, 395, 397, 398, 399, 400, 401, 403, 405, 406, 407,
408, 410, 411, 412, 413, 414, 415, 417, 418, 419, 420, 421, 422,
423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435,
436, 437, 438, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450,
452, 453, 454, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465,
466, 468, 469, 471, 472, 473, 474, 475, 476, 477, 478, 479, 482,
483, 485, 486, 487, 488, 489, 491, 492, 494, 495, 496, 500, 501,
502, 503, 505, 506, 507, 509, 510, 511, 512, 513, 514, 516, 517,
518, 519, 521, 523, 525, 526, 527, 528, 529, 530, 531, 532, 533,
534, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547,
549, 550, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563,
564, 565, 566, 567, 569, 570, 571, 572, 573, 574, 575, 577, 580,
581, 582, 583, 586, 587, 590, 591, 592, 593, 595, 596, 597, 598,
599, 600, 601, 602, 603, 605, 607, 608, 609, 611, 612, 613, 614,
615, 616, 617, 619, 622, 623, 625, 627, 630, 631, 632, 633, 634,
635, 637, 638, 639, 642, 643, 644, 645, 646, 647, 649, 650, 651,
652, 654, 655, 658, 659, 660, 661, 662, 663, 664, 666, 667, 668,
669, 670, 672, 674, 675, 676, 677, 678, 680, 681, 682, 684, 685,
686, 687, 688, 689, 691, 692, 694, 695, 696, 697, 699, 700, 701,
702, 703, 704, 705, 707, 708, 709, 711, 712, 713, 714, 715, 716,
717, 723, 725, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738,
739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751,
752, 753, 754, 755, 756, 758, 759, 760, 761, 762, 763, 764, 765,
766, 767, 769, 770, 771, 772, 774, 775, 780, 781, 782, 783, 784,
785, 787, 788, 790, 793, 795, 797, 799, 800, 801, 805, 808, 809,
811, 812, 813, 816, 819, 820, 821, 822, 823, 825, 826, 827, 829,
831, 832, 833, 834, 835, 837, 838, 839, 840, 841, 842, 844, 845,
846, 847, 848, 850, 851, 852, 854, 855, 856, 858, 860, 861, 862,
864, 867, 868, 870, 871, 873, 874]
print(len(top_feats))<choose_model_class>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
def create_model(n_top_feats, learning_rate):
hidden_units = 512
b_size = int(np.ceil(0.6 * n_top_feats))
m_size = int(np.ceil(0.5 * n_top_feats))
s_size = int(np.ceil(0.4 * n_top_feats))
inp1 = tf.keras.layers.Input(shape =(b_size,))
x1 = tf.keras.layers.BatchNormalization()(inp1)
 for _ in range(3):
x1 = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units, activation='elu'))(x1)
 x1 = tf.keras.layers.Dropout(0.4 )(x1)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 inp2 = tf.keras.layers.Input(shape =(m_size,))
x2 = tf.keras.layers.BatchNormalization()(inp2)
 for _ in range(3):
x2 = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units, activation='elu'))(x2)
 x2 = tf.keras.layers.Dropout(0.35 )(x2)
 x2 = tf.keras.layers.BatchNormalization()(x2)
 inp3 = tf.keras.layers.Input(shape =(m_size,))
x3 = tf.keras.layers.BatchNormalization()(inp3)
 for _ in range(3):
x3 = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units, activation='elu'))(x3)
 x3 = tf.keras.layers.Dropout(0.35 )(x3)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 inp4 = tf.keras.layers.Input(shape =(s_size,))
x4 = tf.keras.layers.BatchNormalization()(inp4)
 for _ in range(3):
x4 = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units, activation='elu'))(x4)
 x4 = tf.keras.layers.Dropout(0.3 )(x4)
 x4 = tf.keras.layers.BatchNormalization()(x4)
 x1 = tf.keras.layers.Lambda(lambda x: x*0.4 )(x1)
 x2 = tf.keras.layers.Lambda(lambda x: x*0.2 )(x2)
 x3 = tf.keras.layers.Lambda(lambda x: x*0.2 )(x3)
 x4 = tf.keras.layers.Lambda(lambda x: x*0.2 )(x4)
 x = tf.keras.layers.Add()([x1, x2, x3, x4])
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 for _ in range(2):
x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation='elu'))(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = [inp1, inp2, inp3, inp4], outputs = out)
 model.compile(optimizer = tfa.optimizers.Lookahead(tf.optimizers.Adam(learning_rate), sync_period = 10),
loss = 'binary_crossentropy')
 return model<compute_test_metric>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
print(f'Split NN OOF Metric: {log_loss_metric(train_targets, res)}')
 res.loc[train['cp_type'] == 1, train_targets.columns] = 0
sub.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'Split NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res)}' )<save_to_csv>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
sub.to_csv('submission.csv', index = False )<set_options>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
warnings.filterwarnings('ignore' )<load_from_csv>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<categorify>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
def preprocess(df):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
train_targets = train_targets.loc[train['cp_type']==0].reset_index(drop=True)
 train = train.loc[train['cp_type']==0].reset_index(drop=True )<define_variables>
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16
)
def set_seed(seed):
torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed )<choose_model_class>
test_pred = model.predict(test_input )
<define_search_model><EOS> 
submission=pd.DataFrame()
submission['id']=test['id']
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )