Kaggle competition: Natural Language Processing with Disaster Tweets
feature_score = pd.DataFrame(train.columns, columns = ['feature'])
 feature_score['LGB'] = modelL.feature_importance()<predict_on_test>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 sample = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )
y_preds_lgb = modelL.predict(test, num_iteration=modelL.best_iteration )<prepare_x_and_y>
sns.countplot(train.text.duplicated() )
data_tr = xgb.DMatrix(Xtrain, label=Ztrain)
 data_cv = xgb.DMatrix(Xval , label=Zval)
 data_train = xgb.DMatrix(train)
 data_test = xgb.DMatrix(test)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]<train_model>
duplicate_index = train[train.text.duplicated() ].index
train.drop(index = duplicate_index, inplace = True)
 train.reset_index(drop = True, inplace = True )
parms = {'max_depth':5,
'objective':'reg:logistic',
'eval_metric':'error',
'learning_rate':0.01,
'subsample':0.8,
'colsample_bylevel':0.9,
'min_child_weight': 2,
'seed': 0}
modelx = xgb.train(parms, data_tr, num_boost_round=2000, evals = evallist,
early_stopping_rounds=300, maximize=False,
verbose_eval=100)
 print('score = %1.5f, n_boost_round =%d.'%(modelx.best_score,modelx.best_iteration))<feature_engineering>
shortforms = {"ain't": "am not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "He had",
"he'd've": "He would have",
"he'll": "He will",
"he'll've": "He will have",
"he's": "He is",
"how'd": "How did",
"how'd'y": "How do you",
"how'll": "How will",
"how's": "How is",
"i'd": "I had",
"i'd've": "I would have",
"i'll": "I will",
"i'll've": "I will have",
"i'm": "I am",
"i've": "I have",
"isn't": "is not",
"it'd": "It had",
"it'd've": "It would have",
"it'll": "It will",
"it'll've": "It will have",
"it's": "It is",
".it's": "It is",
"let's": "Let us",
"ma'am": "Madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "She had",
"she'd've": "She would have",
"she'll": "She will",
"she'll've": "She will have",
"she's": "She is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so is",
"that'd": "that had",
"that'd've": "that would have",
"that's": "that is",
"there'd": "There had",
"there'd've": "There would have",
"there's": "There has",
"they'd": "They had",
"they'd've": "They would have",
"they'll": "They will",
"they'll've": "They will have",
"they're": "They are",
"they've": "They have",
"to've": "to have",
"wasn't": "was not",
"we'd": "We had",
"we'd've": "We would have",
"we'll": "We will",
"we'll've": "We will have",
"we're": "We are",
"we've": "We have",
"weren't": "were not",
"what'll": "What will",
"what'll've": "What will have",
"what're": "What are",
"what's": "What is",
"what've": "What have",
"when's": "When is",
"when've": "When have",
"where'd": "Where did",
"where's": "Where is",
"where've": "Where have",
"who'll": "Who will",
"who'll've": "Who will have",
"who's": "Who is",
"who've": "Who have",
"why's": "Why is",
"why've": "Why have",
"will've": "ill have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "You all",
"y'all'd": "You all would",
"y'all'd've": "You all would have",
"y'all're": "You all are",
"y'all've": "You all have",
"you'd": "You had",
"you'd've": "You would have",
"you'll": "You will",
"you'll've": "You will have",
"you're": "You are",
"you've": "You have"
}
feature_score['XGB'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))<predict_on_test>
def cleaner(text):
text = str(text ).lower()
text = re.sub(r'<*?>',' ',text)
 text = re.sub(r'https?://\S+|www\.\S+',' ',text)
 text = ' '.join([shortforms[word]
if word in shortforms.keys() else word
for word in text.split() ])
 text = str(text ).lower()
text = re.sub(r'^\s','',text)
 text = re.sub(r'\s+',' ',text)
 return(text )
y_preds_xgb = modelx.predict(data_test )<normalization>
%%time
train['cleaner_text'] = train.text.progress_apply(lambda x: cleaner(x))
test['cleaner_text'] = test.text.progress_apply(lambda x: cleaner(x))
Scaler_train = preprocessing.MinMaxScaler().fit(train)
 train = pd.DataFrame(Scaler_train.transform(train), columns=train.columns, index=train.index)
 test = pd.DataFrame(Scaler_train.transform(test), columns=test.columns, index=test.index )<train_on_grid>
case = 'roberta-base'
tokenizer = RobertaTokenizer.from_pretrained(case)
 config = AutoConfig.from_pretrained(case, output_attentions = True, output_hidden_states = True)
 model = TFAutoModel.from_pretrained(case, config = config)
 bert = TFRobertaMainLayer(config )
linreg = LinearRegression()
linreg.fit(train, target )<load_pretrained>
%%time
def convert2token(all_text):
token_id, attention_id = [], []
for i, sent in tqdm.tqdm(enumerate(all_text)) :
token_dict = tokenizer.encode_plus(sent, max_length=60, pad_to_max_length=True, return_attention_mask=True,
return_tensors='tf', add_special_tokens= True)
 token_id.append(token_dict['input_ids'])
 attention_id.append(token_dict['attention_mask'])
 token_id = np.array(token_id, dtype='int32')
 attention_id = np.array(attention_id, dtype='int32')
 return(token_id, attention_id)
 train_token_id, train_attention_id = convert2token(train.cleaner_text.values)
 test_token_id, test_attention_id = convert2token(test.cleaner_text.values )
eli5.show_weights(linreg )<merge>
def building_model(need_emb):
inp_1 = tf.keras.layers.Input(shape =(60,), name = 'token_id', dtype = 'int32')
 inp_2 = tf.keras.layers.Input(shape =(60,), name = 'mask_id', dtype = 'int32')
 x1 = tf.keras.layers.Reshape(( 60,))(inp_1)
 x2 = tf.keras.layers.Reshape(( 60,))(inp_2)
 if need_emb:
emb = model(x1, attention_mask = x2)[0]
x = tf.keras.layers.Dense(256, activation = 'relu' )(emb[:,0,:])
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.3 )(x)
 x = tf.keras.layers.Dense(32, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.3 )(x)
 else :
emb = bert(x1, attention_mask = x2)[0]
x = tf.keras.layers.Dropout(0.2 )(emb[:,0,:])
 out = tf.keras.layers.Dense(1, activation = 'sigmoid' )(x)
 Emb_Model = tf.keras.models.Model(inputs = [inp_1, inp_2], outputs = out)
 callback = ModelCheckpoint(filepath = 'best.hdf5', monitor = 'val_loss', save_best_only = True, verbose = 1)
 if need_emb :
for layer in Emb_Model.layers[:5]:
layer.trainable = False
return(Emb_Model, callback )
coeff_linreg["LinRegress"] = coeff_linreg["LinRegress"].abs()
feature_score = pd.merge(feature_score, coeff_linreg, on='feature')
 feature_score = feature_score.fillna(0)
 feature_score = feature_score.set_index('feature')
 feature_score<predict_on_test>
Emb_Model.compile(metrics=['accuracy'], optimizer=tf.keras.optimizers.Adam(learning_rate = 4e-5), loss='binary_crossentropy')
 Emb_Model.fit([np.reshape(train_token_id,(7503,60)) , np.reshape(train_attention_id,(7503,60)) ], train.target, epochs=10,
batch_size=64, validation_split=0.20, shuffle = True )
y_preds_linreg = linreg.predict(test )<prepare_output>
%%time
Emb_Model_Answer = Emb_Model.predict([np.reshape(test_token_id,(3263,60)) , np.reshape(test_attention_id,(3263,60)) ] )
feature_score = pd.DataFrame(
 preprocessing.MinMaxScaler().fit_transform(feature_score),
columns=feature_score.columns,
index=feature_score.index 
)
 feature_score['Mean'] = feature_score.mean(axis=1 )<feature_engineering>
Tune_Bert.compile(metrics=['accuracy'], optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5), loss='binary_crossentropy')
 Tune_Bert.fit([np.reshape(train_token_id,(7503,60)) , np.reshape(train_attention_id,(7503,60)) ], train.target, epochs=10,
batch_size=64, validation_split=0.20, shuffle = True, callbacks = [callback] )
w_lgb = 0.4
w_xgb = 0.5
w_linreg = 1 - w_lgb - w_xgb
w_linreg
feature_score['Merging'] = w_lgb*feature_score['LGB'] + w_xgb*feature_score['XGB'] + w_linreg*feature_score['LinRegress']
feature_score.sort_values('Merging', ascending=False )<feature_engineering>
Tune_Bert.load_weights('best.hdf5')
 Tune_answer = Tune_Bert.predict([np.reshape(test_token_id,(3263,60)) , np.reshape(test_attention_id,(3263,60)) ] )
def features_selection_by_weights(df, threshold):
features_list = df.feature.tolist()
features_best = []
for i in range(len(df)) :
feature_name = features_list[i]
feature_is_best = False
for col in feature_score_columns:
if df.loc[i, col] > threshold:
feature_is_best = True
if feature_is_best:
features_best.append(feature_name)
 return df[df['feature'].isin(features_best)].reset_index(drop=True )<features_selection>
answer_Emb = pd.DataFrame({'id': sample.id, 'target': np.where(Emb_Model_Answer>0.5,1,0 ).reshape(Emb_Model_Answer.shape[0])})
 answer_tune = pd.DataFrame({'id': sample.id, 'target': np.where(Tune_answer>0.5,1,0 ).reshape(Tune_answer.shape[0])} )
<prepare_output><EOS> 
answer_Emb.to_csv('submission_emb.csv', index = False)
 answer_tune.to_csv('submission_tune.csv', index = False )