Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<predict_on_test>
%matplotlib inline
cf.go_offline()
cf.set_config_file(offline=False, world_readable=True)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
etc.fit(train_data, y)
 test_predictions = etc.predict(test_data )<save_to_csv>
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 train_df.head()
solutions = pd.DataFrame({'Id':test_Id, 'Cover_Type':test_predictions})
 solutions.to_csv('submission.csv',index=False )<set_options>
"{:.2f}% of the entries which represent actual disasters have a location".format(( len(train_df[(train_df['haslocation'] == 1)&(train_df['target'] == 1)])/ len(train_df[(train_df['target'] == 1)])* 100))
%matplotlib inline 
<load_from_csv>
"{:.2f}% of the entries which do not represent disasters have a location".format(( len(train_df[(train_df['haslocation'] == 1)&(train_df['target'] == 0)])/ len(train_df[(train_df['target'] == 0)])* 100))
train = pd.read_csv('.. /input/forest-cover-type-prediction/train.csv',
index_col='Id')
 test = pd.read_csv('.. /input/forest-cover-type-prediction/test.csv',
index_col='Id' )<count_values>
train_df.drop(["location", "haslocation"], axis = 1, inplace = True)
 test_df.drop(["location"], axis = 1, inplace = True )
train['Cover_Type'].value_counts()<save_to_csv>
train_df['keyword_sentiment'] = train_df['keyword'].apply(lambda x:(TextBlob(x ).sentiment[0] + 1)/ 2 if type(x)== str else None)
 train_df['tweet_sentiment'] = train_df['text'].apply(lambda x:(TextBlob(x ).sentiment[0] + 1)/ 2)
 test_df['tweet_sentiment'] = test_df['text'].apply(lambda x:(TextBlob(x ).sentiment[0] + 1)/ 2 )
def write_to_submission_file(predicted_labels, out_file,
target='Cover_Type', index_label="Id", init_index=15121):
predicted_df = pd.DataFrame(predicted_labels,
index = np.arange(init_index,
predicted_labels.shape[0] + init_index),
columns=[target])
 predicted_df.to_csv(out_file, index_label=index_label )<split>
train_df['text'] = train_df['keyword'].astype(str)+ ' ' + train_df['text'].astype(str)
 test_df['text'] = test_df['keyword'].astype(str)+ ' ' + test_df['text'].astype(str)
 train_df.drop(["keyword", "haskeyword", "keyword_sentiment"], axis = 1, inplace = True)
 test_df.drop(["keyword"], axis = 1, inplace = True )
X_train, X_valid, y_train, y_valid = train_test_split(
 train.drop('Cover_Type', axis=1), train['Cover_Type'],
test_size=0.3, random_state=17 )<choose_model_class>
train_df['nr_of_char'] = train_df['text'].str.len()
train_df['nr_of_char'] = train_df['nr_of_char'] / train_df['nr_of_char'].max()
train_df[['target', 'nr_of_char']].pivot(columns = 'target', values = 'nr_of_char' ).iplot(kind = 'box')
 test_df['nr_of_char'] = test_df['text'].str.len()
test_df['nr_of_char'] = test_df['nr_of_char'] / test_df['nr_of_char'].max()
logit = LogisticRegression(C=1, solver='lbfgs', max_iter=500,
random_state=17, n_jobs=4,
multi_class='multinomial')
 logit_pipe = Pipeline([('scaler', StandardScaler()), 
('logit', logit)] )<train_model>
train_df['nr_of_words'] = train_df['text'].str.split().str.len()
train_df['nr_of_words'] = train_df['nr_of_words'] / train_df['nr_of_words'].max()
train_df[['target', 'nr_of_words']].pivot(columns = 'target', values = 'nr_of_words' ).iplot(kind = 'box')
 test_df['nr_of_words'] = test_df['text'].str.split().str.len()
test_df['nr_of_words'] = test_df['nr_of_words'] / test_df['nr_of_words'].max()
%%time
logit_pipe.fit(X_train, y_train )<predict_on_test>
train_df['nr_of_unique_words'] = train_df['text'].apply(lambda x: len(set(x.split())))
 train_df['nr_of_unique_words'] = train_df['nr_of_unique_words'] / train_df['nr_of_unique_words'].max()
train_df[['target', 'nr_of_unique_words']].pivot(columns = 'target', values = 'nr_of_unique_words' ).iplot(kind='box')
 test_df['nr_of_unique_words'] = test_df['text'].apply(lambda x: len(set(x.split())))
 test_df['nr_of_unique_words'] = test_df['nr_of_unique_words'] / test_df['nr_of_unique_words'].max()
logit_val_pred = logit_pipe.predict(X_valid )<compute_test_metric>
train_df['nr_of_punctuation'] = train_df['text'].str.split(r"\?|,|\.|\!|"|'" ).str.len()
train_df['nr_of_punctuation'] = train_df['nr_of_punctuation'] / train_df['nr_of_punctuation'].max()
train_df[['target', 'nr_of_punctuation']].pivot(columns = 'target', values = 'nr_of_punctuation' ).iplot(kind = 'box')
 test_df['nr_of_punctuation'] = test_df['text'].str.split(r"\?|,|\.|\!|"|'" ).str.len()
test_df['nr_of_punctuation'] = test_df['nr_of_punctuation'] / test_df['nr_of_punctuation'].max()
accuracy_score(y_valid, logit_val_pred )<choose_model_class>
stop_words = set(stopwords.words('english'))
train_df['nr_of_stopwords'] = train_df['text'].str.split().apply(lambda x: len(set(x)& stop_words))
train_df['nr_of_stopwords'] = train_df['nr_of_stopwords'] / train_df['nr_of_stopwords'].max()
train_df[['target', 'nr_of_stopwords']].pivot(columns = 'target', values = 'nr_of_stopwords' ).iplot(kind = 'box')
 test_df['nr_of_stopwords'] = test_df['text'].str.split().apply(lambda x: len(set(x)& stop_words))
test_df['nr_of_stopwords'] = test_df['nr_of_stopwords'] / test_df['nr_of_stopwords'].max()
first_forest = RandomForestClassifier(
 n_estimators=100, random_state=17, n_jobs=4 )<train_model>
glove_embeddings = np.load('/kaggle/input/embfile/emb/glove.840B.300d.pkl', allow_pickle=True)
 def build_vocab(X):
tweets = X.apply(lambda s: s.split() ).values
vocab = {}
for tweet in tweets:
for word in tweet:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_embeddings_coverage(X, embeddings):
vocab = build_vocab(X)
 covered = {}
oov = {}
n_covered = 0
n_oov = 0
for word in vocab:
try:
covered[word] = embeddings[word]
n_covered += vocab[word]
except:
n_oov += vocab[word]
vocab_coverage = len(covered)/ len(vocab)
 text_coverage =(n_covered /(n_covered + n_oov))
return vocab_coverage, text_coverage
train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(train_df['text'], glove_embeddings)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))
test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(test_df['text'], glove_embeddings)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage))
%%time
first_forest.fit(X_train, y_train )<predict_on_test>
def clean(tweet):
tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", tweet)
 punctuations = '@
for p in punctuations:
tweet = tweet.replace(p, '')
 tweet = tweet.replace('...', '...')
 if '...' not in tweet:
tweet = tweet.replace('.. ', '...')
 tweet = tweet.replace(' ', ' ')
 tweet = tweet.replace(' ', ' ')
 tweet = tweet.lower()
tweet = " ".join(tweet.split())
 return tweet
train_df['text_cleaned'] = train_df['text'].apply(lambda s : clean(s))
test_df['text_cleaned'] = test_df['text'].apply(lambda s : clean(s))
train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(train_df['text_cleaned'], glove_embeddings)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))
test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(test_df['text_cleaned'], glove_embeddings)
 print('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage))
forest_val_pred = first_forest.predict(X_valid )<compute_test_metric>
def create_corpus(df):
corpus = []
for tweet in tqdm(df['text_cleaned']):
words = [word.lower() for word in word_tokenize(tweet)if(( word.isalpha() == 1)and word not in stop_words)]
corpus.append(words)
 return corpus
train_corpus = create_corpus(train_df)
 test_corpus = create_corpus(test_df)
 tokenizer_obj_train = Tokenizer()
tokenizer_obj_train.fit_on_texts(train_corpus)
 seq_train=tokenizer_obj_train.texts_to_sequences(train_corpus)
 tokenizer_obj_test = Tokenizer()
tokenizer_obj_test.fit_on_texts(test_corpus)
 seq_test=tokenizer_obj_test.texts_to_sequences(test_corpus)
 MAX_WORDS = 0
for i in seq_train:
MAX_WORDS = max(MAX_WORDS, len(i))
for i in seq_test:
MAX_WORDS = max(MAX_WORDS, len(i))
train_pad = pad_sequences(seq_train, maxlen = MAX_WORDS, truncating = 'post', padding = 'post')
 test_pad = pad_sequences(seq_test, maxlen = MAX_WORDS, truncating = 'post', padding = 'post')
 tokenizer_obj_train.word_index.update(tokenizer_obj_test.word_index )
accuracy_score(y_valid, forest_val_pred )<create_dataframe>
word_index = tokenizer_obj_train.word_index
print('Number of unique words:',len(word_index))
pd.DataFrame(first_forest.feature_importances_,
index=X_train.columns, columns=['Importance'] ).sort_values(
 by='Importance', ascending=False)[:10]<choose_model_class>
num_words = len(word_index)+ 1
embedding_matrix = np.zeros(( num_words, 300))
counter = 0
for word, i in tqdm(word_index.items()):
emb_vec = glove_embeddings.get(word)
 if emb_vec is not None:
embedding_matrix[i] = emb_vec
else:
counter += 1
del glove_embeddings
counter
lgb_clf = LGBMClassifier(random_state=17 )<train_model>
x_train, x_test, y_train, y_test = train_test_split(train_pad, train_df['target'].values, test_size = 0.25, random_state = 42)
 print("Shape of train", x_train.shape)
 print("Shape of Validation", x_test.shape )
%%time
lgb_clf.fit(X_train, y_train )<compute_test_metric>
inp = Input(shape =(MAX_WORDS,))
x = Embedding(num_words, 300, weights = [embedding_matrix] )(inp)
 x = Bidirectional(LSTM(MAX_WORDS, dropout = 0.2, recurrent_dropout = 0.2, return_sequences = True))(x)
 x = ActivityRegularization(l2 = 0.1 )(x)
 x = GlobalMaxPool1D()(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(4, activation = "relu" )(x)
 x = ActivityRegularization(l2 = 0.1 )(x)
 x = Dropout(0.2 )(x)
 x = Dense(1, activation = "sigmoid" )(x)
 model_LSTM = Model(inputs = inp, outputs = x)
 model_LSTM.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 print(model_LSTM.summary() )
accuracy_score(y_valid, lgb_clf.predict(X_valid))<define_search_space>
checkpoint = ModelCheckpoint(
 'model_LSTM.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 reduce_lr = ReduceLROnPlateau(
 monitor = 'val_loss',
factor = 0.2,
verbose = 1,
patience = 5,
min_lr = 0.001 
)
 model_LSTM.fit(x_train, y_train, batch_size = 32, epochs = 15, validation_data =(x_test, y_test), callbacks = [reduce_lr, checkpoint] )
param_grid = {'num_leaves': [7, 15, 31, 63],
'max_depth': [3, 4, 5, 6, -1]}<choose_model_class>
model_LSTM.load_weights('model_LSTM.h5')
 pred_LSTM = model_LSTM.predict(x_test)
 acc_LSTM = accuracy_score(y_test, np.where(pred_LSTM > 0.5, 1, 0))
f1_LSTM = f1_score(y_test, np.where(pred_LSTM > 0.5, 1, 0))
print(acc_LSTM, f1_LSTM )
grid_searcher = GridSearchCV(estimator=lgb_clf, param_grid=param_grid,
cv=5, verbose=1, n_jobs=4 )<train_model>
inp = Input(shape =(MAX_WORDS,))
x = Embedding(num_words, 300, weights = [embedding_matrix] )(inp)
 x = Conv1D(MAX_WORDS, 3 )(x)
 x = Activation('relu' )(x)
 x = MaxPooling1D(pool_size=2, strides=2 )(x)
 x = Bidirectional(LSTM(MAX_WORDS, dropout = 0.2, recurrent_dropout = 0.2, return_sequences = True))(x)
 x = ActivityRegularization(l2 = 0.1 )(x)
 x = GlobalMaxPool1D()(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(4, activation = "relu" )(x)
 x = ActivityRegularization(l2 = 0.1 )(x)
 x = Dropout(0.2 )(x)
 x = Dense(1, activation = "sigmoid" )(x)
 model_CNN_LSTM = Model(inputs = inp, outputs = x)
 model_CNN_LSTM.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 print(model_CNN_LSTM.summary() )
grid_searcher.fit(X_train, y_train )<find_best_params>
checkpoint = ModelCheckpoint(
 'model_CNN_LSTM.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 model_CNN_LSTM.fit(x_train, y_train, batch_size = 64, epochs = 10, validation_data =(x_test, y_test), callbacks = [reduce_lr, checkpoint] )
grid_searcher.best_params_, grid_searcher.best_score_<compute_test_metric>
model_CNN_LSTM.load_weights('model_CNN_LSTM.h5')
 pred_CNN_LSTM = model_CNN_LSTM.predict(x_test)
 acc_CNN_LSTM = accuracy_score(y_test, np.where(pred_CNN_LSTM > 0.5, 1, 0))
f1_CNN_LSTM = f1_score(y_test, np.where(pred_CNN_LSTM > 0.5, 1, 0))
print(acc_CNN_LSTM, f1_CNN_LSTM )
accuracy_score(y_valid, grid_searcher.predict(X_valid))<train_on_grid>
numerical_features = train_df[['tweet_sentiment', 'nr_of_char', 'nr_of_words', 'nr_of_unique_words', 'nr_of_punctuation', 'nr_of_stopwords']].to_numpy()
num_iterations = 200
lgb_clf2 = LGBMClassifier(random_state=17, max_depth=-1,
num_leaves=63, n_estimators=num_iterations,
n_jobs=1)
 param_grid2 = {'learning_rate': np.logspace(-3, 0, 10)}
grid_searcher2 = GridSearchCV(estimator=lgb_clf2, param_grid=param_grid2,
cv=5, verbose=1, n_jobs=4)
 grid_searcher2.fit(X_train, y_train)
 print(grid_searcher2.best_params_, grid_searcher2.best_score_)
 print(accuracy_score(y_valid, grid_searcher2.predict(X_valid)) )<choose_model_class>
nlp_inp = Input(shape =(MAX_WORDS,))
x = Embedding(num_words, 300, weights = [embedding_matrix] )(nlp_inp)
 x = Bidirectional(LSTM(MAX_WORDS,dropout = 0.2, recurrent_dropout = 0.2, return_sequences = True))(x)
 x = ActivityRegularization(l2 = 0.1 )(x)
 x = GlobalMaxPool1D()(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(4, activation = "relu" )(x)
 x = ActivityRegularization(l2 = 0.1 )(x)
 num_features_inp = Input(shape =(6,), name = 'num_features_inp')
 y = Dense(4, activation = "relu" )(num_features_inp)
 x = ActivityRegularization(l2 = 0.1 )(x)
 z = Concatenate()([x, y])
 z = Dropout(0.2 )(z)
 z = Dense(1, activation = "sigmoid" )(z)
 model_LSTM_FC = Model(inputs = [nlp_inp, num_features_inp], outputs = z)
 model_LSTM_FC.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 print(model_LSTM_FC.summary() )
final_lgb = LGBMClassifier(n_estimators=200, num_leaves=63,
learning_rate=0.2, max_depth=-1,
n_jobs=4 )<train_model>
x_train_text, x_test_text, x_train_num, x_test_num, y_train, y_test = train_test_split(train_pad, numerical_features, train_df['target'].values, test_size=0.25, random_state = 42)
 print("Shape of train", x_train_text.shape)
 print("Shape of Validation", x_test_text.shape)
 print("Shape of train", x_train_num.shape)
 print("Shape of Validation", x_test_num.shape )
%%time
final_lgb.fit(train.drop('Cover_Type', axis=1), train['Cover_Type'] )<predict_on_test>
checkpoint = ModelCheckpoint(
 'model_LSTM_FC.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 model_LSTM_FC.fit([x_train_text, x_train_num], y_train, batch_size=32, epochs=15, validation_data=([x_test_text, x_test_num], y_test), callbacks = [reduce_lr, checkpoint] )
%%time
lgb_final_pred = final_lgb.predict(test )<save_to_csv>
model_LSTM_FC.load_weights('model_LSTM_FC.h5')
 pred_LSTM_FC = model_LSTM_FC.predict([x_test_text, x_test_num])
 acc_LSTM_FC = accuracy_score(y_test, np.where(pred_LSTM_FC > 0.5, 1, 0))
f1_LSTM_FC = f1_score(y_test, np.where(pred_LSTM_FC > 0.5, 1, 0))
print(acc_LSTM_FC, f1_LSTM_FC )
write_to_submission_file(lgb_final_pred,
'lgb_forest_cover_type.csv' )<create_dataframe>
print("Accuracy and F1-Score of the LSTM: ", acc_LSTM, f1_LSTM)
 print("Accuracy and F1-Score of the CNN_LSTM: ", acc_CNN_LSTM, f1_CNN_LSTM)
 print("Accuracy and F1-Score of the LSTM_FC: ", acc_LSTM_FC, f1_LSTM_FC )
pd.DataFrame(final_lgb.feature_importances_,
index=X_train.columns, columns=['Importance'] ).sort_values(
 by='Importance', ascending=False)[:10]<import_modules>
!pip install transformers
!pip install simpletransformers
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense,Dropout<import_modules>
columns = train_df[["text_cleaned", "target"]]
train_df_V2 = columns.copy()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
rename = {"text_cleaned": "text", "target": "labels"}
train_df_V2.rename(columns = rename, inplace=True )
train = pd.read_csv(".. /input/train.csv")
 test =pd.read_csv(".. /input/test.csv")
<count_missing_values>
train_x_y = train_df_V2.sample(frac = 0.75, random_state = 42)
 test_x_y = pd.concat([train_df_V2, train_x_y] ).drop_duplicates(keep=False )
train.isnull().sum()<count_missing_values>
model_args = ClassificationArgs()
model_args.use_early_stopping = True
model_args.early_stopping_delta = 0.01
model_args.early_stopping_metric = "mcc"
model_args.early_stopping_metric_minimize = False
model_args.early_stopping_patience = 5
model_args.evaluate_during_training_steps = 1000
model_args.reprocess_input_data = True
model_args.overwrite_output_dir = True
model_args.no_save = True
model_bert = ClassificationModel("bert", "bert-base-uncased", args=model_args, use_cuda=False)
 model_bert.train_model(train_x_y )
train.isnull().sum()<count_missing_values>
pred_bert, out_bert = model_bert.predict(test_x_y['text'].values)
 acc_bert = accuracy_score(test_x_y['labels'].to_numpy() , pred_bert)
 f1_bert = f1_score(test_x_y['labels'].to_numpy() , pred_bert)
 print(acc_bert, f1_bert )
test.isnull().sum()<count_missing_values>
model_bert = ClassificationModel("bert", "bert-base-uncased", args=model_args, use_cuda=False)
 model_bert.train_model(train_df_V2)
 final_pred_bert, final_out_bert = model_bert.predict(test_df['text_cleaned'].values)
 submit = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submit['target'] = final_pred_bert
submit.to_csv('submission.csv', index=False )