Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
np.random.seed(1)
 nltk.download('stopwords')
 tf.random.set_seed(1)
 pd.set_option('display.max_colwidth', 500)
 warnings.filterwarnings('ignore' )
all_feat = [col for col in df_sample.columns if col not in ["sig_id"]]
df_sample[all_feat] = test_preds_all.mean(axis=0)
 df_sample.loc[df_test['cp_type']==0, df_sample.columns[1:]] = 0
df_sample.to_csv('submission.csv', index=None)
 df_submit = pd.read_csv('./submission.csv')
 df_submit.head()<import_modules>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 print("Train Shape :", train.shape)
 print("Test Shape :", test.shape )
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
from tqdm.notebook import tqdm<load_from_csv>
train.isnull().sum()
train = pd.read_csv('.. /input/prep-lishmoa/data/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test = pd.read_csv('.. /input/prep-lishmoa/data/test_features.csv')
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<drop_column>
train.isnull().sum()
del train_targets['sig_id']
del train['sig_id']
del test['sig_id']<choose_model_class>
train['target'].value_counts(normalize = True )
def create_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss='binary_crossentropy' 
)
 return model<choose_model_class>
train.keyword.value_counts()
def build_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1400, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(840, activation="sigmoid")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(588, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=5),
loss='binary_crossentropy', metrics=["accuracy"] 
)
 return model<normalization>
train.location.value_counts()
seed = tf.random.set_seed(42)
 def iter_shuffled(X, columns_to_shuffle=None, pre_shuffle=False,
random_state=None):
rng = check_random_state(random_state)
 if columns_to_shuffle is None:
columns_to_shuffle = range(X.shape[1])
 if pre_shuffle:
X_shuffled = X.copy()
rng.shuffle(X_shuffled)
 X_res = X.copy()
for columns in tqdm(columns_to_shuffle):
if pre_shuffle:
X_res[:, columns] = X_shuffled[:, columns]
else:
rng.shuffle(X_res[:, columns])
 yield X_res
X_res[:, columns] = X[:, columns]
def get_score_importances(
 score_func,
X,
y,
n_iter=5,
columns_to_shuffle=None,
random_state=None 
):
rng = check_random_state(random_state)
 base_score = score_func(X, y)
 scores_decreases = []
for i in range(n_iter):
scores_shuffled = _get_scores_shufled(
 score_func, X, y, columns_to_shuffle=columns_to_shuffle,
random_state=rng, base_score=base_score 
)
 scores_decreases.append(scores_shuffled)
 return base_score, scores_decreases
def _get_scores_shufled(score_func, X, y, base_score, columns_to_shuffle=None,
random_state=None):
Xs = iter_shuffled(X, columns_to_shuffle, random_state=random_state)
 res = []
for X_shuffled in Xs:
res.append(-score_func(X_shuffled, y)+ base_score)
 return res
def metric(y_true, y_pred):
metrics = []
for i in range(y_pred.shape[1]):
if y_true[:, i].sum() > 1:
metrics.append(log_loss(y_true[:, i], y_pred[:, i]))
return np.mean(metrics)
 perm_imp = np.zeros(train.shape[1])
 for n,(tr, te)in enumerate(KFold(n_splits=2, random_state=seed, shuffle=True ).split(train_targets)) :
print(f'Fold {n}')
 model = create_model(len(list(train.columns)))
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 model.fit(train.values[tr],
train_targets.values[tr],
validation_data=(train.values[te], train_targets.values[te]),
epochs=40, batch_size=128,
callbacks=[reduce_lr_loss], verbose=2 
)
 def _score(X, y):
pred = model.predict(X)
 return metric(y, pred)
 base_score, local_imp = get_score_importances(_score, train.values[te], train_targets.values[te], n_iter=1, random_state=0)
 perm_imp += np.mean(local_imp, axis=0)
 print('')
 break
top_feats = np.argwhere(perm_imp < 0 ).flatten()<compute_test_metric>
real_tweets = train[train['target']==1]['text']
real_tweets.values[0:5]
def metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels=[0,1]))
return np.mean(metrics)
<compute_test_metric>
fake_tweets = train[train['target']==0]['text']
fake_tweets.values[0:5]
print(f'OOF Metric: {metric(train_targets, res)}' )<feature_engineering>
def clean_text(text):
text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text
ss.loc[test['cp_type']==1, train_targets.columns] = 0<save_to_csv>
train['cleaned_text'] = train['text'].apply(lambda x: clean_text(x))
test['cleaned_text'] = test['text'].apply(lambda x: clean_text(x))
train['cleaned_text'].head()
ss.to_csv('submission.csv', index=False )<set_options>
!pip install nlppreprocess
nlp = NLP()
train['stopwords_cleaned'] = train['cleaned_text'].apply(nlp.process)
 test['stopwords_cleaned'] = test['cleaned_text'].apply(nlp.process )
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )<init_hyperparams>
en_model = spacy.load('en', disable=['parser', 'ner'])
 def lemmatization(texts):
output = []
for i in texts:
s = [token.lemma_ for token in en_model(i)]
output.append(' '.join(s))
return output
n_comp_GENES = 463
n_comp_CELLS = 60
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )<create_dataframe>
train['lemmatized_text'] = lemmatization(train['stopwords_cleaned'])
 test['lemmatized_text'] = lemmatization(test['stopwords_cleaned'] )
commits_df = pd.DataFrame(columns = ['n_commit', 'n_comp_GENES', 'n_comp_CELLS', 'train_features','VarianceThreshold_for_FS', 'Dropout_Model', 'LB_score', 'CV_logloss'] )<feature_engineering>
x_train, x_test, y_train, y_test = train_test_split(train['stopwords_cleaned'], train['target'],
test_size = 0.2, random_state = 1 )
n=0
commits_df.loc[n, 'n_commit'] = 0
commits_df.loc[n, 'n_comp_GENES'] = 600
commits_df.loc[n, 'n_comp_CELLS'] = 50
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.8
commits_df.loc[n, 'train_features'] = 1245
commits_df.loc[n, 'Dropout_Model'] = 0.2619422201258426
commits_df.loc[n, 'CV_logloss'] = 0.01458269555140327
commits_df.loc[n, 'LB_score'] = 0.01839<feature_engineering>
hub_layer = hub.KerasLayer('https://tfhub.dev/google/universal-sentence-encoder/4',
input_shape = [],
output_shape = [512],
dtype = tf.string,
trainable = True)
 model = tf.keras.models.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation = 'relu'))
model.add(tf.keras.layers.Dense(32, activation = 'relu'))
model.add(tf.keras.layers.Dense(1, activation = 'sigmoid'))
model.summary()
n=1
commits_df.loc[n, 'n_commit'] = 4
commits_df.loc[n, 'n_comp_GENES'] = 610
commits_df.loc[n, 'n_comp_CELLS'] = 55
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.82
commits_df.loc[n, 'train_features'] = 1240
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014584545081734047
commits_df.loc[n, 'LB_score'] = 0.01839<feature_engineering>
model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics = ['accuracy'] )
n=2
commits_df.loc[n, 'n_commit'] = 5
commits_df.loc[n, 'n_comp_GENES'] = 670
commits_df.loc[n, 'n_comp_CELLS'] = 67
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.67
commits_df.loc[n, 'train_features'] = 1298
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014588561242139069
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
model.fit(x_train,
y_train,
epochs = 1,
validation_data =(x_test, y_test))
n=3
commits_df.loc[n, 'n_commit'] = 6
commits_df.loc[n, 'n_comp_GENES'] = 450
commits_df.loc[n, 'n_comp_CELLS'] = 45
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.67
commits_df.loc[n, 'train_features'] = 1297
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014586229676302227
commits_df.loc[n, 'LB_score'] = 0.01840<feature_engineering>
pred = model.predict_classes(test['stopwords_cleaned'] )
n=4
commits_df.loc[n, 'n_commit'] = 9
commits_df.loc[n, 'n_comp_GENES'] = 463
commits_df.loc[n, 'n_comp_CELLS'] = 60
commits_df.loc[n, 'VarianceThreshold_for_FS'] = 0.9
commits_df.loc[n, 'train_features'] = 1219
commits_df.loc[n, 'Dropout_Model'] = 0.25
commits_df.loc[n, 'CV_logloss'] = 0.014572358066092783
commits_df.loc[n, 'LB_score'] = 0.01839<feature_engineering>
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission['target'] = pred
submission.to_csv('submission.csv', index=False )
<feature_engineering><EOS> 
submission.to_csv("submission.csv", index = False )