Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<count_missing_values>
%env JOBLIB_TEMP_FOLDER=/tmp
plt.style.use('seaborn')
 sns.set_style('whitegrid')
 %matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
ms_cols = []
train_copy = train
train_copy = train_copy.replace(-1, np.NaN)
 for col in train_copy.columns:
ms_nb = train_copy[col].isnull().sum()
if ms_nb > 0:
ms_cols.append(col)
 print('Column {} has {} records({:.2%})with missing values'.format(col, ms_nb, ms_nb/train_copy.shape[0]))<feature_engineering>
a=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission= pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 train_data=a.text.values
train_labels=a.target.values
test_data=test.text.values
a.head()
drop_cols = ['ps_car_03_cat', 'ps_car_05_cat']
real_train = train.drop(drop_cols, axis=1)
 real_test = test.drop(drop_cols, axis=1)
 meta.loc[drop_cols, 'keep'] = False
mean_imp = Imputer(missing_values=-1, strategy='mean', axis=0)
 mode_imp = Imputer(missing_values=-1, strategy='most_frequent', axis=0)
 real_train['ps_reg_03'] = mean_imp.fit_transform(X=real_train[['ps_reg_03']] ).ravel()
real_train['ps_car_11'] = mode_imp.fit_transform(X=real_train[['ps_car_11']] ).ravel()
real_train['ps_car_12'] = mode_imp.fit_transform(X=real_train[['ps_car_12']] ).ravel()
real_train['ps_car_14'] = mean_imp.fit_transform(X=real_train[['ps_car_14']] ).ravel()
real_test['ps_reg_03'] = mean_imp.fit_transform(X=real_test[['ps_reg_03']] ).ravel()
real_test['ps_car_11'] = mode_imp.fit_transform(X=real_test[['ps_car_11']] ).ravel()
real_test['ps_car_12'] = mode_imp.fit_transform(X=real_test[['ps_car_12']] ).ravel()
real_test['ps_car_14'] = mean_imp.fit_transform(X=real_test[['ps_car_14']] ).ravel()<count_values>
NAcols=a.columns
for col in NAcols:
if a[col].dtype == "object":
a[col] = a[col].fillna("None" )
cat_cols = meta[(meta.level=='nominal')& meta.keep].index
for col in cat_cols:
distinct_values = real_train[col].value_counts().shape[0]
print('Categorical column {} has {} distinct values'.format(col, distinct_values))<categorify>
no_dis=a[a['target']==0]
yes_dis=a[a['target']==1]
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_tst_series, noise_level )<categorify>
def word_counter(data):
df=data['text'].str.lower()
swords=[ "i'm",'the', 'a', 'to', 'in', 'of', 'and', 'I', 'you', 'he', 'she', 'it', 'is', 'on','-','The',
'my', 'with', 'that', 'at','by','it','from', 'be', 'was', 'have', 'are', 'this', 'like',
'A', 'as', 'just', 'your', 'up', 'but', 'me', 'so', 'not', 'has', 'out', '??', 'will',
'via','after', 'an', 'about', 'been', 'get', 'or', 'when', 'all', 'no', 'into', 'over', 'In',
'who', 'we', 'if', 'I', 'can', 'The', 'how', 'them', 'But', 'So', 'Too', 'too', 'did', 'much',
'for', 'his', 'her', 'To', "it's", 'You', 'there', 'If', 'what', 'i', '2', 'they', 'had', 'their',
'one', 'got', 'us', 'man', 'our', 'two', '&amp;', 'were', 'than']
b= collections.Counter([y for x in df.values.flatten() for y in x.split() if y not in swords])
 ca= b.most_common(20)
 c=pd.DataFrame(ca)
 c= c.rename(columns={0: "word", 1: "n_word"})
 fig, ax = plt.subplots(figsize=(6,6))
title_cnt=c.n_word.sort_values(ascending=False ).reset_index()
n= ax.barh(title_cnt.iloc[:,0], title_cnt.iloc[:,1], color='lightgreen', edgecolor='green', height=0.7)
 for s in ['top','bottom','left','right']:
ax.spines[s].set_visible(False)
 ax.xaxis.set_ticks_position('none')
 ax.yaxis.set_ticks_position('none')
 ax.xaxis.set_tick_params(pad=5)
 ax.yaxis.set_tick_params(pad=10)
 ax.grid(b=True, color='grey', linestyle='-.', linewidth=1, alpha=0.3)
 ax.invert_yaxis()
ax.set_title('Disaster tweets: MOST FREQUENT WORDS',
loc='center', pad=10, fontsize=16)
 ax.set_xlabel('
for i in ax.patches:
ax.text(i.get_width() +5, i.get_y() +0.5, str(round(( i.get_width()), 2)) ,
fontsize=10, fontweight='bold', color='grey')
 r=range(20)
 plt.yticks(r, c.word ,weight='bold')
 return plt.show()
train_encoded, test_encoded = target_encode(train["ps_car_11_cat"],
test["ps_car_11_cat"],
target=train.target,
min_samples_leaf=100,
smoothing=10,
noise_level=0.01)
 real_train['ps_car_11_cat_te'] = train_encoded
real_train.drop('ps_car_11_cat', axis=1, inplace=True)
 meta.loc['ps_car_11_cat','keep'] = False
real_test['ps_car_11_cat_te'] = test_encoded
real_test.drop('ps_car_11_cat', axis=1, inplace=True )<drop_column>
word_counter(no_dis )
calc_cols = [col for col in train.columns if 'calc' in col]
real_train_nocalc = real_train.drop(calc_cols, axis=1)
 real_test_nocalc = real_test.drop(calc_cols, axis=1)
 real_train_nocalc.shape<define_variables>
word_counter(yes_dis )
bin_cols = meta[(meta.level == 'binary')&(meta.role != 'target')& meta.keep].index
ones_list = []
zeros_list = []
for col in bin_cols:
zeros_nb =(real_train[col] == 0 ).sum()
ones_nb = real_train.shape[0] - zeros_nb
ones_list.append(ones_nb)
 zeros_list.append(zeros_nb)
 print('Binary column {} has {} records({:.2%})with value zero'.format(col, zeros_nb, zeros_nb/real_train.shape[0]))<drop_column>
hub_layer = hub.KerasLayer("https://tfhub.dev/google/Wiki-words-500-with-normalization/2",
input_shape=[], dtype=tf.string)
 model = tf.keras.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dropout(0.4))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dense(32, activation='relu'))
model.add(tf.keras.layers.Dropout(0.4))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
model.summary()
imbalanced_cols = ['ps_ind_10_bin', 'ps_ind_11_bin', 'ps_ind_13_bin']
real_train = real_train.drop(imbalanced_cols, axis=1)
 real_test = real_test.drop(imbalanced_cols, axis=1)
 real_train_nocalc = real_train_nocalc.drop(imbalanced_cols, axis=1)
 real_test_nocalc = real_test_nocalc.drop(imbalanced_cols, axis=1 )<categorify>
model.compile(Adam(lr=0.0002), loss='binary_crossentropy', metrics=['accuracy'] )
cat_cols = meta[(meta.level=='nominal')& meta.keep].index
print('Nb of columns in train data before dummification: {}'.format(real_train.shape[1]))
real_train = pd.get_dummies(data=real_train, columns=cat_cols, drop_first=True)
 print('Nb of columns in train data before dummification: {}'.format(real_train.shape[1]))
print('Nb of columns in test data before dummification: {}'.format(real_test.shape[1]))
real_test = pd.get_dummies(data=real_test, columns=cat_cols, drop_first=True)
 print('Nb of columns in test data after dummification: {}'.format(real_test.shape[1]))
real_train_nocalc = pd.get_dummies(data=real_train_nocalc, columns=cat_cols, drop_first=True)
 real_test_nocalc = pd.get_dummies(data=real_test_nocalc, columns=cat_cols, drop_first=True )<create_dataframe>
history = model.fit(
 train_data, train_labels,
validation_split=0.3,
epochs=22,
callbacks=callbacks_list,
batch_size=32
)
float_cols = meta[(meta.level=='interval')& meta.keep].index
poly = PolynomialFeatures(degree=2, include_bias=False, interaction_only=False)
 interactions = pd.DataFrame(data=poly.fit_transform(real_train[float_cols]), columns=poly.get_feature_names(float_cols))
interactions.drop(float_cols, axis=1, inplace=True)
 interacted_train = pd.concat(objs=[real_train, interactions], axis=1)
 interacted_test = pd.concat(objs=[real_test, interactions], axis=1 )<split>
model.load_weights('my_model_file.h5')
 prediction = model.predict(test_data)
 submission['target'] = prediction.round().astype(int)
 submission.to_csv('submission.csv', index=False)