Kaggle competition: Natural Language Processing with Disaster Tweets
y = train_data["Survived"]
features = ["Pclass", "Sex", "SibSp", "Parch", "Age"]
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
!pip install bert-for-tf2
!pip install sentencepiece
%matplotlib inline<load_from_csv>
try:
%tensorflow_version 2.x
except Exception:
pass
train= pd.read_csv('/kaggle/input/titanic/train.csv' )<categorify>
FullTokenizer = bert.bert_tokenization.FullTokenizer
bert_layer = hub.KerasLayer("https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1", trainable=False)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = FullTokenizer(vocab_file, do_lower_case )
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass==1:
return 37
elif Pclass == 2:
return 29
else :
return 24
else:
return Age<feature_engineering>
train_cols = ["id", "keyword", "location", "text", "target"]
train = pd.read_csv(
 "/kaggle/input/nlp-getting-started/train.csv",
header=None,
names=train_cols,
skiprows=1,
engine="python",
encoding="latin1" 
)
 test_cols = ["id", "keyword", "location", "text"]
test = pd.read_csv(
 "/kaggle/input/nlp-getting-started/test.csv",
header=None,
names=test_cols,
skiprows=1,
engine="python",
encoding="latin1" 
)
 def fix_keyword(x):
return str(x[1] ).replace('%20', ' ')
 def new_text(x):
return str(x[1])+ ' ' + str(x[2])+ ' ' +str(x[3])
 def clean_tweet(tweet):
tweet = re.sub(r"https?://[A-Za-z0-9./]+", ' ', tweet)
 tweet = re.sub(r"@[A-Za-z0-9]+", ' ', tweet)
 tweet = re.sub(r"[^a-zA-Z0-9!?']", ' ', tweet)
 tweet = re.sub(r" +", ' ', tweet)
 tweet = re.sub(r"\?+", ' Q', tweet)
 tweet = re.sub(r"\!+", ' X', tweet)
 return tweet
def encode_sentence(sent):
return ["[CLS]"] + tokenizer.tokenize(sent)+ ["[SEP]"]
train.keyword = train.apply(fix_keyword, axis=1)
 train['new_text'] = train.apply(new_text, axis=1)
 train_clean = [clean_tweet(tweet)for tweet in train.new_text]
train_inputs = [encode_sentence(sentence)for sentence in train_clean]
train_labels = train.target.values
test.keyword = test.apply(fix_keyword, axis=1)
 test['new_text'] = test.apply(new_text, axis=1)
 test_clean = [clean_tweet(tweet)for tweet in test.new_text]
test_inputs = [encode_sentence(sentence)for sentence in test_clean]
train['Age']=train[['Age','Pclass']].apply(impute_age,axis = 1 )<drop_column>
def get_ids(tokens):
return tokenizer.convert_tokens_to_ids(tokens)
 def get_mask(tokens):
return np.char.not_equal(tokens, "[PAD]" ).astype(int)
 def get_segments(tokens):
seg_ids = []
current_seg_id = 0
for tok in tokens:
seg_ids.append(current_seg_id)
 if tok == "[SEP]":
current_seg_id = 1-current_seg_id
return seg_ids
train.drop('Cabin',inplace = True,axis =1 )<categorify>
data_with_len = [[sent, train_labels[i], len(sent)]
for i, sent in enumerate(train_inputs)]
random.shuffle(data_with_len)
 data_with_len.sort(key=lambda x: x[2])
 train_all = [ 
(
 [
get_ids(sent_lab[0]),
get_mask(sent_lab[0]),
get_segments(sent_lab[0])
 ],
sent_lab[1] 
)
 for sent_lab in data_with_len]
sex = pd.get_dummies(train['Sex'],drop_first=True)
 embark = pd.get_dummies(train['Embarked'],drop_first=True )<concatenate>
all_dataset = tf.data.Dataset.from_generator(lambda: train_all, output_types=(tf.int32, tf.int32))
train = pd.concat([train,sex,embark],axis=1 )<drop_column>
BATCH_SIZE = 32
all_batched = all_dataset.padded_batch(BATCH_SIZE,
padded_shapes=(( 3, None),()),
padding_values=(0, 0))
NB_BATCHES = math.ceil(len(train_all)/ BATCH_SIZE)
 NB_BATCHES_TEST = NB_BATCHES // 10
all_batched.shuffle(NB_BATCHES)
 test_dataset = all_batched.take(NB_BATCHES_TEST)
 train_dataset = all_batched.skip(NB_BATCHES_TEST )
train.drop(['Sex','Embarked','PassengerId','Name','Ticket'],axis=1,inplace=True )<normalization>
class DCNNBERTEmbedding(tf.keras.Model):
def __init__(self,
nb_filters=50,
FFN_units=512,
nb_classes=2,
dropout_rate=0.1,
name="dcnn"):
super(DCNNBERTEmbedding, self ).__init__(name=name)
 self.bert_layer = hub.KerasLayer(
 "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1",
trainable=True)
 self.bigram = layers.Conv1D(filters=nb_filters,
kernel_size=2,
padding="valid",
activation="relu")
 self.trigram = layers.Conv1D(filters=nb_filters,
kernel_size=3,
padding="valid",
activation="relu")
 self.fourgram = layers.Conv1D(filters=nb_filters,
kernel_size=4,
padding="valid",
activation="relu")
 self.pool = layers.GlobalMaxPool1D()
self.dense_1 = layers.Dense(units=FFN_units, activation="relu")
 self.dropout = layers.Dropout(rate=dropout_rate)
 if nb_classes == 2:
self.last_dense = layers.Dense(units=1,
activation="sigmoid")
 else:
self.last_dense = layers.Dense(units=nb_classes,
activation="softmax")
 def embed_with_bert(self, all_tokens):
_, embs = self.bert_layer([all_tokens[:, 0, :],
all_tokens[:, 1, :],
all_tokens[:, 2, :]])
 return embs
def call(self, inputs, training):
x = self.embed_with_bert(inputs)
 x_1 = self.bigram(x)
 x_1 = self.pool(x_1)
 x_2 = self.trigram(x)
 x_2 = self.pool(x_2)
 x_3 = self.fourgram(x)
 x_3 = self.pool(x_3)
 merged = tf.concat([x_1, x_2, x_3], axis=-1)
 merged = self.dense_1(merged)
 merged = self.dropout(merged, training)
 output = self.last_dense(merged)
 return output
st = StandardScaler()<categorify>
NB_FILTERS = 128
FFN_UNITS = 256
NB_CLASSES = 2
DROPOUT_RATE = 0.2
BATCH_SIZE = 32
NB_EPOCHS = 3
feature_scale = ['Age','Fare']
train[feature_scale] = st.fit_transform(train[feature_scale] )<prepare_x_and_y>
Dcnn = DCNNBERTEmbedding(nb_filters=NB_FILTERS,
FFN_units=FFN_UNITS,
nb_classes=NB_CLASSES,
dropout_rate=DROPOUT_RATE)
 if NB_CLASSES == 2:
Dcnn.compile(loss="binary_crossentropy",
optimizer=tf.optimizers.Adam(learning_rate=2e-5),
metrics=["accuracy"])
 else:
Dcnn.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["sparse_categorical_accuracy"])
 Dcnn.fit(all_batched, epochs=NB_EPOCHS )
x = train.drop(['Survived'],axis=1)
 y = train['Survived']<import_modules>
results = Dcnn.evaluate(test_dataset)
 print(results )
from sklearn.model_selection import GridSearchCV
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier<train_model>
cols = ["id", "keyword", "location", "text"]
test = pd.read_csv(
 "/kaggle/input/nlp-getting-started/test.csv",
header=None,
names=cols,
skiprows=1,
engine="python",
encoding="latin1" 
)
 test.keyword = test.apply(fix_keyword, axis=1)
 test['new_text'] = test.apply(new_text, axis=1)
 test_clean = [clean_tweet(tweet)for tweet in test.new_text]
test_inputs = [encode_sentence(sentence)for sentence in test_clean]
tree = DecisionTreeClassifier()
tree.fit(x,y)
 tree.score(x,y )<load_from_csv>
preds = []
for sentence in test_inputs:
input = [[
get_ids(sentence),
get_mask(sentence),
get_segments(sentence)
 ]]
preds.append(int(np.round(Dcnn.predict(input)[0][0])))
 if len(preds)% 100 == 0:
print('Predictions made:', len(preds))
<create_dataframe><EOS> 
test['target'] = preds
submission = test[['id', 'target']]
submission.to_csv('submission.csv', index=False)
 submission.target.hist()