Kaggle competition: Natural Language Processing with Disaster Tweets
x = InceptionV3_base_model.output
x_pool = GlobalAveragePooling2D()(x)
 x_dense = Dense(Number_FC_Neurons,activation='relu' )(x_pool)
 final_pred = Dense(num_classes,activation='softmax' )(x_dense)
 model = Model(inputs=InceptionV3_base_model.input,outputs=final_pred)
 model.summary()<choose_model_class>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
cb_checkpoint = ModelCheckpoint(filepath = '.. /working/best.hd5', monitor = 'val_loss', save_best_only = True, mode = 'auto',)
 cb_stopping = EarlyStopping(monitor='val_loss',patience=10,mode=min,restore_best_weights=True)
 cb_tensorboard = TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True,write_images=False)
 my_callback=[cb_stopping, cb_checkpoint,cb_tensorboard]<choose_model_class>
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier
import tensorflow_hub as hub
import tokenization
from tensorflow.keras.layers import Dense, Input, Bidirectional, SpatialDropout1D, Embedding, add, concatenate
from tensorflow.keras.layers import GRU, GlobalAveragePooling1D, LSTM, GlobalMaxPooling1D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from tensorflow.keras.models import Model
import tensorflow as tf
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS
from sklearn.feature_extraction.text import TfidfVectorizer
layer_to_Freeze=172
for layer in model.layers[:layer_to_Freeze]:
layer.trainable =False
for layer in model.layers[layer_to_Freeze:]:
layer.trainable=True
sgd = SGD(lr = 0.01, decay = 1e-6, momentum = 0.9, nesterov = True)
 model.compile(optimizer=sgd,loss='binary_crossentropy',metrics=['accuracy'])
<train_model>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
history_transfer_learning = model.fit_generator(train_generator,epochs=30,
steps_per_epoch=n_train//batch_size,
validation_data=val_generator,
validation_steps=n_val//batch_size,
verbose=1,
callbacks=my_callback,
class_weight='auto')
<load_pretrained>
df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )
print('Loading best model...')
 best_model = load_model('.. /working/best.hd5')
 print('Best model loaded!')
<set_options>
df.isnull().any()
gc.collect()<compute_test_metric>
comment_words = ' '
stopwords = set(STOPWORDS)
 for tweet in df.loc[df.target == 1].text:
tokens = tweet.split()
for i in range(len(tokens)) :
tokens[i] = tokens[i].lower()
for words in tokens:
comment_words = comment_words + words + ' '
wordcloud = WordCloud(width = 1000, height = 1000,
stopwords = stopwords,
min_font_size = 10 ).generate(comment_words)
 plt.figure(figsize =(15, 15))
plt.imshow(wordcloud)
 plt.title('Most used word in Disaster Tweet(Target = 1)')
 plt.axis("off")
 plt.show()
score = best_model.evaluate_generator(val_generator,verbose=1)
 print('Test loss: ', score[0])
 print('Test accuracy', score[1])
<define_variables>
comment_words = ' '
stopwords = set(STOPWORDS)
 for tweet in df.loc[df.target == 0].text:
tokens = tweet.split()
for i in range(len(tokens)) :
tokens[i] = tokens[i].lower()
for words in tokens:
comment_words = comment_words + words + ' '
wordcloud = WordCloud(width = 1000, height = 1000,
stopwords = stopwords,
min_font_size = 10 ).generate(comment_words)
 plt.figure(figsize =(15, 15))
plt.imshow(wordcloud)
 plt.title('Most used word in Non Disaster Tweet(Target = 0)')
 plt.axis("off")
 plt.show()
test_dir='.. /data/test/'<define_variables>
X_train, X_test, y_train, y_test = train_test_split(df.iloc[:, 3], df.iloc[:, 4], test_size=0.2 )
test_image_gen = ImageDataGenerator(rescale=1/255)
 test_generator = test_image_gen.flow_from_directory(test_dir,target_size=(Image_width,Image_height),batch_size=1,seed=42,class_mode=None,shuffle=False )<predict_on_test>
def tfidf(data):
tfidf_vectorizer = TfidfVectorizer()
train = tfidf_vectorizer.fit_transform(data)
 return train, tfidf_vectorizer
X_train_tfidf, tfidf_vectorizer = tfidf(X_train)
 X_test_tfidf = tfidf_vectorizer.transform(X_test)
 submission_tfidf = tfidf_vectorizer.transform(test.iloc[:, 3] )
y_pred = model.predict_generator(generator=test_generator,verbose=1 )<prepare_output>
model = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0,
learning_rate=0.1, max_delta_step=0, max_depth=7,
min_child_weight=1, missing=None, n_estimators=100, n_jobs=-1,
nthread=None, objective='binary:logistic', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 model.fit(X_train_tfidf, y_train )
submission = pd.DataFrame({'id':pd.Series(test_generator.filenames),'label':pd.Series(y_pred.clip(min=0.02,max=0.98)[:,1])})
 submission['id'] = submission.id.str.extract('(\d+)')
 submission['id']=pd.to_numeric(submission['id'])
<save_to_csv>
model.score(X_test_tfidf, y_test )
submission.to_csv('DogVsCats_submission.csv',index=False )<train_model>
predictions = model.predict(submission_tfidf )
len(model.layers )<set_options>
submission['target'] = predictions
submission
warnings.filterwarnings("ignore", category=DeprecationWarning)
 %matplotlib inline
plt.style.use('ggplot')
 mpl.rcParams['axes.unicode_minus'] = False<load_from_csv>
submission.to_csv('tfidf_submission.csv', index=False )
test = pd.read_csv('.. /input/bike-sharing-demand/test.csv', parse_dates=["datetime"])
 train = pd.read_csv('.. /input/bike-sharing-demand/train.csv', parse_dates=["datetime"] )<count_values>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
train['humidity'].value_counts().head(10 )<concatenate>
def bert_encoder(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segments_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segments_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
data = train.append(test)
 data.reset_index(inplace=True)
 data.drop('index',inplace=True,axis=1 )<feature_engineering>
def build_model(bert_layer, max_len=512):
with strategy.scope() :
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 x = SpatialDropout1D(0.3 )(sequence_output)
 x = Bidirectional(GRU(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(GRU(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([GlobalMaxPooling1D()(x),GlobalAveragePooling1D()(x),])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 dropout = tf.keras.layers.Dropout(0.2 )(hidden)
 result = Dense(1, activation='sigmoid' )(dropout)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=result)
 model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 return model
train["year"] = train["datetime"].dt.year
train["month"] = train["datetime"].dt.month
train["day"] = train["datetime"].dt.day
train["hour"] = train["datetime"].dt.hour
train["dayofweek"] = train["datetime"].dt.dayofweek
train.shape<feature_engineering>
BATCH_SIZE = 16
LSTM_UNITS = 64
EPOCHS = 10
DENSE_HIDDEN_UNITS = 256
model = build_model(bert_layer, max_len=160 )
test["year"] = test["datetime"].dt.year
test["month"] = test["datetime"].dt.month
test["day"] = test["datetime"].dt.day
test["hour"] = test["datetime"].dt.hour
test["dayofweek"] = test["datetime"].dt.dayofweek
test.shape<predict_on_test>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 train_input = bert_encoder(df.iloc[:, 3], tokenizer, max_len=160)
 test_input = bert_encoder(test.iloc[:, 3], tokenizer, max_len=160 )
def predict_windspeed(data):
dataWind0 = data.loc[data['windspeed'] == 0]
dataWindNot0 = data.loc[data['windspeed'] != 0]
wCol = ["season", "weather", "humidity", "month", "temp", "year", "atemp"]
dataWindNot0["windspeed"] = dataWindNot0["windspeed"].astype("str")
 rfModel_wind = RandomForestClassifier()
rfModel_wind.fit(dataWindNot0[wCol], dataWindNot0["windspeed"])
 wind0Values = rfModel_wind.predict(X = dataWind0[wCol])
 predictWind0 = dataWind0
predictWindNot0 = dataWindNot0
predictWind0["windspeed"] = wind0Values
data = predictWindNot0.append(predictWind0)
 data["windspeed"] = data["windspeed"].astype("float")
 data.reset_index(inplace=True)
 data.drop('index', inplace=True, axis=1)
 return data<predict_on_test>
NUM_MODELS = 1
BATCH_SIZE = 16
LSTM_UNITS = 64
EPOCHS = 1
DENSE_HIDDEN_UNITS = 256
checkpoint_predictions = []
checkpoint_val_pred = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(bert_layer, max_len=160)
 for global_epoch in range(EPOCHS):
history = model.fit(
 train_input, df.iloc[: ,4],
batch_size=BATCH_SIZE,
epochs=7,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 2e-6 *(0.6** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(test_input, batch_size=32 ).flatten())
 weights.append(2 ** global_epoch)
 del model
gc.collect()
train = predict_windspeed(train)
 test = predict_windspeed(test )<define_variables>
test_pred = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )