Kaggle competition: Invasive Species Monitoring
<SOS> metric: AUC Kaggle data source: invasive-species-monitoring<train_model>
%matplotlib inline
master = pd.read_csv(".. /input/invasive-species-monitoring/train_labels.csv")
 master.head()
if not already_trained:
tuner.search(
 {
"input_word_ids": bert_xtrain[0],
"input_mask": bert_xtrain[1],
"segment_ids": bert_xtrain[2],
},
y_train,
epochs=3,
batch_size=32,
validation_data=(
 {
"input_word_ids": bert_xval[0],
"input_mask": bert_xval[1],
"segment_ids": bert_xval[2],
},
y_val, 
),
callbacks=[checkpoint_cb, early_stopping_cb, tensorboard_cb], 
)
 else:
None<train_model>
img_path = ".. /input/invasive-species-monitoring/train/"
y = []
file_paths = []
for i in range(len(master)) :
file_paths.append(img_path + str(master.iloc[i][0])+'.jpg')
 y.append(master.iloc[i][1])
 y = np.array(y )
tf.keras.utils.plot_model(bert_model, to_file="bert_model.png", dpi=96 )<load_from_csv>
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 img_path = ".. /input/invasive-species-monitoring/test/"
test_names = []
file_paths = []
for i in range(len(sample_submission)) :
test_names.append(sample_submission.iloc[i][0])
 file_paths.append(img_path + str(int(sample_submission.iloc[i][0])) +'.jpg')
 test_names = np.array(test_names )
df_test = pd.read_csv(test_file)
 clean_text_column("text", df_test)
 bert_test = bert_encode(df_test.clean_text )<predict_on_test>
data_num = len(y)
 random_index = np.random.permutation(data_num)
 x_shuffle = []
y_shuffle = []
for i in range(data_num):
x_shuffle.append(x[random_index[i]])
 y_shuffle.append(y[random_index[i]])
 x = np.array(x_shuffle)
 y = np.array(y_shuffle )
pred = bert_model.predict(bert_test )<save_to_csv>
x_train = x_train.astype('float32')
 x_test = x_test.astype('float32')
 x_train /= 255
x_test /= 255
round_predictions = pred.round().astype("int")
 df_submit = pd.read_csv(sample_submission_file)
 df_submit["target"] = round_predictions
df_submit.to_csv("submission.csv", index=False, header=True )<save_to_csv>
img_rows, img_cols, img_channel = 224, 224, 3
base_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(img_rows, img_cols, img_channel))
round_predictions = pred.round().astype("int")
 df_submit = pd.read_csv(sample_submission_file)
 df_submit["target"] = round_predictions
df_submit.to_csv("submission.csv", index=False, header=True )<save_to_csv>
add_model = Sequential()
add_model.add(Flatten(input_shape=base_model.output_shape[1:]))
add_model.add(Dense(256, activation='relu'))
add_model.add(Dense(1, activation='sigmoid'))
model = Model(inputs=base_model.input, outputs=add_model(base_model.output))
model.compile(loss='binary_crossentropy', optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
 model.summary()
threshold_predictions = pred.round().astype("int")
 df_submit = pd.read_csv(sample_submission_file)
 df_submit["target"] = round_predictions
df_submit.to_csv("submission.csv", index=False, header=True )<load_from_csv>
batch_size = 32
epochs = 50
train_datagen = ImageDataGenerator(
 rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True)
 train_datagen.fit(x_train)
 history = model.fit_generator(
 train_datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=x_train.shape[0] // batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
)
tweets = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
<filter>
print("Training loss: {:.2f} / Validation loss: {:.2f}".\
format(history.history['loss'][-1], history.history['val_loss'][-1]))
print("Training accuracy: {:.2f}% / Validation accuracy: {:.2f}%".\
format(100*history.history['acc'][-1], 100*history.history['val_acc'][-1]))
tweets[tweets.sent_len > 8]
tweets[tweets.word_len > 50]<string_transform>
test_images = test_images.astype('float32')
 test_images /= 255
stop = set(stopwords.words('english'))
corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(tweet)]
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(tweet)]
def count_top_stopwords(corpus):
stopwords_freq = {}
for word in corpus:
if word in stop:
if word in stopwords_freq:
stopwords_freq[word] += 1
else:
stopwords_freq[word] = 1
topwords = sorted(stopwords_freq.items() , key=lambda item: item[1], reverse=True)[:10]
x,y = zip(*topwords)
 return x,y
x0,y0 = count_top_stopwords(corpus0)
 x1,y1 = count_top_stopwords(corpus1)
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 plt.bar(x0,y0, color=color[0])
 plt.title('Top Stopwords for Non-Disaster Tweets')
 plt.subplot(1,2,2)
 plt.bar(x1,y1, color=color[1])
 plt.title('Top Stopwords for Disaster Tweets')
<string_transform>
predictions = model.predict(test_images )
<feature_engineering><EOS> 
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 for i, name in enumerate(test_names):
sample_submission.loc[sample_submission['name'] == name, 'invasive'] = predictions[i]
sample_submission.to_csv("submit.csv", index=False )