Kaggle competition: Invasive Species Monitoring
def lower_case_data(data=""):
data = data.lower()
return data<string_transform>
%matplotlib inline
master = pd.read_csv(".. /input/invasive-species-monitoring/train_labels.csv")
 master.head()
def sentences_with_emojis(id_texts):
sentences = []
indeces = id_texts[0]
texts = id_texts[1]
for index, sentence in zip(indeces, texts):
has_emoji = bool(emoji.get_emoji_regexp().search(sentence))
if has_emoji:
sentences.append(( index, sentence))
if len(sentences)== 0:
return "Sentences are clean and don't have emojis!"
else:
return sentences
def clean_emojis(text):
allchars = [str for str in text]
emoji_list = [c for c in allchars if c in emoji.UNICODE_EMOJI]
clean_text = " ".join(
 [str for str in text.split() if not any(i in str for i in emoji_list)] 
)
 return clean_text<drop_column>
img_path = ".. /input/invasive-species-monitoring/train/"
y = []
file_paths = []
for i in range(len(master)) :
file_paths.append(img_path + str(master.iloc[i][0])+'.jpg')
 y.append(master.iloc[i][1])
 y = np.array(y )
def clean_urls(text):
text = re.sub(r"https?://\S+", "", text)
 return text<string_transform>
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 img_path = ".. /input/invasive-species-monitoring/test/"
test_names = []
file_paths = []
for i in range(len(sample_submission)) :
test_names.append(sample_submission.iloc[i][0])
 file_paths.append(img_path + str(int(sample_submission.iloc[i][0])) +'.jpg')
 test_names = np.array(test_names )
def removeSpecialChar(text):
sentence = []
for s in text:
if s == " ":
sentence.append(s)
 if s.isalnum() :
sentence.append(s)
 return "".join(sentence )<categorify>
data_num = len(y)
 random_index = np.random.permutation(data_num)
 x_shuffle = []
y_shuffle = []
for i in range(data_num):
x_shuffle.append(x[random_index[i]])
 y_shuffle.append(y[random_index[i]])
 x = np.array(x_shuffle)
 y = np.array(y_shuffle )
def checkHtml(text):
return bool(BeautifulSoup(text, "html.parser" ).find())
 html_sentence_map = df["text"].apply(checkHtml ).tolist()
if not any(html_sentence_map):
print("No text containing html found!")
 else:
print("There is some html text!" )<categorify>
x_train = x_train.astype('float32')
 x_test = x_test.astype('float32')
 x_train /= 255
x_test /= 255
def remove_accented_chars(text):
new_text =(
 unicodedata.normalize("NFKD", text)
.encode("ascii", "ignore")
.decode("utf-8", "ignore")
)
 return new_text<string_transform>
img_rows, img_cols, img_channel = 224, 224, 3
base_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(img_rows, img_cols, img_channel))
def remove_words_having_not(words=[""]):
for word in words:
if "not" in word:
words.remove(word)
 return words
stopwords_list = stopwords.words("english")
 print(stopwords_list)
 stopwords_list = remove_words_having_not(
 [removeSpecialChar(contractions.fix(word)) for word in stopwords_list] 
)
 print("
---------------Stopwords after preprocessing---------------
")
 print(stopwords_list )<string_transform>
add_model = Sequential()
add_model.add(Flatten(input_shape=base_model.output_shape[1:]))
add_model.add(Dense(256, activation='relu'))
add_model.add(Dense(1, activation='sigmoid'))
model = Model(inputs=base_model.input, outputs=add_model(base_model.output))
model.compile(loss='binary_crossentropy', optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
 model.summary()
def remove_stopwords(text):
new_sentence = ""
for word in text.split() :
if word not in stopwords_list:
new_sentence += word + " "
return new_sentence<string_transform>
batch_size = 32
epochs = 50
train_datagen = ImageDataGenerator(
 rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True)
 train_datagen.fit(x_train)
 history = model.fit_generator(
 train_datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=x_train.shape[0] // batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
)
def remove_numbers(text=""):
new_sentence = ""
for word in text.split() :
num_free_word = "".join([i for i in word if not i.isdigit() ])
 new_sentence += num_free_word + " "
return new_sentence<string_transform>
print("Training loss: {:.2f} / Validation loss: {:.2f}".\
format(history.history['loss'][-1], history.history['val_loss'][-1]))
print("Training accuracy: {:.2f}% / Validation accuracy: {:.2f}%".\
format(100*history.history['acc'][-1], 100*history.history['val_acc'][-1]))
def lemmatize(text):
lemmatizer = WordNetLemmatizer()
new_sentence = ""
for word in text.split() :
lematized_word = lemmatizer.lemmatize(word)
 new_sentence += lematized_word + " "
return new_sentence<string_transform>
test_images = test_images.astype('float32')
 test_images /= 255
def remove_white_space(text):
return " ".join(text.split() )<feature_engineering>
predictions = model.predict(test_images )
<feature_engineering><EOS> 
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 for i, name in enumerate(test_names):
sample_submission.loc[sample_submission['name'] == name, 'invasive'] = predictions[i]
sample_submission.to_csv("submit.csv", index=False )