YAML Metadata
Warning:
empty or missing yaml metadata in repo card
(https://huggingface.co/docs/hub/model-cards#model-card-metadata)
Visual Semantic with BERT-CNN
This model can be used to assign an object-to-caption semantic relatedness score, which is valuable for (1) caption diverse re-ranking (this work), and (2) (as an application) generating soft labels for filtering out the related/non-related image-to-post when scraping images from the internet (e.g. Instagram).
To take advantage of the overlapping between the visual context and the caption, and to extract global information from each visual (i.e., object, scene, etc) we use BERT as an embedding layer followed by a shallow CNN (tri-gram kernel) (Kim, 2014).
Please refer to Github for more information.
For datasets that are less than 100K please have look at our shallow model
The model is trained with a strict filter of 0.4 similarity distance thresholds between the object and its related caption.
For a quick start please have a look at this demo
For the dataset
# Result with SoTA pre-trained image Captioning BLIP
Comparison result with BLIP (125M pre-trained images) Table 7 COCO Caption Karpathy testset. For the VilBERT model (3.5M pre-trained images) please refer to the paper.
Accuarcy
| Model | B-1 | B-2 | B-3 | B-4 | M | R | C | S | BERTscore |
|---|---|---|---|---|---|---|---|---|---|
| BLIP Beam Search b=3 | .797 | .649 | .514 | .403 | .311 | .606 | 1.365 | .243 | .9484 |
| + BERT-CNN $th=0$ | .798 | .646 | .506 | .392 | .305 | .598 | 1.339 | .238 | .9473 |
| + BERT-CNN $th\geq0.2$ | .798 | .647 | .507 | .393 | .306 | .600 | 1.342 | .238 | .9473 |
| + BERT-CNN $th\geq0.3$ | .802 | .651 | .511 | .397 | .307 | .601 | 1.349 | .238 | .9479 |
| + BERT-CNN $th\geq0.4$ | .806 | .654 | .513 | .397 | .303 | .599 | 1.343 | .235 | .9476 |
Diversity
| Model | Uniq | Voc | mBLeu-1↓ | Div-1 | Div-2 | SBERT-sts |
|---|---|---|---|---|---|---|
| BLIP Beam Search b=3 | 8.60 | 1406 | .461 | .68 | .80 | .8058 |
| + BERT-CNN $th=0$ | 8.49 | 1532 | .457 | .68 | .80 | .8046 |
| + BERT-CNN $th\geq0.2$ | 8.48 | 1486 | .458 | .68 | .80 | .8052 |
| + BERT-CNN $th\geq0.3$ | 8.41 | 1448 | .458 | .68 | .80 | .8060 |
| + BERT-CNN $th\geq0.4$ | 8.30 | 1448 | .455 | .68 | .80 | .8053 |
| human | 9.14 | 3425 | .375 | .74 | .84 | NA |
conda create -n BERT_visual python=3.6 anaconda
conda activate BERT_visual
pip install tensorflow==1.15.0
pip install --upgrade tensorflow_hub==0.7.0
git clone https://github.com/gaphex/bert_experimental/
import tensorflow as tf
import numpy as np
import pandas as pd
import sys
from sklearn.model_selection import train_test_split
sys.path.insert(0, "bert_experimental")
from bert_experimental.finetuning.text_preprocessing import build_preprocessor
from bert_experimental.finetuning.graph_ops import load_graph
df = pd.read_csv("test.tsv", sep='\t')
texts = []
delimiter = " ||| "
for vis, cap in zip(df.visual.tolist(), df.caption.tolist()):
texts.append(delimiter.join((str(vis), str(cap))))
texts = np.array(texts)
trX, tsX = train_test_split(texts, shuffle=False, test_size=0.01)
restored_graph = load_graph("frozen_graph.pb")
graph_ops = restored_graph.get_operations()
input_op, output_op = graph_ops[0].name, graph_ops[-1].name
print(input_op, output_op)
x = restored_graph.get_tensor_by_name(input_op + ':0')
y = restored_graph.get_tensor_by_name(output_op + ':0')
preprocessor = build_preprocessor("vocab.txt", 64)
py_func = tf.numpy_function(preprocessor, [x], [tf.int32, tf.int32, tf.int32], name='preprocessor')
##predictions
sess = tf.Session(graph=restored_graph)
print(trX[:4])
y = tf.print(y, summarize=-1)
y_out = sess.run(y, feed_dict={
x: trX[:4].reshape((-1,1))
})
print(y_out)
For training and inference
python BERT_CNN.py --train train_0.4.tsv --epochs 5
# -*- coding: utf-8 -*-
#!/bin/env python
import sys
import argparse
import re
import os
import sys
import json
import logging
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow_hub as hub
from BertLayer import BertLayer
from BertLayer import build_preprocessor
from freeze_keras_model import freeze_keras_model
from data_pre import *
from tensorflow import keras
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
from sklearn.model_selection import train_test_split
if not 'bert_repo' in sys.path:
sys.path.insert(0, 'bert_repo')
from modeling import BertModel, BertConfig
from tokenization import FullTokenizer, convert_to_unicode
from extract_features import InputExample, convert_examples_to_features
# get TF logger
log = logging.getLogger('tensorflow')
log.handlers = []
parser=argparse.ArgumentParser()
parser.add_argument('--train', default='train.tsv', help='beam serach', type=str,required=False)
parser.add_argument('--num_bert_layer', default='12', help='truned layers', type=int,required=False)
parser.add_argument('--batch_size', default='128', help='truned layers', type=int,required=False)
parser.add_argument('--epochs', default='5', help='', type=int,required=False)
parser.add_argument('--seq_len', default='64', help='', type=int,required=False)
parser.add_argument('--CNN_kernel_size', default='3', help='', type=int,required=False)
parser.add_argument('--CNN_filters', default='32', help='', type=int,required=False)
args = parser.parse_args()
# Downlaod the pre-trained model
#!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
#!unzip uncased_L-12_H-768_A-12.zip
# tf.Module
def build_module_fn(config_path, vocab_path, do_lower_case=True):
def bert_module_fn(is_training):
"""Spec function for a token embedding module."""
input_ids = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_ids")
input_mask = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_mask")
token_type = tf.placeholder(shape=[None, None], dtype=tf.int32, name="segment_ids")
config = BertConfig.from_json_file(config_path)
model = BertModel(config=config, is_training=is_training,
input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type)
seq_output = model.all_encoder_layers[-1]
pool_output = model.get_pooled_output()
config_file = tf.constant(value=config_path, dtype=tf.string, name="config_file")
vocab_file = tf.constant(value=vocab_path, dtype=tf.string, name="vocab_file")
lower_case = tf.constant(do_lower_case)
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, config_file)
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, vocab_file)
input_map = {"input_ids": input_ids,
"input_mask": input_mask,
"segment_ids": token_type}
output_map = {"pooled_output": pool_output,
"sequence_output": seq_output}
output_info_map = {"vocab_file": vocab_file,
"do_lower_case": lower_case}
hub.add_signature(name="tokens", inputs=input_map, outputs=output_map)
hub.add_signature(name="tokenization_info", inputs={}, outputs=output_info_map)
return bert_module_fn
#MODEL_DIR = "uncased_L-12_H-768_A-12"
config_path = "/{}/bert_config.json".format(MODEL_DIR)
vocab_path = "/{}/vocab.txt".format(MODEL_DIR)
tags_and_args = []
for is_training in (True, False):
tags = set()
if is_training:
tags.add("train")
tags_and_args.append((tags, dict(is_training=is_training)))
module_fn = build_module_fn(config_path, vocab_path)
spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args)
spec.export("bert-module",
checkpoint_path="/{}/bert_model.ckpt".format(MODEL_DIR))
class BertLayer(tf.keras.layers.Layer):
def __init__(self, bert_path, seq_len=64, n_tune_layers=3,
pooling="cls", do_preprocessing=True, verbose=False,
tune_embeddings=False, trainable=True, **kwargs):
self.trainable = trainable
self.n_tune_layers = n_tune_layers
self.tune_embeddings = tune_embeddings
self.do_preprocessing = do_preprocessing
self.verbose = verbose
self.seq_len = seq_len
self.pooling = pooling
self.bert_path = bert_path
self.var_per_encoder = 16
if self.pooling not in ["cls", "mean", None]:
raise NameError(
f"Undefined pooling type (must be either 'cls', 'mean', or None, but is {self.pooling}"
)
super(BertLayer, self).__init__(**kwargs)
def build(self, input_shape):
self.bert = hub.Module(self.build_abspath(self.bert_path),
trainable=self.trainable, name=f"{self.name}_module")
trainable_layers = []
if self.tune_embeddings:
trainable_layers.append("embeddings")
if self.pooling == "cls":
trainable_layers.append("pooler")
if self.n_tune_layers > 0:
encoder_var_names = [var.name for var in self.bert.variables if 'encoder' in var.name]
n_encoder_layers = int(len(encoder_var_names) / self.var_per_encoder)
for i in range(self.n_tune_layers):
trainable_layers.append(f"encoder/layer_{str(n_encoder_layers - 1 - i)}/")
# Add module variables to layer's trainable weights
for var in self.bert.variables:
if any([l in var.name for l in trainable_layers]):
self._trainable_weights.append(var)
else:
self._non_trainable_weights.append(var)
if self.verbose:
print("*** TRAINABLE VARS *** ")
for var in self._trainable_weights:
print(var)
self.build_preprocessor()
self.initialize_module()
super(BertLayer, self).build(input_shape)
def build_abspath(self, path):
if path.startswith("https://") or path.startswith("gs://"):
return path
else:
return os.path.abspath(path)
def build_preprocessor(self):
sess = tf.keras.backend.get_session()
tokenization_info = self.bert(signature="tokenization_info", as_dict=True)
vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],
tokenization_info["do_lower_case"]])
self.preprocessor = build_preprocessor(vocab_file, self.seq_len, do_lower_case)
def initialize_module(self):
sess = tf.keras.backend.get_session()
vars_initialized = sess.run([tf.is_variable_initialized(var)
for var in self.bert.variables])
uninitialized = []
for var, is_initialized in zip(self.bert.variables, vars_initialized):
if not is_initialized:
uninitialized.append(var)
if len(uninitialized):
sess.run(tf.variables_initializer(uninitialized))
def call(self, input):
if self.do_preprocessing:
input = tf.numpy_function(self.preprocessor,
[input], [tf.int32, tf.int32, tf.int32],
name='preprocessor')
for feature in input:
feature.set_shape((None, self.seq_len))
input_ids, input_mask, segment_ids = input
bert_inputs = dict(
input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids
)
output = self.bert(inputs=bert_inputs, signature="tokens", as_dict=True)
if self.pooling == "cls":
pooled = output["pooled_output"]
else:
result = output["sequence_output"]
input_mask = tf.cast(input_mask, tf.float32)
mul_mask = lambda x, m: x * tf.expand_dims(m, axis=-1)
masked_reduce_mean = lambda x, m: tf.reduce_sum(mul_mask(x, m), axis=1) / (
tf.reduce_sum(m, axis=1, keepdims=True) + 1e-10)
if self.pooling == "mean":
pooled = masked_reduce_mean(result, input_mask)
else:
pooled = mul_mask(result, input_mask)
return pooled
def get_config(self):
config_dict = {
"bert_path": self.bert_path,
"seq_len": self.seq_len,
"pooling": self.pooling,
"n_tune_layers": self.n_tune_layers,
"tune_embeddings": self.tune_embeddings,
"do_preprocessing": self.do_preprocessing,
"verbose": self.verbose
}
super(BertLayer, self).get_config()
return config_dict
# read the train data
df = pd.read_csv(args.train, sep='\t')
labels = df.is_related.values
texts = []
delimiter = " ||| "
for vis, cap in zip(df.visual.tolist(), df.caption.tolist()):
texts.append(delimiter.join((str(vis), str(cap))))
texts = np.array(texts)
trX, tsX, trY, tsY = train_test_split(texts, labels, shuffle=True, test_size=0.2)
# Buliding the model
embedding_size = 768
# input
inp = tf.keras.Input(shape=(1,), dtype=tf.string)
# BERT encoder
# For CLS with linear layer
#encoder = BertLayer(bert_path="./bert-module/", seq_len=48, tune_embeddings=False,
# pooling='cls', n_tune_layers=3, verbose=False)
# CNN Layers
encoder = BertLayer(bert_path="./bert-module/", seq_len=args.seq_len, tune_embeddings=False, pooling=None, n_tune_layers=args.num_bert_layer, verbose=False)
cnn_out = tf.keras.layers.Conv1D(args.CNN_filters, args.CNN_kernel_size, padding='VALID', activation=tf.nn.relu)(encoder(inp))
pool = tf.keras.layers.MaxPooling1D(pool_size=2)(cnn_out)
flat = tf.keras.layers.Flatten()(pool)
pred = tf.keras.layers.Dense(1, activation="sigmoid")(flat)
model = tf.keras.models.Model(inputs=[inp], outputs=[pred])
model.summary()
model.compile(
optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5, ),
loss="binary_crossentropy",
metrics=["accuracy"])
# fit the data
import logging
logging.getLogger("tensorflow").setLevel(logging.WARNING)
saver = keras.callbacks.ModelCheckpoint("bert_CNN_tuned.hdf5")
model.fit(trX, trY, validation_data=[tsX, tsY], batch_size=args.batch_size, epochs=args.epochs, callbacks=[saver])
#save the model
model.predict(trX[:10])
import json
json.dump(model.to_json(), open("model.json", "w"))
model = tf.keras.models.model_from_json(json.load(open("model.json")),
custom_objects={"BertLayer": BertLayer})
model.load_weights("bert_CNN_tuned.hdf5")
model.predict(trX[:10])
# For fast inference and less RAM usesage as post-processing we need to "freezing" the model.
from tensorflow.python.framework.graph_util import convert_variables_to_constants
from tensorflow.python.tools.optimize_for_inference_lib import optimize_for_inference
def freeze_keras_model(model, export_path=None, clear_devices=True):
sess = tf.keras.backend.get_session()
graph = sess.graph
with graph.as_default():
input_tensors = model.inputs
output_tensors = model.outputs
dtypes = [t.dtype.as_datatype_enum for t in input_tensors]
input_ops = [t.name.rsplit(":", maxsplit=1)[0] for t in input_tensors]
output_ops = [t.name.rsplit(":", maxsplit=1)[0] for t in output_tensors]
tmp_g = graph.as_graph_def()
if clear_devices:
for node in tmp_g.node:
node.device = ""
tmp_g = optimize_for_inference(
tmp_g, input_ops, output_ops, dtypes, False)
tmp_g = convert_variables_to_constants(sess, tmp_g, output_ops)
if export_path is not None:
with tf.gfile.GFile(export_path, "wb") as f:
f.write(tmp_g.SerializeToString())
return tmp_g
# freeze and save the model
frozen_graph = freeze_keras_model(model, export_path="frozen_graph.pb")
# inference
#!git clone https://github.com/gaphex/bert_experimental/
import tensorflow as tf
import numpy as np
import sys
sys.path.insert(0, "bert_experimental")
from bert_experimental.finetuning.text_preprocessing import build_preprocessor
from bert_experimental.finetuning.graph_ops import load_graph
restored_graph = load_graph("frozen_graph.pb")
graph_ops = restored_graph.get_operations()
input_op, output_op = graph_ops[0].name, graph_ops[-1].name
print(input_op, output_op)
x = restored_graph.get_tensor_by_name(input_op + ':0')
y = restored_graph.get_tensor_by_name(output_op + ':0')
preprocessor = build_preprocessor("vocab.txt", 64)
py_func = tf.numpy_function(preprocessor, [x], [tf.int32, tf.int32, tf.int32], name='preprocessor')
py_func = tf.numpy_function(preprocessor, [x], [tf.int32, tf.int32, tf.int32])
# predictions
sess = tf.Session(graph=restored_graph)
trX[:10]
y_out = sess.run(y, feed_dict={
x: trX[:10].reshape((-1,1))
})
print(y_out)