Repository: google-research/text-to-text-transfer-transformer

Issue:
Fill-in-the-Blank Text Generation
Hi,

Would it be possible to share the pre-trained models and/or scripts for "Fill-in-the-Blank Text Generation" mentioned in this blog post? https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html


Discussion:
> Hi,
> 
> Would it be possible to share the pre-trained models and/or scripts for "Fill-in-the-Blank Text Generation" mentioned in this blog post? https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html


I don't believe it will be possible to share the pretrained models. However, I can probably add the preprocessor I used. You would need to rebuild the C4 dataset or use something similar to get the best results though.

Resolving patch:
diff --git a/t5/data/preprocessors.py b/t5/data/preprocessors.py
--- a/t5/data/preprocessors.py
+++ b/t5/data/preprocessors.py
@@ -560,6 +560,64 @@ def one_random_break():
   return dataset.unbatch()
 
 
+def fill_in_the_blank_sized(
+    dataset,
+    size_bins=(1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
+    text_key='text',
+    label='fill: '):
+  """Fill in the blank preprocessor that labels blank with a binned size.
+
+  The actual blank size is sampled uniformly from the inclusive range of the min
+  and max bin. The blank is then filled in with the closest bin size to the
+  actual blank size.
+
+  Args:
+    dataset: a tf.data.Dataset, the dataset to preprocess.
+    size_bins: a list, a list of blank sizes to select from when labelling the
+      blank.
+    text_key: a string, the key for the text feature to preprocess in the
+      dataset examples.
+    label: a string, the label to prepend to the inputs.
+
+  Returns:
+    a tf.data.Dataset
+  """
+  bins = sorted(size_bins)
+
+  def my_fn(x):
+    """Apply transformation."""
+    words = x['words']
+    n_words = tf.size(words)
+
+    blank_size = tf.random.uniform(
+        [], minval=bins[0], maxval=tf.math.minimum(n_words, bins[-1]),
+        dtype=tf.dtypes.int32)
+    bin_delta = tf.math.abs(bins - blank_size)
+    bin_ = tf.gather(bins, tf.argmin(bin_delta))
+    blank_start = tf.random.uniform(
+        [], minval=0, maxval=tf.math.maximum(0, n_words-blank_size) + 1,
+        dtype=tf.dtypes.int32)
+
+    pre_blank = tf.strings.reduce_join(words[0:blank_start], separator=' ')
+    post_blank = tf.strings.reduce_join(
+        words[blank_start+blank_size:], separator=' ')
+    blank = tf.strings.format('_{}_', bin_)
+    # We strip to handle cases where blank is at beginning or end.
+    input_ = tf.strings.strip(
+        tf.strings.join([pre_blank, blank, post_blank], ' '))
+    input_ = tf.strings.join([label, input_])
+    target = tf.strings.reduce_join(
+        words[blank_start:blank_start+blank_size], separator=' ')
+    return {
+        'inputs': tf.strings.strip(input_),
+        'targets': tf.strings.strip(target)}
+  dataset = _split_text_to_words(dataset, text_key, min_num_words=2)
+  # Filter out examples with fewer words than the minimum.
+  dataset = dataset.filter(lambda x: tf.size(x['words']) >= bins[0])
+  dataset = dataset.map(my_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
+  return dataset
+
+
 def prefix_lm(dataset,
               text_key='text',
               label='prefix: '):


Test patch:
diff --git a/t5/data/preprocessors_test.py b/t5/data/preprocessors_test.py
--- a/t5/data/preprocessors_test.py
+++ b/t5/data/preprocessors_test.py
@@ -488,6 +488,47 @@ def test_fill_in_the_blank(self):
           [i if i != 'X' else out_split.pop(0) for i in inp])
       self.assertEqual(reconstructed, original)
 
+  def test_fill_in_the_blank_sized(self):
+    def _validate_data(data, valid_bins, og_length=15):
+      # Remove the prefix from the start of the input string
+      self.assertTrue(data['inputs'].startswith('fill: '))
+      inp = data['inputs'].replace('fill: ', '')
+      # Split input into chunks according to blank locations.
+      inp_split = inp.split('_')
+      # Make sure that there is exactly one blank (could be at beginning/end).
+      self.assertLen(inp_split, 3)
+      # Make sure reconstruction is accurate.
+      reconstructed = ''.join([inp_split[0], data['targets']] + inp_split[2:])
+      self.assertEqual(reconstructed, original)
+      # Make sure blank size is correctly chosen.
+      blank_bin = int(inp_split[1])
+      self.assertIn(blank_bin, valid_bins)
+      blank_size = len(data['targets'].split())
+      self.assertGreaterEqual(blank_size, min(og_length, valid_bins[0]))
+      self.assertLessEqual(blank_size, valid_bins[-1])
+      return blank_size, blank_bin
+
+    num_tries = 250
+    original = 'This is a long test with lots of words to see if it works ok.'
+    dataset = tf.data.Dataset.from_tensor_slices(
+        {'text': [original] * num_tries})
+    dataset = prep.fill_in_the_blank_sized(dataset, [1, 4])
+    num_outputs = 0
+    for data in test_utils.dataset_as_text(dataset):
+      blank_size, blank_bin = _validate_data(data, [1, 4])
+      if blank_size <= 2:
+        self.assertEqual(blank_bin, 1)
+      else:
+        self.assertEqual(blank_bin, 4)
+      num_outputs += 1
+    self.assertEqual(num_tries, num_outputs)
+
+    # Check case where bin size is larger than text.
+    dataset = tf.data.Dataset.from_tensor_slices(
+        {'text': [original] * num_tries})
+    dataset = prep.fill_in_the_blank_sized(dataset, [1024])
+    self.assertEmpty(list(test_utils.dataset_as_text(dataset)))
+
   def test_prefix_lm(self):
     num_tries = 100
     original = 'This is a long test with lots of words to see if it works ok.'