nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: OpenNMT/OpenNMT-tf
Pull request #761 (2020-12-16 06:40:23)

Another type fix to make the training summary JSON serializable
Related to 74e8a965.


Patch:
diff --git a/opennmt/training.py b/opennmt/training.py
index dd238948..57b9a4d8 100644
--- a/opennmt/training.py
+++ b/opennmt/training.py
@@ -107,7 +107,7 @@ class Trainer:
 
         step = iterations.numpy()
         reset_throughput = False
-        self._training_stats.update_on_step(step, loss.numpy())
+        self._training_stats.update_on_step(step, loss)
         if step % report_steps == 0:
           self._training_stats.log(self._is_master)
           reset_throughput = True
@@ -511,6 +511,10 @@ class TrainingStats:
       step: The current training step.
       loss: The loss for this step.
     """
+    # Convert Numpy or Tensor values to Python.
+    step = int(step)
+    loss = float(loss)
+
     self._last_step = step
     self._last_loss = loss
     self._average_loss = (self._average_loss * self._num_updates + loss) / (self._num_updates + 1)


Test patch:
diff --git a/opennmt/tests/training_test.py b/opennmt/tests/training_test.py
index 6189df44..c07a4b92 100644
--- a/opennmt/tests/training_test.py
+++ b/opennmt/tests/training_test.py
@@ -80,10 +80,10 @@ class TrainingTest(tf.test.TestCase):
     stats = training.TrainingStats(model, optimizer, warmup_steps=2)
 
     def _step(source_length, target_length, step, loss):
-      source_features = {"length": source_length}
-      target_features = {"length": target_length}
+      source_features = {"length": tf.constant(source_length, dtype=tf.int32)}
+      target_features = {"length": tf.constant(target_length, dtype=tf.int32)}
       stats.update_on_example(source_features, target_features)
-      stats.update_on_step(step, loss)
+      stats.update_on_step(tf.constant(step, dtype=tf.int64), tf.constant(loss))
 
     def _is_json_serializable(summary):
       json.dumps(summary)
@@ -96,7 +96,7 @@ class TrainingTest(tf.test.TestCase):
     self.assertTrue(_is_json_serializable(summary))
     self.assertEqual(summary["learning_rate"], 1.0)
     self.assertEqual(summary["step"], 10)
-    self.assertEqual(summary["loss"], 9.6)
+    self.assertAllClose(summary["loss"], 9.6)
 
     # Throughput values are ignored in the 2 first steps.
     self.assertEqual(summary["steps_per_sec"], 0)
@@ -122,8 +122,8 @@ class TrainingTest(tf.test.TestCase):
     self.assertTrue(_is_json_serializable(summary))
     self.assertEqual(summary["last_learning_rate"], 1.0)
     self.assertEqual(summary["last_step"], 15)
-    self.assertEqual(summary["last_loss"], 9.4)
-    self.assertEqual(summary["average_loss"], 9.6)
+    self.assertAllClose(summary["last_loss"], 9.4)
+    self.assertAllClose(summary["average_loss"], 9.6)
     self.assertEqual(summary["num_steps"], 3)
Repository: OpenNMT/OpenNMT-tf
Pull request #784 (2021-02-09 10:42:06)

Fix error on 0-length inputs in matmul_with_relative_representations


Patch:
diff --git a/opennmt/layers/transformer.py b/opennmt/layers/transformer.py
index 55cea979..aa608af0 100644
--- a/opennmt/layers/transformer.py
+++ b/opennmt/layers/transformer.py
@@ -106,11 +106,11 @@ def matmul_with_relative_representations(a, b, transpose_b=False):
     Returns:
       Tensor with shape :math:`[B, H, T, T]`.
     """
-    batch, head, time, _ = misc.shape_list(a)
+    batch, head, time, depth = misc.shape_list(a)
     a = tf.transpose(a, perm=[2, 0, 1, 3])
-    a = tf.reshape(a, [time, batch * head, -1])
+    a = tf.reshape(a, [time, batch * head, depth])
     c = tf.matmul(a, b, transpose_b=transpose_b)
-    c = tf.reshape(c, [time, batch, head, -1])
+    c = tf.reshape(c, [time, batch, head, c.shape[-1] or tf.shape(c)[-1]])
     c = tf.transpose(c, perm=[1, 2, 0, 3])
     return c
 


Test patch:
diff --git a/opennmt/tests/transformer_test.py b/opennmt/tests/transformer_test.py
index c37d6fa5..264c36e7 100644
--- a/opennmt/tests/transformer_test.py
+++ b/opennmt/tests/transformer_test.py
@@ -146,6 +146,12 @@ class TransformerTest(tf.test.TestCase):
         mask = tf.sequence_mask([9, 7])
         attention(x, mask=mask)
 
+    def testMultiHeadSelfAttentionRelativePositionsEmpty(self):
+        attention = transformer.MultiHeadAttention(4, 20, maximum_relative_position=6)
+        x = tf.random.uniform([1, 0, 10])
+        mask = tf.sequence_mask([0])
+        attention(x, mask=mask)
+
     def testMultiHeadSelfAttentionRelativePositionsWithCache(self):
         attention = transformer.MultiHeadAttention(4, 20, maximum_relative_position=6)
         x = tf.random.uniform([4, 1, 10])
Repository: OpenNMT/OpenNMT-tf
Pull request #788 (2021-02-12 10:17:06)

Support initializing the decoder output_layer in its constructor


Patch:
diff --git a/README.md b/README.md
index 8a2c7129..7c67a150 100644
--- a/README.md
+++ b/README.md
@@ -154,8 +154,7 @@ runner.train(num_devices=2, with_eval=True)
 Here is another example using OpenNMT-tf to run efficient beam search with a self-attentional decoder:
 
 ```python
-decoder = opennmt.decoders.SelfAttentionDecoder(num_layers=6)
-decoder.initialize(vocab_size=32000)
+decoder = opennmt.decoders.SelfAttentionDecoder(num_layers=6, vocab_size=32000)
 
 initial_state = decoder.initial_state(
     memory=memory, memory_sequence_length=memory_sequence_length
diff --git a/opennmt/decoders/decoder.py b/opennmt/decoders/decoder.py
index 905458e0..cf566d0a 100644
--- a/opennmt/decoders/decoder.py
+++ b/opennmt/decoders/decoder.py
@@ -62,11 +62,17 @@ def get_sampling_probability(step, read_probability=None, schedule_type=None, k=
 class Decoder(tf.keras.layers.Layer):
     """Base class for decoders."""
 
-    def __init__(self, num_sources=1, **kwargs):
+    def __init__(self, num_sources=1, vocab_size=None, output_layer=None, **kwargs):
         """Initializes the decoder parameters.
 
+        If you don't set one of :obj:`vocab_size` or `output_layer` here,
+        you should later call the method :meth:`opennmt.decoders.Decoder.initialize`
+        to initialize this decoder instance.
+
         Args:
           num_sources: The number of source contexts expected by this decoder.
+          vocab_size: The output vocabulary size (optional if :obj:`output_layer` is set).
+          output_layer: The output projection layer (optional).
           **kwargs: Additional layer arguments.
 
         Raises:
@@ -84,6 +90,8 @@ class Decoder(tf.keras.layers.Layer):
         self.output_layer = None
         self.memory = None
         self.memory_sequence_length = None
+        if vocab_size is not None or output_layer is not None:
+            self.initialize(vocab_size=vocab_size, output_layer=output_layer)
 
     @property
     def minimum_sources(self):
@@ -116,6 +124,8 @@ class Decoder(tf.keras.layers.Layer):
         Raises:
           ValueError: if both :obj:`vocab_size` and :obj:`output_layer` are not set.
         """
+        if self.initialized:
+            return
         if output_layer is not None:
             self.output_layer = output_layer
         else:


Test patch:
diff --git a/opennmt/tests/decoder_test.py b/opennmt/tests/decoder_test.py
index 4055145d..c03051ec 100644
--- a/opennmt/tests/decoder_test.py
+++ b/opennmt/tests/decoder_test.py
@@ -149,8 +149,13 @@ class DecoderTest(tf.test.TestCase):
 
     def testSelfAttentionDecoder(self):
         decoder = decoders.SelfAttentionDecoder(
-            2, num_units=6, num_heads=2, ffn_inner_dim=12
+            num_layers=2,
+            num_units=6,
+            num_heads=2,
+            ffn_inner_dim=12,
+            vocab_size=10,
         )
+        self.assertTrue(decoder.initialized)
         self._testDecoder(decoder)
 
     def testSelfAttentionDecoderMultiSource(self):
Repository: OpenNMT/OpenNMT-tf
Pull request #800 (2021-03-02 08:48:51)

Build model in infer_tflite method
Since this method is not calling the model `__call__` entrypoint, it should call `build()` by itself.

See https://github.com/OpenNMT/OpenNMT-tf/pull/789#issuecomment-788556988


Patch:
diff --git a/opennmt/models/sequence_to_sequence.py b/opennmt/models/sequence_to_sequence.py
index 065db29e..6e322305 100644
--- a/opennmt/models/sequence_to_sequence.py
+++ b/opennmt/models/sequence_to_sequence.py
@@ -198,6 +198,8 @@ class SequenceToSequence(model.SequenceGenerator):
         Args:
           ids: A 1-dimensional tensor with the ids of the sentence you want to predict
         """
+        if not self.built:
+            self.build(None)
         ids = tf.expand_dims(ids, axis=0)
         source_inputs = self.features_inputter.tflite_call(ids)
         source_length = tf.convert_to_tensor([tf.math.count_nonzero(ids)])


Test patch:
diff --git a/opennmt/tests/tflite_test.py b/opennmt/tests/tflite_test.py
index 7a04bd32..50d5ab98 100644
--- a/opennmt/tests/tflite_test.py
+++ b/opennmt/tests/tflite_test.py
@@ -18,7 +18,6 @@ def _create_vocab(temp_dir):
 def _make_model(model_template, vocab):
     model = model_template()
     model.initialize(dict(source_vocabulary=vocab, target_vocabulary=vocab))
-    model.create_variables()
     return model
 
 
@@ -37,10 +36,7 @@ def _get_predictions(model, dataset, vocab_path):
 
     _, pred = model(elem)
     pred_ids = tf.squeeze(tokens_to_ids.lookup(pred["tokens"]))
-    tflite_concrete_fn = tf.function(
-        model.infer_tflite,
-        input_signature=[tf.TensorSpec([None], dtype=tf.dtypes.int32, name="ids")],
-    ).get_concrete_function()
+    tflite_concrete_fn = model.tflite_function().get_concrete_function()
     tflite_pred_ids = tflite_concrete_fn(elem_ids)
 
     # Modify tflite ids tensor to be the same size as normal model output
Repository: OpenNMT/OpenNMT-tf
Pull request #857 (2021-07-09 12:28:57)

Test that WordEmbedder can accept a batch of elements


Patch:
diff --git a/opennmt/inputters/text_inputter.py b/opennmt/inputters/text_inputter.py
index c2b8ae7b..033e0027 100644
--- a/opennmt/inputters/text_inputter.py
+++ b/opennmt/inputters/text_inputter.py
@@ -279,6 +279,7 @@ class TextInputter(Inputter):
             return features
         if "text" in features:
             element = features.pop("text")
+        element = tf.convert_to_tensor(element, dtype=tf.string)
         tokens = self.tokenizer.tokenize(element, training=training)
         if isinstance(tokens, tf.RaggedTensor):
             length = tokens.row_lengths()


Test patch:
diff --git a/opennmt/tests/inputter_test.py b/opennmt/tests/inputter_test.py
index adc2d1fc..8f2f0a4a 100644
--- a/opennmt/tests/inputter_test.py
+++ b/opennmt/tests/inputter_test.py
@@ -347,6 +347,14 @@ class InputterTest(tf.test.TestCase):
         with self.assertRaisesRegex(RuntimeError, "initialize"):
             embedder.make_features("Hello world !")
 
+    def testWordEmbedderBatchElement(self):
+        vocab_file = self._makeTextFile("vocab.txt", list(map(str, range(10))))
+        embedder = text_inputter.WordEmbedder(32)
+        embedder.initialize(dict(vocabulary=vocab_file))
+        features = embedder.make_features(["1 2 3", "1 2 3 4"])
+        self.assertAllEqual(features["length"], [3, 4])
+        self.assertAllEqual(features["ids"].shape.as_list(), [2, 4])
+
     def testCharConvEmbedder(self):
         vocab_file = self._makeTextFile("vocab.txt", ["h", "e", "l", "w", "o"])
         data_file = self._makeTextFile("data.txt", ["hello world !"])
Repository: OpenNMT/OpenNMT-tf
Pull request #863 (2021-07-27 09:33:30)

Fix SequenceRecordInputter.make_features on a batch of elements


Patch:
diff --git a/opennmt/inputters/record_inputter.py b/opennmt/inputters/record_inputter.py
index 8c27b4a9..9c40d1b4 100644
--- a/opennmt/inputters/record_inputter.py
+++ b/opennmt/inputters/record_inputter.py
@@ -39,7 +39,7 @@ class SequenceRecordInputter(Inputter):
             features = {}
         if "tensor" in features:
             return features
-        _, feature_lists = tf.io.parse_single_sequence_example(
+        _, feature_lists, lengths = tf.io.parse_sequence_example(
             element,
             sequence_features={
                 "values": tf.io.FixedLenSequenceFeature(
@@ -48,7 +48,7 @@ class SequenceRecordInputter(Inputter):
             },
         )
         tensor = feature_lists["values"]
-        features["length"] = tf.shape(tensor)[0]
+        features["length"] = lengths["values"]
         features["tensor"] = tf.cast(tensor, self.dtype)
         return features
 


Test patch:
diff --git a/opennmt/tests/inputter_test.py b/opennmt/tests/inputter_test.py
index e07c4b98..fce69f15 100644
--- a/opennmt/tests/inputter_test.py
+++ b/opennmt/tests/inputter_test.py
@@ -702,6 +702,28 @@ class InputterTest(tf.test.TestCase):
         self.assertAllEqual([vector], features["tensor"])
         self.assertAllEqual([vector], transformed)
 
+    def testSequenceRecordBatch(self):
+        vectors = [
+            np.random.rand(3, 2),
+            np.random.rand(6, 2),
+            np.random.rand(1, 2),
+        ]
+
+        record_file = os.path.join(self.get_temp_dir(), "data.records")
+        record_inputter.create_sequence_records(vectors, record_file)
+
+        inputter = record_inputter.SequenceRecordInputter(2)
+        dataset = inputter.make_dataset(record_file)
+        dataset = dataset.batch(3)
+        dataset = dataset.map(inputter.make_features)
+
+        features = next(iter(dataset))
+        lengths = features["length"]
+        tensors = features["tensor"]
+        self.assertAllEqual(lengths, [3, 6, 1])
+        for length, tensor, expected_vector in zip(lengths, tensors, vectors):
+            self.assertAllClose(tensor[:length], expected_vector)
+
     def testSequenceRecordWithCompression(self):
         vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32)
         compression = "GZIP"
Repository: OpenNMT/OpenNMT-tf
Pull request #984 (2022-11-21 23:17:54)

chrF and chrF++
Added scorers for chrF and chrF++ metrics from sacrebleu since they are becoming more and more prevalent in the literature.


Patch:
diff --git a/docs/configuration.md b/docs/configuration.md
index 52fa654b..e465721e 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -301,7 +301,7 @@ eval:
   save_eval_predictions: false
   # (optional) Scorer or list of scorers that are called on the saved evaluation
   # predictions.
-  # Available scorers: bleu, rouge, wer, ter, prf
+  # Available scorers: bleu, rouge, wer, ter, prf, chrf, chrf++
   scorers: bleu
 
   # (optional) The width of the length buckets to select batch candidates from.
diff --git a/opennmt/utils/scorers.py b/opennmt/utils/scorers.py
index 5108dbd4..a27d61e6 100644
--- a/opennmt/utils/scorers.py
+++ b/opennmt/utils/scorers.py
@@ -93,6 +93,34 @@ class BLEUScorer(Scorer):
         return bleu.score
 
 
+@register_scorer(name="chrf")
+class CHRFScorer(Scorer):
+    """Scorer using sacreBLEU."""
+
+    def __init__(self):
+        super().__init__("chrf")
+
+    def __call__(self, ref_path, hyp_path):
+        sys_stream = _get_lines(hyp_path)
+        ref_stream = _get_lines(ref_path)
+        chrf = sacrebleu.corpus_chrf(sys_stream, [ref_stream])
+        return chrf.score
+
+
+@register_scorer(name="chrf++")
+class CHRFPPScorer(Scorer):
+    """Scorer using sacreBLEU."""
+
+    def __init__(self):
+        super().__init__("chrf++")
+
+    def __call__(self, ref_path, hyp_path):
+        sys_stream = _get_lines(hyp_path)
+        ref_stream = _get_lines(ref_path)
+        chrf = sacrebleu.corpus_chrf(sys_stream, [ref_stream], word_order=2)
+        return chrf.score
+
+
 @register_scorer(name="wer")
 class WERScorer(Scorer):
     """WER scorer."""


Test patch:
diff --git a/opennmt/tests/scorers_test.py b/opennmt/tests/scorers_test.py
index 1967a784..9c98a054 100644
--- a/opennmt/tests/scorers_test.py
+++ b/opennmt/tests/scorers_test.py
@@ -22,6 +22,18 @@ class ScorersTest(tf.test.TestCase):
         score = self._run_scorer(scorer, refs, refs)
         self.assertEqual(100, int(score))
 
+    def testCHRFScorer(self):
+        refs = ["Hello world !", "How is it going ?"]
+        scorer = scorers.CHRFScorer()
+        score = self._run_scorer(scorer, refs, refs)
+        self.assertEqual(100, int(score))
+
+    def testCHRFPPScorer(self):
+        refs = ["Hello world !", "How is it going ?"]
+        scorer = scorers.CHRFPPScorer()
+        score = self._run_scorer(scorer, refs, refs)
+        self.assertEqual(100, int(score))
+
     def testROUGEScorer(self):
         refs = ["Hello world !", "How is it going ?"]
         scorer = scorers.ROUGEScorer()