Repository: PaddlePaddle/models

Issue:
work around hung
The fix will follow closely. 


Discussion:

Resolving patch:
diff --git a/PaddleNLP/benchmark/transformer/reader.py b/PaddleNLP/benchmark/transformer/reader.py
--- a/PaddleNLP/benchmark/transformer/reader.py
+++ b/PaddleNLP/benchmark/transformer/reader.py
@@ -43,6 +43,12 @@ def create_data_loader(args):
             mode=m, transform_func=transform_func) for m in ["train", "dev"]
     ]
 
+    if args.shuffle or args.shuffle_batch:
+        if args.shuffle_seed == "None" or args.shuffle_seed is None:
+            shuffle_seed = 0
+        else:
+            shuffle_seed = args.shuffle_seed
+
     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,
                       data_source):
         return max(tokens_sofar,
@@ -69,7 +75,8 @@ def _key(size_so_far, minibatch_len):
                 key=trg_key, buffer_size=buffer_size).sort(
                     key=src_key, buffer_size=buffer_size)
         else:
-            sampler = sampler.shuffle()
+            if args.shuffle:
+                sampler = sampler.shuffle(seed=shuffle_seed)
             if args.sort_type == SortType.POOL:
                 buffer_size = args.pool_size
                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)
@@ -83,6 +90,9 @@ def _key(size_so_far, minibatch_len):
         if m == "train":
             batch_sampler = batch_sampler.shard()
 
+        if args.shuffle_batch:
+            batch_sampler.shuffle(seed=shuffle_seed)
+
         data_loader = DataLoader(
             dataset=dataset,
             batch_sampler=batch_sampler,
diff --git a/PaddleNLP/examples/machine_translation/transformer/reader.py b/PaddleNLP/examples/machine_translation/transformer/reader.py
--- a/PaddleNLP/examples/machine_translation/transformer/reader.py
+++ b/PaddleNLP/examples/machine_translation/transformer/reader.py
@@ -43,6 +43,12 @@ def create_data_loader(args):
             mode=m, transform_func=transform_func) for m in ["train", "dev"]
     ]
 
+    if args.shuffle or args.shuffle_batch:
+        if args.shuffle_seed == "None" or args.shuffle_seed is None:
+            shuffle_seed = 0
+        else:
+            shuffle_seed = args.shuffle_seed
+
     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,
                       data_source):
         return max(tokens_sofar,
@@ -69,7 +75,8 @@ def _key(size_so_far, minibatch_len):
                 key=trg_key, buffer_size=buffer_size).sort(
                     key=src_key, buffer_size=buffer_size)
         else:
-            sampler = sampler.shuffle()
+            if args.shuffle:
+                sampler = sampler.shuffle(seed=shuffle_seed)
             if args.sort_type == SortType.POOL:
                 buffer_size = args.pool_size
                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)
@@ -83,6 +90,9 @@ def _key(size_so_far, minibatch_len):
         if m == "train":
             batch_sampler = batch_sampler.shard()
 
+        if args.shuffle_batch:
+            batch_sampler.shuffle(seed=shuffle_seed)
+
         data_loader = DataLoader(
             dataset=dataset,
             batch_sampler=batch_sampler,


Test patch: