Upload split_utils.py with huggingface_hub
Browse files- split_utils.py +2 -2
split_utils.py
CHANGED
@@ -231,13 +231,13 @@ def random_mix_generator(
|
|
231 |
new_stream_name, new_stream_sources, stream_routing, input_streams
|
232 |
):
|
233 |
for old_stream_name in new_stream_sources:
|
234 |
-
|
235 |
random_generator = new_random_generator(sub_seed=old_stream_name)
|
236 |
assert (
|
237 |
old_stream_name in input_streams
|
238 |
), f"'{old_stream_name}' split not found. Possibles options: {input_streams.keys()}"
|
239 |
for item in input_streams[old_stream_name]:
|
240 |
-
choice = random_generator.choices(
|
241 |
if choice == new_stream_name:
|
242 |
yield item
|
243 |
|
|
|
231 |
new_stream_name, new_stream_sources, stream_routing, input_streams
|
232 |
):
|
233 |
for old_stream_name in new_stream_sources:
|
234 |
+
optional_streams, weights = stream_routing[old_stream_name]
|
235 |
random_generator = new_random_generator(sub_seed=old_stream_name)
|
236 |
assert (
|
237 |
old_stream_name in input_streams
|
238 |
), f"'{old_stream_name}' split not found. Possibles options: {input_streams.keys()}"
|
239 |
for item in input_streams[old_stream_name]:
|
240 |
+
choice = random_generator.choices(optional_streams, weights=weights, k=1)[0]
|
241 |
if choice == new_stream_name:
|
242 |
yield item
|
243 |
|