Elron commited on
Commit
e755967
1 Parent(s): f8ed540

Upload splitters.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. splitters.py +3 -3
splitters.py CHANGED
@@ -36,7 +36,7 @@ class SplitRandomMix(Splitter):
36
  def process(self, multi_stream: MultiStream) -> MultiStream:
37
  mapping = {k: parse_random_mix_string(v) for k, v in self.mix.items()}
38
  generators = random_mix_streams(multi_stream, mapping)
39
- return MultiStream.from_generators(generators, streaming=True)
40
 
41
 
42
  class SeparateSplit(Splitter):
@@ -65,7 +65,7 @@ class SeparateSplit(Splitter):
65
  if size:
66
  so_far += size
67
  generators = slice_streams(multi_stream, mapping)
68
- return MultiStream.from_generators(generators, streaming=True)
69
 
70
 
71
  class SliceSplit(Splitter):
@@ -74,7 +74,7 @@ class SliceSplit(Splitter):
74
  def process(self, multi_stream: MultiStream) -> MultiStream:
75
  mapping = {k: parse_slices_string(v) for k, v in self.slices.items()}
76
  generators = slice_streams(multi_stream, mapping)
77
- return MultiStream.from_generators(generators, streaming=True)
78
 
79
 
80
  class Sampler(Artifact):
 
36
  def process(self, multi_stream: MultiStream) -> MultiStream:
37
  mapping = {k: parse_random_mix_string(v) for k, v in self.mix.items()}
38
  generators = random_mix_streams(multi_stream, mapping)
39
+ return MultiStream.from_generators(generators)
40
 
41
 
42
  class SeparateSplit(Splitter):
 
65
  if size:
66
  so_far += size
67
  generators = slice_streams(multi_stream, mapping)
68
+ return MultiStream.from_generators(generators)
69
 
70
 
71
  class SliceSplit(Splitter):
 
74
  def process(self, multi_stream: MultiStream) -> MultiStream:
75
  mapping = {k: parse_slices_string(v) for k, v in self.slices.items()}
76
  generators = slice_streams(multi_stream, mapping)
77
+ return MultiStream.from_generators(generators)
78
 
79
 
80
  class Sampler(Artifact):