Adding random sampling

mc4/mc4.py

@@ -293,6 +293,8 @@ class Mc4(datasets.GeneratorBasedBuilder):
             self.pp_model = kenlm.Model(self.perplexity_model)
             if self.sampling_method == "gaussian":
                 self.should_keep_doc = self._should_keep_doc_gaussian
+            elif self.sampling_method == "random":
+                self.should_keep_doc = self._should_keep_doc_random
             else:
                 self.should_keep_doc = self._should_keep_doc_gaussian
 
@@ -332,6 +334,9 @@ class Mc4(datasets.GeneratorBasedBuilder):
         weighted_perplexity = factor * np.exp(-9/2*((perplexity-m)/m)**2)
         return np.random.uniform() < weighted_perplexity
 
+    def _should_keep_doc_random(self, doc, factor=None, boundaries=None):
+        return np.random() <= 0.5
+
     def _info(self):
         return datasets.DatasetInfo(
             description=_DESCRIPTION,