Repository: huggingface/datasets
File: tests/test_arrow_dataset.py (Python)

Code before:
        # before processing the dataset examples

        class ExampleCounter:
            def __init__(self, batched=False):
                self.batched = batched
                # state
                self.cnt = 0

            def __call__(self, example):
                if self.batched:
                    self.cnt += len(example)
                else:
                    self.cnt += 1

        with tempfile.TemporaryDirectory() as tmp_dir:
            with self._create_dummy_dataset(in_memory, tmp_dir) as dset:
                ex_cnt = ExampleCounter()
                dset.map(ex_cnt)
                self.assertEqual(ex_cnt.cnt, len(dset))

                ex_cnt = ExampleCounter(batched=True)
                dset.map(ex_cnt)
                self.assertEqual(ex_cnt.cnt, len(dset))

    def test_map_crash_subprocess(self, in_memory):
        # be sure that a crash in one of the subprocess will not
        # hang dataset.map() call forever

        def do_crash(row):
            import os

            os.kill(os.getpid(), 9)
            return row

        with tempfile.TemporaryDirectory() as tmp_dir:
            with self._create_dummy_dataset(in_memory, tmp_dir) as dset:
                with pytest.raises(RuntimeError) as excinfo:
                    dset.map(do_crash, num_proc=2)
                assert str(excinfo.value) == (
                    "One of the subprocesses has abruptly died during map operation."
                    "To debug the error, disable multiprocessing."
                )

    def test_filter(self, in_memory):
        # keep only first five examples

        with tempfile.TemporaryDirectory() as tmp_dir:
            with self._create_dummy_dataset(in_memory, tmp_dir) as dset:
                fingerprint = dset._fingerprint
                with dset.filter(lambda x, i: i < 5, with_indices=True) as dset_filter_first_five:
                    self.assertEqual(len(dset_filter_first_five), 5)

Review comment (bug):
maybe skip the test on windows ? it seems to raise a PermissionError
```suggestion
    @require_not_windows
    def test_map_crash_subprocess(self, in_memory):
```

Code after:
        # before processing the dataset examples

        class ExampleCounter:
            def __init__(self, batched=False):
                self.batched = batched
                # state
                self.cnt = 0

            def __call__(self, example):
                if self.batched:
                    self.cnt += len(example)
                else:
                    self.cnt += 1

        with tempfile.TemporaryDirectory() as tmp_dir:
            with self._create_dummy_dataset(in_memory, tmp_dir) as dset:
                ex_cnt = ExampleCounter()
                dset.map(ex_cnt)
                self.assertEqual(ex_cnt.cnt, len(dset))

                ex_cnt = ExampleCounter(batched=True)
                dset.map(ex_cnt)
                self.assertEqual(ex_cnt.cnt, len(dset))

    @require_not_windows
    def test_map_crash_subprocess(self, in_memory):
        # be sure that a crash in one of the subprocess will not
        # hang dataset.map() call forever

        def do_crash(row):
            import os

            os.kill(os.getpid(), 9)
            return row

        with tempfile.TemporaryDirectory() as tmp_dir:
            with self._create_dummy_dataset(in_memory, tmp_dir) as dset:
                with pytest.raises(RuntimeError) as excinfo:
                    dset.map(do_crash, num_proc=2)
                assert str(excinfo.value) == (
                    "One of the subprocesses has abruptly died during map operation."
                    "To debug the error, disable multiprocessing."
                )

    def test_filter(self, in_memory):
        # keep only first five examples

        with tempfile.TemporaryDirectory() as tmp_dir:
            with self._create_dummy_dataset(in_memory, tmp_dir) as dset:
                fingerprint = dset._fingerprint
                with dset.filter(lambda x, i: i < 5, with_indices=True) as dset_filter_first_five: