Repository: Lightning-AI/pytorch-lightning

Issue:
Bugfix/_has_len
Ready for review

## What does this PR do?

function _has_len in trainer/data_loading.py catches 'NotImplementedError' and returns False

When using torchtext.data.Iterator with a parameter batch_size_fn the __len__ function raises a  NotImplementedError which is not caught by _has_len function in pytorch-lightning.
The fix is very simple by just returning False if a NotImplementedError is raised.

This change does not introduce dependencies.

Fixes #2277

# Before submitting

- [X] Was this discussed/approved via a Github issue? (no need for typos and docs improvements)
- [X] Did you read the [contributor guideline](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md), Pull Request section?
- [X] Did you make sure your PR does only one thing, instead of bundling different changes together? Otherwise, we ask you create a separate PR for every change.
- [X] Did you make sure to update the documentation with your changes?
- [X] Did you write any new necessary tests?
- [X] Did you verify new and existing tests pass locally with your changes?

One test failed a "doctest" on my machine, but this was already failing in the master branch before my changes: pytorch_lightning/loggers/trains.py::pytorch_lightning.loggers.trains.TrainsLogger

- [ ] If you made a notable change (that affects users), did you update the [CHANGELOG](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/CHANGELOG.md)?

Not sure if this is a notable fix, it certainly improves working with torchtext and therefore affects users.

<!-- For CHANGELOG separate each item in unreleased section by a blank line to reduce collisions -->

A suggestion for the CHANGELOG:
Fixed how _has_len handels 'NotImplementedError' e.g. raised by torchtext.data.Iterator (#2277)

## PR review    
Anyone in the community is free to review the PR once the tests have passed.     
If we didn't discuss your PR in Github issues there's a high chance it will not be merged.

## Did you have fun?
Make sure you had fun coding 🙃



Discussion:
Great, thanks for merging!

@williamFalcon not sure if that fix warrants a line in the change log. What are your thoughts on this and the suggested line in the PR?

Resolving patch:
diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py
--- a/pytorch_lightning/trainer/data_loading.py
+++ b/pytorch_lightning/trainer/data_loading.py
@@ -285,7 +285,11 @@ def _reset_eval_dataloader(
         # datasets could be none, 1 or 2+
         if len(dataloaders) != 0:
             for i, dataloader in enumerate(dataloaders):
-                num_batches = 0
+                try:
+                    num_batches = len(dataloader)
+                except (TypeError, NotImplementedError):
+                    num_batches = float('inf')
+
                 self._worker_check(dataloader, f'{mode} dataloader {i}')
 
                 # percent or num_steps
@@ -294,8 +298,6 @@ def _reset_eval_dataloader(
                 if num_batches != float('inf'):
                     self._check_batch_limits(f'limit_{mode}_batches')
 
-                    num_batches = len(dataloader)
-
                     # limit num batches either as a percent or num steps
                     if isinstance(limit_eval_batches, float):
                         num_batches = int(num_batches * limit_eval_batches)


Test patch:
diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py
--- a/tests/trainer/test_dataloaders.py
+++ b/tests/trainer/test_dataloaders.py
@@ -358,7 +358,6 @@ def test_mixing_of_dataloader_options(tmpdir, ckpt_path):
         f'`test_dataloaders` not initiated properly, got {trainer.test_dataloaders}'
 
 
-@pytest.mark.skip('TODO: speed up this test')
 def test_train_inf_dataloader_error(tmpdir):
     """Test inf train data loader (e.g. IterableDataset)"""
     model = EvalModelTemplate()
@@ -370,7 +369,6 @@ def test_train_inf_dataloader_error(tmpdir):
         trainer.fit(model)
 
 
-@pytest.mark.skip('TODO: speed up this test')
 def test_val_inf_dataloader_error(tmpdir):
     """Test inf train data loader (e.g. IterableDataset)"""
     model = EvalModelTemplate()
@@ -382,7 +380,6 @@ def test_val_inf_dataloader_error(tmpdir):
         trainer.fit(model)
 
 
-@pytest.mark.skip('TODO: speed up this test')
 def test_test_inf_dataloader_error(tmpdir):
     """Test inf train data loader (e.g. IterableDataset)"""
     model = EvalModelTemplate()
@@ -395,7 +392,6 @@ def test_test_inf_dataloader_error(tmpdir):
 
 
 @pytest.mark.parametrize('check_interval', [50, 1.0])
-@pytest.mark.skip('TODO: speed up this test')
 def test_inf_train_dataloader(tmpdir, check_interval):
     """Test inf train data loader (e.g. IterableDataset)"""
 
@@ -413,7 +409,6 @@ def test_inf_train_dataloader(tmpdir, check_interval):
 
 
 @pytest.mark.parametrize('check_interval', [1.0])
-@pytest.mark.skip('TODO: speed up this test')
 def test_inf_val_dataloader(tmpdir, check_interval):
     """Test inf val data loader (e.g. IterableDataset)"""
 
@@ -604,7 +599,6 @@ def test_val_dataloader_not_implemented_error(tmpdir, check_interval):
         val_check_interval=check_interval,
     )
     result = trainer.fit(model)
-
     # verify training completed
     assert result == 1
 
diff --git a/tests/trainer/test_lr_finder.py b/tests/trainer/test_lr_finder.py
--- a/tests/trainer/test_lr_finder.py
+++ b/tests/trainer/test_lr_finder.py
@@ -133,7 +133,6 @@ def test_call_to_trainer_method(tmpdir):
         'Learning rate was not altered after running learning rate finder'
 
 
-@pytest.mark.skip('TODO: speed up this test')
 def test_accumulation_and_early_stopping(tmpdir):
     """ Test that early stopping of learning rate finder works, and that
         accumulation also works for this feature """
@@ -155,7 +154,7 @@ def test_accumulation_and_early_stopping(tmpdir):
         'Learning rate was not altered after running learning rate finder'
     assert len(lrfinder.results['lr']) == 100, \
         'Early stopping for learning rate finder did not work'
-    assert lrfinder._total_batch_idx == 100 * 2, \
+    assert lrfinder._total_batch_idx == 190, \
         'Accumulation parameter did not work'