Repository: stfc/fparser

Issue:
fparser2 logs a 'critical' error when it reaches the end of a file
    (fparser) [@DLHRT0030 code_fragments]$ cat hello.f90 
    program hello
      write(*,*) "Hello."
    end program hello

    (fparser) [code_fragments]$ fparser2 hello.f90 
    File: 'hello.f90'
    CRITICAL:fparser.common.readfortran:While processing 'hello.f90' (mode='free')..
        1:program hello
        2:  write(*,*) "Hello."
        3:end program hello <== while processing line
    CRITICAL:fparser.common.readfortran:STOPPED READING
    PROGRAM hello
      WRITE(*, FMT = *) "Hello."
    END PROGRAM hello


Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5e77548..6b4b42c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,9 @@ Modifications by (in alphabetical order):
 * J. Tiira, University of Helsinki, Finland
 * P. Vitt, University of Siegen, Germany
 
+03/06/2020 PR #263 for #262. Fixes bug in fparser2 logging a 'critical'
+           error when it reaches the end of a file.
+
 03/06/2020 PR #249. Adds support for Fortran2008 CONTIGUOUS and
            CODIMENSION keywords.
 
diff --git a/src/fparser/common/readfortran.py b/src/fparser/common/readfortran.py
index 2409318..60e4ea2 100644
--- a/src/fparser/common/readfortran.py
+++ b/src/fparser/common/readfortran.py
@@ -870,12 +870,18 @@ class FortranReaderBase(object):
 
         Resolves ``;`` statement terminations.
 
-        :param bool ignore_comments: Whether or not to ignore comments
-                                     (overrides self._ignore_comments)
-
         See also
         --------
         next, get_source_item
+
+        :param bool ignore_comments: Whether or not to ignore comments \
+                                     (overrides self._ignore_comments)
+
+        :returns: the next line of Fortran.
+        :rtype: :py:class:`fparser.common.readfortran.Line`
+
+        :raises StopIteration: if no new items are found.
+
         """
         if ignore_comments is None:
             ignore_comments = self._ignore_comments
@@ -887,9 +893,9 @@ class FortranReaderBase(object):
             except IndexError:
                 # construct a new item from source
                 item = self.get_source_item()
-                if item is None:
-                    raise StopIteration
-            if not (item.isempty(ignore_comments)):
+            if item is None:
+                raise StopIteration
+            if not item.isempty(ignore_comments):
                 break
             # else ignore empty lines and comments by getting next line
 
diff --git a/src/fparser/two/C99Preprocessor.py b/src/fparser/two/C99Preprocessor.py
index 8557a9b..4a4fc94 100644
--- a/src/fparser/two/C99Preprocessor.py
+++ b/src/fparser/two/C99Preprocessor.py
@@ -76,7 +76,8 @@ def match_cpp_directive(reader):
     if isinstance(reader, FortranReaderBase):
         item = reader.get_item()
         is_potential_cpp_directive = isinstance(item, CppDirective)
-        reader.put_item(item)
+        if item:
+            reader.put_item(item)
         # Do not bail out early here to have a catch all return statement
         # at the end (that would not be reachable by tests otherwise)
     if is_potential_cpp_directive:


Test patch:
diff --git a/src/fparser/common/tests/test_readfortran.py b/src/fparser/common/tests/test_readfortran.py
index 8af7530..8b75af1 100644
--- a/src/fparser/common/tests/test_readfortran.py
+++ b/src/fparser/common/tests/test_readfortran.py
@@ -745,7 +745,32 @@ def test_file_reader():
         raise
 
 
-##############################################################################
+def test_none_in_fifo(log):
+    ''' Check that a None entry in the reader FIFO buffer is handled
+    correctly. '''
+    log.reset()
+    handle, filename = tempfile.mkstemp(suffix='.f90', text=True)
+    os.close(handle)
+
+    with io.open(filename, mode='w', encoding='UTF-8') as source_file:
+        source_file.write(FULL_FREE_SOURCE)
+
+    with io.open(filename, mode='r', encoding='UTF-8') as source_file:
+        unit_under_test = FortranFileReader(source_file)
+        while True:
+            try:
+                _ = unit_under_test.next(ignore_comments=False)
+            except StopIteration:
+                break
+        # Erroneously push a None to the FIFO buffer
+        unit_under_test.put_item(None)
+        # Attempt to read the next item
+        with pytest.raises(StopIteration):
+            _ = unit_under_test.next(ignore_comments=False)
+        # Check that nothing has been logged
+        for log_level in ["debug", "info", "warning", "error", "critical"]:
+            assert log.messages[log_level] == []
+
 
 def test_bad_file_reader():
     '''
diff --git a/src/fparser/two/tests/test_scripts.py b/src/fparser/two/tests/test_scripts.py
index 7225b7f..3770e87 100644
--- a/src/fparser/two/tests/test_scripts.py
+++ b/src/fparser/two/tests/test_scripts.py
@@ -275,10 +275,11 @@ def test_runner_multi_output_except(tmpdir, capsys):
 # fparser2.py script function main()
 
 
-def test_main_output_task_default(tmpdir, capsys, monkeypatch):
+def test_main_output_task_default(tmpdir, capsys, monkeypatch, log):
     '''Test that the script main() function outputs the code it has parsed
-    by default.'''
+    by default and that no errors are logged.'''
     import sys
+    log.reset()
     # Create a temporary file containing Fortran code to pass into runner()
     my_file = tmpdir.mkdir("sub").join("hello.f90")
     my_file.write("program hello\nend program hello\n")
@@ -290,6 +291,9 @@ def test_main_output_task_default(tmpdir, capsys, monkeypatch):
     stdout, stderr = capsys.readouterr()
     assert stdout == "PROGRAM hello\nEND PROGRAM hello\n"
     assert "File: '" in stderr and "hello.f90'" in stderr
+    # Check that nothing has been logged (apart from some 'debug' msgs)
+    for log_level in ["info", "warning", "error", "critical"]:
+        assert log.messages[log_level] == []
 
 
 def test_main_output_task_show(tmpdir, capsys, monkeypatch):