Repository: numpy/numpy

Issue:
numpy 1.12.0 seems to break f2py
I now get lots of output of the form

```
(...)
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c: At top level:
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:1052:21: error: expected expression before â,â token
   {"namezone",2,{{++,8}},NPY_CHAR},
                     ^
//tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2468:15: note: (near initialization for âf2py_vars_def[848].dims.d[0]â)
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2474:18: error: expected expression before â,â token
   {"ionbmax",2,{{,8}},NPY_CHAR},
                  ^
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2474:18: error: initializer element is not constant
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2474:18: note: (near initialization for âf2py_vars_def[854].dims.d[0]â)
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2501:15: error: invalid type argument of unary â*â (have âintâ)
   {"dbuf",1,{{*(+2)}},NPY_DOUBLE},
               ^~~~~
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2501:15: error: initializer element is not constant
   {"dbuf",1,{{*(+2)}},NPY_DOUBLE},
               ^
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2501:15: note: (near initialization for âf2py_vars_def[881].dims.d[0]â)
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:2503:18: error: expected expression before â}â token
   {"ppnold",1,{{*}},NPY_DOUBLE},
                  ^
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:1667:19: error: wrong type argument to unary minus
   {"idumhead",1,{{-}},NPY_INT},
                   ^
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:3547:14: note: (near initialization for âf2py_zbrn_def[13].dims.d[0]â)
/tmp/tmp8i0j4nc_/src.linux-x86_64-3.6/_keplermodule.c:3699:19: error: expected expression before â,â token
   {"uuidhist",3,{{,,16}},NPY_CHAR},
                   ^
(...)
error: Command "gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/tmp/tmpgnfiujz_/src.linux-x86_64-3.6 -I/home/alex/Python/lib/python3.6/site-packages/numpy/core/include -I/home/alex/Python/include/python3.6m -c /tmp/tmpgnfiujz_/src.linux-x86_64-3.6/_keplermodule.c -o /tmp/tmpgnfiujz_/tmp/tmpgnfiujz_/src.linux-x86_64-3.6/_keplermodule.o" failed with exit status 1
```

in f2py in NumPy version 1.12.0 from PyPi and I can no longer compile the code.  For comparison, all worked fine in NumPy 1.11.3.

I am using Python 3.6.0, compiled with optimisation from the sources on Python.org on Fedora 25, gcc version 6.3.1 20161221 (Red Hat 6.3.1-1).

Seems to me something in the parsing is no longer working, not sure about the origin of all the special characters in the output either.


f2py: parameter type inference fails because incomplete handling of kind specs
The following valid code snippet originally from pymc/gibbsit.f  (cf  https://github.com/pymc-devs/pymc/blob/master/pymc/gibbsit.f )  exhibits the problem: 
```
...
      program   gibbsit

      integer   maxiterates
      parameter (maxiterates=50000)

      integer   maxseries
      parameter (maxseries=20)

      integer   maxqcnt
      parameter (maxqcnt=20)

      integer   wasize
      parameter (wasize=maxiterates*2)
```
showing up for f2py version 2 in numpy 1.12.0 as: 
```
   Line #196 in pymc/gibbsit.f:"      parameter (wasize=maxiterates*2)"
        get_parameters: got "unexpected EOF while parsing (<string>, line 0)" on ''
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/tmp/pip-build-en5WK5/pymc/setup.py", line 124, in <module>
        **(config_dict))
      File "/usr/local/lib/python2.7/dist-packages/numpy/distutils/core.py", line 169, in setup
        return old_setup(**new_attr)
      File "/usr/lib/python2.7/distutils/core.py", line 151, in setup
        dist.run_commands()
      File "/usr/lib/python2.7/distutils/dist.py", line 953, in run_commands
        self.run_command(cmd)
      File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command
        cmd_obj.run()
      File "/usr/local/lib/python2.7/dist-packages/numpy/distutils/command/egg_info.py", line 18, in run
        self.run_command("build_src")
      File "/usr/lib/python2.7/distutils/cmd.py", line 326, in run_command
        self.distribution.run_command(command)
      File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command
        cmd_obj.run()
      File "/usr/local/lib/python2.7/dist-packages/numpy/distutils/command/build_src.py", line 148, in run
        self.build_sources()
      File "/usr/local/lib/python2.7/dist-packages/numpy/distutils/command/build_src.py", line 165, in build_sources
        self.build_extension_sources(ext)
      File "/usr/local/lib/python2.7/dist-packages/numpy/distutils/command/build_src.py", line 327, in build_extension_sources
        sources = self.f2py_sources(sources, ext)
      File "/usr/local/lib/python2.7/dist-packages/numpy/distutils/command/build_src.py", line 564, in f2py_sources
        ['-m', ext_name]+f_sources)
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/f2py2e.py", line 408, in run_main
        postlist = callcrackfortran(files, options)
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/f2py2e.py", line 329, in callcrackfortran
        postlist = crackfortran.crackfortran(files)
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/crackfortran.py", line 3245, in crackfortran
        readfortrancode(files, crackline)
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/crackfortran.py", line 511, in readfortrancode
        dowithline(finalline)
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/crackfortran.py", line 782, in crackline
        analyzeline(m, pat[1], line)
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/crackfortran.py", line 1188, in analyzeline
        t = determineexprtype(initexpr, params)
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/crackfortran.py", line 2971, in determineexprtype
        return _ensure_exprdict(vars[e])
      File "/usr/local/lib/python2.7/dist-packages/numpy/f2py/crackfortran.py", line 2947, in _ensure_exprdict
        raise AssertionError(repr(r))
    AssertionError: ''
```

a pull request with a proposal to fix it will follow.


Discussion:
Can you share the Fortran source?
The FORTRAN code of this project is too big to be useful.
I would have to make an isolated test case, but was hoping there is something that may pop up immediately wheb the f2py code from 1.11.3 and 1.12.0 is compared.
If it's a general issue that always occurs, it should have been already 
noticed in prereleases. A possibility is that this is Python 3.6 only, 
but also then there's the f2py test suite which presumably does not 
catch it.

can you bisect numpy to get the commit that caused it? might be an easier first step to finding the issue from your big source.
No, I have the same trouble with python 3.5.2 using numpy 1.12.0 whereas 1.11.3 works and all ese is the same.  It uniquely seems to be the upgraded numpy version.
A number of f2py-using codebases work for me with Numpy 1.12.0, so 
what's broken is probably some feature that does not appear in them.
.
There are only a few changes in f2py, d0eea4a784ac23 and 
d156cafb3b9163d73 look most likely.

yes, it seems they are all related to character arrays

```NPY_CHAR```

something of the format

```
      INTEGER, PARAMETER :: int32 = SELECTED_INT_KIND(8)
      integer(kind=int32), parameter :: ntdatv = 52
      character*10
     &     unittime(ntdatv)
      common/namecom/
     &     unittime
```
Or, it's due to parsing of PARAMETERs. Minimal repro:
```
! foo.f90
! f2py -c foo.f90
subroutine foo(x)
       integer, parameter :: k = 52
       integer x(k)
end subroutine
```
So d0eea4a784a is possibly at fault.

f2py is also broken in Python 2.7.12 and it looks like it is indeed due to PARAMETER parsing. Lines in the generated pyf that used to look like this:
`double precision dimension(278,9) :: elox`
now look like this:
`double precision dimension(,) :: elox`
when the array dimensions are PARAMETERs in the Fortran. I get a bunch of warnings about the PARAMETER processing like:
"can't multiply sequence by non-int of type 'str'" in evaluating 'mxx*mxy'
"cannot concatenate 'str' and 'int' objects" in evaluating 'ng+3'
"unsupported operand type(s) for -: 'str' and 'int'" in evaluating 'int((np-3)/2)'


Resolving patch:
diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py
--- a/numpy/f2py/crackfortran.py
+++ b/numpy/f2py/crackfortran.py
@@ -2451,7 +2451,8 @@ def get_parameters(vars, global_params={}):
                 if not selected_kind_re.match(v):
                     v_ = v.split('_')
                     # In case there are additive parameters
-                    v = ''.join(v_[:-1]).lower().replace(v_[-1].lower(), '')
+                    if len(v_) > 1: 
+                        v = ''.join(v_[:-1]).lower().replace(v_[-1].lower(), '')
 
             # Currently this will not work for complex numbers.
             # There is missing code for extracting a complex number,


Test patch:
diff --git a/numpy/f2py/tests/src/parameter/constant_compound.f90 b/numpy/f2py/tests/src/parameter/constant_compound.f90
new file mode 100644
--- /dev/null
+++ b/numpy/f2py/tests/src/parameter/constant_compound.f90
@@ -0,0 +1,15 @@
+! Check that parameters are correct intercepted.
+! Constants with comma separations are commonly
+! used, for instance Pi = 3._dp
+subroutine foo_compound_int(x)
+  implicit none
+  integer, parameter :: ii = selected_int_kind(9)
+  integer(ii), intent(inout) :: x
+  dimension x(3)
+  integer(ii), parameter :: three = 3_ii
+  integer(ii), parameter :: two = 2_ii
+  integer(ii), parameter :: six = three * 1_ii * two
+
+  x(1) = x(1) + x(2) + x(3) * six
+  return
+end subroutine
diff --git a/numpy/f2py/tests/src/parameter/constant_non_compound.f90 b/numpy/f2py/tests/src/parameter/constant_non_compound.f90
new file mode 100644
--- /dev/null
+++ b/numpy/f2py/tests/src/parameter/constant_non_compound.f90
@@ -0,0 +1,23 @@
+! Check that parameters are correct intercepted.
+! Specifically that types of constants without 
+! compound kind specs are correctly inferred
+! adapted Gibbs iteration code from pymc 
+! for this test case 
+subroutine foo_non_compound_int(x)
+  implicit none
+  integer, parameter :: ii = selected_int_kind(9)
+
+  integer(ii)   maxiterates
+  parameter (maxiterates=2)
+
+  integer(ii)   maxseries
+  parameter (maxseries=2)
+
+  integer(ii)   wasize
+  parameter (wasize=maxiterates*maxseries)
+  integer(ii), intent(inout) :: x
+  dimension x(wasize)
+
+  x(1) = x(1) + x(2) + x(3) + x(4) * wasize
+  return
+end subroutine
diff --git a/numpy/f2py/tests/test_parameter.py b/numpy/f2py/tests/test_parameter.py
--- a/numpy/f2py/tests/test_parameter.py
+++ b/numpy/f2py/tests/test_parameter.py
@@ -18,6 +18,8 @@ class TestParameters(util.F2PyTest):
     sources = [_path('src', 'parameter', 'constant_real.f90'),
                _path('src', 'parameter', 'constant_integer.f90'),
                _path('src', 'parameter', 'constant_both.f90'),
+               _path('src', 'parameter', 'constant_compound.f90'),
+               _path('src', 'parameter', 'constant_non_compound.f90'),
     ]
 
     @dec.slow
@@ -42,6 +44,24 @@ def test_constant_real_double(self):
         self.module.foo_double(x)
         assert_equal(x, [0 + 1 + 2*3, 1, 2])
 
+    @dec.slow
+    def test_constant_compound_int(self):
+        # non-contiguous should raise error
+        x = np.arange(6, dtype=np.int32)[::2]
+        assert_raises(ValueError, self.module.foo_compound_int, x)
+
+        # check values with contiguous array
+        x = np.arange(3, dtype=np.int32)
+        self.module.foo_compound_int(x)
+        assert_equal(x, [0 + 1 + 2*6, 1, 2])
+
+    @dec.slow
+    def test_constant_non_compound_int(self):
+        # check values
+        x = np.arange(4, dtype=np.int32)
+        self.module.foo_non_compound_int(x)
+        assert_equal(x, [0 + 1 + 2 + 3*4, 1, 2, 3])
+
     @dec.slow
     def test_constant_integer_int(self):
         # non-contiguous should raise error