Repository: hylang/hy

Issue:
Failure to add two simple integers on the newly released Python 3.5.1
Observe:

``` hy
hy 0.11.0 using CPython(default) 3.5.1 on Linux
=> (+ 2 2)
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 2278, in hy_compile
    compiler = HyASTCompiler(module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 378, in __init__
    load_stdlib()
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 73, in load_stdlib
    mod = importlib.import_module(module)
  File "/usr/local/lib/python3.5/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 986, in _gcd_import
  File "<frozen importlib._bootstrap>", line 969, in _find_and_load
  File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 664, in _load_unlocked
  File "<frozen importlib._bootstrap>", line 634, in _load_backward_compatible
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 180, in load_module
    self.path)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 75, in import_file_to_module
    _ast = import_file_to_ast(fpath, module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 67, in import_file_to_ast
    return hy_compile(import_file_to_hst(fpath), module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 2279, in hy_compile
    result = compiler.compile(tree)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 428, in compile
    ret = self.compile_atom(_type, tree)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 420, in compile_atom
    ret = _compile_table[atom_type](self, atom)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 596, in compile_raw_list
    ret = self._compile_branch(entries)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 483, in _compile_branch
    return _branch(self.compile(expr) for expr in exprs)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 313, in _branch
    results = list(results)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 483, in <genexpr>
    return _branch(self.compile(expr) for expr in exprs)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 440, in compile
    raise_empty(HyCompileError, e, sys.exc_info()[2])
  File "<string>", line 1, in raise_empty
hy.errors.HyCompileError: Internal Compiler Bug 😱
⤷ TypeError: required field "lineno" missing from arg
Compilation traceback:
File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 428, in compile
    ret = self.compile_atom(_type, tree)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 420, in compile_atom
    ret = _compile_table[atom_type](self, atom)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 1767, in compile_expression
    expression = macroexpand(expression, self.module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/macros.py", line 170, in macroexpand
    load_macros(module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/macros.py", line 154, in load_macros
    _import(module)
  File "/usr/local/lib/python3.5/site-packages/hy/macros.py", line 151, in _import
    __import__(module)
  File "<frozen importlib._bootstrap>", line 969, in _find_and_load
  File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 664, in _load_unlocked
  File "<frozen importlib._bootstrap>", line 634, in _load_backward_compatible
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 180, in load_module
    self.path)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 78, in import_file_to_module
    eval(ast_compile(_ast, fpath, "exec"), mod.__dict__)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 43, in ast_compile
    return compile(ast, filename, mode, flags)
=> 
```

I'll be testing `master` (instead of 0.11.0) in a moment, but figured this was worth filing an issue about regardless in case we need to push 0.11.1 or something (if it is already fixed in master).



Discussion:
Looks like it's the same on `master`:

``` hy
hy 0.11.0 using CPython(default) 3.5.1 on Linux
=> (+ 2 2)
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 2585, in hy_compile
    compiler = HyASTCompiler(module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 383, in __init__
    load_stdlib()
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 76, in load_stdlib
    mod = importlib.import_module(module)
  File "/usr/local/lib/python3.5/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 986, in _gcd_import
  File "<frozen importlib._bootstrap>", line 969, in _find_and_load
  File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 664, in _load_unlocked
  File "<frozen importlib._bootstrap>", line 634, in _load_backward_compatible
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 188, in load_module
    self.path)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 76, in import_file_to_module
    _ast = import_file_to_ast(fpath, module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 68, in import_file_to_ast
    return hy_compile(import_file_to_hst(fpath), module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 2586, in hy_compile
    result = compiler.compile(tree)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 433, in compile
    ret = self.compile_atom(_type, tree)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 425, in compile_atom
    ret = _compile_table[atom_type](self, atom)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 627, in compile_raw_list
    ret = self._compile_branch(entries)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 489, in _compile_branch
    return _branch(self.compile(expr) for expr in exprs)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 316, in _branch
    results = list(results)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 489, in <genexpr>
    return _branch(self.compile(expr) for expr in exprs)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 445, in compile
    raise_empty(HyCompileError, e, sys.exc_info()[2])
  File "<string>", line 1, in raise_empty
hy.errors.HyCompileError: Internal Compiler Bug 😱
⤷ TypeError: required field "lineno" missing from arg
Compilation traceback:
File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 433, in compile
    ret = self.compile_atom(_type, tree)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 425, in compile_atom
    ret = _compile_table[atom_type](self, atom)
  File "/usr/local/lib/python3.5/site-packages/hy/compiler.py", line 1992, in compile_expression
    expression = macroexpand(expression, self.module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/macros.py", line 143, in macroexpand
    load_macros(module_name)
  File "/usr/local/lib/python3.5/site-packages/hy/macros.py", line 117, in load_macros
    _import(module)
  File "/usr/local/lib/python3.5/site-packages/hy/macros.py", line 114, in _import
    __import__(module)
  File "<frozen importlib._bootstrap>", line 969, in _find_and_load
  File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 664, in _load_unlocked
  File "<frozen importlib._bootstrap>", line 634, in _load_backward_compatible
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 188, in load_module
    self.path)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 79, in import_file_to_module
    eval(ast_compile(_ast, fpath, "exec"), mod.__dict__)
  File "/usr/local/lib/python3.5/site-packages/hy/importer.py", line 44, in ast_compile
    return compile(ast, filename, mode, flags)
=> 
```

I'm definitely in way over my head here, but it _appears_ that `TypeError: required field "lineno" missing from arg` is the real blocking error (so perhaps a new missing field in our AST generation or something?)

The thing that seems related from the changelog of 3.5.1 is `Issue #25555: Fix parser and AST: fill lineno and col_offset of “arg” node when compiling AST from Python objects.` (https://bugs.python.org/issue25555)

What if you add the `lineno` and `col_offset` fields [here](https://github.com/hylang/hy/blob/master/hy/compiler.py#L2265) and [here](https://github.com/hylang/hy/blob/master/hy/compiler.py#L2268)?

Introducing a list and using `for x in ...` as above?  Having trouble with what to make the values (although testing just setting them both to `0` for now). :innocent:

Adding those fields (set to `0` for now) does appear to fix the explosion:

``` hy
hy 0.11.0 using CPython(default) 3.5.1 on Linux
=> (+ 2 2)
4
=> 
```

So now it appears to be just a matter of knowing what they ought to be set to. :smile:

I think the error was actually from loading `hy.core`, since it's loaded when you first use it.


Resolving patch:
diff --git a/hy/compiler.py b/hy/compiler.py
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -532,7 +532,7 @@ def _parse_lambda_list(self, exprs):
                     raise HyTypeError(expr,
                                       "There can only be one "
                                       "&rest argument")
-                varargs = str(expr)
+                varargs = expr
             elif lambda_keyword == "&key":
                 if type(expr) != HyDict:
                     raise HyTypeError(expr,
@@ -586,7 +586,7 @@ def _parse_lambda_list(self, exprs):
                     raise HyTypeError(expr,
                                       "There can only be one "
                                       "&kwargs argument")
-                kwargs = str(expr)
+                kwargs = expr
 
         return ret, args, defaults, varargs, kwonlyargs, kwonlydefaults, kwargs
 
@@ -2262,10 +2262,14 @@ def compile_function_def(self, expression):
             # list because it's really just an internal parsing thing.
 
             if kwargs:
-                kwargs = ast.arg(arg=kwargs, annotation=None)
+                kwargs = ast.arg(arg=ast_str(kwargs), annotation=None,
+                                 lineno=kwargs.start_line,
+                                 col_offset=kwargs.start_column)
 
             if stararg:
-                stararg = ast.arg(arg=stararg, annotation=None)
+                stararg = ast.arg(arg=ast_str(stararg), annotation=None,
+                                  lineno=stararg.start_line,
+                                  col_offset=stararg.start_column)
 
             # Let's find a better home for these guys.
         else:
@@ -2280,6 +2284,12 @@ def compile_function_def(self, expression):
                                        col_offset=x.start_column)
                               for x in kwonlyargs]
 
+            if kwargs:
+                kwargs = ast_str(kwargs)
+
+            if stararg:
+                stararg = ast_str(stararg)
+
         args = ast.arguments(
             args=args,
             vararg=stararg,


Test patch: