Repository: hylang/hy

Issue:
Python 3.7 support
Heh, instead of finishing the type annotation work, I've gone down the rabbit hole of seeing how it works on Python 3.7:

Known issues:

- [ ] importer is broken as the pyc header data structure has changed, adding a flags field to enable hash based checking on top of timestamp: PEP 552. Emitting another long of 0s preserved the legacy behaviour.
- [ ] docstrings are no longer the first statement of a function, they're stored in the AST: bpo-29463
- [ ] something broke in macro expansion (test_bin_hy_byte_compile breaks, not sure why)
- [ ] astor breaks because it uses `async` as a keyword argument https://github.com/berkerpeksag/astor/issues/86

Beyond that, all other unit tests passed.


Discussion:
> astor breaks because it uses `async` as a keyword argument

We should probably give the Astor folks an issue for that, if they don't already have one.
Yeah, was planning on doing it later tonight. Just got some work stuff I need to wrap up first
We should consider Python 3.7 support, and testing of it on Travis, a blocker for 0.15.0. It'll be one of the first things I work on next.

Resolving patch:
diff --git a/hy/compiler.py b/hy/compiler.py
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -12,7 +12,7 @@
 
 import hy.macros
 from hy._compat import (
-    str_type, string_types, bytes_type, long_type, PY3, PY35,
+    str_type, string_types, bytes_type, long_type, PY3, PY35, PY37,
     raise_empty)
 from hy.macros import require, macroexpand, tag_macroexpand
 import hy.importer
@@ -1949,10 +1949,11 @@ def compile_function_def(self, expression):
                 ) + expression
                 expression = expression.replace(arg[0])
 
-        # Docstrings must come at the start, so ensure that happens even if we
-        # generate anonymous variables.
-        if docstring is not None:
+        # Before Python 3.7, docstrings must come at the start, so ensure that
+        # happens even if we generate anonymous variables.
+        if docstring is not None and not PY37:
             expression.insert(0, docstring)
+            docstring = None
 
         if PY3:
             # Python 3.4+ requires that args are an ast.arg object, rather
@@ -1989,7 +1990,7 @@ def compile_function_def(self, expression):
             defaults=defaults)
 
         body = self._compile_branch(expression)
-        if not force_functiondef and not body.stmts:
+        if not force_functiondef and not body.stmts and docstring is None:
             ret += asty.Lambda(expression, args=args, body=body.force_expr)
             return ret
 
@@ -2012,7 +2013,9 @@ def compile_function_def(self, expression):
                     name=name,
                     args=args,
                     body=body.stmts,
-                    decorator_list=[])
+                    decorator_list=[],
+                    docstring=(None if docstring is None else
+                        str_type(docstring)))
 
         ast_name = asty.Name(expression, id=name, ctx=ast.Load())
 
@@ -2225,9 +2228,16 @@ def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
     if not get_expr:
         result += result.expr_as_stmt()
 
+    module_docstring = None
+    if (PY37 and result.stmts and
+            isinstance(result.stmts[0], ast.Expr) and
+            isinstance(result.stmts[0].value, ast.Str)):
+        module_docstring = result.stmts.pop(0).value.s
+
     body = compiler.imports_as_stmts(tree) + result.stmts
 
-    ret = root(body=body)
+    ret = root(body=body, docstring=(
+        None if module_docstring is None else module_docstring))
 
     if get_expr:
         expr = ast.Expression(body=expr)
diff --git a/hy/importer.py b/hy/importer.py
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -77,6 +77,9 @@ def import_file_to_module(module_name, fpath, loader=None):
             # The first 4 bytes are the magic number for the version of Python
             # that compiled this bytecode.
             bytecode_magic = bc_f.read(4)
+            # Python 3.7 introduced a new flags entry in the header structure.
+            if PY37:
+                bc_f.read(4)
             # The next 4 bytes, interpreted as a little-endian 32-bit integer,
             # are the mtime of the corresponding source file.
             bytecode_mtime, = struct.unpack('<i', bc_f.read(4))
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ def run(self):
                         "." + filename[:-len(".hy")])
         install.run(self)
 
-install_requires = ['rply>=0.7.5', 'astor>=0.6', 'clint>=0.4']
+install_requires = ['rply>=0.7.5', 'astor', 'clint>=0.4']
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 
@@ -40,6 +40,9 @@ def run(self):
     name=PKG,
     version=__version__,
     install_requires=install_requires,
+    dependency_links=[
+        'git+https://github.com/berkerpeksag/astor.git#egg=astor-0.7.0'
+    ],
     cmdclass=dict(install=Install),
     entry_points={
         'console_scripts': [
@@ -81,6 +84,7 @@ def run(self):
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Topic :: Software Development :: Code Generators",
         "Topic :: Software Development :: Compilers",
         "Topic :: Software Development :: Libraries",


Test patch:
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -53,7 +53,7 @@ def cant_compile(expr):
 
 
 def s(x):
-    return can_compile(x).body[0].value.s
+    return can_compile('"module docstring" ' + x).body[-1].value.s
 
 
 def test_ast_bad_type():
@@ -476,13 +476,12 @@ def test_ast_unicode_strings():
 
     def _compile_string(s):
         hy_s = HyString(s)
-        hy_s.start_line = hy_s.end_line = 0
-        hy_s.start_column = hy_s.end_column = 0
 
-        code = hy_compile(hy_s, "__main__")
+        code = hy_compile([hy_s], "__main__")
+        # We put hy_s in a list so it isn't interpreted as a docstring.
 
-        # code == ast.Module(body=[ast.Expr(value=ast.Str(s=xxx))])
-        return code.body[0].value.s
+        # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))])
+        return code.body[0].value.elts[0].s
 
     assert _compile_string("test") == "test"
     assert _compile_string("\u03b1\u03b2") == "\u03b1\u03b2"
diff --git a/tests/native_tests/contrib/hy_repr.hy b/tests/native_tests/contrib/hy_repr.hy
--- a/tests/native_tests/contrib/hy_repr.hy
+++ b/tests/native_tests/contrib/hy_repr.hy
@@ -3,7 +3,7 @@
 ;; license. See the LICENSE.
 
 (import
-  [hy._compat [PY3 PY36]]
+  [hy._compat [PY3 PY36 PY37]]
   [math [isnan]]
   [hy.contrib.hy-repr [hy-repr hy-repr-register]])
 
@@ -163,8 +163,8 @@
   (setv mo (re.search "b+" "aaaabbbccc"))
   (assert (= (hy-repr mo)
     (.format
-      #[[<{}.SRE_Match object; :span {} :match "bbb">]]
-      (. (type mo) __module__)
+      #[[<{} object; :span {} :match "bbb">]]
+      (if PY37 "re.Match" (+ (. (type mo) __module__) ".SRE_Match"))
       (if PY3 "(, 4 7)" "(, (int 4) (int 7))")))))
 
 (defn test-hy-repr-custom []
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -11,7 +11,7 @@
         pytest)
 (import sys)
 
-(import [hy._compat [PY3 PY35]])
+(import [hy._compat [PY3 PY35 PY37]])
 
 (defn test-sys-argv []
   "NATIVE: test sys.argv"
@@ -1606,18 +1606,21 @@
 
 (defn test-disassemble []
   "NATIVE: Test the disassemble function"
-  (if PY35
-    (assert (= (disassemble '(do (leaky) (leaky) (macros)))
-               "Module(
+  (assert (= (disassemble '(do (leaky) (leaky) (macros))) (cond
+    [PY37 "Module(
     body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
-        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))])"))
-    (assert (= (disassemble '(do (leaky) (leaky) (macros)))
-               "Module(
+        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))],
+    docstring=None)"]
+    [PY35 "Module(
+    body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
+        Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
+        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))])"]
+    [True "Module(
     body=[
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),
-        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[], starargs=None, kwargs=None))])")))
+        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[], starargs=None, kwargs=None))])"])))
   (assert (= (disassemble '(do (leaky) (leaky) (macros)) True)
              "leaky()
 leaky()
@@ -1785,6 +1788,11 @@ macros()
   (assert (none? (. f4 __doc__)))
   (assert (= (f4 [1 2]) "not a docstring")))
 
+(defn test-module-docstring []
+  (import [tests.resources.module-docstring-example :as m])
+  (assert (= m.__doc__ "This is the module docstring."))
+  (assert (= m.foo 5)))
+
 (defn test-relative-import []
   "Make sure relative imports work properly"
   (import [..resources [tlib]])
diff --git a/tests/resources/module_docstring_example.hy b/tests/resources/module_docstring_example.hy
new file mode 100644
--- /dev/null
+++ b/tests/resources/module_docstring_example.hy
@@ -0,0 +1,3 @@
+"This is the module docstring."
+
+(setv foo 5)