Repository: pypa/setuptools (python)

Issue:
setup.py develop fails to install non ascii scripts
Using a minimal setup.py:

``` python
#!/usr/bin/python
from setuptools import setup
setup(name="test",
      scripts=['bin/non_ascii'],
      )
```

With bin/non_ascii as:

``` sh
#!/bin/sh
# á

non_python_fn() {
}
```

setup develop fails with:

``` sh
$ ./setup.py develop --user
running develop
running egg_info
writing test.egg-info/PKG-INFO
writing top-level names to test.egg-info/top_level.txt
writing dependency_links to test.egg-info/dependency_links.txt
reading manifest file 'test.egg-info/SOURCES.txt'
writing manifest file 'test.egg-info/SOURCES.txt'
running build_ext
Creating /home/user/.local/lib/python2.7/site-packages/test.egg-link (link to .)
test 0.0.0 is already the active version in easy-install.pth
Installing non_ascii script to /home/user/.local/bin
Traceback (most recent call last):
  File "./setup.py", line 5, in <module>
    scripts=['bin/non_ascii'],
  File "/usr/lib/python2.7/distutils/core.py", line 151, in setup
    dist.run_commands()
  File "/usr/lib/python2.7/distutils/dist.py", line 953, in run_commands
    self.run_command(cmd)
  File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command
    cmd_obj.run()
  File "/usr/lib/python2.7/dist-packages/setuptools/command/develop.py", line 34, in run
    self.install_for_development()
  File "/usr/lib/python2.7/dist-packages/setuptools/command/develop.py", line 133, in install_for_development
    self.process_distribution(None, self.dist, not self.no_deps)
  File "/usr/lib/python2.7/dist-packages/setuptools/command/easy_install.py", line 720, in process_distribution
    self.install_egg_scripts(dist)
  File "/usr/lib/python2.7/dist-packages/setuptools/command/develop.py", line 169, in install_egg_scripts
    self.install_script(dist, script_name, script_text, script_path)
  File "/usr/lib/python2.7/dist-packages/setuptools/command/easy_install.py", line 798, in install_script
    self.write_script(script_name, _to_ascii(script_text), 'b')
  File "/usr/lib/python2.7/dist-packages/setuptools/command/easy_install.py", line 830, in write_script
    f.write(contents)
UnicodeEncodeError: 'ascii' codec can't encode character u'\xe1' in position 12: ordinal not in range(128)
```

Same goes for python3:

``` sh
$ python3 ./setup.py develop --user
running develop
running egg_info
writing dependency_links to test.egg-info/dependency_links.txt
writing test.egg-info/PKG-INFO
writing top-level names to test.egg-info/top_level.txt
reading manifest file 'test.egg-info/SOURCES.txt'
writing manifest file 'test.egg-info/SOURCES.txt'
running build_ext
Creating /home/user/.local/lib/python3.5/site-packages/test.egg-link (link to .)
Adding test 0.0.0 to easy-install.pth file
Traceback (most recent call last):
  File "./setup.py", line 5, in <module>
    scripts=['bin/non_ascii'],
  File "/usr/lib/python3.5/distutils/core.py", line 148, in setup
    dist.run_commands()
  File "/usr/lib/python3.5/distutils/dist.py", line 955, in run_commands
    self.run_command(cmd)
  File "/usr/lib/python3.5/distutils/dist.py", line 974, in run_command
    cmd_obj.run()
  File "/usr/lib/python3/dist-packages/setuptools/command/develop.py", line 34, in run
    self.install_for_development()
  File "/usr/lib/python3/dist-packages/setuptools/command/develop.py", line 133, in install_for_development
    self.process_distribution(None, self.dist, not self.no_deps)
  File "/usr/lib/python3/dist-packages/setuptools/command/easy_install.py", line 720, in process_distribution
    self.install_egg_scripts(dist)
  File "/usr/lib/python3/dist-packages/setuptools/command/develop.py", line 169, in install_egg_scripts
    self.install_script(dist, script_name, script_text, script_path)
  File "/usr/lib/python3/dist-packages/setuptools/command/easy_install.py", line 798, in install_script
    self.write_script(script_name, _to_ascii(script_text), 'b')
  File "/usr/lib/python3/dist-packages/setuptools/command/easy_install.py", line 98, in _to_ascii
    return s.encode('ascii')
UnicodeEncodeError: 'ascii' codec can't encode character '\xe1' in position 12: ordinal not in range(128)
```

Happy hacking,


Pull request description:
Support for scripts with unicode content
## Summary of changes

Makes the `_to_ascii()` function able to handle script's contents in unicode format.

Closes #761 

### Pull Request Checklist
- [x] Changes have tests
- [x] News fragment added in changelog.d

Patch:
diff --git a/changelog.d/1389.change.rst b/changelog.d/1389.change.rst
new file mode 100644
index 000000000..1177b9249
--- /dev/null
+++ b/changelog.d/1389.change.rst
@@ -0,0 +1,1 @@
+Scripts which have unicode content are now supported
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index 05508cebe..dd17cc13f 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -96,7 +96,7 @@ def samefile(p1, p2):
 
 if six.PY2:
 
-    def _to_ascii(s):
+    def _to_bytes(s):
         return s
 
     def isascii(s):
@@ -107,8 +107,8 @@ if six.PY2:
             return False
 else:
 
-    def _to_ascii(s):
-        return s.encode('ascii')
+    def _to_bytes(s):
+        return s.encode('utf8')
 
     def isascii(s):
         try:
@@ -805,7 +805,7 @@ class easy_install(Command):
         if is_script:
             body = self._load_template(dev_path) % locals()
             script_text = ScriptWriter.get_header(script_text) + body
-        self.write_script(script_name, _to_ascii(script_text), 'b')
+        self.write_script(script_name, _to_bytes(script_text), 'b')
 
     @staticmethod
     def _load_template(dev_path):


Test patch:
diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py
index 345d283c3..80a654973 100644
--- a/setuptools/tests/test_easy_install.py
+++ b/setuptools/tests/test_easy_install.py
@@ -185,6 +185,58 @@ class TestEasyInstallTest:
         cmd.ensure_finalized()
         cmd.easy_install(sdist_unicode)
 
+    @pytest.fixture
+    def sdist_unicode_in_script(self, tmpdir):
+        files = [
+            (
+                "setup.py",
+                DALS("""
+                    import setuptools
+                    setuptools.setup(
+                        name="setuptools-test-unicode",
+                        version="1.0",
+                        packages=["mypkg"],
+                        include_package_data=True,
+                        scripts=['mypkg/unicode_in_script'],
+                    )
+                    """),
+            ),
+            ("mypkg/__init__.py", ""),
+            (
+                "mypkg/unicode_in_script",
+                DALS(
+                    """
+                    #!/bin/sh
+                    # \xc3\xa1
+
+                    non_python_fn() {
+                    }
+                """),
+            ),
+        ]
+        sdist_name = "setuptools-test-unicode-script-1.0.zip"
+        sdist = tmpdir / sdist_name
+        # can't use make_sdist, because the issue only occurs
+        #  with zip sdists.
+        sdist_zip = zipfile.ZipFile(str(sdist), "w")
+        for filename, content in files:
+            sdist_zip.writestr(filename, content)
+        sdist_zip.close()
+        return str(sdist)
+
+    @fail_on_ascii
+    def test_unicode_content_in_sdist(self, sdist_unicode_in_script, tmpdir, monkeypatch):
+        """
+        The install command should execute correctly even if
+        the package has unicode in scripts.
+        """
+        dist = Distribution({"script_args": ["easy_install"]})
+        target = (tmpdir / "target").ensure_dir()
+        cmd = ei.easy_install(dist, install_dir=str(target), args=["x"])
+        monkeypatch.setitem(os.environ, "PYTHONPATH", str(target))
+        cmd.ensure_finalized()
+        cmd.easy_install(sdist_unicode_in_script)
+
     @pytest.fixture
     def sdist_script(self, tmpdir):
         files = [