Repository: pex-tool/pex

Issue:
UnicodeDecodeError when packaging after upgrading to v2.1.46 
Hi, I am trying to generate a pex file with the following command:

```
pex . -r requirements.txt -c __main__.py -o /home/jenkins/example.pex --disable-cache
```

This command runs fine in `v2.1.45` or older. However, when I try using `v2.1.46` or newer, I see the following stacktrace:
```
Traceback (most recent call last):
  File "/usr/local/bin/pex", line 8, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.6/dist-packages/pex/bin/pex.py", line 1149, in main
    pex_builder = build_pex(options.requirements, options, cache=ENV.PEX_ROOT)
  File "/usr/local/lib/python3.6/dist-packages/pex/bin/pex.py", line 1045, in build_pex
    ignore_errors=options.ignore_errors,
  File "/usr/local/lib/python3.6/dist-packages/pex/resolver.py", line 935, in resolve
    ignore_errors=ignore_errors, workspace=workspace, max_parallel_jobs=max_parallel_jobs
  File "/usr/local/lib/python3.6/dist-packages/pex/resolver.py", line 683, in install_distributions
    max_jobs=max_parallel_jobs,
  File "/usr/local/lib/python3.6/dist-packages/pex/jobs.py", line 431, in execute_parallel
    yield spawn_result.spawned_job.await_result()
  File "/usr/local/lib/python3.6/dist-packages/pex/jobs.py", line 185, in await_result
    job.wait()
  File "/usr/local/lib/python3.6/dist-packages/pex/jobs.py", line 76, in wait
    self._finalize_job()
  File "/usr/local/lib/python3.6/dist-packages/pex/jobs.py", line 110, in _finalize_job
    self._finalizer()
  File "/usr/local/lib/python3.6/dist-packages/pex/pip.py", line 719, in _fixup_install
    modified_scripts = list(cls._fixup_scripts(install_dir))
  File "/usr/local/lib/python3.6/dist-packages/pex/pip.py", line 836, in _fixup_scripts
    for line in script_fi:
  File "/usr/lib/python3.6/fileinput.py", line 250, in __next__
    line = self._readline()
  File "/usr/lib/python3.6/fileinput.py", line 364, in _readline
    return self._readline()
  File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 91: ordinal not in range(128)
Traceback (most recent call last):
  File "/tmp/tmpmxiteky_/venvs/813aff41909de48f9fc4117c2598a7e12f48f668/061a3a67f6a451d33806aa2696288de4ee1b175c/pex", line 157, in <module>
  File "/usr/lib/python3.6/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/__pex_patched_pip__.py", line 21, in <module>
  File "/usr/lib/python3.6/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/__main__.py", line 23, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/main.py", line 10, in <module>
ModuleNotFoundError: No module named 'pip._internal.cli.autocompletion'
Traceback (most recent call last):
  File "/tmp/tmpmxiteky_/venvs/813aff41909de48f9fc4117c2598a7e12f48f668/061a3a67f6a451d33806aa2696288de4ee1b175c/pex", line 157, in <module>
  File "/usr/lib/python3.6/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/__pex_patched_pip__.py", line 21, in <module>
  File "/usr/lib/python3.6/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/__main__.py", line 23, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/main.py", line 10, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/autocompletion.py", line 9, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/main_parser.py", line 7, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/cmdoptions.py", line 25, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/progress_bars.py", line 12, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/utils/logging.py", line 18, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/utils/misc.py", line 21, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_vendor/pkg_resources/__init__.py", line 79, in <module>
ImportError: cannot import name 'py31compat'
Traceback (most recent call last):
  File "/tmp/tmpmxiteky_/venvs/813aff41909de48f9fc4117c2598a7e12f48f668/061a3a67f6a451d33806aa2696288de4ee1b175c/pex", line 157, in <module>
  File "/usr/lib/python3.6/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/__pex_patched_pip__.py", line 21, in <module>
  File "/usr/lib/python3.6/runpy.py", line 205, in run_module
    return _run_module_code(code, init_globals, run_name, mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/__main__.py", line 23, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/main.py", line 10, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/autocompletion.py", line 9, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/main_parser.py", line 7, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/cmdoptions.py", line 25, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/cli/progress_bars.py", line 12, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/utils/logging.py", line 18, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_internal/utils/misc.py", line 21, in <module>
  File "/tmp/tmpmxiteky_/venvs/short/57f9d172/lib/python3.6/site-packages/pip/_vendor/pkg_resources/__init__.py", line 79, in <module>
ImportError: cannot import name 'py31compat'
```

I am running `pip==21.2.4` and `setuptools==58.1.0` inside a Docker container `ubuntu:trusty`.

I can't see anything obvious in the release notes for `v2.1.46` that could cause this issue.

Let me know if you need any further details.

Thanks in advance,
Juan



Discussion:
@juanitosvq Can you share the contents of your requirements.txt, if it doesn't contain sensitive information?
@juanitosvq & @kaos I have a simple repro. This only happens under Python 3.5 and 3.6 when I unset `LANG` and it appears to happen only for empty scripts of which the docutils project ships one:
```console
$ for py in pypy{,3} python{2.7,3.{5,6,7,8,9}}; do rm -rf ~/.pex && LANG= $py -mpex docutils==0.17.1 -odocutils.pex 2>/tmp/$py && echo "Success with $py" || echo "Failed with $py" | cat - <(tail -5 /tmp/$py); done
Success with pypy
Failed with pypy3
  File "/opt/pypy3/lib-python/3/fileinput.py", line 366, in _readline
    return self._readline()
  File "/opt/pypy3/lib-python/3/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 89: ordinal not in range(128)
Success with python2.7
Failed with python3.5
  File "/usr/lib/python3.5/fileinput.py", line 362, in _readline
    return self._readline()
  File "/usr/lib/python3.5/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 93: ordinal not in range(128)
Failed with python3.6
  File "/usr/lib/python3.6/fileinput.py", line 364, in _readline
    return self._readline()
  File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 93: ordinal not in range(128)
Success with python3.7
Success with python3.8
Success with python3.9
```

That's with:
```diff
diff --git a/pex/pip.py b/pex/pip.py
index c6a18d2..f4528c2 100644
--- a/pex/pip.py
+++ b/pex/pip.py
@@ -935,7 +935,10 @@ class Pip(object):
         install_dir,  # type: str
     ):
         # type: (...) -> None
-        modified_scripts = list(cls._fixup_scripts(install_dir))
+        modified_scripts = []
+        for modified_script in cls._fixup_scripts(install_dir):
+            print("... re-wrote {}".format(modified_script), file=sys.stderr)
+            modified_scripts.append(modified_script)
         cls._fixup_record(dist, install_dir, modified_scripts=modified_scripts)
 
     @staticmethod
@@ -1051,7 +1054,8 @@ class Pip(object):
         if not scripts:
             return
 
-        with closing(fileinput.input(files=scripts, inplace=True)) as script_fi:
+        print(">>> re-writing scripts:\n{}".format("\n".join(sorted(scripts))), file=sys.stderr)
+        with closing(fileinput.input(files=sorted(scripts), inplace=True)) as script_fi:
             for line in script_fi:
                 if script_fi.isfirstline():
                     # Ensure python shebangs are reproducible. The only place these can be used is
```
And a success like:
```console
$ rm -rf ~/.pex && LANG= python2.7 -mpex docutils==0.17.1 -odocutils.pex
>>> re-writing scripts:
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2html.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2html4.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2html5.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2latex.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2man.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2odt.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2odt_prepstyles.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2pseudoxml.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2s5.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2xetex.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rst2xml.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.d08abf03ce6d4c5fbc6a31549b1e180d/bin/rstpep2html.py
... re-wrote bin/rst2html.py
... re-wrote bin/rst2html4.py
... re-wrote bin/rst2html5.py
... re-wrote bin/rst2latex.py
... re-wrote bin/rst2man.py
... re-wrote bin/rst2odt.py
... re-wrote bin/rst2odt_prepstyles.py
... re-wrote bin/rst2pseudoxml.py
... re-wrote bin/rst2s5.py
... re-wrote bin/rst2xetex.py
... re-wrote bin/rst2xml.py
... re-wrote bin/rstpep2html.py
```
And a failure like:
```console
$ rm -rf ~/.pex && LANG= python3.6 -mpex docutils==0.17.1 -odocutils.pex
>>> re-writing scripts:
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2html.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2html4.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2html5.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2latex.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2man.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2odt.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2odt_prepstyles.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2pseudoxml.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2s5.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2xetex.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rst2xml.py
/home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/rstpep2html.py
... re-wrote bin/rst2html.py
... re-wrote bin/rst2html4.py
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/__main__.py", line 8, in <module>
    __name__ == "__main__" and pex.main()
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/bin/pex.py", line 672, in main
    env=env,
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/bin/pex.py", line 692, in do_main
    cache=ENV.PEX_ROOT,
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/bin/pex.py", line 596, in build_pex
    ignore_errors=options.ignore_errors,
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/resolver.py", line 944, in resolve
    ignore_errors=ignore_errors, workspace=workspace, max_parallel_jobs=max_parallel_jobs
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/resolver.py", line 690, in install_distributions
    max_jobs=max_parallel_jobs,
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/jobs.py", line 447, in execute_parallel
    yield spawn_result.spawned_job.await_result()
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/jobs.py", line 201, in await_result
    job.wait()
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/jobs.py", line 76, in wait
    self._finalize_job()
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/jobs.py", line 110, in _finalize_job
    self._finalizer()
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/pip.py", line 939, in _fixup_install
    for modified_script in cls._fixup_scripts(install_dir):
  File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/pip.py", line 1059, in _fixup_scripts
    for line in script_fi:
  File "/usr/lib/python3.6/fileinput.py", line 250, in __next__
    line = self._readline()
  File "/usr/lib/python3.6/fileinput.py", line 364, in _readline
    return self._readline()
  File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 93: ordinal not in range(128)
```
Where the failure file is `rst2html5.py` which is empty:
```console
$ ls -l /home/jsirois/.pex/installed_wheels/fbf7e9f9398442c425ca6bef22b87aef81cf2618/docutils-0.17.1-py2.py3-none-any.whl.4ad32fe397c2476db5b899d35ddf4335/bin/
total 44
-rwxr-xr-x 1 jsirois jsirois  704 Oct  3 13:19 rst2html4.py
-rwxr-xr-x 1 jsirois jsirois    0 Oct  3 13:19 rst2html5.py
-rwxr-xr-x 1 jsirois jsirois  584 Oct  3 13:19 rst2html.py
-rwxr-xr-x 1 jsirois jsirois  828 Oct  3 13:19 rst2latex.py
-rwxr-xr-x 1 jsirois jsirois  636 Oct  3 13:19 rst2man.py
-rwxr-xr-x 1 jsirois jsirois 1763 Oct  3 13:19 rst2odt_prepstyles.py
-rwxr-xr-x 1 jsirois jsirois  801 Oct  3 13:19 rst2odt.py
-rwxr-xr-x 1 jsirois jsirois  638 Oct  3 13:19 rst2pseudoxml.py
-rwxr-xr-x 1 jsirois jsirois  674 Oct  3 13:19 rst2s5.py
-rwxr-xr-x 1 jsirois jsirois  908 Oct  3 13:19 rst2xetex.py
-rwxr-xr-x 1 jsirois jsirois  639 Oct  3 13:19 rst2xml.py
-rwxr-xr-x 1 jsirois jsirois  707 Oct  3 13:19 rstpep2html.py
```
Thanks for looking into this, everyone! Looks like @jsirois found the issue, but leaving my `requirements.txt` here for reference:
```
marshmallow==2.13.5
Flask==0.12.2
Flask-API==1.0
Flask-HTTPAuth==3.2.3
tinydb==3.7.0
apispec==0.29.0
PyYAML==3.12
boto3==1.4.7
requests==2.18.4
APScheduler==3.8.0
backoff==1.5.0
werkzeug==1.0.1
```
Thanks @juanitosvq. I'll check that requirement set out as well to confirm a fix.

To correct my observations above, `rst2html5.py` is not empty - originally - it's only empty after the fileinput `inplace=True` botch job. That input file is in fact unicode-containing (`:Copyright: © 2015 Günter Milde.`):
```console
$ test.venv/bin/pip download docutils==0.17.1
Collecting docutils==0.17.1
  Using cached https://files.pythonhosted.org/packages/4c/5e/6003a0d1f37725ec2ebd4046b657abb9372202655f96e76795dca8c0063c/docutils-0.17.1-py2.py3-none-any.whl
  Saved ./docutils-0.17.1-py2.py3-none-any.whl
Successfully downloaded docutils
$ unzip -q docutils-0.17.1-py2.py3-none-any.whl
$ file docutils-0.17.1.data/scripts/*
docutils-0.17.1.data/scripts/rst2html4.py:          Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2html5.py:          Python script, Unicode text, UTF-8 text executable
docutils-0.17.1.data/scripts/rst2html.py:           Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2latex.py:          Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2man.py:            Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2odt_prepstyles.py: Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2odt.py:            Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2pseudoxml.py:      Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2s5.py:             Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2xetex.py:          Python script, ASCII text executable
docutils-0.17.1.data/scripts/rst2xml.py:            Python script, ASCII text executable
docutils-0.17.1.data/scripts/rstpep2html.py:        Python script, ASCII text executable
$ head -3 docutils-0.17.1.data/scripts/rst2html5.py
#!python
# -*- coding: utf8 -*-
# :Copyright: © 2015 Günter Milde.
```
Ok, yeah - both issues were triggered by docutils 0.17.1 and its `rst2html5.py` script.
@juanitosvq for you the path was: boto3==1.4.7 -> botocore==1.7.48 -> docutils==0.17.1
@kaos for you it was: twine==3.4.2 -> readme-renderer==30.0 -> docutils==0.17.1


Resolving patch:
diff --git a/pex/compatibility.py b/pex/compatibility.py
--- a/pex/compatibility.py
+++ b/pex/compatibility.py
@@ -7,7 +7,9 @@
 from __future__ import absolute_import
 
 import os
+import sys
 from abc import ABCMeta
+from io import BytesIO
 from sys import version_info as sys_version_info
 
 from pex.typing import TYPE_CHECKING, cast
@@ -146,3 +148,8 @@ def cpu_count():
 
 # Universal newlines is the default in Python 3.
 MODE_READ_UNIVERSAL_NEWLINES = "rU" if PY2 else "r"
+
+
+def get_stdout_bytes_buffer():
+    # type: () -> BytesIO
+    return cast(BytesIO, getattr(sys.stdout, "buffer", sys.stdout))
diff --git a/pex/pip.py b/pex/pip.py
--- a/pex/pip.py
+++ b/pex/pip.py
@@ -22,7 +22,7 @@
 
 from pex import dist_metadata, third_party
 from pex.common import atomic_directory, is_python_script, safe_mkdtemp
-from pex.compatibility import MODE_READ_UNIVERSAL_NEWLINES, urlparse
+from pex.compatibility import MODE_READ_UNIVERSAL_NEWLINES, get_stdout_bytes_buffer, urlparse
 from pex.dist_metadata import ProjectNameAndVersion
 from pex.distribution_target import DistributionTarget
 from pex.fetcher import URLFetcher
@@ -1051,17 +1051,18 @@ def _fixup_scripts(
         if not scripts:
             return
 
-        with closing(fileinput.input(files=scripts, inplace=True)) as script_fi:
+        with closing(fileinput.input(files=scripts, inplace=True, mode="rb")) as script_fi:
             for line in script_fi:
+                buffer = get_stdout_bytes_buffer()
                 if script_fi.isfirstline():
                     # Ensure python shebangs are reproducible. The only place these can be used is
                     # in venv mode PEXes where the `#!python` placeholder shebang will be re-written
                     # to use the venv's python interpreter.
-                    print("#!python")
+                    buffer.write(b"#!python\n")
                     yield os.path.relpath(script_fi.filename(), install_dir)
                 else:
                     # N.B.: These lines include the newline already.
-                    sys.stdout.write(line)
+                    buffer.write(cast(bytes, line))
 
     def spawn_install_wheel(
         self,
diff --git a/pex/tools/commands/virtualenv.py b/pex/tools/commands/virtualenv.py
--- a/pex/tools/commands/virtualenv.py
+++ b/pex/tools/commands/virtualenv.py
@@ -10,10 +10,11 @@
 from contextlib import closing
 
 from pex.common import AtomicDirectory, is_exe, safe_mkdir
+from pex.compatibility import get_stdout_bytes_buffer
 from pex.interpreter import PythonInterpreter
 from pex.third_party.pkg_resources import resource_string
 from pex.tracer import TRACER
-from pex.typing import TYPE_CHECKING
+from pex.typing import TYPE_CHECKING, cast
 from pex.util import named_temporary_file
 
 if TYPE_CHECKING:
@@ -226,20 +227,25 @@ def rewrite_scripts(
                 continue
             python_scripts.append(executable)
         if python_scripts:
-            with closing(fileinput.input(files=sorted(python_scripts), inplace=True)) as fi:
+            with closing(
+                fileinput.input(files=sorted(python_scripts), inplace=True, mode="rb")
+            ) as fi:
                 # N.B.: `fileinput` is strange, but useful: the context manager above monkey-patches
                 # sys.stdout to print to the corresponding original input file, which is has moved
                 # aside.
                 for line in fi:
+                    buffer = get_stdout_bytes_buffer()
                     if fi.isfirstline():
                         shebang = [python or self._interpreter.binary]
                         if python_args:
                             shebang.append(python_args)
-                        print("#!{shebang}".format(shebang=" ".join(shebang)))
+                        buffer.write(
+                            "#!{shebang}\n".format(shebang=" ".join(shebang)).encode("utf-8")
+                        )
                         yield fi.filename()
                     else:
                         # N.B.: These lines include the newline already.
-                        sys.stdout.write(line)
+                        buffer.write(cast(bytes, line))
 
     def install_pip(self):
         # type: () -> None


Test patch:
diff --git a/tests/integration/test_issue_1479.py b/tests/integration/test_issue_1479.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_issue_1479.py
@@ -0,0 +1,46 @@
+# coding=utf-8
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os.path
+import platform
+import subprocess
+import sys
+
+from pex.common import safe_rmtree
+from pex.testing import make_env, run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def test_unicode_script_shebang_rewrite_docutils(tmpdir):
+    # type: (Any) -> None
+
+    # The docutils distribution contains many scripts, one of which, rst2html5.py, contains
+    # non-ascii unicode characters which could trip up script shebang re-writing in environments
+    # without a default encoding accepting those characters.
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    env = make_env(LANG=None, PEX_ROOT=pex_root)
+
+    docutils_pex = os.path.join(str(tmpdir), "docutils.pex")
+
+    run_pex_command(
+        args=[
+            "docutils==0.17.1",
+            "-c",
+            "rst2html5.py",
+            "-o",
+            docutils_pex,
+            "--venv",
+        ],
+        env=env,
+    ).assert_success()
+
+    safe_rmtree(pex_root)
+    output = subprocess.check_output(args=[docutils_pex, "-V"], env=env)
+    assert "rst2html5.py (Docutils 0.17.1 [release], Python {version}, on {platform})\n".format(
+        version=platform.python_version(), platform=sys.platform
+    ) == output.decode("utf-8")