Repository: spack/spack

Issue:
Python 2.6 support broken on Blue Waters
Now that #12207 is merged, most commands work fine on BW. However, I discovered a new bug with our Python 2.6 support.

### Steps to reproduce the issue

```console
$ spack install zlib
```

### Error Message

```console
$ spack -d install zlib
...
==> [2019-08-03-14:35:35.825712] Installing zlib
==> [2019-08-03-14:35:35.825810] Searching for binary cache of zlib
==> [2019-08-03-14:35:35.829746] Warning: No Spack mirrors are currently configured
==> [2019-08-03-14:35:35.830998] No binary for zlib found: installing from source
==> [2019-08-03-14:35:35.831189] READ LOCK: /mnt/a/u/sciteam/stewart1/spack/opt/spack/.spack-db/lock[0:0] [Acquiring]
==> [2019-08-03-14:35:35.831872] READ LOCK: /mnt/a/u/sciteam/stewart1/spack/opt/spack/.spack-db/lock[0:0] [Acquired]
==> [2019-08-03-14:35:36.319365] READ LOCK: /mnt/a/u/sciteam/stewart1/spack/opt/spack/.spack-db/lock[0:0] [Released]
==> [2019-08-03-14:35:46.138265] '/usr/bin/env' '-i' '/bin/bash' '-lc' 'echo $CRAY_CPU_TARGET'
==> [2019-08-03-14:35:48.474273] Found default module:interlagos
==> [2019-08-03-14:35:48.825811] A dependency has updated CPATH, this may lead pkg-config to assume that the package is part of the system includes and omit it when invoked with '--cflags'.
==> [2019-08-03-14:35:48.900257] Module name: gcc/8.2.0
==> [2019-08-03-14:35:48.900589] Package directory variable prefix: GCC
==> [2019-08-03-14:35:49.020036] Module name: gcc/8.2.0
==> [2019-08-03-14:35:49.020348] Package directory variable prefix: GCC
==> [2019-08-03-14:35:59.163396] '/usr/bin/env' '-i' '/bin/bash' '-lc' 'echo $CRAY_CPU_TARGET'
==> [2019-08-03-14:35:59.222729] Error: TypeError: execve() argument 1 must be encoded string without NULL bytes, not str

/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/package.py:994, in architecture:
        991        """Get the spack.architecture.Arch object that represents the
        992        environment in which this package will be built."""
        993        if not self.spec.concrete:
  >>    994            raise ValueError("Can only get the arch for concrete package.")
        995        return spack.architecture.arch_for_spec(self.spec.architecture)


Traceback (most recent call last):
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/build_environment.py", line 765, in child_process
    setup_package(pkg, dirty=dirty)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/build_environment.py", line 715, in setup_package
    if pkg.architecture.target.module_name:
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/package.py", line 995, in architecture
    return spack.architecture.arch_for_spec(self.spec.architecture)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/architecture.py", line 342, in arch_for_spec
    arch_plat = get_platform(arch_spec.platform)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/architecture.py", line 322, in get_platform
    return p()
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/platforms/cray.py", line 63, in __init__
    _target = self._default_target_from_env()
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/platforms/cray.py", line 122, in _default_target_from_env
    output=str, error=os.devnull)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/util/executable.py", line 167, in __call__
    env=env)
  File "/usr/lib64/python2.6/subprocess.py", line 623, in __init__
    errread, errwrite)
  File "/usr/lib64/python2.6/subprocess.py", line 1141, in _execute_child
    raise child_exception
TypeError: execve() argument 1 must be encoded string without NULL bytes, not str
==> [2019-08-03-14:35:59.224063] READ LOCK: /mnt/a/u/sciteam/stewart1/spack/opt/spack/.spack-db/lock[0:0] [Acquiring]
==> [2019-08-03-14:35:59.225934] READ LOCK: /mnt/a/u/sciteam/stewart1/spack/opt/spack/.spack-db/lock[0:0] [Acquired]
==> [2019-08-03-14:35:59.750742] READ LOCK: /mnt/a/u/sciteam/stewart1/spack/opt/spack/.spack-db/lock[0:0] [Released]
==> [2019-08-03-14:35:59.759357] ChildError: TypeError: execve() argument 1 must be encoded string without NULL bytes, not str
/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/package.py:994, in architecture:
        991        """Get the spack.architecture.Arch object that represents the
        992        environment in which this package will be built."""
        993        if not self.spec.concrete:
  >>    994            raise ValueError("Can only get the arch for concrete package.")
        995        return spack.architecture.arch_for_spec(self.spec.architecture)

Traceback (most recent call last):
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/build_environment.py", line 765, in child_process
    setup_package(pkg, dirty=dirty)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/build_environment.py", line 715, in setup_package
    if pkg.architecture.target.module_name:
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/package.py", line 995, in architecture
    return spack.architecture.arch_for_spec(self.spec.architecture)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/architecture.py", line 342, in arch_for_spec
    arch_plat = get_platform(arch_spec.platform)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/architecture.py", line 322, in get_platform
    return p()
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/platforms/cray.py", line 63, in __init__
    _target = self._default_target_from_env()
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/platforms/cray.py", line 122, in _default_target_from_env
    output=str, error=os.devnull)
  File "/mnt/a/u/sciteam/stewart1/spack/lib/spack/spack/util/executable.py", line 167, in __call__
    env=env)
  File "/usr/lib64/python2.6/subprocess.py", line 623, in __init__
    errread, errwrite)
  File "/usr/lib64/python2.6/subprocess.py", line 1141, in _execute_child
    raise child_exception
TypeError: execve() argument 1 must be encoded string without NULL bytes, not str
```

### Information on your system

This problem occurs on Cray CNL5 (Blue Waters) when running Spack with the system Python 2.6.9.

A workaround for this bug is to `module load bwpy` and use a newer version of Python.

@tgamblin @alalazo 


Discussion:

Resolving patch:
diff --git a/lib/spack/spack/parse.py b/lib/spack/spack/parse.py
--- a/lib/spack/spack/parse.py
+++ b/lib/spack/spack/parse.py
@@ -143,7 +143,7 @@ def expect(self, id):
 
     def setup(self, text):
         if isinstance(text, string_types):
-            text = shlex.split(text)
+            text = shlex.split(str(text))
         self.text = text
         self.push_tokens(self.lexer.lex(text))
 
diff --git a/lib/spack/spack/util/editor.py b/lib/spack/spack/util/editor.py
--- a/lib/spack/spack/util/editor.py
+++ b/lib/spack/spack/util/editor.py
@@ -41,7 +41,7 @@ def _find_exe_from_env_var(var):
         return None, []
 
     # split env var into executable and args if needed
-    args = shlex.split(exe)
+    args = shlex.split(str(exe))
     if not args:
         return None, []
 
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -20,7 +20,7 @@ class Executable(object):
     """Class representing a program that can be run on the command line."""
 
     def __init__(self, name):
-        self.exe = shlex.split(name)
+        self.exe = shlex.split(str(name))
         self.default_env = {}
         self.returncode = None
 


Test patch:
diff --git a/lib/spack/spack/test/cmd/release_jobs.py b/lib/spack/spack/test/cmd/release_jobs.py
--- a/lib/spack/spack/test/cmd/release_jobs.py
+++ b/lib/spack/spack/test/cmd/release_jobs.py
@@ -3,10 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import pytest
-
 import json
-import sys
 
 from jsonschema import validate
 
@@ -39,11 +36,6 @@ def test_specs_deps(tmpdir, config):
     validate(deps_object, specs_deps_schema)
 
 
-@pytest.mark.skipif(
-    sys.version_info[:2] < (2, 7),
-    reason="For some reason in Python2.6 we get a utf-32 string "
-           "that can't be parsed"
-)
 def test_specs_staging(config):
     """Make sure we achieve the best possible staging for the following
 spec DAG::
diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py
--- a/lib/spack/spack/test/spec_syntax.py
+++ b/lib/spack/spack/test/spec_syntax.py
@@ -97,7 +97,7 @@ def check_parse(self, expected, spec=None):
 
     def check_lex(self, tokens, spec):
         """Check that the provided spec parses to the provided token list."""
-        spec = shlex.split(spec)
+        spec = shlex.split(str(spec))
         lex_output = sp.SpecLexer().lex(spec)
         for tok, spec_tok in zip(tokens, lex_output):
             if tok.type == sp.ID or tok.type == sp.VAL: