# Buggy code
def launch_jobs(
    launcher: RayAWSLauncher, local_tmp_dir: str, sweep_dir: Path
) -> Sequence[JobReturn]:
    ray_up(launcher.ray_yaml_path)
    with tempfile.TemporaryDirectory() as local_tmp_download_dir:
        with ray_tmp_dir(
            launcher.ray_yaml_path, launcher.docker_enabled
        ) as remote_tmp_dir:
            ray_rsync_up(
                launcher.ray_yaml_path, os.path.join(local_tmp_dir, ""), remote_tmp_dir
            )

            script_path = os.path.join(os.path.dirname(__file__), "_remote_invoke.py")
            ray_rsync_up(launcher.ray_yaml_path, script_path, remote_tmp_dir)

            if launcher.sync_up.source_dir:
                source_dir = _get_abs_code_dir(launcher.sync_up.source_dir)
                target_dir = (
                    launcher.sync_up.target_dir
                    if launcher.sync_up.target_dir
                    else remote_tmp_dir
                )
                rsync(
                    launcher.ray_yaml_path,
                    launcher.sync_up.include,
                    launcher.sync_up.exclude,
                    os.path.join(source_dir, ""),
                    target_dir,
                )

            ray_exec(
                launcher.ray_yaml_path,
                launcher.docker_enabled,
                os.path.join(remote_tmp_dir, "_remote_invoke.py"),
                remote_tmp_dir,
            )

            ray_rsync_down(
                launcher.ray_yaml_path,
                os.path.join(remote_tmp_dir, JOB_RETURN_PICKLE),
                local_tmp_download_dir,
            )

            sync_down_cfg = launcher.sync_down

            if (
                sync_down_cfg.target_dir
                or sync_down_cfg.source_dir
                or sync_down_cfg.include
                or sync_down_cfg.exclude
            ):
                source_dir = (
                    sync_down_cfg.source_dir if sync_down_cfg.source_dir else sweep_dir
                )
                target_dir = (
                    sync_down_cfg.source_dir if sync_down_cfg.source_dir else sweep_dir
                )
                target_dir = Path(_get_abs_code_dir(target_dir))
                target_dir.mkdir(parents=True, exist_ok=True)

                rsync(
                    launcher.ray_yaml_path,
                    launcher.sync_down.include,
                    launcher.sync_down.exclude,
                    os.path.join(source_dir),
                    str(target_dir),
                    up=False,
                )
                log.info(
                    f"Syncing outputs from remote dir: {source_dir} to local dir: {target_dir.absolute()} "
                )

        if launcher.stop_cluster:
            log.info("Stopping cluster now. (stop_cluster=true)")
            if launcher.ray_cfg.cluster.provider.cache_stopped_nodes:
                log.info("NOT deleting the cluster (provider.cache_stopped_nodes=true)")
            else:
                log.info("Deleted the cluster (provider.cache_stopped_nodes=false)")
            ray_down(launcher.ray_yaml_path)
        else:
            log.warning(
                "NOT stopping cluster, this may incur extra cost for you. (stop_cluster=false)"
            )

        with open(os.path.join(local_tmp_download_dir, JOB_RETURN_PICKLE), "rb") as f:
            job_returns = pickle.load(f)  # nosec
            assert isinstance(job_returns, List)
            for run in job_returns:
                assert isinstance(run, JobReturn)
            return job_returns


# Traceback (AssertionError)
[2020-12-10 13:18:50,056][HYDRA] Ray Launcher is launching 1 jobs,
[2020-12-10 13:18:50,056][HYDRA]        #0 :
[2020-12-10 13:18:50,335][HYDRA] Pickle for jobs: /tmp/tmpb4ropuxo/job_spec.pkl
[2020-12-10 13:18:50,338][HYDRA] Saving RayClusterConf in a temp yaml file: /tmp/tmp2uz4ynig.yaml.
[2020-12-10 13:18:50,340][HYDRA] Running command: ['ray', 'up', '-y', '/tmp/tmp2uz4ynig.yaml']
[2020-12-10 13:25:39,406][HYDRA] Output: /usr/bin/xauth:  file /home/ubuntu/.Xauthority does not exist
18:20:11 up 0 min,  1 user,  load average: 1.59, 0.41, 0.14
sending incremental file list
ray-bootstrap-1_t_b3_e

sent 899 bytes  received 35 bytes  1,868.00 bytes/sec
total size is 2,314  speedup is 2.48
sending incremental file list
hydra_key-1.pem

sent 1,393 bytes  received 35 bytes  2,856.00 bytes/sec
total size is 1,674  speedup is 1.17
latest-cpu: Pulling from rayproject/ray-ml
6a5697faee43: Pull complete
ba13d3bc422b: Pull complete
a254829d9e55: Pull complete
7d7be32e453d: Pull complete
6a0d739f1b58: Pull complete
48fe103a5137: Pull complete
6ba3ed757423: Pull complete
35d28d3e311c: Pull complete
d866c872c6c4: Pull complete
04f4ed2a074b: Pull complete
f90845af33ae: Pull complete
3dc6dece1a5a: Pull complete
Digest: sha256:49277bb5129de9db9705e8d3a5a94703bdfb13f421e4709410801d8e7df48713
Status: Downloaded newer image for rayproject/ray-ml:latest-cpu
docker.io/rayproject/ray-ml:latest-cpu
NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver. Make sure that the latest NVIDIA driver is installed and running.

2d262bdb8554f1e75e16cc2363b33325d8051122dad1b3166187c3a4a2395b79
Collecting omegaconf==2.0.3
Downloading omegaconf-2.0.3-py3-none-any.whl (36 kB)
Requirement already satisfied: PyYAML>=5.1.* in /root/anaconda3/lib/python3.7/site-packages (from omegaconf==2.0.3) (5.3.1)
Requirement already satisfied: typing-extensions in /root/anaconda3/lib/python3.7/site-packages (from omegaconf==2.0.3) (3.7.4.3)
Installing collected packages: omegaconf
Successfully installed omegaconf-2.0.3
Collecting hydra-core==1.0.3
Downloading hydra_core-1.0.3-py3-none-any.whl (122 kB)
|████████████████████████████████| 122 kB 4.8 MB/s
Collecting importlib-resources; python_version < "3.9"
Downloading importlib_resources-3.3.0-py2.py3-none-any.whl (26 kB)
Collecting antlr4-python3-runtime==4.8
Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)
|████████████████████████████████| 112 kB 5.8 MB/s
Requirement already satisfied: omegaconf>=2.0.2 in /root/anaconda3/lib/python3.7/site-packages (from hydra-core==1.0.3) (2.0.3)
Requirement already satisfied: zipp>=0.4; python_version < "3.8" in /root/anaconda3/lib/python3.7/site-packages (from importlib-resources; python_version < "3.9"->hydra-core==1.0.3) (3.4.0)
Requirement already satisfied: PyYAML>=5.1.* in /root/anaconda3/lib/python3.7/site-packages (from omegaconf>=2.0.2->hydra-core==1.0.3) (5.3.1)
Requirement already satisfied: typing-extensions in /root/anaconda3/lib/python3.7/site-packages (from omegaconf>=2.0.2->hydra-core==1.0.3) (3.7.4.3)
Building wheels for collected packages: antlr4-python3-runtime
Building wheel for antlr4-python3-runtime (setup.py) ... done
Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=142054 sha256=a70b51a8a90ef4a4e8e28f44e1357d75c0306f1023ccef17bbef6d021260eaed
Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c
Successfully built antlr4-python3-runtime
Installing collected packages: importlib-resources, antlr4-python3-runtime, hydra-core
Successfully installed antlr4-python3-runtime-4.8 hydra-core-1.0.3 importlib-resources-3.3.0
Collecting hydra-ray-launcher==0.1.1
Downloading hydra_ray_launcher-0.1.1-py3-none-any.whl (13 kB)
Collecting boto3==1.15.6
Downloading boto3-1.15.6-py2.py3-none-any.whl (129 kB)
|████████████████████████████████| 129 kB 4.5 MB/s
Requirement already satisfied: hydra-core>=1.0.0 in /root/anaconda3/lib/python3.7/site-packages (from hydra-ray-launcher==0.1.1) (1.0.3)
Requirement already satisfied: ray>=1.0.0 in /root/anaconda3/lib/python3.7/site-packages (from hydra-ray-launcher==0.1.1) (1.0.1.post1)
Collecting pickle5
Downloading pickle5-0.0.11.tar.gz (132 kB)
|████████████████████████████████| 132 kB 4.9 MB/s
Requirement already satisfied: cloudpickle>=1.6.0 in /root/anaconda3/lib/python3.7/site-packages (from hydra-ray-launcher==0.1.1) (1.6.0)
Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /root/anaconda3/lib/python3.7/site-packages (from boto3==1.15.6->hydra-ray-launcher==0.1.1) (0.10.0)
Collecting botocore<1.19.0,>=1.18.6
Downloading botocore-1.18.18-py2.py3-none-any.whl (6.7 MB)
|████████████████████████████████| 6.7 MB 4.7 MB/s
Collecting s3transfer<0.4.0,>=0.3.0
Downloading s3transfer-0.3.3-py2.py3-none-any.whl (69 kB)
|████████████████████████████████| 69 kB 797 kB/s
Requirement already satisfied: antlr4-python3-runtime==4.8 in /root/anaconda3/lib/python3.7/site-packages (from hydra-core>=1.0.0->hydra-ray-launcher==0.1.1) (4.8)
Requirement already satisfied: importlib-resources; python_version < "3.9" in /root/anaconda3/lib/python3.7/site-packages (from hydra-core>=1.0.0->hydra-ray-launcher==0.1.1) (3.3.0)
Requirement already satisfied: omegaconf>=2.0.2 in /root/anaconda3/lib/python3.7/site-packages (from hydra-core>=1.0.0->hydra-ray-launcher==0.1.1) (2.0.3)
Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.0.0)
Requirement already satisfied: google in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.0.0)
Requirement already satisfied: requests in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (2.24.0)
Requirement already satisfied: pyyaml in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (5.3.1)
Requirement already satisfied: numpy>=1.16 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.19.4)
Requirement already satisfied: colorama in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.4.4)
Requirement already satisfied: aiohttp-cors in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.7.0)
Requirement already satisfied: aioredis in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.3.1)
Requirement already satisfied: prometheus-client>=0.7.1 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.9.0)
Requirement already satisfied: colorful in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.5.4)
Requirement already satisfied: filelock in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.0.12)
Requirement already satisfied: grpcio>=1.28.1 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.33.2)
Requirement already satisfied: click>=7.0 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (7.1.2)
Requirement already satisfied: opencensus in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.7.11)
Requirement already satisfied: gpustat in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.6.0)
Requirement already satisfied: jsonschema in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.2.0)
Requirement already satisfied: py-spy>=0.2.0 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.3.3)
Requirement already satisfied: redis<3.5.0,>=3.3.2 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.4.1)
Requirement already satisfied: aiohttp in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.7.3)
Requirement already satisfied: protobuf>=3.8.0 in /root/anaconda3/lib/python3.7/site-packages (from ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.14.0)
Requirement already satisfied: urllib3<1.26,>=1.20; python_version != "3.4" in /root/anaconda3/lib/python3.7/site-packages (from botocore<1.19.0,>=1.18.6->boto3==1.15.6->hydra-ray-launcher==0.1.1) (1.25.11)
Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /root/anaconda3/lib/python3.7/site-packages (from botocore<1.19.0,>=1.18.6->boto3==1.15.6->hydra-ray-launcher==0.1.1) (2.8.1)
Requirement already satisfied: zipp>=0.4; python_version < "3.8" in /root/anaconda3/lib/python3.7/site-packages (from importlib-resources; python_version < "3.9"->hydra-core>=1.0.0->hydra-ray-launcher==0.1.1) (3.4.0)
Requirement already satisfied: typing-extensions in /root/anaconda3/lib/python3.7/site-packages (from omegaconf>=2.0.2->hydra-core>=1.0.0->hydra-ray-launcher==0.1.1) (3.7.4.3)
Requirement already satisfied: beautifulsoup4 in /root/anaconda3/lib/python3.7/site-packages (from google->ray>=1.0.0->hydra-ray-launcher==0.1.1) (4.9.3)
Requirement already satisfied: idna<3,>=2.5 in /root/anaconda3/lib/python3.7/site-packages (from requests->ray>=1.0.0->hydra-ray-launcher==0.1.1) (2.9)
Requirement already satisfied: chardet<4,>=3.0.2 in /root/anaconda3/lib/python3.7/site-packages (from requests->ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.0.4)
Requirement already satisfied: certifi>=2017.4.17 in /root/anaconda3/lib/python3.7/site-packages (from requests->ray>=1.0.0->hydra-ray-launcher==0.1.1) (2020.6.20)
Requirement already satisfied: hiredis in /root/anaconda3/lib/python3.7/site-packages (from aioredis->ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.1.0)
Requirement already satisfied: async-timeout in /root/anaconda3/lib/python3.7/site-packages (from aioredis->ray>=1.0.0->hydra-ray-launcher==0.1.1) (3.0.1)
Requirement already satisfied: six>=1.5.2 in /root/anaconda3/lib/python3.7/site-packages (from grpcio>=1.28.1->ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.15.0)
Requirement already satisfied: google-api-core<2.0.0,>=1.0.0 in /root/anaconda3/lib/python3.7/site-packages (from opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.23.0)
Requirement already satisfied: opencensus-context==0.1.2 in /root/anaconda3/lib/python3.7/site-packages (from opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.1.2)
Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /root/anaconda3/lib/python3.7/site-packages (from gpustat->ray>=1.0.0->hydra-ray-launcher==0.1.1) (7.352.0)
Requirement already satisfied: blessings>=1.6 in /root/anaconda3/lib/python3.7/site-packages (from gpustat->ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.7)
Requirement already satisfied: psutil in /root/anaconda3/lib/python3.7/site-packages (from gpustat->ray>=1.0.0->hydra-ray-launcher==0.1.1) (5.7.3)
Requirement already satisfied: pyrsistent>=0.14.0 in /root/anaconda3/lib/python3.7/site-packages (from jsonschema->ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.17.3)
Requirement already satisfied: attrs>=17.4.0 in /root/anaconda3/lib/python3.7/site-packages (from jsonschema->ray>=1.0.0->hydra-ray-launcher==0.1.1) (20.3.0)
Requirement already satisfied: importlib-metadata; python_version < "3.8" in /root/anaconda3/lib/python3.7/site-packages (from jsonschema->ray>=1.0.0->hydra-ray-launcher==0.1.1) (2.0.0)
Requirement already satisfied: setuptools in /root/anaconda3/lib/python3.7/site-packages (from jsonschema->ray>=1.0.0->hydra-ray-launcher==0.1.1) (50.3.1.post20201107)
Requirement already satisfied: multidict<7.0,>=4.5 in /root/anaconda3/lib/python3.7/site-packages (from aiohttp->ray>=1.0.0->hydra-ray-launcher==0.1.1) (5.0.2)
Requirement already satisfied: yarl<2.0,>=1.0 in /root/anaconda3/lib/python3.7/site-packages (from aiohttp->ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.6.3)
Requirement already satisfied: soupsieve>1.2; python_version >= "3.0" in /root/anaconda3/lib/python3.7/site-packages (from beautifulsoup4->google->ray>=1.0.0->hydra-ray-launcher==0.1.1) (2.0.1)
Requirement already satisfied: google-auth<2.0dev,>=1.21.1 in /root/anaconda3/lib/python3.7/site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.23.0)
Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /root/anaconda3/lib/python3.7/site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (1.52.0)
Requirement already satisfied: pytz in /root/anaconda3/lib/python3.7/site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (2020.4)
Requirement already satisfied: rsa<5,>=3.1.4; python_version >= "3.5" in /root/anaconda3/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (4.6)
Requirement already satisfied: pyasn1-modules>=0.2.1 in /root/anaconda3/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.2.8)
Requirement already satisfied: cachetools<5.0,>=2.0.0 in /root/anaconda3/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (4.1.1)
Requirement already satisfied: pyasn1>=0.1.3 in /root/anaconda3/lib/python3.7/site-packages (from rsa<5,>=3.1.4; python_version >= "3.5"->google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0->hydra-ray-launcher==0.1.1) (0.4.8)
Building wheels for collected packages: pickle5
Building wheel for pickle5 (setup.py) ... error
ERROR: Command errored out with exit status 1:
command: /root/anaconda3/bin/python -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-we50x98m/pickle5/setup.py'"'"'; __file__='"'"'/tmp/pip-install-we50x98m/pickle5/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' bdist_wheel -d /tmp/pip-wheel-_z2c2oz0
cwd: /tmp/pip-install-we50x98m/pickle5/
Complete output (21 lines):
running bdist_wheel
running build
running build_py
creating build
creating build/lib.linux-x86_64-3.7
creating build/lib.linux-x86_64-3.7/pickle5
copying pickle5/pickle.py -> build/lib.linux-x86_64-3.7/pickle5
copying pickle5/pickletools.py -> build/lib.linux-x86_64-3.7/pickle5
copying pickle5/__init__.py -> build/lib.linux-x86_64-3.7/pickle5
creating build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/pickletester.py -> build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/test_picklebuffer.py -> build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/test_pickle.py -> build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/__init__.py -> build/lib.linux-x86_64-3.7/pickle5/test
running build_ext
building 'pickle5._pickle' extension
creating build/temp.linux-x86_64-3.7
creating build/temp.linux-x86_64-3.7/pickle5
gcc -pthread -B /root/anaconda3/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/root/anaconda3/include/python3.7m -c pickle5/_pickle.c -o build/temp.linux-x86_64-3.7/pickle5/_pickle.o -std=c99
unable to execute 'gcc': No such file or directory
error: command 'gcc' failed with exit status 1
----------------------------------------
ERROR: Failed building wheel for pickle5
Running setup.py clean for pickle5
Failed to build pickle5
Installing collected packages: botocore, s3transfer, boto3, pickle5, hydra-ray-launcher
Attempting uninstall: botocore
Found existing installation: botocore 1.8.50
Uninstalling botocore-1.8.50:
Successfully uninstalled botocore-1.8.50
Attempting uninstall: s3transfer
Found existing installation: s3transfer 0.1.13
Uninstalling s3transfer-0.1.13:
Successfully uninstalled s3transfer-0.1.13
Attempting uninstall: boto3
Found existing installation: boto3 1.4.8
Uninstalling boto3-1.4.8:
Successfully uninstalled boto3-1.4.8
Running setup.py install for pickle5 ... error
ERROR: Command errored out with exit status 1:
command: /root/anaconda3/bin/python -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-we50x98m/pickle5/setup.py'"'"'; __file__='"'"'/tmp/pip-install-we50x98m/pickle5/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record /tmp/pip-record-kj3mznv1/install-record.txt --single-version-externally-managed --compile --install-headers /root/anaconda3/include/python3.7m/pickle5
cwd: /tmp/pip-install-we50x98m/pickle5/
Complete output (21 lines):
running install
running build
running build_py
creating build
creating build/lib.linux-x86_64-3.7
creating build/lib.linux-x86_64-3.7/pickle5
copying pickle5/pickle.py -> build/lib.linux-x86_64-3.7/pickle5
copying pickle5/pickletools.py -> build/lib.linux-x86_64-3.7/pickle5
copying pickle5/__init__.py -> build/lib.linux-x86_64-3.7/pickle5
creating build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/pickletester.py -> build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/test_picklebuffer.py -> build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/test_pickle.py -> build/lib.linux-x86_64-3.7/pickle5/test
copying pickle5/test/__init__.py -> build/lib.linux-x86_64-3.7/pickle5/test
running build_ext
building 'pickle5._pickle' extension
creating build/temp.linux-x86_64-3.7
creating build/temp.linux-x86_64-3.7/pickle5
gcc -pthread -B /root/anaconda3/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/root/anaconda3/include/python3.7m -c pickle5/_pickle.c -o build/temp.linux-x86_64-3.7/pickle5/_pickle.o -std=c99
unable to execute 'gcc': No such file or directory
error: command 'gcc' failed with exit status 1
----------------------------------------
ERROR: Command errored out with exit status 1: /root/anaconda3/bin/python -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-we50x98m/pickle5/setup.py'"'"'; __file__='"'"'/tmp/pip-install-we50x98m/pickle5/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record /tmp/pip-record-kj3mznv1/install-record.txt --single-version-externally-managed --compile --install-headers /root/anaconda3/include/python3.7m/pickle5 Check the logs for full command output.
Cluster: hydra-ray-cluster

Checking AWS environment settings
AWS config
IAM Profile: ray-head-v1
EC2 Key pair (head &amp; workers): hydra_key-1 [default]
VPC Subnets (head &amp; workers): subnet-7df0af07, subnet-db16f7b0 [default]
EC2 Security groups (head &amp; workers): sg-07777bf2f9d59ad1c [default]
EC2 AMI (head &amp; workers): ami-08e16447bd5caf26a

No head node found. Launching a new cluster. Confirm [y/N]: y [automatic, due to --yes]

Acquiring an up-to-date head node
Launched 1 nodes [subnet_id=subnet-7df0af07]
Launched instance i-0f356b75af10f43b3 [state=pending, info=pending]
Launched a new head node
Fetching the new head node

<1/1> Setting up head node
Prepared bootstrap config
New status: waiting-for-ssh
[1/6] Waiting for SSH to become available
Running `uptime` as a test.
Waiting for IP
Not yet available, retrying in 10 seconds
Not yet available, retrying in 10 seconds
Received: 18.222.27.12
SSH still not available SSH command failed., retrying in 5 seconds.
SSH still not available SSH command failed., retrying in 5 seconds.
SSH still not available SSH command failed., retrying in 5 seconds.
SSH still not available SSH command failed., retrying in 5 seconds.
SSH still not available SSH command failed., retrying in 5 seconds.
Success.
Updating cluster configuration. [hash=a08b593e3cc7e8cff4536186fc51fb9763b37243]
New status: syncing-files
[3/6] Processing file mounts
[4/6] No worker file mounts to sync
New status: setting-up
[3/6] No initialization commands to run.
[4/6] Running setup commands
(0/6) pip3 install omegaconf==2.0.3
(1/6) pip3 install hydra-core==1.0.3
(2/6) pip3 install hydra-ray-launche...
New status: update-failed
!!!
SSH command failed.
!!!

Failed to setup head node.
Error: ssh: connect to host 18.222.27.12 port 22: Connection timed out
ssh: connect to host 18.222.27.12 port 22: Connection timed out
ssh: connect to host 18.222.27.12 port 22: Connection timed out
ssh: connect to host 18.222.27.12 port 22: Connection refused
ssh: connect to host 18.222.27.12 port 22: Connection refused
Warning: Permanently added '18.222.27.12' (ECDSA) to the list of known hosts.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
2020-12-10 13:24:39,917 WARNING command_runner.py:773 -- Nvidia Container Runtime is present, but no GPUs found.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
Shared connection to 18.222.27.12 closed.
[2020-12-10 13:25:40,197][HYDRA] Running command: ['ray', 'exec', '--docker', '/tmp/tmp2uz4ynig.yaml', 'echo $(mktemp -d)']
[2020-12-10 13:25:40,445][HYDRA] Output:
Error: Usage: ray exec [OPTIONS] CLUSTER_CONFIG_FILE CMD
Try 'ray exec --help' for help.

Error: no such option: --docker
Traceback (most recent call last):
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra/_internal/utils.py", line 198, in run_and_report
return func()
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra/_internal/utils.py", line 358, in <lambda>
overrides=args.overrides,
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra/_internal/hydra.py", line 136, in multirun
return sweeper.sweep(arguments=task_overrides)
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra/_internal/core_plugins/basic_sweeper.py", line 154, in sweep
results = self.launcher.launch(batch, initial_job_idx=initial_job_idx)
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra_plugins/hydra_ray_launcher/ray_aws_launcher.py", line 59, in launch
launcher=self, job_overrides=job_overrides, initial_job_idx=initial_job_idx
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra_plugins/hydra_ray_launcher/_core_aws.py", line 104, in launch
launcher, local_tmp_dir, Path(HydraConfig.get().sweep.dir)
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra_plugins/hydra_ray_launcher/_core_aws.py", line 115, in launch_jobs
launcher.ray_yaml_path, launcher.docker_enabled
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/sam/miniconda3/envs/gnosis/lib/python3.6/site-packages/hydra_plugins/hydra_ray_launcher/_launcher_util.py", line 96, in ray_tmp_dir
assert len(tmppath) == 1, f"tmppath is : {tmppath}"
AssertionError: tmppath is : []

# Fixed code
def launch_jobs(
    launcher: RayAWSLauncher, local_tmp_dir: str, sweep_dir: Path
) -> Sequence[JobReturn]:
    ray_up(launcher.ray_yaml_path)
    with tempfile.TemporaryDirectory() as local_tmp_download_dir:
        with ray_tmp_dir(
            launcher.ray_yaml_path, launcher.ray_cfg.run_env.name
        ) as remote_tmp_dir:
            ray_rsync_up(
                launcher.ray_yaml_path, os.path.join(local_tmp_dir, ""), remote_tmp_dir
            )

            script_path = os.path.join(os.path.dirname(__file__), "_remote_invoke.py")
            remote_script_path = os.path.join(remote_tmp_dir, "_remote_invoke.py")
            ray_rsync_up(launcher.ray_yaml_path, script_path, remote_script_path)

            if launcher.sync_up.source_dir:
                source_dir = _get_abs_code_dir(launcher.sync_up.source_dir)
                target_dir = (
                    launcher.sync_up.target_dir
                    if launcher.sync_up.target_dir
                    else remote_tmp_dir
                )
                rsync(
                    launcher.ray_yaml_path,
                    launcher.sync_up.include,
                    launcher.sync_up.exclude,
                    os.path.join(source_dir, ""),
                    target_dir,
                )

            ray_exec(
                launcher.ray_yaml_path,
                launcher.ray_cfg.run_env.name,
                remote_script_path,
                remote_tmp_dir,
            )

            ray_rsync_down(
                launcher.ray_yaml_path,
                os.path.join(remote_tmp_dir, JOB_RETURN_PICKLE),
                local_tmp_download_dir,
            )

            sync_down_cfg = launcher.sync_down

            if (
                sync_down_cfg.target_dir
                or sync_down_cfg.source_dir
                or sync_down_cfg.include
                or sync_down_cfg.exclude
            ):
                source_dir = (
                    sync_down_cfg.source_dir if sync_down_cfg.source_dir else sweep_dir
                )
                target_dir = (
                    sync_down_cfg.source_dir if sync_down_cfg.source_dir else sweep_dir
                )
                target_dir = Path(_get_abs_code_dir(target_dir))
                target_dir.mkdir(parents=True, exist_ok=True)

                rsync(
                    launcher.ray_yaml_path,
                    launcher.sync_down.include,
                    launcher.sync_down.exclude,
                    os.path.join(source_dir),
                    str(target_dir),
                    up=False,
                )
                log.info(
                    f"Syncing outputs from remote dir: {source_dir} to local dir: {target_dir.absolute()} "
                )

        if launcher.stop_cluster:
            log.info("Stopping cluster now. (stop_cluster=true)")
            if launcher.ray_cfg.cluster.provider.cache_stopped_nodes:
                log.info("NOT deleting the cluster (provider.cache_stopped_nodes=true)")
            else:
                log.info("Deleted the cluster (provider.cache_stopped_nodes=false)")
            ray_down(launcher.ray_yaml_path)
        else:
            log.warning(
                "NOT stopping cluster, this may incur extra cost for you. (stop_cluster=false)"
            )

        with open(os.path.join(local_tmp_download_dir, JOB_RETURN_PICKLE), "rb") as f:
            job_returns = pickle.load(f)  # nosec
            assert isinstance(job_returns, List)
            for run in job_returns:
                assert isinstance(run, JobReturn)
            return job_returns