Repository: Parsl/parsl

Issue:
Radical Pilot test failure in CI
**Describe the bug**
Since around Friday (according to @WardLT ), Parsl CI has been failing with this radical failure:

```
parsl/tests/conftest.py:180: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
parsl/tests/configs/local_radical.py:4: in <module>
    from parsl.executors.radical import RadicalPilotExecutor
parsl/executors/radical/__init__.py:1: in <module>
    from parsl.executors.radical.executor import RadicalPilotExecutor
parsl/executors/radical/executor.py:20: in <module>
    from .rpex_resources import ResourceConfig
parsl/executors/radical/rpex_resources.py:8: in <module>
    import radical.pilot as rp
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    
    __copyright__ = "Copyright 2013-2014, http://radical.rutgers.edu/"
    __license__   = "MIT"
    
    # ------------------------------------------------------------------------------
    # we *first* import radical.utils, so that the monkeypatching of the logger has
    # a chance to kick in before the logging module is pulled by any other 3rd party
    # module, and also to monkeypatch `os.fork()` for the `atfork` functionality
    import radical.utils as _ru
    
    # ------------------------------------------------------------------------------
    # constants and types
    from .states     import *
    from .constants  import *
    
    
    # ------------------------------------------------------------------------------
    # import API
    from .session                   import Session
    from .proxy                     import Proxy
    
    from .task_manager              import TaskManager
    from .task                      import Task
    from .raptor_tasks              import RaptorMaster, RaptorWorker
    from .pytask                    import PythonTask
    from .task_description          import TaskDescription
    from .task_description          import TASK_EXECUTABLE
    from .task_description          import TASK_METH, TASK_METHOD
    from .task_description          import TASK_FUNC, TASK_FUNCTION
    from .task_description          import TASK_EXEC, TASK_EVAL
    from .task_description          import TASK_PROC, TASK_SHELL
    from .task_description          import RAPTOR_MASTER, RAPTOR_WORKER
    from .task_description          import AGENT_SERVICE
    from .resource_config           import ResourceConfig
    
    from .pilot_manager             import PilotManager
    from .pilot                     import Pilot
    from .pilot_description         import PilotDescription
    
    pythontask = PythonTask.pythontask
    
    
    # ------------------------------------------------------------------------------
    # make submodules available -- mostly for internal use
    from . import utils
    from . import tmgr
    from . import pmgr
    from . import agent
    
    from .agent  import Agent_0
    from .agent  import Agent_n
    
    from .raptor import Master, Worker
    
    
    # ------------------------------------------------------------------------------
    #
    # get version info
    #
    import os as _os
    
>   version_short, version_detail, version_base, version_branch, \
            sdist_name, sdist_path = _ru.get_version(_os.path.dirname(__file__))
E   ValueError: not enough values to unpack (expected 6, got 5)

.venv/lib/python3.11/site-packages/radical/pilot/__init__.py:62: ValueError
```

cc @AymenFJA @andre-merzky
Radical Pilot test failure in CI
**Describe the bug**
Since around Friday (according to @WardLT ), Parsl CI has been failing with this radical failure:

```
parsl/tests/conftest.py:180: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
parsl/tests/configs/local_radical.py:4: in <module>
    from parsl.executors.radical import RadicalPilotExecutor
parsl/executors/radical/__init__.py:1: in <module>
    from parsl.executors.radical.executor import RadicalPilotExecutor
parsl/executors/radical/executor.py:20: in <module>
    from .rpex_resources import ResourceConfig
parsl/executors/radical/rpex_resources.py:8: in <module>
    import radical.pilot as rp
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    
    __copyright__ = "Copyright 2013-2014, http://radical.rutgers.edu/"
    __license__   = "MIT"
    
    # ------------------------------------------------------------------------------
    # we *first* import radical.utils, so that the monkeypatching of the logger has
    # a chance to kick in before the logging module is pulled by any other 3rd party
    # module, and also to monkeypatch `os.fork()` for the `atfork` functionality
    import radical.utils as _ru
    
    # ------------------------------------------------------------------------------
    # constants and types
    from .states     import *
    from .constants  import *
    
    
    # ------------------------------------------------------------------------------
    # import API
    from .session                   import Session
    from .proxy                     import Proxy
    
    from .task_manager              import TaskManager
    from .task                      import Task
    from .raptor_tasks              import RaptorMaster, RaptorWorker
    from .pytask                    import PythonTask
    from .task_description          import TaskDescription
    from .task_description          import TASK_EXECUTABLE
    from .task_description          import TASK_METH, TASK_METHOD
    from .task_description          import TASK_FUNC, TASK_FUNCTION
    from .task_description          import TASK_EXEC, TASK_EVAL
    from .task_description          import TASK_PROC, TASK_SHELL
    from .task_description          import RAPTOR_MASTER, RAPTOR_WORKER
    from .task_description          import AGENT_SERVICE
    from .resource_config           import ResourceConfig
    
    from .pilot_manager             import PilotManager
    from .pilot                     import Pilot
    from .pilot_description         import PilotDescription
    
    pythontask = PythonTask.pythontask
    
    
    # ------------------------------------------------------------------------------
    # make submodules available -- mostly for internal use
    from . import utils
    from . import tmgr
    from . import pmgr
    from . import agent
    
    from .agent  import Agent_0
    from .agent  import Agent_n
    
    from .raptor import Master, Worker
    
    
    # ------------------------------------------------------------------------------
    #
    # get version info
    #
    import os as _os
    
>   version_short, version_detail, version_base, version_branch, \
            sdist_name, sdist_path = _ru.get_version(_os.path.dirname(__file__))
E   ValueError: not enough values to unpack (expected 6, got 5)

.venv/lib/python3.11/site-packages/radical/pilot/__init__.py:62: ValueError
```

cc @AymenFJA @andre-merzky
Radical Pilot test failure in CI
**Describe the bug**
Since around Friday (according to @WardLT ), Parsl CI has been failing with this radical failure:

```
parsl/tests/conftest.py:180: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
parsl/tests/configs/local_radical.py:4: in <module>
    from parsl.executors.radical import RadicalPilotExecutor
parsl/executors/radical/__init__.py:1: in <module>
    from parsl.executors.radical.executor import RadicalPilotExecutor
parsl/executors/radical/executor.py:20: in <module>
    from .rpex_resources import ResourceConfig
parsl/executors/radical/rpex_resources.py:8: in <module>
    import radical.pilot as rp
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    
    __copyright__ = "Copyright 2013-2014, http://radical.rutgers.edu/"
    __license__   = "MIT"
    
    # ------------------------------------------------------------------------------
    # we *first* import radical.utils, so that the monkeypatching of the logger has
    # a chance to kick in before the logging module is pulled by any other 3rd party
    # module, and also to monkeypatch `os.fork()` for the `atfork` functionality
    import radical.utils as _ru
    
    # ------------------------------------------------------------------------------
    # constants and types
    from .states     import *
    from .constants  import *
    
    
    # ------------------------------------------------------------------------------
    # import API
    from .session                   import Session
    from .proxy                     import Proxy
    
    from .task_manager              import TaskManager
    from .task                      import Task
    from .raptor_tasks              import RaptorMaster, RaptorWorker
    from .pytask                    import PythonTask
    from .task_description          import TaskDescription
    from .task_description          import TASK_EXECUTABLE
    from .task_description          import TASK_METH, TASK_METHOD
    from .task_description          import TASK_FUNC, TASK_FUNCTION
    from .task_description          import TASK_EXEC, TASK_EVAL
    from .task_description          import TASK_PROC, TASK_SHELL
    from .task_description          import RAPTOR_MASTER, RAPTOR_WORKER
    from .task_description          import AGENT_SERVICE
    from .resource_config           import ResourceConfig
    
    from .pilot_manager             import PilotManager
    from .pilot                     import Pilot
    from .pilot_description         import PilotDescription
    
    pythontask = PythonTask.pythontask
    
    
    # ------------------------------------------------------------------------------
    # make submodules available -- mostly for internal use
    from . import utils
    from . import tmgr
    from . import pmgr
    from . import agent
    
    from .agent  import Agent_0
    from .agent  import Agent_n
    
    from .raptor import Master, Worker
    
    
    # ------------------------------------------------------------------------------
    #
    # get version info
    #
    import os as _os
    
>   version_short, version_detail, version_base, version_branch, \
            sdist_name, sdist_path = _ru.get_version(_os.path.dirname(__file__))
E   ValueError: not enough values to unpack (expected 6, got 5)

.venv/lib/python3.11/site-packages/radical/pilot/__init__.py:62: ValueError
```

cc @AymenFJA @andre-merzky


Discussion:
The radical pilot tests were hanging on my laptop, but with an upgrade of some of the dependencies, I now see the above problem too.

I tried upgrading to radical 1.60 (from the pinned 1.52.1) and that does not work for me either.


With radical pilot 1.60, I get this error:

```
parsl/tests/configs/local_radical.py:4: in <module>
    from parsl.executors.radical import RadicalPilotExecutor
parsl/executors/radical/__init__.py:1: in <module>
    from parsl.executors.radical.executor import RadicalPilotExecutor
parsl/executors/radical/executor.py:20: in <module>
    from .rpex_resources import ResourceConfig
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    import sys
    import json
    
    from typing import List
    
    _setup_paths: List[str]
    try:
        import radical.pilot as rp
        import radical.utils as ru
    except ImportError:
        _setup_paths = []
    else:
>       _setup_paths = [rp.sdist_path,
                        ru.sdist_path]
E       AttributeError: module 'radical.pilot' has no attribute 'sdist_path'

parsl/executors/radical/rpex_resources.py:13: AttributeError
```
PR #3431 pins an earlier version of radical.utils. Maybe a better thing to do is upgrade the whole radical stack as used by parsl, and pin more radical stuff if that's how things are meant to be versioned.
The radical pilot tests were hanging on my laptop, but with an upgrade of some of the dependencies, I now see the above problem too.

I tried upgrading to radical 1.60 (from the pinned 1.52.1) and that does not work for me either.


With radical pilot 1.60, I get this error:

```
parsl/tests/configs/local_radical.py:4: in <module>
    from parsl.executors.radical import RadicalPilotExecutor
parsl/executors/radical/__init__.py:1: in <module>
    from parsl.executors.radical.executor import RadicalPilotExecutor
parsl/executors/radical/executor.py:20: in <module>
    from .rpex_resources import ResourceConfig
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    import sys
    import json
    
    from typing import List
    
    _setup_paths: List[str]
    try:
        import radical.pilot as rp
        import radical.utils as ru
    except ImportError:
        _setup_paths = []
    else:
>       _setup_paths = [rp.sdist_path,
                        ru.sdist_path]
E       AttributeError: module 'radical.pilot' has no attribute 'sdist_path'

parsl/executors/radical/rpex_resources.py:13: AttributeError
```
PR #3431 pins an earlier version of radical.utils. Maybe a better thing to do is upgrade the whole radical stack as used by parsl, and pin more radical stuff if that's how things are meant to be versioned.
The radical pilot tests were hanging on my laptop, but with an upgrade of some of the dependencies, I now see the above problem too.

I tried upgrading to radical 1.60 (from the pinned 1.52.1) and that does not work for me either.


With radical pilot 1.60, I get this error:

```
parsl/tests/configs/local_radical.py:4: in <module>
    from parsl.executors.radical import RadicalPilotExecutor
parsl/executors/radical/__init__.py:1: in <module>
    from parsl.executors.radical.executor import RadicalPilotExecutor
parsl/executors/radical/executor.py:20: in <module>
    from .rpex_resources import ResourceConfig
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    import sys
    import json
    
    from typing import List
    
    _setup_paths: List[str]
    try:
        import radical.pilot as rp
        import radical.utils as ru
    except ImportError:
        _setup_paths = []
    else:
>       _setup_paths = [rp.sdist_path,
                        ru.sdist_path]
E       AttributeError: module 'radical.pilot' has no attribute 'sdist_path'

parsl/executors/radical/rpex_resources.py:13: AttributeError
```
PR #3431 pins an earlier version of radical.utils. Maybe a better thing to do is upgrade the whole radical stack as used by parsl, and pin more radical stuff if that's how things are meant to be versioned.

Resolving patch:
diff --git a/parsl/executors/radical/rpex_resources.py b/parsl/executors/radical/rpex_resources.py
--- a/parsl/executors/radical/rpex_resources.py
+++ b/parsl/executors/radical/rpex_resources.py
@@ -3,15 +3,11 @@
 
 from typing import List
 
-_setup_paths: List[str]
+_setup_paths: List[str] = []
 try:
     import radical.pilot as rp
-    import radical.utils as ru
 except ImportError:
-    _setup_paths = []
-else:
-    _setup_paths = [rp.sdist_path,
-                    ru.sdist_path]
+    pass
 
 
 MPI = "mpi"
@@ -77,7 +73,7 @@ class ResourceConfig:
 
     pilot_env_setup : list
         List of setup commands/packages for the pilot environment.
-        Default setup includes "parsl", rp.sdist_path, and ru.sdist_path.
+        Default is an empty list.
 
     python_v : str
         The Python version to be used in the pilot environment.
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -34,7 +34,7 @@
     'workqueue': ['work_queue'],
     'flux': ['pyyaml', 'cffi', 'jsonschema'],
     'proxystore': ['proxystore'],
-    'radical-pilot': ['radical.pilot==1.52.1', 'radical.utils==1.52'],
+    'radical-pilot': ['radical.pilot==1.60', 'radical.utils==1.60'],
     # Disabling psi-j since github direct links are not allowed by pypi
     # 'psij': ['psi-j-parsl@git+https://github.com/ExaWorks/psi-j-parsl']
 }


Test patch: