internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: Parsl/parsl

Issue:
File creation fails if no executor has been loaded
```python
In [53]: file = File('first.txt')
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-53-b5dc63ca042a> in <module>()
----> 1 file = File('first.txt')

~/ci/parsl/parsl/data_provider/files.py in __init__(self, url, dman, cache, caching_dir)
     40         self.path = parsed_url.path
     41         self.filename = os.path.basename(self.path)
---> 42         self.dman = dman if dman else DataManager.get_data_manager()
     43         self.data_future = {}
     44         if self.scheme == 'globus':

~/ci/parsl/parsl/data_provider/data_manager.py in get_data_manager(cls, max_threads, executors)
     24     def get_data_manager(cls, max_threads=None, executors=None):
     25         if cls.default_data_manager is None:
---> 26             cls.default_data_manager = DataManager(max_threads=max_threads, executors=executors)
     27         return cls.default_data_manager
     28

~/ci/parsl/parsl/data_provider/data_manager.py in __init__(self, max_threads, executors)
     37
     38         self.label = 'data_manager'
---> 39         self.executors = {e.label: e for e in executors}
     40         self.max_threads = max_threads
     41         self.files = []

TypeError: 'NoneType' object is not iterable
```


Discussion:

Resolving patch:
diff --git a/parsl/data_provider/data_manager.py b/parsl/data_provider/data_manager.py
--- a/parsl/data_provider/data_manager.py
+++ b/parsl/data_provider/data_manager.py
@@ -36,6 +36,8 @@ def __init__(self, max_threads=10, executors=None):
         self._scaling_enabled = False
 
         self.label = 'data_manager'
+        if executors is None:
+            executors = []
         self.executors = {e.label: e for e in executors}
         self.max_threads = max_threads
         self.files = []


Test patch:
Repository: Parsl/parsl

Issue:
Fix ambiguity between `parsl.app.python_app` and `parsl.app.app.python_app`
Reported by @djf604.

Especially when using an IDE to look for the decorator name, it is confusing that we have a module and the decorator named the same thing.


Discussion:

Resolving patch:
diff --git a/parsl/app/app_factory.py b/parsl/app/app_factory.py
--- a/parsl/app/app_factory.py
+++ b/parsl/app/app_factory.py
@@ -4,8 +4,8 @@
 from inspect import getsource
 from hashlib import md5
 from inspect import signature
-from parsl.app.bash_app import BashApp
-from parsl.app.python_app import PythonApp
+from parsl.app.bash import BashApp
+from parsl.app.python import PythonApp
 from parsl.app.errors import InvalidAppTypeError
 
 logger = logging.getLogger(__name__)
diff --git a/parsl/app/bash_app.py b/parsl/app/bash.py
similarity index 100%
rename from parsl/app/bash_app.py
rename to parsl/app/bash.py
diff --git a/parsl/app/python_app.py b/parsl/app/python.py
similarity index 100%
rename from parsl/app/python_app.py
rename to parsl/app/python.py


Test patch:
Repository: Parsl/parsl

Issue:
confusing error message with simple configs
Using this config:

```
config = Config(executors=[ThreadPoolExecutor()])
```

parsl startup logs this message to the logger:

```
$ ./c.py 
2018-07-11 08:04:42 parsl.config:66 [DEBUG]  Checkpoint period only has an effect with checkpoint_mode='periodic'
```

This appears to be in config creation, as it happens even without passing that config to create a DataFlowKernel.

This might cause user confusion that something is broken/misconfigured.



Discussion:

Resolving patch:
diff --git a/parsl/config.py b/parsl/config.py
--- a/parsl/config.py
+++ b/parsl/config.py
@@ -48,7 +48,7 @@ def __init__(self,
                  app_cache=True,
                  checkpoint_files=None,
                  checkpoint_mode=None,
-                 checkpoint_period="00:30:00",
+                 checkpoint_period=None,
                  data_management_max_threads=10,
                  lazy_errors=True,
                  retries=0,
@@ -62,8 +62,17 @@ def __init__(self,
         self.app_cache = app_cache
         self.checkpoint_files = checkpoint_files
         self.checkpoint_mode = checkpoint_mode
-        if checkpoint_mode is not 'periodic' and checkpoint_period is not None:
-            logger.debug("Checkpoint period only has an effect with checkpoint_mode='periodic'")
+        if checkpoint_period is not None:
+            if checkpoint_mode is None:
+                logger.debug('The requested `checkpoint_period={}` will have no effect because `checkpoint_mode=None`'.format(
+                    checkpoint_period)
+                )
+            elif checkpoint_mode is not 'periodic':
+                logger.debug("Requested checkpoint period of {} only has an effect with checkpoint_mode='periodic'".format(
+                    checkpoint_period)
+                )
+        if checkpoint_mode is 'periodic' and checkpoint_period is None:
+            checkpoint_period = "00:30:00"
         self.checkpoint_period = checkpoint_period
         self.data_management_max_threads = data_management_max_threads
         self.lazy_errors = lazy_errors


Test patch:
Repository: Parsl/parsl

Issue:
IPP `engine_dir` has no effect if indicated dir does not exist
Reported by @daheise. 


Discussion:

Resolving patch:
diff --git a/parsl/executors/ipp.py b/parsl/executors/ipp.py
--- a/parsl/executors/ipp.py
+++ b/parsl/executors/ipp.py
@@ -160,7 +160,8 @@ def compose_launch_cmd(self, filepath, engine_dir, container_image):
             logger.error("Could not open engine_json : ", self.engine_file)
             raise e
 
-        return """cd {0}
+        return """mkdir -p {0}
+cd {0}
 cat <<EOF > ipengine.{uid}.json
 {1}
 EOF
@@ -190,7 +191,8 @@ def compose_containerized_launch_cmd(self, filepath, engine_dir, container_image
             logger.error("Could not open engine_json : ", self.engine_file)
             raise e
 
-        return """cd {0}
+        return """mkdir -p {0}
+cd {0}
 cat <<EOF > ipengine.{uid}.json
 {1}
 EOF


Test patch:
Repository: Parsl/parsl

Issue:
Improve datafuture representation
Here is an example: `<DataFuture at 0x7f305c0b1da0 state=finished returned /home/annawoodard/pnpfit/results/best-fit-cuB.root_file>`

I do not think we should append `_file` to the end of the filepath, it makes it confusing what the actual filepath is.


Discussion:

Resolving patch:
diff --git a/parsl/app/futures.py b/parsl/app/futures.py
--- a/parsl/app/futures.py
+++ b/parsl/app/futures.py
@@ -203,7 +203,7 @@ def __repr__(self):
                             self.__class__.__name__,
                             id(self),
                             _STATE_TO_DESCRIPTION_MAP[parent._state],
-                            self.filepath + '_file')
+                            self.filepath)
                 return '<%s at %#x state=%s>' % (
                     self.__class__.__name__,
                     id(self),


Test patch:
Repository: Parsl/parsl

Issue:
Tweak logging to show info per-executor
If a user has configured multiple executors, they will see multiple `Tasks: X Slots: X Parallelism: X` logging lines for each executor. We need to label the executor here so it is clear what each line refers to.

Idea and report from @djf604.


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/strategy.py b/parsl/dataflow/strategy.py
--- a/parsl/dataflow/strategy.py
+++ b/parsl/dataflow/strategy.py
@@ -2,6 +2,8 @@
 import time
 import math
 
+from parsl.executors.ipp import IPyParallelExecutor
+
 logger = logging.getLogger(__name__)
 
 
@@ -178,15 +180,18 @@ def _strategy_simple(self, tasks, *args, kind=None, **kwargs):
             nodes_per_block = executor.provider.nodes_per_block
             parallelism = executor.provider.parallelism
 
-            active_blocks = sum([1 for x in status if x in ('RUNNING',
-                                                            'SUBMITTING',
-                                                            'PENDING')])
+            running = sum([1 for x in status if x == 'RUNNING'])
+            submitting = sum([1 for x in status if x == 'SUBMITTING'])
+            pending = sum([1 for x in status if x == 'PENDING'])
+            active_blocks = running + submitting + pending
             active_slots = active_blocks * tasks_per_node * nodes_per_block
 
-            # import pdb; pdb.set_trace()
-            logger.debug("Tasks:{} Slots:{} Parallelism:{}".format(len(active_tasks),
-                                                                   active_slots,
-                                                                   parallelism))
+            if isinstance(executor, IPyParallelExecutor):
+                logger.debug('Executor {} has {} active tasks, {}/{}/{} running/submitted/pending blocks, and {} connected engines'.format(
+                    label, len(active_tasks), running, submitting, pending, len(executor.executor)))
+            else:
+                logger.debug('Executor {} has {} active tasks and {}/{}/{} running/submitted/pending blocks'.format(
+                    label, len(active_tasks), running, submitting, pending))
 
             # Case 1
             # No tasks.


Test patch:
Repository: Parsl/parsl

Issue:
Inform user checkpointed results are being used
Reported by @djf604.

We should add a logging line to inform users when checkpointed results are being re-used.


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -339,6 +339,7 @@ def launch_task(self, task_id, executable, *args, **kwargs):
         """
         hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])
         if hit:
+            logger.info("Reusing cached result for task {}".format(task_id))
             self.handle_update(task_id, memo_fu, memo_cbk=True)
             return memo_fu
 


Test patch:
Repository: Parsl/parsl

Issue:
Fix import error
```
ImportError: cannot import name 'BashApp' from 'parsl.app.python' (/home/annawoodard/parsl/parsl/app/python.py)
```

It looks like I introduced this bug in 3d0e2d1e69ad27a133b0c40a42472ae43876d5f2.


Discussion:

Resolving patch:
diff --git a/parsl/app/app.py b/parsl/app/app.py
--- a/parsl/app/app.py
+++ b/parsl/app/app.py
@@ -181,7 +181,7 @@ def bash_app(function=None, data_flow_kernel=None, walltime=60, cache=False, exe
     cache : bool
         Enable caching of the app call. Default is False.
     """
-    from parsl.app.python import BashApp
+    from parsl.app.bash import BashApp
 
     def decorator(func):
         def wrapper(f):


Test patch:
Repository: Parsl/parsl

Issue:
Do not CD to engine_dir
Currently in the IPP submit script we CD to the `engine_dir`. This breaks staging stuff. We should just send engine-related files there rather than changing the PWD.


Discussion:
This was causing files downloaded with `http` to be sent to the `runinfo/NNN` directory.

Resolving patch:
diff --git a/parsl/executors/ipp.py b/parsl/executors/ipp.py
--- a/parsl/executors/ipp.py
+++ b/parsl/executors/ipp.py
@@ -155,13 +155,12 @@ def compose_launch_cmd(self, filepath, engine_dir, container_image):
             raise e
 
         return """mkdir -p {0}
-cd {0}
-cat <<EOF > ipengine.{uid}.json
+cat <<EOF > {0}/ipengine.{uid}.json
 {1}
 EOF
 
-mkdir -p 'engine_logs'
-ipengine --file=ipengine.{uid}.json {debug_option} >> engine_logs/$JOBNAME.log 2>&1
+mkdir -p '{0}/engine_logs'
+ipengine --file={0}/ipengine.{uid}.json {debug_option} >> {0}/engine_logs/$JOBNAME.log 2>&1
 """.format(engine_dir, engine_json, debug_option=self.debug_option, uid=uid)
 
     def compose_containerized_launch_cmd(self, filepath, engine_dir, container_image):


Test patch:
Repository: Parsl/parsl

Issue:
DataManager calls a wrong dfk
When multiple dkfs are used, it is possible that DataManager calls a dkf that was not specified as an argument of an app whose input/output files are remote file(s).


Discussion:

Resolving patch:
diff --git a/parsl/data_provider/data_manager.py b/parsl/data_provider/data_manager.py
--- a/parsl/data_provider/data_manager.py
+++ b/parsl/data_provider/data_manager.py
@@ -47,13 +47,14 @@ class DataManager(ParslExecutor):
     to it, and DataFutures are returned.
     """
 
-    default_data_manager = None
-
     @classmethod
-    def get_data_manager(cls, max_threads=None, executors=None):
-        if cls.default_data_manager is None:
-            cls.default_data_manager = DataManager(max_threads=max_threads, executors=executors)
-        return cls.default_data_manager
+    def get_data_manager(cls):
+        """Return the DataManager of the currently loaded DataFlowKernel.
+        """
+        from parsl.dataflow.dflow import DataFlowKernelLoader
+        dfk = DataFlowKernelLoader.dfk()
+
+        return dfk.executors['data_manager']
 
     def __init__(self, max_threads=10, executors=None):
         """Initialize the DataManager.
diff --git a/parsl/data_provider/files.py b/parsl/data_provider/files.py
--- a/parsl/data_provider/files.py
+++ b/parsl/data_provider/files.py
@@ -20,6 +20,14 @@ class File(str):
 
     This class captures various attributes of a file, and relies on client-side and
     worker-side systems to enable to appropriate transfer of files.
+
+    Note that an error will be raised if one tries to create a File without an
+    associated DataManager. That DataManager may be specified explicitly
+    as a parameter to the File constructor, or may be implicitly specified via
+    a previously loaded Parsl config.
+
+    A File which is not associated with a DataManager is ill-defined.
+
     """
 
     def __init__(self, url, dman=None, cache=False, caching_dir="."):
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -144,10 +144,7 @@ def __init__(self, config=Config()):
         self._checkpoint_timer = None
         self.checkpoint_mode = config.checkpoint_mode
 
-        data_manager = DataManager.get_data_manager(
-            max_threads=config.data_management_max_threads,
-            executors=config.executors
-        )
+        data_manager = DataManager(max_threads=config.data_management_max_threads, executors=config.executors)
         self.executors = {e.label: e for e in config.executors + [data_manager]}
         for executor in self.executors.values():
             executor.run_dir = self.run_dir


Test patch:
diff --git a/parsl/tests/test_data/test_file.py b/parsl/tests/test_data/test_file.py
--- a/parsl/tests/test_data/test_file.py
+++ b/parsl/tests/test_data/test_file.py
@@ -4,6 +4,11 @@
 
 from parsl.data_provider.files import File
 
+import parsl
+from parsl.tests.configs.local_threads import config
+parsl.clear()
+dfk = parsl.load(config)
+
 
 @pytest.mark.local
 def test_files():
Repository: Parsl/parsl

Issue:
Fix import error
```
>>> from parsl.dataflow.states import FINAL_FAILED_STATES
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/annawoodard/parsl/parsl/__init__.py", line 37, in <module>
    from parsl.dataflow.dflow import DataFlowKernel, DataFlowKernelLoader
  File "/home/annawoodard/parsl/parsl/dataflow/dflow.py", line 31, in <module>
    from parsl.dataflow.usage_tracking.usage import UsageTracker
  File "/home/annawoodard/parsl/parsl/dataflow/usage_tracking/usage.py", line 13, in <module>
    from parsl.dataflow.states import FINAL_FAILED_STATES
ImportError: cannot import name 'FINAL_FAILED_STATES'
```


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/usage_tracking/usage.py b/parsl/dataflow/usage_tracking/usage.py
--- a/parsl/dataflow/usage_tracking/usage.py
+++ b/parsl/dataflow/usage_tracking/usage.py
@@ -10,7 +10,7 @@
 import platform
 import multiprocessing as mp
 
-from parsl.dataflow.states import FINAL_FAILED_STATES
+from parsl.dataflow.states import FINAL_FAILURE_STATES
 from parsl.version import VERSION as PARSL_VERSION
 
 logger = logging.getLogger(__name__)
@@ -185,9 +185,8 @@ def construct_end_message(self):
 
         site_count = len([x for x in self.dfk.config.executors if x.managed])
 
-        failed_states = FINAL_FAILED_STATES
         app_fails = len([t for t in self.dfk.tasks if
-                         self.dfk.tasks[t]['status'] in failed_states])
+                         self.dfk.tasks[t]['status'] in FINAL_FAILURE_STATES])
 
         message = {'uuid': self.uuid,
                    'end': time.time(),


Test patch:
Repository: Parsl/parsl

Issue:
Do not hardcode directory to `rundir` for globus tokens
Currently the token file is hardcoded to `TOKEN_FILE = 'runinfo/.globus.json'`. This will break if the user uses a non-default rundir. I vote not to couple it to the rundir so that it can be re-used between different scripts with different rundirs without requiring re-authentication, for example, `$HOME/.parsl/globus.json`.
Do not hardcode directory to `rundir` for globus tokens
Currently the token file is hardcoded to `TOKEN_FILE = 'runinfo/.globus.json'`. This will break if the user uses a non-default rundir. I vote not to couple it to the rundir so that it can be re-used between different scripts with different rundirs without requiring re-authentication, for example, `$HOME/.parsl/globus.json`.


Discussion:

Resolving patch:
diff --git a/parsl/data_provider/globus.py b/parsl/data_provider/globus.py
--- a/parsl/data_provider/globus.py
+++ b/parsl/data_provider/globus.py
@@ -1,6 +1,7 @@
 import logging
 import json
 import globus_sdk
+import os
 
 
 logger = logging.getLogger(__name__)
@@ -22,8 +23,10 @@
 SCOPES = ('openid '
           'urn:globus:auth:scope:transfer.api.globus.org:all')
 
-TOKEN_FILE = 'runinfo/.globus.json'
-
+token_path = os.path.join(os.path.expanduser('~'), '.parsl')
+if not os.path.isdir(token_path):
+    os.mkdir(token_path)
+TOKEN_FILE = os.path.join(token_path, '.globus.json')
 
 get_input = getattr(__builtins__, 'raw_input', input)
 


Test patch:
Repository: Parsl/parsl

Issue:
Make `GlobusScheme` inherit from `RepresentationMixin`
Otherwise, the config printed in the log is not copy-and-pasteable:

```
        storage_access=[<parsl.data_provider.scheme.GlobusScheme object at 0x7f48d021fbe0>], 
        working_dir=None
```


Make `GlobusScheme` inherit from `RepresentationMixin`
Otherwise, the config printed in the log is not copy-and-pasteable:

```
        storage_access=[<parsl.data_provider.scheme.GlobusScheme object at 0x7f48d021fbe0>], 
        working_dir=None
```




Discussion:

Resolving patch:
diff --git a/parsl/data_provider/scheme.py b/parsl/data_provider/scheme.py
--- a/parsl/data_provider/scheme.py
+++ b/parsl/data_provider/scheme.py
@@ -1,5 +1,7 @@
+from parsl.utils import RepresentationMixin
 
-class GlobusScheme(object):
+
+class GlobusScheme(RepresentationMixin):
     """Specification for accessing data on a remote executor via Globus.
 
     Parameters


Test patch:
Repository: Parsl/parsl

Issue:
TorqueProvider fails on NSCC 
The following patch is required in order to run the `TorqueProvider` on NSCC:
```
[nscc04] ~/libsubmit >git diff
diff --git a/libsubmit/providers/torque/template.py b/libsubmit/providers/torque/template.py
index a00ce7c..056c648 100644
--- a/libsubmit/providers/torque/template.py
+++ b/libsubmit/providers/torque/template.py
@@ -8,7 +8,6 @@ template_string = '''#!/bin/bash
 #PBS -l nodes=${nodes_per_block}:ppn=${tasks_per_node}
 #PBS -o ${submit_script_dir}/${jobname}.submit.stdout
 #PBS -e ${submit_script_dir}/${jobname}.submit.stderr
-#PBS -v WORKER_LOGGING_LEVEL
 ${overrides}

 export JOBNAME="${jobname}"
```

Otherwise, the job fails with `qsub: cannot send environment with the job`. Could we just merge the patch, or should we make this configurable somehow?


Discussion:
This is odd. Both PBSPro and Torque support -v syntax for setting env variables.
http://docs.adaptivecomputing.com/torque/4-0-2/Content/topics/commands/qsub.htm
Page 35: https://www.pbsworks.com/pdfs/PBSUserGuide13.0.pdf

I'm worried that we might be overlooking a scheduler specific oddity.
@yadudoc what is this actually used for? I do not see it mentioned anywhere else in parsl or libsubmit. It appears to be a vestige and safe to remove.
When you specify the worker logging level to the IPP executor, that option has to make it through the submit script down to the point where the ipengines are launched. This is done via an env variable.

Ref: https://github.com/Parsl/parsl/blob/master/parsl/executors/ipp.py#L78
That option is added to the command line ipengine invocation in the submit script as `"--log-level={}".format(self.engine_debug_level)`. `WORKER_LOGGING_LEVEL` does not appear in the ipyparallel source either. Who is reading that env variable?

[https://github.com/Parsl/parsl/blob/master/parsl/executors/ipp.py#L95](https://github.com/Parsl/parsl/blob/master/parsl/executors/ipp.py#L95)


Good point. I think we can safely remove this option.
awesome, thanks!

Resolving patch:
diff --git a/parsl/providers/torque/template.py b/parsl/providers/torque/template.py
--- a/parsl/providers/torque/template.py
+++ b/parsl/providers/torque/template.py
@@ -8,7 +8,6 @@
 #PBS -l nodes=${nodes_per_block}:ppn=${tasks_per_node}
 #PBS -o ${submit_script_dir}/${jobname}.submit.stdout
 #PBS -e ${submit_script_dir}/${jobname}.submit.stderr
-#PBS -v WORKER_LOGGING_LEVEL
 ${overrides}
 
 export JOBNAME="${jobname}"


Test patch:
Repository: Parsl/parsl

Issue:
Error building docs with new env

```
Running Sphinx v1.8.1
loading pickled environment... failed: build environment version not current

Theme error:
sphinx_rtd_theme is no longer a hard dependency since version 1.4.0. Please install it manually.(pip install sphinx_rtd_theme)
```


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@
         'aws' : ['boto3'],
         'jetstream' : ['python-novaclient'],
         'extreme_scale' : ['mpi4py'],
-        'docs' : ['nbsphinx'],
+        'docs' : ['nbsphinx', 'sphinx_rtd_theme'],
         'google_cloud' : ['google-auth', 'google-api-python-client']
         },
     classifiers = [


Test patch:
Repository: Parsl/parsl

Issue:
Improve documentation for launchers
For example, the docstrings for Srun and SrunMPI launchers are the same. We need to include enough information for a user to determine which one they need.


Discussion:

Resolving patch:
diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py
--- a/parsl/executors/high_throughput/executor.py
+++ b/parsl/executors/high_throughput/executor.py
@@ -302,7 +302,11 @@ def weakref_cb(self, q=None):
         q.put(None)
 
     def _start_local_queue_process(self):
+        """ Starts the interchange process locally
 
+        Starts the interchange process locally and uses an internal command queue to
+        get the worker task and result ports that the interchange has bound to.
+        """
         comm_q = Queue(maxsize=10)
         self.queue_proc = Process(target=interchange.starter,
                                   args=(comm_q,),
diff --git a/parsl/launchers/launchers.py b/parsl/launchers/launchers.py
--- a/parsl/launchers/launchers.py
+++ b/parsl/launchers/launchers.py
@@ -212,9 +212,11 @@ def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
 
 
 class SrunMPILauncher(Launcher):
-    """Worker launcher that wraps the user's command with the SRUN launch framework
-    to launch multiple cmd invocations in parallel on a single job allocation.
+    """Launches as many workers as MPI tasks to be executed concurrently within a block.
 
+    Use this launcher instead of SrunLauncher if each block will execute multiple MPI applications
+    at the same time. Workers should be launched with independent Srun calls so as to setup the
+    environment for MPI application launch.
     """
     def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
         """


Test patch: