internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: Parsl/parsl

Issue:
Add "all" install option
We have several subpackages now. I think it would be good if we had an option to install all subpackages. 


Discussion:
Based on this [issue](https://github.com/pypa/pip/issues/4340) posted in the pip repos, there's no plan to support `all` as a magic keyword in pip. So, we should add this to parsl via an "extras_require" option in setup.py.

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -24,10 +24,17 @@
     extras_require = {
         'db_logging' : ['CMRESHandler', 'psutil', 'sqlalchemy'],
         'aws' : ['boto3'],
-        'jetstream' : ['python-novaclient'],
+        # Jetstream is deprecated since the interface has not been maintained.
+        # 'jetstream' : ['python-novaclient'],
         'extreme_scale' : ['mpi4py'],
         'docs' : ['nbsphinx', 'sphinx_rtd_theme'],
-        'google_cloud' : ['google-auth', 'google-api-python-client']
+        'google_cloud' : ['google-auth', 'google-api-python-client'],
+        'all' : ['CMRESHandler', 'psutil', 'sqlalchemy',
+                 'boto3',
+                 'mpi4py',
+                 'nbsphinx', 'sphinx_rtd_theme',
+                 'google-auth', 'google-api-python-client']
+
         },
     classifiers = [
         # Maturity


Test patch:
Repository: Parsl/parsl

Issue:
SSHChannel fails with host-based authentication
Systems using host-based authentication (without a key or a password) fail with:
```
paramiko.ssh_exception.SSHException: No authentication methods available
```

Reported by @jmoon1506


Discussion:

Resolving patch:
diff --git a/parsl/channels/ssh/ssh.py b/parsl/channels/ssh/ssh.py
--- a/parsl/channels/ssh/ssh.py
+++ b/parsl/channels/ssh/ssh.py
@@ -10,6 +10,12 @@
 logger = logging.getLogger(__name__)
 
 
+class HostAuthSSHClient(paramiko.SSHClient):
+    def _auth(self, username, *args):
+        self._transport.auth_none(username)
+        return
+
+
 class SSHChannel(Channel, RepresentationMixin):
     ''' SSH persistent channel. This enables remote execution on sites
     accessible via ssh. It is assumed that the user has setup host keys
@@ -20,7 +26,7 @@ class SSHChannel(Channel, RepresentationMixin):
 
     '''
 
-    def __init__(self, hostname, username=None, password=None, script_dir=None, envs=None, **kwargs):
+    def __init__(self, hostname, username=None, password=None, script_dir=None, envs=None, host_auth=False, **kwargs):
         ''' Initialize a persistent connection to the remote system.
         We should know at this point whether ssh connectivity is possible
 
@@ -42,8 +48,12 @@ def __init__(self, hostname, username=None, password=None, script_dir=None, envs
         self.password = password
         self.kwargs = kwargs
         self.script_dir = script_dir
+        self.host_auth = host_auth
 
-        self.ssh_client = paramiko.SSHClient()
+        if host_auth:
+            self.ssh_client = HostAuthSSHClient()
+        else:
+            self.ssh_client = paramiko.SSHClient()
         self.ssh_client.load_system_host_keys()
         self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 


Test patch:
Repository: Parsl/parsl

Issue:
With TorqueProvider, submit stderr/stdout does not go to runinfo
This happens on both NSCC and Blue Waters. The submit script has

```
#PBS -o /mnt/a/u/sciteam/woodard/simple-tests/runinfo/001/submit_scripts/parsl.parsl.auto.1542146393.457273.submit.stdout
#PBS -e /mnt/a/u/sciteam/woodard/simple-tests/runinfo/001/submit_scripts/parsl.parsl.auto.1542146393.457273.submit.stderr
```

but the stdout goes to `$HOME/parsl.parsl.auto.1542146393.457273.o9212235`


Discussion:

Resolving patch:
diff --git a/parsl/providers/torque/template.py b/parsl/providers/torque/template.py
--- a/parsl/providers/torque/template.py
+++ b/parsl/providers/torque/template.py
@@ -3,7 +3,6 @@
 #PBS -S /bin/bash
 #PBS -N ${jobname}
 #PBS -m n
-#PBS -k eo
 #PBS -l walltime=$walltime
 #PBS -l nodes=${nodes_per_block}:ppn=${tasks_per_node}
 #PBS -o ${submit_script_dir}/${jobname}.submit.stdout


Test patch:
Repository: Parsl/parsl

Issue:
Add midway worker bug to FAQ
We should try to give some hints in the FAQ that would help someone encountering #633.


Discussion:
While we should go ahead and document troubleshooting these issues, we still need confirmation on whether this was a networking issue on midway. I'm waiting on a midway support ticket on this.

Resolving patch:
diff --git a/parsl/configs/htex_local.py b/parsl/configs/htex_local.py
--- a/parsl/configs/htex_local.py
+++ b/parsl/configs/htex_local.py
@@ -1,6 +1,5 @@
 from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
 
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
@@ -8,17 +7,13 @@
 config = Config(
     executors=[
         HighThroughputExecutor(
-            label="htex_Local",
-            worker_debug=True,
+            label="htex_local",
             cores_per_worker=1,
             provider=LocalProvider(
                 channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
-                tasks_per_node=1,  # For HighThroughputExecutor, this option should in most cases be 1
-                launcher=SimpleLauncher(),
             ),
         )
     ],
-    strategy=None,
 )
diff --git a/parsl/executors/extreme_scale/executor.py b/parsl/executors/extreme_scale/executor.py
--- a/parsl/executors/extreme_scale/executor.py
+++ b/parsl/executors/extreme_scale/executor.py
@@ -78,6 +78,9 @@ class ExtremeScaleExecutor(HighThroughputExecutor, RepresentationMixin):
 
     launch_cmd : str
         Command line string to launch the mpi_worker_pool from the provider.
+        The command line string will be formatted with appropriate values for the following values (debug, task_url, result_url,
+        ranks_per_node, nodes_per_block, heartbeat_period ,heartbeat_threshold, logdir). For eg:
+        launch_cmd="mpiexec -np {ranks_per_node} mpi_worker_pool.py {debug} --task_url={task_url} --result_url={result_url}"
 
     address : string
         An address to connect to the main Parsl process which is reachable from the network in which
diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py
--- a/parsl/executors/high_throughput/executor.py
+++ b/parsl/executors/high_throughput/executor.py
@@ -74,7 +74,10 @@ class HighThroughputExecutor(ParslExecutor, RepresentationMixin):
         Label for this executor instance.
 
     launch_cmd : str
-        Command line string to launch the process_worker_pool from the provider.
+        Command line string to launch the process_worker_pool from the provider. The command line string
+        will be formatted with appropriate values for the following values (debug, task_url, result_url,
+        cores_per_worker, nodes_per_block, heartbeat_period ,heartbeat_threshold, logdir). For eg:
+        launch_cmd="process_worker_pool.py {debug} -c {cores_per_worker} --task_url={task_url} --result_url={result_url}"
 
     address : string
         An address to connect to the main Parsl process which is reachable from the network in which


Test patch:
Repository: Parsl/parsl

Issue:
Kubernetes option missing in setup.py
The option to install kubernetes as an optional extra is missing from our setup.py script.

reported by Ben Galewsky.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -27,6 +27,7 @@
         'visualize': ['dash', 'dash-html-components', 'dash-core-components', 'pandas'],
         'db_logging' : ['CMRESHandler', 'psutil', 'sqlalchemy'],
         'aws' : ['boto3'],
+        'kubernetes' : ['kubernetes'],
         # Jetstream is deprecated since the interface has not been maintained.
         # 'jetstream' : ['python-novaclient'],
         'extreme_scale' : ['mpi4py'],
@@ -36,6 +37,7 @@
         'all' : ['CMRESHandler', 'psutil', 'sqlalchemy',
                  'dash', 'dash-html-components', 'dash-core-components', 'pandas',
                  'boto3',
+                 'kubernetes',
                  'mpi4py',
                  'nbsphinx', 'sphinx_rtd_theme',
                  'google-auth', 'google-api-python-client',


Test patch:
Repository: Parsl/parsl

Issue:
Two attributes never used in parsl/executors/high_throughput/interchange.py
Seems the two attributes below are not used anywhere. https://github.com/Parsl/parsl/blob/6701688f6d9bfe8e5a4d1ef9183f8fec930db5de/parsl/executors/high_throughput/interchange.py#L171 and https://github.com/Parsl/parsl/blob/6701688f6d9bfe8e5a4d1ef9183f8fec930db5de/parsl/executors/high_throughput/interchange.py#L173



Discussion:
I think that is true.

It would be reasonable to submit a PR removing them.

Resolving patch:
diff --git a/parsl/executors/high_throughput/interchange.py b/parsl/executors/high_throughput/interchange.py
--- a/parsl/executors/high_throughput/interchange.py
+++ b/parsl/executors/high_throughput/interchange.py
@@ -168,9 +168,7 @@ def __init__(self,
         logger.info("Bound to ports {},{} for incoming worker connections".format(
             self.worker_task_port, self.worker_result_port))
 
-        self._task_queue = []
         self._ready_manager_queue = {}
-        self.max_task_queue_size = 10 ** 5
 
         self.heartbeat_threshold = heartbeat_threshold
 


Test patch:
Repository: Parsl/parsl

Issue:
Make usage tracking opt-in



Discussion:

Resolving patch:
diff --git a/parsl/config.py b/parsl/config.py
--- a/parsl/config.py
+++ b/parsl/config.py
@@ -43,7 +43,8 @@ class Config(RepresentationMixin):
         Strategy to use for scaling resources according to workflow needs. Can be 'simple' or `None`. If `None`, dynamic
         scaling will be disabled. Default is 'simple'.
     usage_tracking : bool, optional
-        Enable usage tracking. Default is True.
+        Set this field to True to Opt-in to Parsl's usage tracking system. Parsl only collects minimal, non personally-identifiable,
+        information used for reporting to our funding agencies. Default is False.
     """
     def __init__(self,
                  executors=None,
@@ -57,7 +58,7 @@ def __init__(self,
                  run_dir='runinfo',
                  strategy='simple',
                  monitoring_config=None,
-                 usage_tracking=True):
+                 usage_tracking=False):
         if executors is None:
             executors = [ThreadPoolExecutor()]
         self.executors = executors


Test patch:
Repository: Parsl/parsl

Issue:
Executor idle timer not reset
TLDR; I think there's a bug in assessing timeout for idle executors.

Parsl log: https://gist.github.com/macintoshpie/1ae87cd93e45e532513ea2833a3f8e60

### Setup
Running sequence of 5 tasks that each last about 3 hours on NSCC Aspire1, on a single executor.

### Problem
The first task completes after about 3 hours but then the executor is killed and the parsl head errors out.
See this line for where things go awry: https://gist.github.com/macintoshpie/1ae87cd93e45e532513ea2833a3f8e60#file-parsl-log-L5197

### Proposed bug
You can see in [this line](https://gist.github.com/macintoshpie/1ae87cd93e45e532513ea2833a3f8e60#file-parsl-log-L88) that the `idle_time` is set early in the process. There's no logging that the kill timer is being reset/updated later on either. Once the first task finishes, it meets the conditions to enter a potential timeout state (`active_tasks == 0 && active_blocks > min_blocks`). It looks at the very old `idle_time` set earlier and kills the executor immediately.

The solution is to reset the timer once it's no longer in the possible timeout state. Here's the problematic area of code: https://github.com/Parsl/parsl/blob/b4c55567b468278e15516e89373bc0a72d445317/parsl/dataflow/strategy.py#L209-L228


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/strategy.py b/parsl/dataflow/strategy.py
--- a/parsl/dataflow/strategy.py
+++ b/parsl/dataflow/strategy.py
@@ -204,6 +204,10 @@ def _strategy_simple(self, tasks, *args, kind=None, **kwargs):
                 logger.debug('Executor {} has {} active tasks and {}/{}/{} running/submitted/pending blocks'.format(
                     label, active_tasks, running, submitting, pending))
 
+            # reset kill timer if executor has active tasks
+            if active_tasks > 0 and self.executors[executor.label]['idle_since']:
+                self.executors[executor.label]['idle_since'] = None
+
             # Case 1
             # No tasks.
             if active_tasks == 0:


Test patch:
Repository: Parsl/parsl

Issue:
Condor provider should set default channel
When running the condor provider without explicitly setting the channel:
```
  File "/afs/hep.wisc.edu/home/awoodard/.local/lib/python3.6/site-packages/parsl-0.7.2-py3.6.egg/parsl/dataflow/dflow.py", line 762, in add_executors
    if executor.provider.channel.script_dir is None:
AttributeError: 'NoneType' object has no attribute 'script_dir'
```




Discussion:

Resolving patch:
diff --git a/parsl/providers/condor/condor.py b/parsl/providers/condor/condor.py
--- a/parsl/providers/condor/condor.py
+++ b/parsl/providers/condor/condor.py
@@ -3,6 +3,7 @@
 import re
 import time
 
+from parsl.channels import LocalChannel
 from parsl.utils import RepresentationMixin
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.condor.template import template_string
@@ -59,7 +60,7 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
         :class:`~parsl.launchers.SingleNodeLauncher` (the default),
     """
     def __init__(self,
-                 channel=None,
+                 channel=LocalChannel(),
                  nodes_per_block=1,
                  init_blocks=1,
                  min_blocks=0,


Test patch:
Repository: Parsl/parsl

Issue:
Add mpirun launcher
Needed at NSCC, which doesn't have aprun.


Discussion:

Resolving patch:
diff --git a/parsl/launchers/__init__.py b/parsl/launchers/__init__.py
--- a/parsl/launchers/__init__.py
+++ b/parsl/launchers/__init__.py
@@ -1,6 +1,6 @@
 from parsl.launchers.launchers import SimpleLauncher, SingleNodeLauncher, \
     SrunLauncher, AprunLauncher, SrunMPILauncher, AprunLauncher, \
-    GnuParallelLauncher, MpiExecLauncher
+    GnuParallelLauncher, MpiExecLauncher, MpiRunLauncher
 
 __all__ = ['SimpleLauncher',
            'SingleNodeLauncher',
@@ -9,4 +9,5 @@
            'SrunMPILauncher',
            'AprunLauncher',
            'GnuParallelLauncher',
-           'MpiExecLauncher']
+           'MpiExecLauncher',
+           'MpiRunLauncher']
diff --git a/parsl/launchers/launchers.py b/parsl/launchers/launchers.py
--- a/parsl/launchers/launchers.py
+++ b/parsl/launchers/launchers.py
@@ -172,6 +172,47 @@ def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
         return x
 
 
+class MpiRunLauncher(Launcher):
+    """ Worker launcher that wraps the user's command with the framework to
+    launch multiple command invocations via mpirun.
+
+    This wrapper sets the bash env variable CORES to the number of cores on the
+    machine.
+
+    This launcher makes the following assumptions:
+    - mpirun is installed and can be located in $PATH
+    - The provider makes available the $PBS_NODEFILE environment variable
+    """
+    def __init__(self, bash_location='/bin/bash'):
+        self.bash_location = bash_location
+
+    def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):
+        """
+        Args:
+        - command (string): The command string to be launched
+        - task_block (string) : bash evaluated string.
+
+        KWargs:
+        - walltime (int) : This is not used by this launcher.
+        """
+        task_blocks = tasks_per_node * nodes_per_block
+
+        x = '''export CORES=$(getconf _NPROCESSORS_ONLN)
+echo "Found cores : $CORES"
+WORKERCOUNT={3}
+
+cat << MPIRUN_EOF > cmd_$JOBNAME.sh
+{0}
+MPIRUN_EOF
+chmod u+x cmd_$JOBNAME.sh
+
+mpirun -np $WORKERCOUNT {4} cmd_$JOBNAME.sh
+
+echo "All workers done"
+'''.format(command, tasks_per_node, nodes_per_block, task_blocks, self.bash_location)
+        return x
+
+
 class SrunLauncher(Launcher):
     """ Worker launcher that wraps the user's command with the SRUN launch framework
     to launch multiple cmd invocations in parallel on a single job allocation.


Test patch:
Repository: Parsl/parsl

Issue:
Provide a way by which users can quickly find the associated log file of a failed task
At the moment users see 'task 10 failed', etc, in the parsl log. Their first course of action should be to check the associated stdout/stderr. But that isn't straightforward to do, because they do not have access to a mapping between the task id and the stdout/stderr file name. I think we should either append the task id to the filename they specify, or print the path to the file in the parsl log when we report a task failure.


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -281,6 +281,11 @@ def handle_exec_update(self, task_id, future):
             logger.info("Task {} completed".format(task_id))
             self.tasks[task_id]['time_returned'] = datetime.datetime.now()
 
+        if self.tasks[task_id]['app_fu'].stdout is not None:
+            logger.info("Standard output for task {} available at {}".format(task_id, self.tasks[task_id]['app_fu'].stdout))
+        if self.tasks[task_id]['app_fu'].stderr is not None:
+            logger.info("Standard error for task {} available at {}".format(task_id, self.tasks[task_id]['app_fu'].stderr))
+
         if self.monitoring:
             task_log_info = self._create_task_log_info(task_id, 'lazy')
             self.monitoring.send(MessageType.TASK_INFO, task_log_info)


Test patch:
Repository: Parsl/parsl

Issue:
Properly handle func names for bash apps
At the moment bash apps are always `remote_side_bash_executor` in the monitoring DB because that is what is actually running. We should pass through the name of the app's function instead.


Discussion:
I changed the title of this issue because the problem is more general. We also log all bash app tasks with app name `remote_side_bash_executor`

Resolving patch:
diff --git a/parsl/app/bash.py b/parsl/app/bash.py
--- a/parsl/app/bash.py
+++ b/parsl/app/bash.py
@@ -1,6 +1,7 @@
 import logging
-
+from functools import update_wrapper
 from inspect import signature, Parameter
+
 from parsl.app.errors import wrap_error
 from parsl.app.futures import DataFuture
 from parsl.app.app import AppBase
@@ -150,7 +151,8 @@ def __call__(self, *args, **kwargs):
         else:
             dfk = self.data_flow_kernel
 
-        app_fut = dfk.submit(wrap_error(remote_side_bash_executor), self.func, *args,
+        app_fut = dfk.submit(wrap_error(update_wrapper(remote_side_bash_executor, self.func)),
+                             self.func, *args,
                              executors=self.executors,
                              fn_hash=self.func_hash,
                              cache=self.cache,


Test patch:
Repository: Parsl/parsl

Issue:
Log app args
For easier debugging, we should log the arguments apps are called with.

Requested by @mjwilde 


Discussion:

Resolving patch:
diff --git a/parsl/app/bash.py b/parsl/app/bash.py
--- a/parsl/app/bash.py
+++ b/parsl/app/bash.py
@@ -77,6 +77,9 @@ def open_std_fd(fdname):
     std_err = open_std_fd('stderr')
     timeout = kwargs.get('walltime')
 
+    if std_err is not None:
+        print('--> executable follows <--\n{}\n--> end executable <--'.format(executable), file=std_err)
+
     returncode = None
     try:
         proc = subprocess.Popen(executable, stdout=std_out, stderr=std_err, shell=True, executable='/bin/bash')


Test patch:
Repository: Parsl/parsl

Issue:
Fix workflow name in monitoring DB
At the moment, it is always 'typeguard.py'. This looks related to #875.


Discussion:
cc @benclifford 

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -108,7 +108,7 @@ def __init__(self, config=Config()):
         else:
             for frame in inspect.stack():
                 fname = os.path.basename(str(frame.filename))
-                parsl_file_names = ['dflow.py']
+                parsl_file_names = ['dflow.py', 'typeguard.py']
                 # Find first file name not considered a parsl file
                 if fname not in parsl_file_names:
                     self.workflow_name = fname


Test patch:
Repository: Parsl/parsl

Issue:
visualization merge missing app.py
The current merge of visualization to master is missing `app.py`.


Discussion:

Resolving patch:
diff --git a/parsl/__init__.py b/parsl/__init__.py
--- a/parsl/__init__.py
+++ b/parsl/__init__.py
@@ -22,6 +22,11 @@
 2. set_file_logger
     This sets the logging to a file. This is ideal for reporting issues to the dev team.
 
+Constants
+---------
+AUTO_LOGNAME
+    Special value that indicates Parsl should construct a filename for logging.
+
 """
 import logging
 import typeguard
@@ -42,6 +47,8 @@
 __author__ = 'The Parsl Team'
 __version__ = VERSION
 
+AUTO_LOGNAME = -1
+
 __all__ = [
 
     # decorators
@@ -56,6 +63,7 @@
     # logging
     'set_stream_logger',
     'set_file_logger',
+    'AUTO_LOGNAME',
 
     # executors
     'ThreadPoolExecutor',
diff --git a/parsl/app/bash.py b/parsl/app/bash.py
--- a/parsl/app/bash.py
+++ b/parsl/app/bash.py
@@ -67,8 +67,11 @@ def open_std_fd(fdname):
             fname, mode = stdfspec
         else:
             raise pe.BadStdStreamFile("std descriptor %s has unexpected type %s" % (fdname, str(type(stdfspec))), TypeError('Bad Tuple Type'))
+
         try:
             fd = open(fname, mode)
+            if os.path.dirname(fname):
+                os.makedirs(os.path.dirname(fname), exist_ok=True)
         except Exception as e:
             raise pe.BadStdStreamFile(fname, e)
         return fd
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -631,6 +631,20 @@ def submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwar
 
         # Transform remote input files to data futures
         args, kwargs = self._add_input_deps(executor, args, kwargs)
+        label = kwargs.get('label')
+        for kw in ['stdout', 'stderr']:
+            if kw in kwargs:
+                if kwargs[kw] == parsl.AUTO_LOGNAME:
+                    kwargs[kw] = os.path.join(
+                            self.run_dir,
+                            'task_logs',
+                            str(int(task_id / 10000)).zfill(4),  # limit logs to 10k entries per directory
+                            'task_{}_{}{}.{}'.format(
+                                str(task_id).zfill(4),
+                                func.__name__,
+                                '' if label is None else '_{}'.format(label),
+                                kw)
+                    )
 
         task_def = {'depends': None,
                     'executor': executor,


Test patch: