internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: Parsl/parsl

Issue:
Fix `ModuleNotFoundError: No module named 'monitoring'`
Looks like this bug was introduced with the recent merge of monitoring back into the parsl repo.

```
Traceback (most recent call last):
  File "/Users/awoodard/software/miniconda3/bin/parsl-visualize", line 11, in <module>
    load_entry_point('parsl==0.7.2', 'console_scripts', 'parsl-visualize')()
  File "/Users/awoodard/software/miniconda3/lib/python3.7/site-packages/pkg_resources/__init__.py", line 484, in load_entry_point
    return get_distribution(dist).load_entry_point(group, name)
  File "/Users/awoodard/software/miniconda3/lib/python3.7/site-packages/pkg_resources/__init__.py", line 2707, in load_entry_point
    return ep.load()
  File "/Users/awoodard/software/miniconda3/lib/python3.7/site-packages/pkg_resources/__init__.py", line 2325, in load
    return self.resolve()
  File "/Users/awoodard/software/miniconda3/lib/python3.7/site-packages/pkg_resources/__init__.py", line 2331, in resolve
    module = __import__(self.module_name, fromlist=['__name__'], level=0)
ModuleNotFoundError: No module named 'monitoring'
```


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,6 @@
     entry_points={'console_scripts':
       [
        'parsl-globus-auth=parsl.data_provider.globus:cli_run',
-       'parsl-visualize=monitoring.visualization.app:cli_run',
+       'parsl-visualize=parsl.monitoring.visualization.app:cli_run',
       ]}
 )


Test patch:
Repository: Parsl/parsl

Issue:
visualization merge missing app.py
The current merge of visualization to master is missing `app.py`.


Discussion:

Resolving patch:
diff --git a/parsl/monitoring/visualization/app.py b/parsl/monitoring/visualization/app.py
new file mode 100644
--- /dev/null
+++ b/parsl/monitoring/visualization/app.py
@@ -0,0 +1,36 @@
+from flask import Flask
+from parsl.monitoring.visualization.models import db
+import argparse
+
+
+def cli_run():
+    """ Instantiates the Monitoring viz server
+    """
+    parser = argparse.ArgumentParser(description='Parsl visualization tool')
+    parser.add_argument('db_path', type=str,
+                        help='Database path in the format sqlite:///<absolute_path_to_db>')
+    parser.add_argument('--port', type=int, default=8080,
+                        help='Port at which the monitoring Viz Server is hosted. Default: 8080')
+    parser.add_argument("-d", "--debug", action='store_true',
+                        help="Enable debug logging")
+    parser.add_argument("-e", "--external", action='store_true',
+                        help="Enable hosting on Internet")
+    args = parser.parse_args()
+
+    app = Flask(__name__)
+    app.config['SQLALCHEMY_DATABASE_URI'] = args.db_path
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    db.init_app(app)
+
+    with app.app_context():
+        db.create_all()
+        from parsl.monitoring.visualization import views
+        views.dummy = False
+        host = '127.0.0.1'
+        if args.external:
+            host = '0.0.0.0'
+        app.run(host=host, port=args.port, debug=args.debug)
+
+
+if __name__ == "__main__":
+    cli_run()


Test patch:
Repository: Parsl/parsl

Issue:
bash_apps fail due to missing dir paths
I'm running into bash_app errors when I'm using `stdout=parsl.AUTO_LOGNAME`.
Here's the traceback:
```
2019-05-31 11:22:06 parsl.dataflow.dflow:251 [ERROR]  Task 0 failed
Traceback (most recent call last):
  File "/home/yadu/miniconda3/envs/mpi_executor/lib/python3.6/site-packages/parsl/dataflow/dflow.py", line 248, in handle_exec_update
    res.reraise()
  File "/home/yadu/miniconda3/envs/mpi_executor/lib/python3.6/site-packages/parsl/app/errors.py", line 163, in reraise
    reraise(t, v, tb)
  File "/home/yadu/.local/lib/python3.6/site-packages/six.py", line 692, in reraise
    raise value.with_traceback(tb)
  File "/home/yadu/miniconda3/envs/mpi_executor/lib/python3.6/site-packages/parsl/app/errors.py", line 172, in wrapper
    return func(*args, **kwargs)
  File "/home/yadu/miniconda3/envs/mpi_executor/lib/python3.6/site-packages/parsl/app/bash.py", line 79, in remote_side_bash_executor
    std_out = open_std_fd('stdout')
  File "/home/yadu/miniconda3/envs/mpi_executor/lib/python3.6/site-packages/parsl/app/bash.py", line 76, in open_std_fd
    raise pe.BadStdStreamFile(fname, e)
parsl.app.errors.BadStdStreamFile: FilePath: [/home/yadu/src/parsl/parsl/tests/manual_tests/runinfo/050/task_logs/0000/task_0000_sleeper_bash.stdout] Exception: [Errno 2] No such file or directory: '/home/yadu/src/parsl/parsl/tests/manual_tests/runinfo/050/task_logs/0000/task_0000_sleeper_bash.stdout'
```

I suspect this is coming from the our remote side bash wrapper, trying to open the stdout/stderr stream before we do makedirs on the dirname here 
 : https://github.com/Parsl/parsl/blob/master/parsl/app/bash.py#L72


Discussion:

Resolving patch:
diff --git a/parsl/app/bash.py b/parsl/app/bash.py
--- a/parsl/app/bash.py
+++ b/parsl/app/bash.py
@@ -69,9 +69,9 @@ def open_std_fd(fdname):
             raise pe.BadStdStreamFile("std descriptor %s has unexpected type %s" % (fdname, str(type(stdfspec))), TypeError('Bad Tuple Type'))
 
         try:
-            fd = open(fname, mode)
             if os.path.dirname(fname):
                 os.makedirs(os.path.dirname(fname), exist_ok=True)
+            fd = open(fname, mode)
         except Exception as e:
             raise pe.BadStdStreamFile(fname, e)
         return fd


Test patch:
Repository: Parsl/parsl

Issue:
Improve datatime precision in monitoring.db
e.g., `workflow_duration`, etc.


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -114,7 +114,7 @@ def __init__(self, config=Config()):
                     self.workflow_name = fname
                     break
 
-        self.workflow_version = str(self.time_began)
+        self.workflow_version = str(self.time_began.replace(microsecond=0))
         if self.monitoring is not None and self.monitoring.workflow_version is not None:
             self.workflow_version = self.monitoring.workflow_version
 
diff --git a/parsl/monitoring/visualization/views.py b/parsl/monitoring/visualization/views.py
--- a/parsl/monitoring/visualization/views.py
+++ b/parsl/monitoring/visualization/views.py
@@ -9,6 +9,22 @@
 
 dummy = True
 
+import datetime
+
+
+def format_time(value):
+    if value is None:
+        return value
+    elif isinstance(value, float):
+        return str(datetime.timedelta(seconds=round(value)))
+    elif isinstance(value, datetime.datetime):
+        return value.replace(microsecond=0)
+    else:
+        return "Incorrect time format found (neither float nor datetime.datetime object)"
+
+
+app.jinja_env.filters['timeformat'] = format_time
+
 
 @app.route('/')
 def index():


Test patch:
Repository: Parsl/parsl

Issue:
ugly status message in local provider
This log message should probably unwrap the dict_values and [list] - in commit b9ecc1342e1b6ce795d942c4b9df4c841f00193d

```
2019-06-11 08:40:29.773 parsl.providers.local.local:92 [DEBUG]  Checking status of: dict_values([50510])
```


Discussion:

Resolving patch:
diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py
--- a/parsl/executors/high_throughput/executor.py
+++ b/parsl/executors/high_throughput/executor.py
@@ -597,7 +597,7 @@ def status(self):
 
         status = []
         if self.provider:
-            status = self.provider.status(self.blocks.values())
+            status = self.provider.status(list(self.blocks.values()))
 
         return status
 


Test patch:
Repository: Parsl/parsl

Issue:
Condor provider has a hard-coded `cmd_timeout`. 
This timeout should be configurable and probably default to something much longer.

https://github.com/Parsl/parsl/blob/1d8c9e35752274af6ae2ce2f07107474ece4561c/parsl/providers/condor/condor.py#L225

cc @ZhuozhaoLi who noted this in a comment to #889 


Discussion:
Also reported by @lgray 

Resolving patch:
diff --git a/parsl/providers/condor/condor.py b/parsl/providers/condor/condor.py
--- a/parsl/providers/condor/condor.py
+++ b/parsl/providers/condor/condor.py
@@ -60,6 +60,8 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
     launcher : Launcher
         Launcher for this provider. Possible launchers include
         :class:`~parsl.launchers.SingleNodeLauncher` (the default),
+    cmd_timeout : int
+        Timeout for commands made to the scheduler in seconds
     """
     def __init__(self,
                  channel=LocalChannel(),
@@ -75,7 +77,8 @@ def __init__(self,
                  walltime="00:10:00",
                  worker_init='',
                  launcher=SingleNodeLauncher(),
-                 requirements=''):
+                 requirements='',
+                 cmd_timeout=60):
 
         label = 'condor'
         super().__init__(label,
@@ -86,8 +89,8 @@ def __init__(self,
                          max_blocks,
                          parallelism,
                          walltime,
-                         launcher)
-
+                         launcher,
+                         cmd_timeout=cmd_timeout)
         self.provisioned_blocks = 0
 
         self.environment = environment if environment is not None else {}
@@ -227,7 +230,7 @@ def submit(self, command, blocksize, tasks_per_node, job_name="parsl.auto"):
         channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
 
         cmd = "condor_submit {0}".format(channel_script_path)
-        retcode, stdout, stderr = super().execute_wait(cmd, 30)
+        retcode, stdout, stderr = super().execute_wait(cmd)
         logger.debug("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
 
         job_id = []
@@ -264,7 +267,7 @@ def cancel(self, job_ids):
         job_id_list = ' '.join(job_ids)
         cmd = "condor_rm {0}; condor_rm -forcex {0}".format(job_id_list)
         logger.debug("Attempting removal of jobs : {0}".format(cmd))
-        retcode, stdout, stderr = self.channel.execute_wait(cmd, 30)
+        retcode, stdout, stderr = super().execute_wait(cmd)
         rets = None
         if retcode == 0:
             for jid in job_ids:


Test patch:
Repository: Parsl/parsl

Issue:
remote_side_bash_executor global logger conflates logs from multiple tasks/sources
`remote_side_bash_executor` configures and uses the global `logging` logger, rather than one scoped to that function. Once it has configured a log file for output, all subsequent global logs from any further `remote_side_bash_executor` invocation in that process, as well as other uses of logging, such as htex `process_worker_pool.py`, end up in earlier configured log files.

This results in `/tmp/bashexec` logs containing a confused assortment of logs from different sources, rather than being focuses on a single bash execution.


Discussion:
PR #1045 contains some draft work addressing this.

Resolving patch:
diff --git a/parsl/app/bash.py b/parsl/app/bash.py
--- a/parsl/app/bash.py
+++ b/parsl/app/bash.py
@@ -1,4 +1,3 @@
-import logging
 from functools import update_wrapper
 from inspect import signature, Parameter
 
@@ -7,8 +6,6 @@
 from parsl.app.app import AppBase
 from parsl.dataflow.dflow import DataFlowKernelLoader
 
-logger = logging.getLogger(__name__)
-
 
 def remote_side_bash_executor(func, *args, **kwargs):
     """Execute the bash app type function and return the command line string.
@@ -21,8 +18,20 @@ def remote_side_bash_executor(func, *args, **kwargs):
     import subprocess
     import logging
     import parsl.app.errors as pe
+    from parsl import set_file_logger
+
+    logbase = "/tmp"
+    format_string = "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s]  %(message)s"
+
+    # make this name unique per invocation so that each invocation can
+    # log to its own file. It would be better to include the task_id here
+    # but that is awkward to wire through at the moment as apps do not
+    # have access to that execution context.
+    t = time.time()
 
-    logging.basicConfig(filename='/tmp/bashexec.{0}.log'.format(time.time()), level=logging.DEBUG)
+    logname = __name__ + "." + str(t)
+    logger = logging.getLogger(logname)
+    set_file_logger(filename='{0}/bashexec.{1}.log'.format(logbase, t), name=logname, level=logging.DEBUG, format_string=format_string)
 
     func_name = func.__name__
 
@@ -44,10 +53,10 @@ def remote_side_bash_executor(func, *args, **kwargs):
     except IndexError as e:
         raise pe.AppBadFormatting("App formatting failed for app '{}' with IndexError: {}".format(func_name, e))
     except Exception as e:
-        logging.error("Caught exception during formatting of app '{}': {}".format(func_name, e))
+        logger.error("Caught exception during formatting of app '{}': {}".format(func_name, e))
         raise e
 
-    logging.debug("Executable: %s", executable)
+    logger.debug("Executable: %s", executable)
 
     # Updating stdout, stderr if values passed at call time.
 


Test patch:
Repository: Parsl/parsl

Issue:
WorkQueue executor should be an optional extra
Recently we merged #771 which added WorkQueue as an executor. Currently this is added to parsl core and not as an optional extra. But, the `work_queue` module that this PR depends on is not available on PyPI yet and is to be installed manually. So unless the module was manually install parsl master would raise `ModuleNotFoundError` as soon as it is imported.


Discussion:

Resolving patch:
diff --git a/parsl/executors/workqueue/executor.py b/parsl/executors/workqueue/executor.py
--- a/parsl/executors/workqueue/executor.py
+++ b/parsl/executors/workqueue/executor.py
@@ -14,17 +14,24 @@
 from parsl.executors.errors import ExecutorError
 from parsl.executors.base import ParslExecutor
 from parsl.data_provider.files import File
+from parsl.providers.error import OptionalModuleMissing
 from parsl.executors.workqueue import workqueue_worker
 
-from work_queue import WorkQueue
-from work_queue import Task
-from work_queue import WORK_QUEUE_DEFAULT_PORT
-from work_queue import WORK_QUEUE_INPUT
-from work_queue import WORK_QUEUE_OUTPUT
-from work_queue import WORK_QUEUE_RESULT_SUCCESS
-from work_queue import WORK_QUEUE_RESULT_OUTPUT_MISSING
-from work_queue import cctools_debug_flags_set
-from work_queue import cctools_debug_config_file
+try:
+    from work_queue import WorkQueue
+    from work_queue import Task
+    from work_queue import WORK_QUEUE_DEFAULT_PORT
+    from work_queue import WORK_QUEUE_INPUT
+    from work_queue import WORK_QUEUE_OUTPUT
+    from work_queue import WORK_QUEUE_RESULT_SUCCESS
+    from work_queue import WORK_QUEUE_RESULT_OUTPUT_MISSING
+    from work_queue import cctools_debug_flags_set
+    from work_queue import cctools_debug_config_file
+except ImportError:
+    _work_queue_enabled = False
+    WORK_QUEUE_DEFAULT_PORT = 0
+else:
+    _work_queue_enabled = True
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +50,6 @@ def WorkQueueSubmitThread(task_queue=multiprocessing.Queue(),
                           project_password=None,
                           project_password_file=None,
                           project_name=None):
-
     logger.debug("Starting WorkQueue Submit/Wait Process")
 
     orig_ppid = os.getppid()
@@ -355,6 +361,8 @@ def __init__(self,
                  shared_fs=False,
                  init_command="",
                  see_worker_output=False):
+        if not _work_queue_enabled:
+            raise OptionalModuleMissing(['work_queue'], "WorkQueueExecutor requires the work_queue module.")
 
         self.label = label
         self.managed = managed
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,8 @@
     'docs' : ['nbsphinx', 'sphinx_rtd_theme'],
     'google_cloud' : ['google-auth', 'google-api-python-client'],
     'gssapi' : ['python-gssapi'],
-    'azure' : ['azure', 'msrestazure']
+    'azure' : ['azure', 'msrestazure'],
+    'workqueue': ['work_queue'],
 }
 extras_require['all'] = sum(extras_require.values(), [])
 


Test patch:
Repository: Parsl/parsl

Issue:
Monitoring should be an optional extra
We should push this out as 0.8.1. This is needed by the nersc DESC stack.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -7,6 +7,17 @@
     install_requires = f.readlines()
 
 extras_require = {
+    'monitoring' : [
+        'sqlalchemy>=1.3.0,!=1.3.4',
+        'sqlalchemy_utils',
+        'pydot',
+        'networkx',
+        'Flask>=1.0.2',
+        'flask_sqlalchemy',
+        'pandas',
+        'plotly',
+        'python-daemon'
+    ],
     'aws' : ['boto3'],
     'kubernetes' : ['kubernetes'],
     'oauth_ssh' : ['oauth-ssh>=0.9'],


Test patch:
Repository: Parsl/parsl

Issue:
MonitoringHub does not work without setting a hub port
We should set a default `hub_port` or a range


Discussion:

Resolving patch:
diff --git a/parsl/monitoring/monitoring.py b/parsl/monitoring/monitoring.py
--- a/parsl/monitoring/monitoring.py
+++ b/parsl/monitoring/monitoring.py
@@ -345,9 +345,6 @@ def __init__(self,
                                         level=logging_level)
         self.logger.debug("Hub starting")
 
-        if not hub_port:
-            self.logger.critical("At this point the hub port must be set")
-
         self.hub_port = hub_port
         self.hub_address = hub_address
         self.atexit_timeout = atexit_timeout
@@ -356,15 +353,19 @@ def __init__(self,
         self.loop_freq = 10.0  # milliseconds
 
         # Initialize the UDP socket
-        self.logger.debug("Intiializing the UDP socket on 0.0.0.0:{}".format(hub_port))
         try:
             self.sock = socket.socket(socket.AF_INET,
                                       socket.SOCK_DGRAM,
                                       socket.IPPROTO_UDP)
 
             # We are trying to bind to all interfaces with 0.0.0.0
-            self.sock.bind(('0.0.0.0', hub_port))
+            if not self.hub_port:
+                self.sock.bind(('0.0.0.0', 0))
+                self.hub_port = self.sock.getsockname()[1]
+            else:
+                self.sock.bind(('0.0.0.0', self.hub_port))
             self.sock.settimeout(self.loop_freq / 1000)
+            self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.hub_port))
         except OSError:
             self.logger.critical("The port is already in use")
             self.hub_port = -1


Test patch:
Repository: Parsl/parsl

Issue:
Kubernetes missing from configuration guide
@MatthewBM reports that the configuration section is missing docs for Kubernetes.



Discussion:

Resolving patch:
diff --git a/parsl/configs/kubernetes_htex.py b/parsl/configs/kubernetes_htex.py
new file mode 100644
--- /dev/null
+++ b/parsl/configs/kubernetes_htex.py
@@ -0,0 +1,27 @@
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import KubernetesProvider
+from parsl.addresses import address_by_route
+
+
+config = Config(
+    executors=[
+        HighThroughputExecutor(
+            label='kube-htex',
+            cores_per_worker=1,
+            max_workers=1,
+            worker_logdir_root='runinfo',
+            address=address_by_route(),  # Address for the pod worker to connect back
+            provider=KubernetesProvider(
+                namespace="default",
+                image='CONTAINER_LOCATION',  # Specify where to download the image
+                nodes_per_block=1,
+                init_blocks=1,
+                max_blocks=10,  # Maximum number of pods to scale up
+                worker_init="""pip install parsl""",  # install Parsl when the pod starts
+                secret="kube-secret",  # The secret key to download the image
+                pod_name='test-pod',  # Should follow the Kubernetes naming rules
+            ),
+        ),
+    ]
+)


Test patch:
Repository: Parsl/parsl

Issue:
Cumulative CPU time loses time when subprocesses end
Monitoring accumulates time used by a process, for example, `psutil_process_time_user` both for the executing process and for any still-alive children.

However it does not keep the time used by any exited children -  when a child process exits, the CPU time for that child disappears from `psutil_process_time_user`, and the CPU time used reduces.

It would probably be a more useful metric to have this field accumulate all child process cpu usage, even exited children, so that it is monotonically increasing, and so that the final value gives a decent estimate of the CPU used by all processes spawned in the task.

This maybe related to #1033, which is visualization related.


Discussion:
If that is the case, we need to fix that. Perhaps use a local dict to store the values of exited child processes 

Resolving patch:
diff --git a/parsl/monitoring/monitoring.py b/parsl/monitoring/monitoring.py
--- a/parsl/monitoring/monitoring.py
+++ b/parsl/monitoring/monitoring.py
@@ -459,7 +459,10 @@ def monitor(pid, task_id, monitoring_hub_url, run_id, sleep_dur=10):
     pm.cpu_percent()
 
     first_msg = True
-
+    children_user_time = {}
+    children_system_time = {}
+    total_children_user_time = 0.0
+    total_children_system_time = 0.0
     while True:
         logging.debug("start of monitoring loop")
         try:
@@ -492,8 +495,12 @@ def monitor(pid, task_id, monitoring_hub_url, run_id, sleep_dur=10):
             for child in children:
                 for k, v in child.as_dict(attrs=summable_values).items():
                     d['psutil_process_' + str(k)] += v
-                d['psutil_process_time_user'] += child.cpu_times().user
-                d['psutil_process_time_system'] += child.cpu_times().system
+                child_user_time = child.cpu_times().user
+                child_system_time = child.cpu_times().system
+                total_children_user_time += child_user_time - children_user_time.get(child.pid, 0)
+                total_children_system_time += child_system_time - children_system_time.get(child.pid, 0)
+                children_user_time[child.pid] = child_user_time
+                children_system_time[child.pid] = child_system_time
                 d['psutil_process_memory_virtual'] += child.memory_info().vms
                 d['psutil_process_memory_resident'] += child.memory_info().rss
                 try:
@@ -504,6 +511,8 @@ def monitor(pid, task_id, monitoring_hub_url, run_id, sleep_dur=10):
                     logging.exception("Exception reading IO counters for child {k}. Recorded IO usage may be incomplete".format(k=k), exc_info=True)
                     d['psutil_process_disk_write'] += 0
                     d['psutil_process_disk_read'] += 0
+            d['psutil_process_time_user'] += total_children_user_time
+            d['psutil_process_time_system'] += total_children_system_time
             logging.debug("sending message")
             radio.send(MessageType.TASK_INFO, task_id, d)
             first_msg = False


Test patch: