# Buggy code
def wait_for_current_tasks(self):
    """Waits for all tasks in the task list to be completed, by waiting for their
    AppFuture to be completed. This method will not necessarily wait for any tasks
    added after cleanup has started (such as data stageout?)
    """

    logger.info("Waiting for all remaining tasks to complete")
    for task_id in self.tasks:
        # .exception() is a less exception throwing way of
        # waiting for completion than .result()
        fut = self.tasks[task_id]["app_fu"]
        if not fut.done():
            logger.debug("Waiting for task {} to complete".format(task_id))
            fut.exception()
    logger.info("All remaining tasks completed")


# Traceback (RuntimeError)
2020-03-14 14:38:50 parsl.dataflow.memoization:159 [DEBUG]  Ignoring these kwargs for checkpointing: []
Traceback (most recent call last):
File "/home/users/nus/csipav/Documents/GDC-Pipeline.git/run.py", line 187, in <module>
2020-03-14 14:38:50 parsl.dataflow.memoization:202 [DEBUG]  Task 1 has memoization hash 6888eb84e8d55e06d97d5fb5d0b61eb6
2020-03-14 14:38:50 parsl.dataflow.memoization:210 [INFO]  Task 1 had no result in cache
main(sys.argv[1:])
File "/home/users/nus/csipav/Documents/GDC-Pipeline.git/run.py", line 183, in main
2020-03-14 14:38:50 parsl.executors.high_throughput.executor:529 [DEBUG]  Pushing function <function MonitoringHub.monitor_wrapper.<locals>.wrapped at 0x2aaab86a5200> to queue with args (<function split_vcf_by_chromosome at 0x2aaab84929e0>, "{'MuSE': '/home/users/nus/csipav/anaconda3/envs/gdc/bin/MuSE', 'ace2sam': '/home/users/nus/csipav/an...", "<<class 'parsl.data_provider.files.File'> at 0x2aaab88eec90 url=/home/users/nus/csipav/scratch/Parsl...", "'/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/TCGA-13-0910/1/annotations/...", "'/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/TCGA-13-0910/1/strelka2-ana...")
2020-03-14 14:38:50 parsl.monitoring.monitoring:261 [DEBUG]  Sending message MessageType.TASK_INFO, {'task_func_name': 'split_vcf_by_chromosome', 'task_fn_hash': '5f63d27122e82d22e85ec201359e25a7', 'task_memoize': True, 'task_hashsum': '6888eb84e8d55e06d97d5fb5d0b61eb6', 'task_fail_count': 0, 'task_status': <States.launched: 7>, 'task_id': 1, 'task_time_submitted': datetime.datetime(2020, 3, 14, 14, 38, 50, 14435), 'task_time_returned': None, 'task_executor': 'htex', 'run_id': '85606c50-cec0-4f13-b4af-b05d6791b9c3', 'timestamp': datetime.datetime(2020, 3, 14, 14, 38, 50, 27280), 'task_status_name': 'launched', 'tasks_failed_count': 0, 'tasks_completed_count': 12, 'task_inputs': 'None', 'task_outputs': "['/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/TCGA-13-0910/1/annotations/variants.indels_filtered.vcf.gz/split_chr/.complete']", 'task_stdin': None, 'task_stdout': '/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/runinfo/000/task_logs/0000/task_0001_split_vcf_by_chromosome_TCGA-13-0910_1-split_vcf_by_chromosome-variants.indels_filtered.vcf.gz.stdout', 'task_stderr': '/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/runinfo/000/task_logs/0000/task_0001_split_vcf_by_chromosome_TCGA-13-0910_1-split_vcf_by_chromosome-variants.indels_filtered.vcf.gz.stderr', 'task_fail_history': '', 'task_depends': '0', 'task_elapsed_time': None, 'task_fail_mode': 'lazy'}
run_gdc_pipeline(gdc_config)
File "/home/users/nus/csipav/Documents/GDC-Pipeline.git/run.py", line 105, in run_gdc_pipeline
2020-03-14 14:38:50 parsl.dataflow.dflow:499 [INFO]  Task 1 launched on executor htex
parsl.wait_for_current_tasks()
File "/home/users/nus/csipav/anaconda3/envs/gdc/lib/python3.7/site-packages/parsl-0.9.0-py3.7.egg/parsl/dataflow/dflow.py", line 1147, in wait_for_current_tasks
2020-03-14 14:38:50 parsl.dataflow.memoization:159 [DEBUG]  Ignoring these kwargs for checkpointing: []
2020-03-14 14:38:50 parsl.dataflow.memoization:202 [DEBUG]  Task 28 has memoization hash 5b56fba6029bd95a9fb9d02f405e2282
2020-03-14 14:38:50 parsl.dataflow.memoization:210 [INFO]  Task 28 had no result in cache
2020-03-14 14:38:50 parsl.executors.high_throughput.executor:529 [DEBUG]  Pushing function <function MonitoringHub.monitor_wrapper.<locals>.wrapped at 0x2aaae8a3f830> to queue with args (<function split_vcf_by_chromosome at 0x2aaab84929e0>, "{'MuSE': '/home/users/nus/csipav/anaconda3/envs/gdc/bin/MuSE', 'ace2sam': '/home/users/nus/csipav/an...", "<<class 'parsl.data_provider.files.File'> at 0x2aaadc04d890 url=/home/users/nus/csipav/scratch/Parsl...", "'/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/TCGA-13-0910/1/annotations/...", "'/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/TCGA-13-0910/1/strelka2-ana...")
2020-03-14 14:38:50 parsl.monitoring.monitoring:261 [DEBUG]  Sending message MessageType.TASK_INFO, {'task_func_name': 'split_vcf_by_chromosome', 'task_fn_hash': '5f63d27122e82d22e85ec201359e25a7', 'task_memoize': True, 'task_hashsum': '5b56fba6029bd95a9fb9d02f405e2282', 'task_fail_count': 0, 'task_status': <States.launched: 7>, 'task_id': 28, 'task_time_submitted': datetime.datetime(2020, 3, 14, 14, 38, 50, 28401), 'task_time_returned': None, 'task_executor': 'htex', 'run_id': '85606c50-cec0-4f13-b4af-b05d6791b9c3', 'timestamp': datetime.datetime(2020, 3, 14, 14, 38, 50, 39742), 'task_status_name': 'launched', 'tasks_failed_count': 0, 'tasks_completed_count': 12, 'task_inputs': 'None', 'task_outputs': "['/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/TCGA-13-0910/1/annotations/variants.snvs_filtered.vcf.gz/split_chr/.complete']", 'task_stdin': None, 'task_stdout': '/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/runinfo/000/task_logs/0000/task_0028_split_vcf_by_chromosome_TCGA-13-0910_1-split_vcf_by_chromosome-variants.snvs_filtered.vcf.gz.stdout', 'task_stderr': '/home/users/nus/csipav/scratch/Parsl-workflows/GDC-Pipeline-Output/OV_1/runinfo/000/task_logs/0000/task_0028_split_vcf_by_chromosome_TCGA-13-0910_1-split_vcf_by_chromosome-variants.snvs_filtered.vcf.gz.stderr', 'task_fail_history': '', 'task_depends': '0', 'task_elapsed_time': None, 'task_fail_mode': 'lazy'}
cls.dfk().wait_for_current_tasks()
File "/home/users/nus/csipav/anaconda3/envs/gdc/lib/python3.7/site-packages/parsl-0.9.0-py3.7.egg/parsl/dataflow/dflow.py", line 882, in wait_for_current_tasks
2020-03-14 14:38:50 parsl.dataflow.dflow:499 [INFO]  Task 28 launched on executor htex
2020-03-14 14:38:50 parsl.dataflow.memoization:159 [DEBUG]  Ignoring these kwargs for checkpointing: []
for task_id in self.tasks:
RuntimeError: dictionary changed size during iteration
2020-03-14 14:38:50 parsl.dataflow.dflow:900 [INFO]  DFK cleanup initiated

# Fixed code
def wait_for_current_tasks(self):
    """Waits for all tasks in the task list to be completed, by waiting for their
    AppFuture to be completed. This method will not necessarily wait for any tasks
    added after cleanup has started (such as data stageout?)
    """

    logger.info("Waiting for all remaining tasks to complete")
    for task_id in list(self.tasks):
        # .exception() is a less exception throwing way of
        # waiting for completion than .result()
        if task_id not in self.tasks:
            logger.debug("Task {} no longer in task list".format(task_id))
        else:
            fut = self.tasks[task_id]["app_fu"]
            if not fut.done():
                logger.debug("Waiting for task {} to complete".format(task_id))
                fut.exception()
    logger.info("All remaining tasks completed")