Repository: Parsl/parsl

Issue:
Block number re-use
**Describe the bug**
Parsl will reuse block the latest block number, if that block ends and is recreated.
This is confusing when examining logs and monitoring, trying to understand which actual block is being referred to.

This is specifically caused some confusion when looking at logs for a long-standing CI hang in `--config local` tests.

**To Reproduce**
Watch an execution which scales down a block and scales up again.

**Expected behavior**
Blocks should be uniquely numbered - for example by a monotonically increasing counter.
A clear and concise description of what you expected to happen.


Discussion:

Resolving patch:
diff --git a/parsl/executors/status_handling.py b/parsl/executors/status_handling.py
--- a/parsl/executors/status_handling.py
+++ b/parsl/executors/status_handling.py
@@ -9,6 +9,7 @@
 from parsl.executors.base import ParslExecutor
 from parsl.executors.errors import ScalingFailed
 from parsl.providers.provider_base import JobStatus, ExecutionProvider, JobState
+from parsl.utils import AtomicIDCounter
 
 
 logger = logging.getLogger(__name__)
@@ -48,7 +49,9 @@ def __init__(self, provider: ExecutionProvider):
         self._simulated_status: Dict[Any, JobStatus] = {}
         self._executor_bad_state = threading.Event()
         self._executor_exception: Optional[Exception] = None
-        self._generated_block_id_counter = 1
+
+        self._block_id_counter = AtomicIDCounter()
+
         self._tasks = {}  # type: Dict[object, Future]
         self.blocks = {}  # type: Dict[str, str]
         self.block_mapping = {}  # type: Dict[str, str]
@@ -84,8 +87,8 @@ def _fail_job_async(self, block_id: Any, message: str):
         as failed and report it in status()
         """
         if block_id is None:
-            block_id = "failed-block-{}".format(self._generated_block_id_counter)
-            self._generated_block_id_counter += 1
+            block_id = str(self._block_id_counter.get_id())
+            logger.info(f"Allocated block ID {block_id} for simulated failure")
         self._simulated_status[block_id] = JobStatus(JobState.FAILED, message)
 
     @abstractproperty
@@ -160,8 +163,10 @@ def scale_out(self, blocks: int = 1) -> List[str]:
         if not self.provider:
             raise (ScalingFailed(None, "No execution provider available"))
         block_ids = []
+        logger.info(f"Scaling out by {blocks} blocks")
         for i in range(blocks):
-            block_id = str(len(self.blocks))
+            block_id = str(self._block_id_counter.get_id())
+            logger.info(f"Allocated block ID {block_id}")
             try:
                 job_id = self._launch_block(block_id)
                 self.blocks[block_id] = job_id
diff --git a/parsl/utils.py b/parsl/utils.py
--- a/parsl/utils.py
+++ b/parsl/utils.py
@@ -3,6 +3,7 @@
 import os
 import shlex
 import subprocess
+import threading
 import time
 import typeguard
 from contextlib import contextmanager
@@ -239,3 +240,18 @@ def assemble_line(args: List[str], kwargs: Dict[str, object]) -> str:
             return assemble_line(args, kwargs)
         else:
             return assemble_multiline(args, kwargs)
+
+
+class AtomicIDCounter:
+    """A class to allocate counter-style IDs, in a thread-safe way.
+    """
+
+    def __init__(self):
+        self.count = 0
+        self.lock = threading.Lock()
+
+    def get_id(self):
+        with self.lock:
+            new_id = self.count
+            self.count += 1
+            return new_id


Test patch: