Repository: MatterMiners/tardis

Issue:
Rare crash in `get_resource_ratios` when Slot variables are not defined yet
@wiene and I observe rare crashes with this trace (line numbers of the 0.5.0 version):
```
2021-02-20 01:41:11 [ERROR][cobald.runtime.runner.asyncio]: runner aborted: <cobald.daemon.runners.asyncio_runner.AsyncioRunner object at 0x7fbb5d697978>
Traceback (most recent call last):
  File "/opt/cobald/lib64/python3.6/site-packages/cobald/daemon/runners/base_runner.py", line 62, in run
    self._run()
  File "/opt/cobald/lib64/python3.6/site-packages/cobald/daemon/runners/asyncio_runner.py", line 28, in _run
    self.event_loop.run_until_complete(self._run_payloads())
  File "/usr/lib64/python3.6/asyncio/base_events.py", line 484, in run_until_complete
    return future.result()
  File "/opt/cobald/lib64/python3.6/site-packages/cobald/daemon/runners/asyncio_runner.py", line 36, in _run_payloads
    await self._reap_payloads()
  File "/opt/cobald/lib64/python3.6/site-packages/cobald/daemon/runners/asyncio_runner.py", line 58, in _reap_payloads
    raise task.exception()
  File "/opt/cobald/lib64/python3.6/site-packages/cobald/daemon/runners/async_tools.py", line 7, in raise_return
    value = await payload()
  File "/opt/cobald/lib64/python3.6/site-packages/tardis/resources/drone.py", line 94, in run
    await current_state.run(self)
  File "/opt/cobald/lib64/python3.6/site-packages/tardis/resources/dronestates.py", line 171, in run
    drone_uuid=drone.resource_attributes["drone_uuid"]
  File "/opt/cobald/lib64/python3.6/site-packages/tardis/agents/batchsystemagent.py", line 20, in get_allocation
    return await self._batch_system_adapter.get_allocation(drone_uuid)
  File "/opt/cobald/lib64/python3.6/site-packages/tardis/adapters/batchsystems/htcondor.py", line 195, in get_allocation
    return max(await self.get_resource_ratios(drone_uuid), default=0.0)
  File "/opt/cobald/lib64/python3.6/site-packages/tardis/adapters/batchsystems/htcondor.py", line 181, in <genexpr>
    if key in self.ratios.keys()
ValueError: could not convert string to float: 'error'
```

So that happens in here:
https://github.com/MatterMiners/tardis/blob/1139be80d885305ef8a62fca7ca49d8e15ca334e/tardis/adapters/batchsystems/htcondor.py#L172-L182

We set the following `BatchSystem.ratios` for our HTCondor LBS:
```YAML
    cpu_ratio: Real(TotalSlotCpus-Cpus)/TotalSlotCpus
    memory_ratio: Real(TotalSlotMemory-Memory)/TotalSlotMemory
    cpu_usage: IfThenElse(AverageCPUsUsage=?=undefined, 0, Real(AverageCPUsUsage))
```

This seems to happen always close / synchronous to the `schedd` negotiating with the LBS, i.e. new drones starting. Presumably, `Cpus` or `Memory` may not be well-defined for short time fractions in HTCondor (maybe when the `pslot` is in the process of being formed?). 

Of course, this could be worked around with an `ifThenElse`, but maybe catching an `error` string in TARDIS and retrying later in that case might be more reliable :smile: . 

Discussion:

Patch:
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 1f95158..c8aad5a 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,4 +1,4 @@
-.. Created by changelog.py at 2021-03-23, command
+.. Created by changelog.py at 2021-03-24, command
    '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.9/bin/changelog docs/source/changes compile --output=docs/source/changelog.rst'
    based on the format of 'https://keepachangelog.com/'
 
@@ -6,7 +6,7 @@
 CHANGELOG
 #########
 
-[Unreleased] - 2021-03-23
+[Unreleased] - 2021-03-24
 =========================
 
 Added
@@ -17,6 +17,7 @@ Added
 Fixed
 -----
 
+* Fixes a bug that get_resource_ratios raised a ValueError
 * Fixes a bug that the drone_minimum_lifetime parameter is not working as described in the documentation
 * Fixes a bug in the HTCondor Site Adapter which leads to wrong requirements when using non HTCondor OBS
 
diff --git a/docs/source/changes/175.fix_resource_ratios.yaml b/docs/source/changes/175.fix_resource_ratios.yaml
new file mode 100644
index 0000000..1ba3409
--- /dev/null
+++ b/docs/source/changes/175.fix_resource_ratios.yaml
@@ -0,0 +1,10 @@
+category: fixed
+summary: "Fixes a bug that get_resource_ratios raised a ValueError"
+description: |
+    In case one of the resource ratios is `undefined` or even has the value `error`
+    a `ValueError` or `TypeError` could occur. In case one of those errors occurs,
+    an empty list is returned.
+pull requests:
+  - 175
+issues:
+  - 168
diff --git a/tardis/adapters/batchsystems/htcondor.py b/tardis/adapters/batchsystems/htcondor.py
index 0bb702b..67ec25b 100644
--- a/tardis/adapters/batchsystems/htcondor.py
+++ b/tardis/adapters/batchsystems/htcondor.py
@@ -172,14 +172,13 @@ class HTCondorAdapter(BatchSystemAdapter):
         await self._htcondor_status.update_status()
         try:
             htcondor_status = self._htcondor_status[drone_uuid]
-        except KeyError:
-            return {}
-        else:
-            return (
+            return [
                 float(value)
                 for key, value in htcondor_status.items()
                 if key in self.ratios.keys()
-            )
+            ]
+        except (KeyError, ValueError, TypeError):
+            return []
 
     async def get_allocation(self, drone_uuid: str) -> float:
         """


Test patch:
diff --git a/tests/adapters_t/batchsystems_t/test_htcondor.py b/tests/adapters_t/batchsystems_t/test_htcondor.py
index cc4c41f..0a0e2a8 100644
--- a/tests/adapters_t/batchsystems_t/test_htcondor.py
+++ b/tests/adapters_t/batchsystems_t/test_htcondor.py
@@ -57,6 +57,8 @@ class TestHTCondorAdapter(TestCase):
                 f"test_drained\tslot1@test\tDrained\tIdle\tundefined\t{self.cpu_ratio}\t{self.memory_ratio}",  # noqa: B950
                 f"test_owner\tslot1@test\tOwner\tIdle\tundefined\t{self.cpu_ratio}\t{self.memory_ratio}",  # noqa: B950
                 f"test_uuid_plus\tslot1@test_uuid@test\tUnclaimed\tIdle\ttest_uuid\t{self.cpu_ratio}\t{self.memory_ratio}",  # noqa: B950
+                f"test_undefined\tslot1@test\tUnclaimed\tIdle\tundefined\tundefined\t{self.memory_ratio}",  # noqa: B950
+                f"test_error\tslot1@test\tUnclaimed\tIdle\tundefined\terror\t{self.memory_ratio}",  # noqa: B950
                 "exoscale-26d361290f\tslot1@exoscale-26d361290f\tUnclaimed\tIdle\tundefined\t0.125\t0.125",  # noqa: B950
             ]
         )
@@ -151,12 +153,31 @@ class TestHTCondorAdapter(TestCase):
             [self.cpu_ratio, self.memory_ratio],
         )
         self.mock_async_run_command.assert_called_with(self.command)
+        self.mock_async_run_command.reset_mock()
 
         self.assertEqual(
             run_async(
                 self.htcondor_adapter.get_resource_ratios, drone_uuid="not_exists"
             ),
-            {},
+            [],
+        )
+        self.mock_async_run_command.assert_not_called()
+        self.mock_async_run_command.reset_mock()
+
+        self.assertEqual(
+            run_async(
+                self.htcondor_adapter.get_resource_ratios, drone_uuid="test_undefined"
+            ),
+            [],
+        )
+        self.mock_async_run_command.assert_not_called()
+        self.mock_async_run_command.reset_mock()
+
+        self.assertEqual(
+            run_async(
+                self.htcondor_adapter.get_resource_ratios, drone_uuid="test_error"
+            ),
+            [],
         )
 
     def test_get_resource_ratios_without_options(self):