Repository: twosixlabs/armory
Issue: WER metric fails during ASR run

When running:
```
armory run scenario_configs/asr_librispeech_targeted.json --check
```
There is an error when finalizing metrics:
```
2022-10-05 16:22:19 18s ERROR    armory.scenarios.scenario:evaluate:399 Encountered error during scenario evaluation.
Traceback (most recent call last):                                                        
File "/opt/conda/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
│         │     └ {'__name__': '__main__', '__doc__': '\\nMain script for running scenarios. Users will run a scenario by calling:\\n    armory r...
│         └ <code object <module> at 0x7fbb2e9ae190, file "/workspace/armory/scenarios/main.py", line 1>
└ <function _run_code at 0x7fbc1f0c19d0>                                                                                                                                             
File "/opt/conda/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)                                                                   
│     └ {'__name__': '__main__', '__doc__': '\\nMain script for running scenarios. Users will run a scenario by calling:\\n    armory r...
└ <code object <module> at 0x7fbb2e9ae190, file "/workspace/armory/scenarios/main.py", line 1>
File "/workspace/armory/scenarios/main.py", line 228, in <module>
run_config(                                                                               
└ <function run_config at 0x7fbb2d8610d0>                                                 
File "/workspace/armory/scenarios/main.py", line 144, in run_config                                                                                                                  scenario.evaluate()                                                                                                                                                                  │        └ <function Scenario.evaluate at 0x7fbb2d4ab5e0>                                                                                                                            └ <armory.scenarios.audio_asr.AutomaticSpeechRecognition object at 0x7fbc1e7699a0>                                                                                                   > File "/workspace/armory/scenarios/scenario.py", line 390, in evaluate                                                                                                              self._evaluate()                                                                                                                                                                     │    └ <function Scenario._evaluate at 0x7fbb2d4ab550>                                                                                                                               └ <armory.scenarios.audio_asr.AutomaticSpeechRecognition object at 0x7fbc1e7699a0>                                                                                                   File "/workspace/armory/scenarios/scenario.py", line 381, in _evaluate                                                                                                               self.finalize_results()                                                                                                                                                              │    └ <function Scenario.finalize_results at 0x7fbb2d4ab4c0>                                                                                                                        └ <armory.scenarios.audio_asr.AutomaticSpeechRecognition object at 0x7fbc1e7699a0>                                                                                                   
File "/workspace/armory/scenarios/scenario.py", line 369, in finalize_results                                                                                                        
self.metric_results = self.metrics_logger.results()                                                                                                                                  
│                     │    │              └ <function MetricsLogger.results at 0x7fbb2d704e50>
│                     │    └ <armory.instrument.config.MetricsLogger object at 0x7fbb0dd49220>
│                     └ <armory.scenarios.audio_asr.AutomaticSpeechRecognition object at 0x7fbc1e7699a0>
└ <armory.scenarios.audio_asr.AutomaticSpeechRecognition object at 0x7fbc1e7699a0>
File "/workspace/armory/instrument/config.py", line 118, in results
get_hub().close()
└ <function get_hub at 0x7fbb2d703040>
File "/workspace/armory/instrument/instrument.py", line 444, in close
meter.finalize()
│     └ <function Meter.finalize at 0x7fbb2d703f70>
└ <armory.instrument.instrument.Meter object at 0x7fbb0dcbbd30>
File "/workspace/armory/instrument/instrument.py", line 599, in finalize
result = self.final(self._results, **s
elf.final_kwargs)
         │    │     │    │           │    └ {}
         │    │     │    │           └ <armory.instrument.instrument.Meter object at 0x7fbb0dcbbd30>
         │    │     │    └ [array([ 5., 33.])]
         │    │     └ <armory.instrument.instrument.Meter object at 0x7fbb0dcbbd30>
         │    └ <function total_wer at 0x7fbb2d766280>
         └ <armory.instrument.instrument.Meter object at 0x7fbb0dcbbd30>
File "/workspace/armory/metrics/task.py", line 196, in total_wer
raise ValueError("Inputs must be tuples of size 2: (edit distance, length)")
ValueError: Inputs must be tuples of size 2: (edit distance, length)
```

Changed files:
armory/metrics/task.py, tests/unit/test_task_metrics.py

diff --git a/armory/metrics/task.py b/armory/metrics/task.py
index 11dd7d5e..d94ed1d7 100644
--- a/armory/metrics/task.py
+++ b/armory/metrics/task.py
@@ -192,7 +192,7 @@ def total_wer(sample_wers):
     """
 
     if isinstance(sample_wers, list):
-        if not all(isinstance(wer_tuple, tuple) for wer_tuple in sample_wers):
+        if not all(len(wer_tuple) == 2 for wer_tuple in sample_wers):
             raise ValueError("Inputs must be tuples of size 2: (edit distance, length)")
     elif isinstance(sample_wers, np.ndarray):
         if sample_wers.ndim != 2 or sample_wers.shape[-1] != 2:
diff --git a/tests/unit/test_task_metrics.py b/tests/unit/test_task_metrics.py
index 5b680cca..5b54cc80 100644
--- a/tests/unit/test_task_metrics.py
+++ b/tests/unit/test_task_metrics.py
@@ -252,7 +252,9 @@ def test_word_error_rate():  # and total
     with pytest.raises(TypeError):
         task.word_error_rate("hi there", 3)
 
-    for err in [1, (3, 4)], [(1, 2, 3)], [(1,)]:
+    with pytest.raises(TypeError):
+        task.total_wer([1, (3, 4)])
+    for err in [(1, 2, 3)], [(1,)]:
         with pytest.raises(ValueError):
             task.total_wer(err)