Repository: soundata/soundata

Issue:
How are we gonna handle multiple annotators ?
I started writing the loader of `sonyc_ust` which has multiple annotators for each file, and binary labels for a set of classes. I was wondering how do we wanna handle this. I have some ideas but I'm happy to hear what others think:

For the binary labels, since the dataset has a defined taxonomy, I'm gonna get the vector `[0, 1, 0, ... ,1]` corresponding to activations of the different classes `[class_1, class_2, ... , class_N]` from each clip into tags, so the clip will have a list of only the classes present in that clip (e.g. `[class_2, class_N]`. This allows to explore clip-wise metadata easy. For exploring the big picture of the dataset, I refer to #22.

For handling multiple annotators, I see two possible solutions:

Right now `clips.tags = annotations.Tags` which internally has a list of labels and a list of confidence values. 

1) One option is to return a list of `annotations.Tags` and have a separated property for the annotator IDs, ordered in the same way:

```
clip.tags = [annotations.Tags, annotations.Tags, ... ]
clip.annotators = [annotator_1, annotator_2, ... ]  # these are str
```
Good: the datasets will always have the same property `tags`. Bad: `tags` is used in different ways in different datasets and is not so clear the link between annotators and their annotations.

2) We introduce a `muti_annotator` annotation type or property with `tuples` matching annotator ID and annotation:

```
clip.multi_annotator = [(annotation, annotator_ID), (annotation, annotator_ID), ...]
```
In this case `annotation` is `annotations.Tags`.

Good: it's very clear the dataset has a ground truth with multiple annotators. Bad: the annotation type gets a bit buried down in the code, so we have to be super clear in the docs or try to make it very intuitive somehow.

Thoughts? @justinsalamon @mrocamora @pzinemanas @mcartwright

Discussion:

Patch:
diff --git a/setup.py b/setup.py
index 3cfe0ad..099c5f5 100644
--- a/setup.py
+++ b/setup.py
@@ -37,7 +37,6 @@ if __name__ == "__main__":
             "numpy>=1.16",
             "jams",
             "requests",
-            "pretty_midi >= 0.2.8",
         ],
         extras_require={
             "tests": [
diff --git a/soundata/annotations.py b/soundata/annotations.py
index 6fb2928..b003f5a 100644
--- a/soundata/annotations.py
+++ b/soundata/annotations.py
@@ -53,259 +53,38 @@ class Events(Annotation):
         self.confidence = confidence
 
 
-class BeatData(Annotation):
-    """BeatData class
+class MultiAnnotator(Annotation):
+    """Multiple annotator class.
+    This class should be used for datasets with multiple annotators (e.g. multiple annotators per clip).
 
     Attributes:
-        times (np.ndarray): array of time stamps (as floats) in seconds
-            with positive, strictly increasing values
-        positions (np.ndarray or None): array of beat positions (as ints)
-            e.g. 1, 2, 3, 4
-
+        annotators (list): list with annotator ids
+        labels (list): list of annotations (e.g. [annotations.Tags, annotations.Tags]
     """
 
-    def __init__(self, times, positions=None):
-        validate_array_like(times, np.ndarray, float)
-        validate_array_like(positions, np.ndarray, int, none_allowed=True)
-        validate_lengths_equal([times, positions])
-        validate_times(times)
-
-        self.times = times
-        self.positions = positions
-
+    def __init__(self, annotators, labels) -> None:
+        validate_array_like(annotators, list, str)
+        validate_array_like(labels, list, Annotation, check_child=True)
+        validate_lengths_equal([annotators, labels])
 
-class SectionData(Annotation):
-    """SectionData class
-
-    Attributes:
-        intervals (np.ndarray): (n x 2) array of intervals
-            (as floats) in seconds in the form [start_time, end_time]
-            times should be positive and intervals should have
-            non-negative duration
-        labels (list or None): list of labels (as strings)
-
-    """
-
-    def __init__(self, intervals, labels=None):
-        validate_array_like(intervals, np.ndarray, float)
-        validate_array_like(labels, list, str, none_allowed=True)
-        validate_lengths_equal([intervals, labels])
-        validate_intervals(intervals)
-
-        self.intervals = intervals
+        self.annotators = annotators
         self.labels = labels
 
 
-class NoteData(Annotation):
-    """NoteData class
-
-    Attributes:
-        intervals (np.ndarray): (n x 2) array of intervals
-            (as floats) in seconds in the form [start_time, end_time]
-            with positive time stamps and end_time >= start_time.
-        notes (np.ndarray): array of notes (as floats) in Hz
-        confidence (np.ndarray or None): array of confidence values
-            between 0 and 1
-
-    """
-
-    def __init__(self, intervals, notes, confidence=None):
-        validate_array_like(intervals, np.ndarray, float)
-        validate_array_like(notes, np.ndarray, float)
-        validate_array_like(confidence, np.ndarray, float, none_allowed=True)
-        validate_lengths_equal([intervals, notes, confidence])
-        validate_intervals(intervals)
-        validate_confidence(confidence)
-
-        self.intervals = intervals
-        self.notes = notes
-        self.confidence = confidence
-
-
-class ChordData(Annotation):
-    """ChordData class
-
-    Attributes:
-        intervals (np.ndarray or None): (n x 2) array of intervals
-            (as floats) in seconds in the form [start_time, end_time]
-            with positive time stamps and end_time >= start_time.
-        labels (list): list chord labels (as strings)
-        confidence (np.ndarray or None): array of confidence values
-            between 0 and 1
-
-    """
-
-    def __init__(self, intervals, labels, confidence=None):
-        validate_array_like(intervals, np.ndarray, float)
-        validate_array_like(labels, list, str)
-        validate_array_like(confidence, np.ndarray, float, none_allowed=True)
-        validate_lengths_equal([intervals, labels, confidence])
-        validate_intervals(intervals)
-        validate_confidence(confidence)
-
-        self.intervals = intervals
-        self.labels = labels
-        self.confidence = confidence
-
-
-class F0Data(Annotation):
-    """F0Data class
-
-    Attributes:
-        times (np.ndarray): array of time stamps (as floats) in seconds
-            with positive, strictly increasing values
-        frequencies (np.ndarray): array of frequency values (as floats)
-            in Hz
-        confidence (np.ndarray or None): array of confidence values
-            between 0 and 1
-
-    """
-
-    def __init__(self, times, frequencies, confidence=None):
-        validate_array_like(times, np.ndarray, float)
-        validate_array_like(frequencies, np.ndarray, float)
-        validate_array_like(confidence, np.ndarray, float, none_allowed=True)
-        validate_lengths_equal([times, frequencies, confidence])
-        validate_times(times)
-        validate_confidence(confidence)
-
-        self.times = times
-        self.frequencies = frequencies
-        self.confidence = confidence
-
-
-class MultiF0Data(Annotation):
-    """MultiF0Data class
-
-    Attributes:
-        times (np.ndarray): array of time stamps (as floats) in seconds
-            with positive, strictly increasing values
-        frequency_list (list): list of lists of frequency values (as floats)
-            in Hz
-        confidence_list (list or None): list of lists of confidence values
-            between 0 and 1
-
-    """
-
-    def __init__(self, times, frequency_list, confidence_list=None):
-        validate_array_like(times, np.ndarray, float)
-        validate_array_like(frequency_list, list, list)
-        validate_array_like(confidence_list, list, list, none_allowed=True)
-        validate_lengths_equal([times, frequency_list, confidence_list])
-        validate_times(times)
-
-        self.times = times
-        self.frequency_list = frequency_list
-        self.confidence_list = confidence_list
-
-
-class KeyData(Annotation):
-    """KeyData class
-
-    Attributes:
-        intervals (np.ndarray): (n x 2) array of intervals
-            (as floats) in seconds in the form [start_time, end_time]
-            with positive time stamps and end_time >= start_time.
-        keys (list): list key labels (as strings)
-
-    """
-
-    def __init__(self, intervals, keys):
-        validate_array_like(intervals, np.ndarray, float)
-        validate_array_like(keys, list, str)
-        validate_lengths_equal([intervals, keys])
-        validate_intervals(intervals)
-
-        self.intervals = intervals
-        self.keys = keys
-
-
-class LyricData(Annotation):
-    """LyricData class
-
-    Attributes:
-        intervals (np.ndarray): (n x 2) array of intervals
-            (as floats) in seconds in the form [start_time, end_time]
-            with positive time stamps and end_time >= start_time.
-        lyrics (list): list of lyrics (as strings)
-        pronunciations (list or None): list of pronunciations (as strings)
-
-    """
-
-    def __init__(self, intervals, lyrics, pronunciations=None):
-        validate_array_like(intervals, np.ndarray, float)
-        validate_array_like(lyrics, list, str)
-        validate_array_like(pronunciations, list, str, none_allowed=True)
-        validate_lengths_equal([intervals, lyrics, pronunciations])
-        validate_intervals(intervals)
-
-        self.intervals = intervals
-        self.lyrics = lyrics
-        self.pronunciations = pronunciations
-
-
-class TempoData(Annotation):
-    """TempoData class
-
-    Attributes:
-        intervals (np.ndarray): (n x 2) array of intervals
-            (as floats) in seconds in the form [start_time, end_time]
-            with positive time stamps and end_time >= start_time.
-        value (list): array of tempo values (as floats)
-        confidence (np.ndarray or None): array of confidence values
-            between 0 and 1
-
-    """
-
-    def __init__(self, intervals, value, confidence=None):
-        validate_array_like(intervals, np.ndarray, float)
-        validate_array_like(value, np.ndarray, float)
-        validate_array_like(confidence, np.ndarray, float, none_allowed=True)
-        validate_lengths_equal([intervals, value, confidence])
-        validate_intervals(intervals)
-        validate_confidence(confidence)
-
-        self.intervals = intervals
-        self.value = value
-        self.confidence = confidence
-
-
-class EventData(Annotation):
-    """TempoData class
-
-    Attributes:
-        intervals (np.ndarray): (n x 2) array of intervals
-            (as floats) in seconds in the form [start_time, end_time]
-            with positive time stamps and end_time >= start_time.
-        events (list): list of event labels (as strings)
-
-    """
-
-    def __init__(self, intervals, events):
-        validate_array_like(intervals, np.ndarray, float)
-        validate_array_like(events, list, str)
-        validate_lengths_equal([intervals, events])
-        validate_intervals(intervals)
-
-        self.intervals = intervals
-        self.events = events
-
-
-def validate_array_like(array_like, expected_type, expected_dtype, none_allowed=False):
+def validate_array_like(
+    array_like, expected_type, expected_dtype, check_child=False, none_allowed=False
+):
     """Validate that array-like object is well formed
-
     If array_like is None, validation passes automatically.
-
     Args:
         array_like (array-like): object to validate
         expected_type (type): expected type, either list or np.ndarray
         expected_dtype (type): expected dtype
+        check_child (bool): if True, checks if all elements of array are children of expected_dtype
         none_allowed (bool): if True, allows array to be None
-
     Raises:
         TypeError: if type/dtype does not match expected_type/expected_dtype
         ValueError: if array
-
     """
     if array_like is None:
         if none_allowed:
@@ -326,6 +105,12 @@ def validate_array_like(array_like, expected_type, expected_dtype, none_allowed=
     if expected_type == list and not all(
         isinstance(n, expected_dtype) for n in array_like
     ):
+        if check_child:
+            if not all(issubclass(type(n), expected_dtype) for n in array_like):
+                raise TypeError(
+                    f"List elements should all be instances of {expected_dtype} class"
+                )
+
         raise TypeError(f"List elements should all have type {expected_dtype}")
 
     if expected_type == np.ndarray and array_like.dtype != expected_dtype:


Test patch:
diff --git a/tests/test_annotations.py b/tests/test_annotations.py
index 4a8d04e..f935bfa 100644
--- a/tests/test_annotations.py
+++ b/tests/test_annotations.py
@@ -16,8 +16,10 @@ def test_repr():
     test_track = TestAnnotation()
     assert test_track.__repr__() == """TestAnnotation(a, b)"""
 
-    beat_data = annotations.BeatData(np.array([1.0, 2.0]))
-    assert beat_data.__repr__() == "BeatData(positions, times)"
+    event_data = annotations.Events(
+        np.array([[1.0, 2.0], [3.0, 4.0]]), ["Siren", "Dog"]
+    )
+    assert event_data.__repr__() == "Events(confidence, intervals, labels)"
 
 
 def test_tags():
@@ -57,160 +59,33 @@ def test_events():
     pytest.raises(ValueError, annotations.Events, intervals, labels, bad_confidence)
 
 
-def test_beat_data():
-    times = np.array([1.0, 2.0])
-    positions = np.array([3, 4])
-    beat_data = annotations.BeatData(times)
-    assert np.allclose(beat_data.times, times)
-    assert beat_data.positions is None
-    beat_data2 = annotations.BeatData(times, positions)
-    assert np.allclose(beat_data2.times, times)
-    assert np.allclose(beat_data2.positions, positions)
-
-    with pytest.raises(ValueError):
-        annotations.BeatData(None, None)
-
-    with pytest.raises(TypeError):
-        annotations.BeatData([1.0, 2.0])
-
-    with pytest.raises(TypeError):
-        annotations.BeatData(times, [3, 4])
-
-    with pytest.raises(TypeError):
-        annotations.BeatData(times.astype(int))
-
-    with pytest.raises(ValueError):
-        annotations.BeatData(times, np.array([1]))
-
-
-def test_section_data():
-    intervals = np.array([[1.0, 2.0], [1.5, 3.0], [2.0, 3.0]])
-    labels = ["a", "b", "c"]
-    section_data = annotations.SectionData(intervals)
-    assert np.allclose(section_data.intervals, intervals)
-    assert section_data.labels is None
-    section_data2 = annotations.SectionData(intervals, labels)
-    assert np.allclose(section_data2.intervals, intervals)
-    assert section_data2.labels == labels
-
-    with pytest.raises(ValueError):
-        annotations.SectionData(None, None)
-
-    with pytest.raises(TypeError):
-        annotations.SectionData([1.0, 2.0])
-
-    with pytest.raises(TypeError):
-        annotations.SectionData(intervals, np.array(labels))
-
-    with pytest.raises(TypeError):
-        annotations.SectionData(intervals.astype(int))
-
-    with pytest.raises(ValueError):
-        annotations.SectionData(intervals, ["a"])
-
-    with pytest.raises(ValueError):
-        annotations.SectionData(np.array([1.0, 2.0, 3.0]))
-
-    with pytest.raises(ValueError):
-        annotations.SectionData(np.array([[1.0, 2.0], [2.0, 1.0]]))
-
-
-def test_note_data():
-    intervals = np.array([[1.0, 2.0], [1.5, 3.0], [2.0, 3.0]])
-    notes = np.array([100.0, 150.0, 120.0])
-    confidence = np.array([0.1, 0.4, 0.2])
-    note_data = annotations.NoteData(intervals, notes)
-    assert np.allclose(note_data.intervals, intervals)
-    assert np.allclose(note_data.notes, notes)
-    assert note_data.confidence is None
-    note_data2 = annotations.NoteData(intervals, notes, confidence)
-    assert np.allclose(note_data2.intervals, intervals)
-    assert np.allclose(note_data2.notes, notes)
-    assert np.allclose(note_data2.confidence, confidence)
-
-    with pytest.raises(ValueError):
-        annotations.NoteData(None, notes)
-
-    with pytest.raises(ValueError):
-        annotations.NoteData(intervals, None)
-
-    with pytest.raises(TypeError):
-        annotations.NoteData([1.0, 2.0], notes)
-
-    with pytest.raises(TypeError):
-        annotations.NoteData(intervals, [3.0, 4.0])
-
-    with pytest.raises(TypeError):
-        annotations.NoteData(intervals.astype(int), notes)
-
-    with pytest.raises(ValueError):
-        annotations.NoteData(intervals, np.array([1.0]))
-
-
-def test_chord_data():
-    intervals = np.array([[1.0, 2.0], [1.5, 3.0], [2.0, 3.0]])
-    labels = ["E:m", "A", "G:7"]
-    confidence = np.array([0.1, 0.4, 0.2])
-    chord_data = annotations.ChordData(intervals, labels, confidence)
-    assert np.allclose(chord_data.intervals, intervals)
-    assert chord_data.labels == labels
-    assert np.allclose(chord_data.confidence, confidence)
-
-
-def test_f0_data():
-    times = np.array([1.0, 2.0, 3.0])
-    frequencies = np.array([100.0, 150.0, 120.0])
-    confidence = np.array([0.1, 0.4, 0.2])
-    f0_data = annotations.F0Data(times, frequencies, confidence)
-    assert np.allclose(f0_data.times, times)
-    assert np.allclose(f0_data.frequencies, frequencies)
-    assert np.allclose(f0_data.confidence, confidence)
-
-
-def test_multif0_data():
-    times = np.array([1.0, 2.0])
-    frequencies = [[100.0], [150.0, 120.0]]
-    confidence = [[0.1], [0.4, 0.2]]
-    f0_data = annotations.MultiF0Data(times, frequencies, confidence)
-    assert np.allclose(f0_data.times, times)
-    assert f0_data.frequency_list is not None
-    assert f0_data.confidence_list is not None
-
-
-def test_key_data():
-    intervals = np.array([[1.0, 2.0], [1.5, 3.0], [2.0, 3.0]])
-    keys = ["Em", "A", "G"]
-    key_data = annotations.KeyData(intervals, keys)
-    assert np.allclose(key_data.intervals, intervals)
-    assert key_data.keys == keys
-
-
-def test_lyric_data():
-    intervals = np.array([[1.0, 2.0], [1.5, 3.0], [2.0, 3.0]])
-    lyrics = ["E:m", "A", "G:7"]
-    pronunciations = ["a", "", "b"]
-    lyric_data = annotations.LyricData(intervals, lyrics, pronunciations)
-    assert np.allclose(lyric_data.intervals, intervals)
-    assert lyric_data.lyrics == lyrics
-    assert lyric_data.pronunciations == pronunciations
-
-
-def test_tempo_data():
-    intervals = np.array([[1.0, 2.0], [1.5, 3.0], [2.0, 3.0]])
-    value = np.array([140.0, 110.0, 111.0])
-    confidence = np.array([0.1, 0.4, 0.2])
-    tempo_data = annotations.TempoData(intervals, value, confidence)
-    assert np.allclose(tempo_data.intervals, intervals)
-    assert np.allclose(tempo_data.value, value)
-    assert np.allclose(tempo_data.confidence, confidence)
-
+def test_multiannotator():
+    # test good data
+    annotators = ["annotator_1", "annotator_2"]
+    labels_1 = ["Siren", "Engine"]
+    labels_2 = ["Siren", "Dog"]
+    confidence_1 = np.array([1.0, 1.0])
+    confidence_2 = np.array([1.0, 1.0])
+    multi_annot = [
+        annotations.Tags(labels_1, confidence_1),
+        annotations.Tags(labels_2, confidence_2),
+    ]
+    tags = annotations.MultiAnnotator(annotators, multi_annot)
+
+    assert tags.labels[0].labels == labels_1
+    assert tags.annotators[1] == "annotator_2"
+    assert np.allclose(tags.labels[1].confidence, confidence_2)
 
-def test_event_data():
-    intervals = np.array([[1.0, 2.0], [1.5, 3.0], [2.0, 3.0]])
-    events = ["E:m", "A", "G:7"]
-    event_data = annotations.EventData(intervals, events)
-    assert np.allclose(event_data.intervals, intervals)
-    assert event_data.events == events
+    # test bad data
+    bad_labels = ["Siren", "Laughter", 5]
+    pytest.raises(TypeError, annotations.MultiAnnotator, annotators, bad_labels)
+    pytest.raises(TypeError, annotations.MultiAnnotator, [0, 1], multi_annot)
+    pytest.raises(
+        TypeError,
+        annotations.MultiAnnotator,
+        annotators,
+        [["bad", "format"], ["indeed"]],
+    )
 
 
 def test_validate_array_like():